LKMC v3.0

This is a squash commit, the unsquashed development went through many
unstable phases which would break bisects. The unsquashed branch is:
https://github.com/cirosantilli/linux-kernel-module-cheat/tree/v3.0-unsquash

The main improvement of this release was to greatly generalize the testing system.

The key addition was cli_function.py, which allows scripts such as ./run to
be transparently called either from Python or from the command line.

New tests scripts were created using this improved framework: test-baremetal
and test-user-mode.

We were lazy to port some of less important tests to the new setup, TODO's were
added, and we need comes they will be fixed. Getting started is however sacred
as usual and should work.

Other changes include:

-   gem5: update to 7fa4c946386e7207ad5859e8ade0bbfc14000d91

-   run: --tmux-args implies --tmux

-   run: add --userland-args to make userland arguments across QEMU and gem5

    Get rid of --userland-before as a consequence.

-   bring initrd and initramfs back to life

-   build-userland: create --static to make build a bit easier

-   gem5: --gem5-worktree also set --gem5-build-id

-   remove --gem5, use --emulator gem5 everywhere

    Allow passing --emulator multiple times for transparent tests selection
    just like --arch.

-   test-userland: allow selecting just a few tests

-   linux: update to v4.20

-   buildroot: update to 2018.08

    The main motivation for this was to fix the build for Ubuntu 18.10, which
    has glibc 2.28, which broke the 2018.05 build at the m4-host package with:

        #error "Please port gnulib fseeko.c to your platform!

-   getvar --type input

-   failed xen attempt, refactor timer, failed svc attempt, aarch64 use gicv3

-   build-doc: exit 1 on error, add to release testing

-   build: add --apt option to make things easier on other distros

-   build-linux: --no-modules-install

.gitmodules

@@ -21,3 +21,9 @@
 [submodule "submodules/qemu"]
 	path = submodules/qemu
 	url = https://github.com/cirosantilli/qemu
+[submodule "submodules/xen"]
+	path = submodules/xen
+	url = git://xenbits.xen.org/xen.git
+[submodule "submodules/boot-wrapper-aarch64"]
+	path = submodules/boot-wrapper-aarch64
+	url = git://git.kernel.org/pub/scm/linux/kernel/git/mark/boot-wrapper-aarch64.git

arm

@@ -1,3 +0,0 @@
-#!/usr/bin/env bash
-build-crosstool-ng \
-;

baremetal/add.c

@@ -1,13 +1,13 @@
 #include <common.h>
 
 int main(void) {
-	int i, j, k;
+    int i, j, k;
-	i = 1;
+    i = 1;
     /* test-gdb-op1 */
-	j = 2;
+    j = 2;
     /* test-gdb-op2 */
-	k = i + j;
+    k = i + j;
     /* test-gdb-result */
-	if (k != 3)
+    if (k != 3)
-		common_assert_fail();
+        common_assert_fail();
 }

baremetal/arch/aarch64/common_aarch64.h

@@ -0,0 +1,24 @@
+#ifndef COMMON_AARCH64_H
+#define COMMON_AARCH64_H
+
+#include <inttypes.h>
+
+#define SYSREG_READ(type, name) \
+    type sysreg_ ## name ## _read(void) { \
+        type name; \
+        __asm__ __volatile__("mrs %0, " #name : "=r" (name) : : ); \
+        return name; \
+    }
+
+#define SYSREG_WRITE(type, name) \
+    void sysreg_ ## name ## _write(type name) { \
+        __asm__ __volatile__("msr " #name ", %0" : : "r" (name) : ); \
+    }
+
+#define SYSREG_READ_WRITE(name, type) \
+    SYSREG_READ(name, type) \
+    SYSREG_WRITE(name, type)
+
+#define SVC(immediate) __asm__ __volatile__("svc " #immediate : : : )
+
+#endif

baremetal/arch/aarch64/el.c

@@ -4,8 +4,8 @@
 #include <inttypes.h>
 
 int main(void) {
-	register uint64_t x0 __asm__ ("x0");
+    uint64_t el;
-	__asm__ ("mrs x0, CurrentEL;" : : : "%x0");
+    __asm__ ("mrs %0, CurrentEL;" : "=r" (el) : :);
-	printf("%" PRIu64 "\n", x0 >> 2);
+    printf("%" PRIu64 "\n", el >> 2);
-	return 0;
+    return 0;
 }

baremetal/arch/aarch64/multicore.S

@@ -9,6 +9,7 @@ main:
 
     /* Read cpu id into x1.
      * TODO: cores beyond 4th?
+     * Mnemonic: Main Processor ID Register
      */
     mrs x1, mpidr_el1
     ands x1, x1, 3

baremetal/arch/aarch64/svc.c

@@ -0,0 +1,28 @@
+#include <stdio.h>
+#include <inttypes.h>
+
+#include "common_aarch64.h"
+
+/* Masks each of the 4 exception types: Synchronous, System error,
+ * IRQ and FIQ.
+ */
+SYSREG_READ_WRITE(uint32_t, daif)
+
+/* Determines if we use SP0 or SPx. Default: SP0.
+ * See also: https://stackoverflow.com/questions/29393677/armv8-exception-vector-significance-of-el0-sp
+ */
+SYSREG_READ_WRITE(uint32_t, spsel)
+
+/* Jump to this SP if spsel == SPx. */
+SYSREG_READ_WRITE(uint64_t, sp_el1)
+
+int main(void) {
+    printf("daif 0x%" PRIx32 "\n", sysreg_daif_read());
+    printf("spsel 0x%" PRIx32 "\n", sysreg_spsel_read());
+    /* TODO this breaks execution because reading system registers that end
+     * in ELx "trap", leading into an exception on the upper EL.
+     */
+    /*printf("sp_el1 0x%" PRIx64 "\n", sysreg_sp_el1_read());*/
+    /*SVC(0);*/
+    return 0;
+}

baremetal/arch/aarch64/timer.c

@@ -0,0 +1,59 @@
+#include <stdio.h>
+#include <inttypes.h>
+
+#include "common_aarch64.h"
+
+#define CNTV_CTL_ENABLE  (1 << 0)
+#define CNTV_CTL_IMASK   (1 << 1)
+#define CNTV_CTL_ISTATUS (1 << 2)
+
+/* Frequency in Hz. ? */
+SYSREG_READ_WRITE(uint64_t, cntfrq_el0)
+
+/* Current virtual counter value. */
+SYSREG_READ(uint64_t, cntvct_el0)
+
+/* Compare value. See: cntv_ctl_el0_enable. */
+SYSREG_READ_WRITE(uint64_t, cntv_cval_el0)
+
+/* On write, set cntv_cval_el0 = (cntvct_el0 + cntv_tval_el0).
+ * This means that the next interrupt will happen in cntv_tval_el0 cycles.
+ */
+SYSREG_READ_WRITE(uint64_t, cntv_tval_el0)
+
+/* Control register. */
+SYSREG_READ_WRITE(uint32_t, cntv_ctl_el0)
+
+void cntv_ctl_el0_disable(void) {
+    sysreg_cntv_ctl_el0_write(sysreg_cntv_ctl_el0_read() & ~CNTV_CTL_ENABLE);
+}
+
+/* If enabled, when: cntv_ctl > cntv_cval then:
+ *
+ * * if CNTV_CTL_IMASK is clear, raise an interrupt
+ * * set CNTV_CTL_ISTATUS
+ */
+void cntv_ctl_el0_enable(void) {
+    sysreg_cntv_ctl_el0_write(sysreg_cntv_ctl_el0_read() | CNTV_CTL_ENABLE);
+}
+
+int main(void) {
+    /* Initial state. */
+    printf("cntv_ctl_el0 0x%" PRIx32 "\n", sysreg_cntv_ctl_el0_read());
+    printf("cntfrq_el0 0x%" PRIx64 "\n", sysreg_cntfrq_el0_read());
+    printf("cntv_cval_el0 0x%" PRIx64 "\n", sysreg_cntv_cval_el0_read());
+
+    /* Get the counter value many times to watch the time pass. */
+    printf("cntvct_el0 0x%" PRIx64 "\n", sysreg_cntvct_el0_read());
+    printf("cntvct_el0 0x%" PRIx64 "\n", sysreg_cntvct_el0_read());
+    printf("cntvct_el0 0x%" PRIx64 "\n", sysreg_cntvct_el0_read());
+
+#if 0
+    /* TODO crashes gem5. */
+    puts("cntfrq_el0 = 1");
+    sysreg_cntfrq_el0_write(1);
+    printf("cntfrq_el0 0x%" PRIx64 "\n", sysreg_cntfrq_el0_read());
+#endif
+
+    return 0;
+}

baremetal/arch/arm/el.c

@@ -4,8 +4,8 @@
 #include <inttypes.h>
 
 int main(void) {
-	register uint32_t r0 __asm__ ("r0");
+    uint32_t cpsr;
-	__asm__ ("mrs r0, CPSR" : : : "%r0");
+    __asm__ ("mrs %0, CPSR" : "=r" (cpsr) : :);
-	printf("%" PRIu32 "\n", r0 & 0x1F);
+    printf("%" PRIu32 "\n", cpsr & 0x1F);
-	return 0;
+    return 0;
 }

baremetal/exit.c

@@ -2,6 +2,5 @@
 #include <stdlib.h>
 
 int main(void) {
-	exit(0);
+    exit(0);
 }
-

baremetal/hello.c

@@ -1,6 +1,6 @@
 #include <stdio.h>
 
 int main(void) {
-	puts("hello");
+    puts("hello");
-	return 0;
+    return 0;
 }

baremetal/interactive/assert_fail.c

@@ -1,6 +1,6 @@
 #include <common.h>
 
 int main(void) {
-	common_assert_fail();
+    common_assert_fail();
 }
 

baremetal/interactive/exit1.c

@@ -2,5 +2,5 @@
 #include <stdlib.h>
 
 int main(void) {
-	exit(1);
+    exit(1);
 }

baremetal/interactive/infinite_loop.c

@@ -0,0 +1,4 @@
+int main(void) {
+    while(1) {}
+    return 0;
+}

baremetal/lib/syscalls.c

@@ -64,24 +64,24 @@ int _write(int file, char *ptr, int len) {
 void _exit(int status) {
 #if defined(GEM5)
 #if defined(__arm__)
-	__asm__ __volatile__ ("mov r0, #0; mov r1, #0; .inst 0xEE000110 | (0x21 << 16);");
+    __asm__ __volatile__ ("mov r0, #0; mov r1, #0; .inst 0xEE000110 | (0x21 << 16);");
 #elif defined(__aarch64__)
-	__asm__ __volatile__ ("mov x0, #0; .inst 0XFF000110 | (0x21 << 16);");
+    __asm__ __volatile__ ("mov x0, #0; .inst 0XFF000110 | (0x21 << 16);");
 #endif
 #else
 #if defined(__arm__)
     __asm__ __volatile__ ("mov r0, #0x18; ldr r1, =#0x20026; svc 0x00123456");
 #elif defined(__aarch64__)
-	/* TODO actually use the exit value here, just for fun. */
+    /* TODO actually use the exit value here, just for fun. */
     __asm__ __volatile__ (
-		"mov x1, #0x26\n" \
+        "mov x1, #0x26\n" \
-		"movk x1, #2, lsl #16\n" \
+        "movk x1, #2, lsl #16\n" \
-		"str x1, [sp,#0]\n" \
+        "str x1, [sp,#0]\n" \
-		"mov x0, #0\n" \
+        "mov x0, #0\n" \
-		"str x0, [sp,#8]\n" \
+        "str x0, [sp,#8]\n" \
-		"mov x1, sp\n" \
+        "mov x1, sp\n" \
-		"mov w0, #0x18\n" \
+        "mov w0, #0x18\n" \
-		"hlt 0xf000\n"
+        "hlt 0xf000\n"
     );
 #endif
 #endif

bench-all

@@ -100,9 +100,9 @@ if "$bench_gem5_build"; then
   common_arch="$default_arch"
   gem5_build_id=bench
   common_gem5_build_dir="$("$getvar" --arch "$common_arch" --gem5-build-id "$gem5_build_id" gem5_build_dir)"
-  common_gem5_src_dir="$("$getvar" --arch "$common_arch" --gem5-build-id "$gem5_build_id" gem5_src_dir)"
+  common_gem5_source_dir="$("$getvar" --arch "$common_arch" --gem5-build-id "$gem5_build_id" gem5_source_dir)"
   results_file="${common_gem5_build_dir}/lkmc-bench-build.txt"
-  git -C "${common_gem5_src_dir}" clean -xdf
+  git -C "${common_gem5_source_dir}" clean -xdf
   rm -f "$results_file"
   "${root_dir}/build-gem5" --arch "$common_arch" --clean --gem5-build-id "$gem5_build_id"
   # TODO understand better: --foreground required otherwise we cannot
@@ -110,15 +110,15 @@ if "$bench_gem5_build"; then
   # bash -c "eval 'timeout 5 sleep 3'"
   "${root_dir}/bench-cmd" "timeout --foreground 900 ./build-gem5 --arch '$common_arch' --gem5-build-id '$gem5_build_id'" "$results_file"
   cp "$results_file" "${new_dir}/gem5-bench-build-${common_arch}.txt"
-  git -C "${common_gem5_src_dir}" clean -xdf
+  git -C "${common_gem5_source_dir}" clean -xdf
   "${root_dir}/build-gem5" --arch "$common_arch" --clean --gem5-build-id "$gem5_build_id"
 fi
 
 if "$bench_linux_boot"; then
   cd "${root_dir}"
-  "${root_dir}/build" --all
+  "${root_dir}/build" --all-archs all
-  "${root_dir}/bench-boot" --size 3
+  "${root_dir}/test-boot" --size 3
-  cp "$(${root_dir}/getvar bench_boot)" "$new_dir"
+  cp "$(${root_dir}/getvar test_boot_benchmark_file)" "$new_dir"
 fi
 
 if "$update_repo"; then

bench-boot

@@ -1,94 +0,0 @@
-#!/usr/bin/env bash
-set -eu
-root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
-test_size=1
-while [ $# -gt 0 ]; do
-  case "$1" in
-    --size)
-      # 1: a few seconds and important
-      # 2: < 5 minutes and important or a few seconds and not too important
-      # 3: all
-      test_size="$2"
-      shift 2
-      ;;
-  esac
-done
-
-if [ $# -gt 1 ]; then
-  extra_args=" $*"
-else
-  extra_args=
-fi
-getvar="${root_dir}/getvar"
-common_bench_boot="$("$getvar" bench_boot)"
-caches='--caches --l2cache --l1d_size=1024kB --l1i_size=1024kB --l2_size=1024kB --l3_size=1024kB'
-
-bench() (
-  "${root_dir}/bench-cmd" "./run --arch ${1}${extra_args}" "$common_bench_boot"
-)
-
-newline() (
-  echo >> "$common_bench_boot"
-)
-
-gem5_insts() (
-  printf "instructions $(./gem5-stat --arch "$1" sim_insts)\n" >> "$common_bench_boot"
-  newline
-)
-
-qemu_insts() (
-  common_arch="$1"
-  ./qemu-trace2txt --arch "$common_arch"
-  common_qemu_trace_txt_file="$("$getvar" --arch "$common_arch" qemu_trace_txt_file)"
-  printf "instructions $(wc -l "${common_qemu_trace_txt_file}" | cut -d' ' -f1)\n" >> "$common_bench_boot"
-  newline
-)
-
-rm -f "${common_bench_boot}"
-
-arch=x86_64
-bench "${arch} --eval '/poweroff.out'"
-newline
-bench "${arch} --eval '/poweroff.out' --kvm"
-newline
-if [ "$test_size" -ge 2 ]; then
-  bench "${arch} --eval '/poweroff.out' --trace exec_tb"
-  qemu_insts "$arch"
-  bench "$arch --eval 'm5 exit' --gem5"
-  gem5_insts "$arch"
-fi
-#bench "$arch --eval 'm5 exit' --gem5 -- --cpu-type=DerivO3CPU ${caches}"
-#gem5_insts "$arch"
-
-arch=arm
-bench "$arch --eval '/poweroff.out'"
-if [ "$test_size" -ge 2 ]; then
-  bench "$arch --eval '/poweroff.out' --trace exec_tb"
-  qemu_insts "$arch"
-  bench "$arch --eval 'm5 exit' --gem5"
-  gem5_insts "$arch"
-fi
-if [ "$test_size" -ge 3 ]; then
-  bench "$arch --eval 'm5 exit' --gem5 -- --cpu-type=HPI ${caches}"
-  gem5_insts "$arch"
-fi
-
-arch=aarch64
-bench "$arch --eval '/poweroff.out'"
-newline
-if [ "$test_size" -ge 2 ]; then
-  bench "$arch --eval '/poweroff.out' --trace exec_tb"
-  qemu_insts "$arch"
-  bench "$arch --eval 'm5 exit' --gem5"
-  gem5_insts "$arch"
-fi
-if [ "$test_size" -ge 3 ]; then
-  bench "$arch --eval 'm5 exit' --gem5 -- --cpu-type=HPI ${caches}"
-  gem5_insts "$arch"
-  #bench "$arch --eval 'm5 exit' --gem5 --gem5-script biglittle"
-  #gem5_insts "$arch"
-  bench "$arch --eval 'm5 exit' --gem5 --gem5-build-type fast"
-  gem5_insts "$arch"
-  bench "$arch --eval 'm5 exit' --gem5 --gem5-build-type debug"
-  gem5_insts "$arch"
-fi

bisect-linux-boot-gem5

@@ -6,25 +6,25 @@ import shutil
 import sys
 
 import common
-build_linux = imp.load_source('build-linux', os.path.join(common.root_dir, 'build_linux'))
+build_linux = imp.load_source('build-linux', os.path.join(kwargs['root_dir'], 'build_linux'))
-run = imp.load_source('run', os.path.join(common.root_dir, 'run'))
+run = imp.load_source('run', os.path.join(kwargs['root_dir'], 'run'))
 
-parser = common.get_argparse(
+parser = self.get_argparse(
     argparse_args={
         'description': '''Bisect the Linux kernel on gem5 boots.
 
 More information at: https://github.com/cirosantilli/linux-kernel-module-cheat#bisection
 '''},
     default_args={
-        'gem5': True,
+        'emulators': ['gem5'],
         'linux_build_id': 'bisect',
     },
 )
-args = common.setup(parser)
+args = self.setup(parser)
 # We need a clean rebuild because rebuilds at different revisions:
 # - may fail
 # - may not actually rebuild all files, e.g. on header changes
-common.rmrf(common.linux_build_dir)
+self.rmrf(kwargs['linux_build_dir'])
 build_linux.LinuxComponent().do_build(args)
 status = run.main(args, {
     'eval': 'm5 exit',

bisect-qemu-linux-boot

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 set -eu
-git submodule update
+git submodule update --recursive
-cd ..
+cd ../..
-./build-qemu --arch arm -Q bisect
+./build-qemu --arch aarch64 --qemu-build-id bisect
-./run --arch arm -E '/poweroff.out' -Q bisect
+./run --arch aarch64 --kernel-cli 'init=/poweroff.out' --qemu-build-id bisect

bst-vs-heap

@@ -1,10 +1,10 @@
 #!/usr/bin/env python3
 import common
-parser = common.get_argparse(
+parser = self.get_argparse(
     argparse_args={'description':'Convert a BST vs heap stat file into a gnuplot input'}
 )
-args = common.setup(parser)
+args = self.setup(parser)
-stats = common.get_stats()
+stats = self.get_stats()
 it = iter(stats)
 i = 1
 for stat in it:

build

@@ -1,17 +1,19 @@
 #!/usr/bin/env python3
 
-import argparse
-import collections
-import platform
 import re
 import os
 
+import cli_function
+import collections
 import common
+import copy
+import shell_helpers
+from shell_helpers import LF
 
-class Component:
+class _Component:
     '''
-    Yes, we are re-inventing a crappy dependency resolution system.
+    Yes, we are re-inventing a crappy dependency resolution system,
-    I can't believe it.
+    reminescent of scons or apt or Buildroot. I can't believe it.
 
     The hard part is that we have optional dependencies as well...
     e.g. buildroot optionally depends on m5 to put m5 in the root filesystem,
@@ -39,229 +41,28 @@ class Component:
         self.submodules_shallow = submodules_shallow or set()
         self.python2_pkgs = python2_pkgs or set()
         self.python3_pkgs = python3_pkgs or set()
+
     def build(self, arch):
         if (
             (self.build_callback is not None) and
             (self.supported_archs is None or arch in self.supported_archs)
         ):
-            self.build_callback(arch)
+            self.build_callback()
 
-def run_cmd(cmd, arch):
+class Main(common.LkmcCliFunction):
-    global args
+    def __init__(self):
-    cmd_abs = cmd.copy()
+        super().__init__(
-    cmd_abs[0] = os.path.join(common.root_dir, cmd[0])
+            description='''\
-    cmd_abs.extend(['--arch', arch])
+Build a component and all its dependencies.
-    if args.extra_args:
-        cmd_abs.append(args.extra_args)
-    common.run_cmd(cmd_abs, dry_run=args.dry_run)
-
-buildroot_component = Component(
-    lambda arch: run_cmd(['build-buildroot'], arch),
-    submodules = {'buildroot'},
-    # https://buildroot.org/downloads/manual/manual.html#requirement
-    apt_get_pkgs={
-        'bash',
-        'bc',
-        'binutils',
-        'build-essential',
-        'bzip2',
-        'cpio',
-        'g++',
-        'gcc',
-        'graphviz',
-        'gzip',
-        'make',
-        'patch',
-        'perl',
-        'python-matplotlib',
-        'python3',
-        'rsync',
-        'sed',
-        'tar',
-        'unzip',
-    },
-)
-
-name_to_component_map = {
-    # Leaves without dependencies.
-    'baremetal-qemu': Component(
-        lambda arch: run_cmd(['build-baremetal', '--qemu'], arch),
-        supported_archs=common.crosstool_ng_supported_archs,
-    ),
-    'baremetal-gem5': Component(
-        lambda arch: run_cmd(['build-baremetal', '--gem5'], arch),
-        supported_archs=common.crosstool_ng_supported_archs,
-    ),
-    'baremetal-gem5-pbx': Component(
-        lambda arch: run_cmd(['build-baremetal', '--gem5', '--machine', 'RealViewPBX'], arch),
-        supported_archs=common.crosstool_ng_supported_archs,
-    ),
-    'buildroot': buildroot_component,
-    'buildroot-gcc': buildroot_component,
-    'copy-overlay': Component(
-        lambda arch: run_cmd(['copy-overlay'], arch),
-    ),
-    'crosstool-ng': Component(
-        lambda arch: run_cmd(['build-crosstool-ng'], arch),
-        supported_archs=common.crosstool_ng_supported_archs,
-        # http://crosstool-ng.github.io/docs/os-setup/
-        apt_get_pkgs={
-            'bison',
-            'docbook2x',
-            'flex',
-            'gawk',
-            'gcc',
-            'gperf',
-            'help2man',
-            'libncurses5-dev',
-            'libtool-bin',
-            'make',
-            'python-dev',
-            'texinfo',
-        },
-        submodules={'crosstool-ng'},
-    ),
-    'gem5': Component(
-        lambda arch: run_cmd(['build-gem5'], arch),
-        # TODO test it out on Docker and answer that question properly:
-        # https://askubuntu.com/questions/350475/how-can-i-install-gem5
-        apt_get_pkgs={
-            'device-tree-compiler',
-            'diod',
-            'libgoogle-perftools-dev',
-            'm4',
-            'protobuf-compiler',
-            'python-dev',
-            'python-pip',
-            # For prebuilt qcow2 unpack.
-            'qemu-utils',
-            'scons',
-            'zlib1g-dev',
-        },
-        python2_pkgs={
-            # Generate graphs of config.ini under m5out.
-            'pydot',
-        },
-        submodules={'gem5'},
-    ),
-    'gem5-debug': Component(
-        lambda arch: run_cmd(['build-gem5', '--gem5-build-type', 'debug'], arch),
-    ),
-    'gem5-fast': Component(
-        lambda arch: run_cmd(['build-gem5', '--gem5-build-type', 'fast'], arch),
-    ),
-    'linux': Component(
-        lambda arch: run_cmd(['build-linux'], arch),
-        submodules_shallow={'linux'},
-        apt_get_pkgs={
-            'bison',
-            'flex',
-            # Without this started failing in kernel 4.15 with:
-            # Makefile:932: *** "Cannot generate ORC metadata for CONFIG_UNWINDER_ORC=y, please install libelf-dev, libelf-devel or elfutils-libelf-devel".  Stop.
-            'libelf-dev',
-        },
-    ),
-    'modules': Component(
-        lambda arch: run_cmd(['build-modules'], arch),
-    ),
-    'm5': Component(
-        lambda arch: run_cmd(['build-m5'], arch),
-        submodules={'gem5'},
-    ),
-    'qemu': Component(
-        lambda arch: run_cmd(['build-qemu'], arch),
-        apt_build_deps={'qemu'},
-        apt_get_pkgs={'libsdl2-dev'},
-        submodules={'qemu'},
-    ),
-    'qemu-user': Component(
-        lambda arch: run_cmd(['build-qemu', '--userland'], arch),
-        apt_build_deps = {'qemu'},
-        apt_get_pkgs={'libsdl2-dev'},
-        submodules = {'qemu'},
-    ),
-    'parsec-benchmark': Component(
-        submodules = {'parsec-benchmark'},
-    ),
-    'userland': Component(
-        lambda arch: run_cmd(['build-userland'], arch),
-    ),
-
-    # Dependency only nodes.
-    'all': Component(dependencies=[
-        'all-linux',
-        'all-baremetal',
-    ]),
-    'all-baremetal': Component(dependencies=[
-            'qemu-baremetal',
-            'gem5-baremetal',
-            'baremetal-gem5-pbx',
-        ],
-        supported_archs=common.crosstool_ng_supported_archs,
-    ),
-    'all-linux': Component(dependencies=[
-        'qemu-gem5-buildroot',
-        'gem5-debug',
-        'gem5-fast',
-        'qemu-user',
-    ]),
-    'baremetal': Component(dependencies=[
-        'baremetal-gem5',
-        'baremetal-qemu',
-    ]),
-    'gem5-buildroot': Component(dependencies=[
-        'buildroot-gcc',
-        'linux',
-        'm5',
-        'overlay',
-        'gem5',
-    ]),
-    'gem5-baremetal': Component(dependencies=[
-        'gem5',
-        'crosstool-ng',
-        'baremetal-gem5',
-    ]),
-    'overlay': Component(dependencies=[
-        'copy-overlay',
-        'modules',
-        'userland',
-        'buildroot',
-    ]),
-    'qemu-baremetal': Component(dependencies=[
-        'qemu',
-        'crosstool-ng',
-        'baremetal-qemu',
-    ]),
-    'qemu-buildroot': Component(dependencies=[
-        'qemu',
-        'buildroot-gcc',
-        'overlay',
-        'linux',
-    ]),
-    'qemu-gem5-buildroot': Component(dependencies=[
-        'qemu',
-        'gem5-buildroot',
-    ]),
-    'release': Component(dependencies=[
-        'qemu-buildroot',
-    ]),
-}
-parser = argparse.ArgumentParser(
-    description= '''\
-Shallow helper to build everything, or a subset of everything conveniently.
 
 Our build-* scripts don't build any dependencies to make iterative
 development fast and more predictable.
 
-While modifying a specific component however, you will likely want to just run the
+It is currently not possible to configure indivitual components from the command line
-individual build-* commands which:
+when you build with this script. TODO.
 
-* build no dependencies, and so are fast and predictable
+Without any args, build only what is necessary for:
-* can take multiple options to custumize the build
-
-Without any args, build only what is necessary for
 https://github.com/cirosantilli/linux-kernel-module-cheat#qemu-buildroot-setup
-for x86_64:
 
 ....
 ./%(prog)s
@@ -273,211 +74,475 @@ This is equivalent to:
 ./%(prog)s --arch x86_64 qemu-buildroot
 ....
 
-If `--arch` is given, build just for the given archs:
+Another important target is `all`:
 
 ....
-./%(prog)s --arch arm --arch aarch64
+./%(prog)s all
 ....
 
-This will build `qemu-buildroot` for arm and aarch64 only, but not `x86_64`.
+This does not trully build ALL configurations: that would be impractical.
+But more precisely: build the reference configuration of each major component.
 
-Clean all Linux kernel builds:
+So e.g.: one config of Linux kenrel, Buildroot, gem5 and qemu.
+Don't do for example all possible gem5 configs: debug, opt and fast,
+as that would be huge. This ensures that every piece of software
+builds in at least one config.
+
+TODO looping over emulators is not currently supported by this script, e.g.:
 
 ....
-./build --all-archs --extra-args=--clean buildroot
+./%(prog)s --arch x86_64 --arch aarch64 all
 ....
+
+Instead, for the targets that are emulator dependent, you must select the
+taret version for the desired emulatore, e.g.:
+
+....
+./build --arch aarch64 baremetal-qemu baremetal-gem5
+....
+
+The reason is that some targets depend on emulator, while others don't,
+so looping over all of them would waste time.
 ''',
-    formatter_class=argparse.RawTextHelpFormatter,
+        )
-)
+        buildroot_component = _Component(
-parser.add_argument('--all', default=False, action='store_true', help='''\
+            self._build_file('build-buildroot'),
-Build absolutely everything for all archs.
+            submodules = {'buildroot'},
-''')
+            # https://buildroot.org/downloads/manual/manual.html#requirement
-group = parser.add_mutually_exclusive_group(required=False)
+            apt_get_pkgs={
-group.add_argument('-A', '--all-archs', default=False, action='store_true', help='''\
+                'bash',
-Build the selected components for all archs.
+                'bc',
-''')
+                'binutils',
-group.add_argument('-a', '--arch', choices=common.arch_choices, default=[], action='append', help='''\
+                'build-essential',
-Build the selected components for this arch. Select multiple archs by
+                'bzip2',
-passing this option multiple times. Default: [{}]
+                'cpio',
-'''.format(common.default_arch))
+                'g++',
-parser.add_argument('-D', '--download-dependencies', default=False, action='store_true', help='''\
+                'gcc',
+                'graphviz',
+                'gzip',
+                'make',
+                'patch',
+                'perl',
+                'python-matplotlib',
+                'python3',
+                'rsync',
+                'sed',
+                'tar',
+                'unzip',
+            },
+        )
+        buildroot_overlay_qemu_component = copy.copy(buildroot_component)
+        # We need to build QEMU before the final Buildroot to get qemu-img.
+        buildroot_overlay_qemu_component.dependencies = ['overlay', 'qemu']
+        buildroot_overlay_gem5_component = copy.copy(buildroot_component)
+        buildroot_overlay_gem5_component.dependencies = ['overlay-gem5']
+        gem5_deps = {
+            # TODO test it out on Docker and answer that question properly:
+            # https://askubuntu.com/questions/350475/how-can-i-install-gem5
+            'apt_get_pkgs': {
+                'device-tree-compiler',
+                'diod',
+                'libgoogle-perftools-dev',
+                'm4',
+                'protobuf-compiler',
+                'python-dev',
+                'python-pip',
+                # For prebuilt qcow2 unpack.
+                'qemu-utils',
+                'scons',
+                'zlib1g-dev',
+            },
+            'python2_pkgs': {
+                # Generate graphs of config.ini under m5out.
+                'pydot',
+            },
+            'submodules': {'gem5'},
+        }
+
+        self.name_to_component_map = {
+            'all': _Component(dependencies=[
+                'qemu-gem5-buildroot',
+                'all-baremetal',
+                'user-mode-qemu',
+                'doc',
+            ]),
+            'all-baremetal': _Component(dependencies=[
+                    'qemu-baremetal',
+                    'gem5-baremetal',
+                    'baremetal-gem5-pbx',
+                ],
+                supported_archs=common.consts['crosstool_ng_supported_archs'],
+            ),
+            'baremetal': _Component(dependencies=[
+                'baremetal-gem5',
+                'baremetal-qemu',
+            ]),
+            'baremetal-qemu': _Component(
+                self._build_file('build-baremetal', emulators=['qemu']),
+                supported_archs=common.consts['crosstool_ng_supported_archs'],
+                dependencies=['crosstool-ng'],
+            ),
+            'baremetal-gem5': _Component(
+                self._build_file('build-baremetal', emulators=['gem5']),
+                supported_archs=common.consts['crosstool_ng_supported_archs'],
+                dependencies=['crosstool-ng'],
+            ),
+            'baremetal-gem5-pbx': _Component(
+                self._build_file('build-baremetal', emulators=['gem5'], machine='RealViewPBX'),
+                supported_archs=common.consts['crosstool_ng_supported_archs'],
+                dependencies=['crosstool-ng'],
+            ),
+            'buildroot': buildroot_component,
+            # We need those to avoid cirtulcar dependencies, since we need to run Buildroot
+            # twice: once to get the toolchain, and a second time to put the overlay into
+            # the root filesystem.
+            'buildroot-overlay-qemu': buildroot_overlay_qemu_component,
+            'buildroot-overlay-gem5': buildroot_overlay_gem5_component,
+            'copy-overlay': _Component(
+                self._build_file('copy-overlay'),
+            ),
+            'crosstool-ng': _Component(
+                self._build_file('build-crosstool-ng'),
+                supported_archs=common.consts['crosstool_ng_supported_archs'],
+                # http://crosstool-ng.github.io/docs/os-setup/
+                apt_get_pkgs={
+                    'bison',
+                    'docbook2x',
+                    'flex',
+                    'gawk',
+                    'gcc',
+                    'gperf',
+                    'help2man',
+                    'libncurses5-dev',
+                    'libtool-bin',
+                    'make',
+                    'python-dev',
+                    'texinfo',
+                },
+                submodules={'crosstool-ng'},
+            ),
+            'doc': _Component(
+                self._build_file('build-doc'),
+            ),
+            'gem5': _Component(
+                self._build_file('build-gem5'),
+                **gem5_deps
+            ),
+            'gem5-baremetal': _Component(dependencies=[
+                'gem5',
+                'baremetal-gem5',
+            ]),
+            'gem5-buildroot': _Component(dependencies=[
+                'buildroot-overlay-gem5',
+                'linux',
+                'gem5',
+            ]),
+            'gem5-debug': _Component(
+                self._build_file('build-gem5', gem5_build_type='debug'),
+                **gem5_deps
+            ),
+            'gem5-fast': _Component(
+                self._build_file('build-gem5', gem5_build_type='fast'),
+                **gem5_deps
+            ),
+            'linux': _Component(
+                self._build_file('build-linux'),
+                dependencies={'buildroot'},
+                submodules_shallow={'linux'},
+                apt_get_pkgs={
+                    'bison',
+                    'flex',
+                    # Without this started failing in kernel 4.15 with:
+                    # Makefile:932: *** "Cannot generate ORC metadata for CONFIG_UNWINDER_ORC=y, please install libelf-dev, libelf-devel or elfutils-libelf-devel".  Stop.
+                    'libelf-dev',
+                },
+            ),
+            'modules': _Component(
+                self._build_file('build-modules'),
+                dependencies=['buildroot', 'linux'],
+            ),
+            'm5': _Component(
+                self._build_file('build-m5'),
+                dependencies=['buildroot'],
+                submodules={'gem5'},
+            ),
+            'overlay': _Component(dependencies=[
+                'copy-overlay',
+                'modules',
+                'userland',
+            ]),
+            'overlay-gem5': _Component(dependencies=[
+                'm5',
+                'overlay',
+            ]),
+            'parsec-benchmark': _Component(
+                submodules={'parsec-benchmark'},
+                dependencies=['buildroot'],
+            ),
+            'qemu': _Component(
+                self._build_file('build-qemu'),
+                apt_build_deps={'qemu'},
+                apt_get_pkgs={'libsdl2-dev'},
+                submodules={'qemu'},
+            ),
+            'qemu-baremetal': _Component(dependencies=[
+                'qemu',
+                'baremetal-qemu',
+            ]),
+            'qemu-buildroot': _Component(dependencies=[
+                'buildroot-overlay-qemu',
+                'linux',
+            ]),
+            'qemu-gem5-buildroot': _Component(dependencies=[
+                'qemu',
+                'gem5-buildroot',
+            ]),
+            'qemu-user': _Component(
+                self._build_file('build-qemu', user_mode=True),
+                apt_build_deps = {'qemu'},
+                apt_get_pkgs={'libsdl2-dev'},
+                submodules={'qemu'},
+            ),
+            'release': _Component(dependencies=[
+                'qemu-buildroot',
+                'doc',
+            ]),
+            'test-gdb': _Component(dependencies=[
+                    'all-baremetal',
+                ],
+                supported_archs=common.consts['crosstool_ng_supported_archs'],
+            ),
+            'test-user-mode': _Component(dependencies=[
+                'test-user-mode-qemu',
+                'test-user-mode-gem5',
+            ]),
+            'test-user-mode-qemu': _Component(dependencies=[
+                'user-mode-qemu',
+                'userland',
+            ]),
+            'test-user-mode-gem5': _Component(dependencies=[
+                'gem5',
+                'userland-gem5',
+            ]),
+            'user-mode-qemu': _Component(
+                dependencies=['qemu-user', 'userland'],
+            ),
+            'userland': _Component(
+                self._build_file('build-userland'),
+                dependencies=['buildroot'],
+            ),
+            'userland-gem5': _Component(
+                self._build_file('build-userland', static=True, userland_build_id='static'),
+                dependencies=['buildroot'],
+            ),
+        }
+        self.component_to_name_map = {self.name_to_component_map[key]:key for key in self.name_to_component_map}
+
+        self.add_argument(
+            '--apt',
+            default=True,
+            help='''\
+Don't run any apt-get commands. To make it easier to use with other archs:
+https://github.com/cirosantilli/linux-kernel-module-cheat#supported-hosts
+'''
+        )
+        self.add_argument(
+            '--download-dependencies',
+            default=False,
+            help='''\
 Also download all dependencies required for a given build: Ubuntu packages,
 Python packages and git submodules.
-''')
+'''
-parser.add_argument('--extra-args', default='', help='''\
+        )
+        self.add_argument(
+            '--print-components',
+            default=False,
+            help='''\
+Print the components that would be built, including dependencies, but don't
+build them, nor show the build commands.
+'''
+        )
+        self.add_argument(
+            '--travis',
+            default=False,
+            help='''\
 Extra args to pass to all scripts.
 '''
-)
+        )
-parser.add_argument('--travis', default=False, action='store_true', help='''\
+        self.add_argument(
-Extra args to pass to all scripts.
+            'components',
-'''
+            choices=list(self.name_to_component_map.keys()) + [[]],
-)
+            default=[],
-parser.add_argument('components', choices=list(name_to_component_map.keys()) + [[]], default=[], nargs='*', help='''\
+            nargs='*',
+            help='''\
 Which components to build. Default: qemu-buildroot
-'''.format(common.default_arch))
+'''
-common.add_dry_run_argument(parser)
+        )
-args = parser.parse_args()
-common.setup_dry_run_arguments(args)
 
-# Decide archs.
+    def _build_file(self, component_file, **extra_args):
-if args.arch == []:
+        '''
-    if args.all or args.all_archs:
+        Build something based on a component file that defines a Main class.
-        archs = common.all_archs.copy()
+        '''
-    else:
+        def f():
-        archs = set([common.default_arch])
+            args = self.get_common_args()
-else:
+            args.update(extra_args)
-    archs = set()
+            args['print_time'] = False
-    for arch in args.arch:
+            self.import_path_main(component_file)(**args)
-        if arch in common.arch_short_to_long_dict:
+        return f
-            arch = common.arch_short_to_long_dict[arch]
-        archs.add(arch)
 
-# Decide components.
+    def timed_main(self):
-components = args.components
+        self.sh = shell_helpers.ShellHelpers(dry_run=self.env['dry_run'])
-if args.all:
-    components = ['all']
-elif components == []:
-    components = ['qemu-buildroot']
-selected_components = []
-selected_component_name_set = set()
-for component_name in components:
-    todo = [component_name]
-    while todo:
-        current_name = todo.pop(0)
-        if current_name not in selected_component_name_set:
-            selected_component_name_set.add(current_name)
-            component = name_to_component_map[current_name]
-            selected_components.append(component)
-            todo.extend(component.dependencies)
 
-if args.download_dependencies:
+        # Decide components.
-    apt_get_pkgs = {
+        components = self.env['components']
-        # Core requirements for this repo.
+        if components == []:
-        'git',
+            components = ['qemu-buildroot']
-        'moreutils', # ts
+        selected_components = []
-        'python3-pip',
+        for component_name in components:
-        'tmux',
+            todo = [component_name]
-        'vinagre',
+            while todo:
-        'wget',
+                current_name = todo.pop(0)
-    }
+                component = self.name_to_component_map[current_name]
-    # E.e. on an ARM host, the package gcc-arm-linux-gnueabihf
+                selected_components.insert(0, component)
-    # is called just gcc.
+                todo.extend(component.dependencies)
-    processor = platform.processor()
+        # Remove duplicates, keep only the first one of each.
-    if processor != 'arm':
+        # https://stackoverflow.com/questions/7961363/removing-duplicates-in-lists/7961390#7961390
-        apt_get_pkgs.update({
+        selected_components = collections.OrderedDict.fromkeys(selected_components)
-            'gcc-arm-linux-gnueabihf',
+
-            'g++-arm-linux-gnueabihf',
+        if self.env['download_dependencies']:
-        })
+            apt_get_pkgs = {
-    if processor != 'aarch64':
+                # Core requirements for this repo.
-        apt_get_pkgs.update({
+                'git',
-            'gcc-aarch64-linux-gnu',
+                'moreutils', # ts
-            'g++-aarch64-linux-gnu',
+                'python3-pip',
-        })
+                'tmux',
-    apt_build_deps = set()
+                'vinagre',
-    submodules = set()
+                'wget',
-    submodules_shallow = set()
-    python2_pkgs = set()
-    python3_pkgs = {
-        'pexpect==4.6.0',
-    }
-    for component in selected_components:
-        apt_get_pkgs.update(component.apt_get_pkgs)
-        apt_build_deps.update(component.apt_build_deps)
-        submodules.update(component.submodules)
-        submodules_shallow.update(component.submodules_shallow)
-        python2_pkgs.update(component.python2_pkgs)
-        python3_pkgs.update(component.python3_pkgs)
-    if apt_get_pkgs or apt_build_deps:
-        if args.travis:
-            interacive_pkgs = {
-                'libsdl2-dev',
             }
-            apt_get_pkgs.difference_update(interacive_pkgs)
+            # E.g. on an ARM host, the package gcc-arm-linux-gnueabihf
-        if common.in_docker:
+            # is called just gcc.
-            sudo = []
+            processor = self.env['host_arch']
-            # https://askubuntu.com/questions/909277/avoiding-user-interaction-with-tzdata-when-installing-certbot-in-a-docker-contai
+            if processor != 'arm':
-            os.environ['DEBIAN_FRONTEND'] = 'noninteractive'
+                apt_get_pkgs.update({
-            # https://askubuntu.com/questions/496549/error-you-must-put-some-source-uris-in-your-sources-list
+                    'gcc-arm-linux-gnueabihf',
-            sources_path = os.path.join('/etc', 'apt', 'sources.list')
+                    'g++-arm-linux-gnueabihf',
-            with open(sources_path, 'r') as f:
+                })
-                sources_txt = f.read()
+            if processor != 'aarch64':
-            sources_txt = re.sub('^# deb-src ', 'deb-src ', sources_txt, flags=re.MULTILINE)
+                apt_get_pkgs.update({
-            with open(sources_path, 'w') as f:
+                    'gcc-aarch64-linux-gnu',
-                f.write(sources_txt)
+                    'g++-aarch64-linux-gnu',
-        else:
+                })
-            sudo = ['sudo']
+            apt_build_deps = set()
-        if common.in_docker or args.travis:
+            submodules = set()
-            y = ['-y']
+            submodules_shallow = set()
-        else:
+            python2_pkgs = set()
-            y = []
+            python3_pkgs = {
-        common.run_cmd(
+                'pexpect==4.6.0',
-            sudo + ['apt-get', 'update', common.Newline]
+            }
-        )
+            for component in selected_components:
-        if apt_get_pkgs:
+                apt_get_pkgs.update(component.apt_get_pkgs)
-            common.run_cmd(
+                apt_build_deps.update(component.apt_build_deps)
-                sudo + ['apt-get', 'install'] + y + [common.Newline] +
+                submodules.update(component.submodules)
-                common.add_newlines(sorted(apt_get_pkgs))
+                submodules_shallow.update(component.submodules_shallow)
-            )
+                python2_pkgs.update(component.python2_pkgs)
-        if apt_build_deps:
+                python3_pkgs.update(component.python3_pkgs)
-            common.run_cmd(
+            if apt_get_pkgs or apt_build_deps:
-                sudo +
+                if self.env['travis']:
-                ['apt-get', 'build-dep'] + y + [common.Newline] +
+                    interacive_pkgs = {
-                common.add_newlines(sorted(apt_build_deps))
+                        'libsdl2-dev',
-            )
+                    }
-    if python2_pkgs:
+                    apt_get_pkgs.difference_update(interacive_pkgs)
-        common.run_cmd(
+                if common.consts['in_docker']:
-            ['python', '-m', 'pip', 'install', '--user', common.Newline] +
+                    sudo = []
-            common.add_newlines(sorted(python2_pkgs))
+                    # https://askubuntu.com/questions/909277/avoiding-user-interaction-with-tzdata-when-installing-certbot-in-a-docker-contai
-        )
+                    os.environ['DEBIAN_FRONTEND'] = 'noninteractive'
-    if python3_pkgs:
+                    # https://askubuntu.com/questions/496549/error-you-must-put-some-source-uris-in-your-sources-list
-        # Not with pip executable directly:
+                    sources_path = os.path.join('/etc', 'apt', 'sources.list')
-        # https://stackoverflow.com/questions/49836676/error-after-upgrading-pip-cannot-import-name-main/51846054#51846054
+                    with open(sources_path, 'r') as f:
-        common.run_cmd(
+                        sources_txt = f.read()
-            ['python3', '-m', 'pip', 'install', '--user', common.Newline] +
+                    sources_txt = re.sub('^# deb-src ', 'deb-src ', sources_txt, flags=re.MULTILINE)
-            common.add_newlines(sorted(python3_pkgs))
+                    with open(sources_path, 'w') as f:
-        )
+                        f.write(sources_txt)
-    git_cmd_common = ['git', 'submodule', 'update', '--init', '--recursive']
+                else:
-    if submodules:
+                    sudo = ['sudo']
-        # == Other nice git options for when distros move to newer Git
+                if common.consts['in_docker'] or self.env['travis']:
-        #
+                    y = ['-y']
-        # Currently not on Ubuntu 16.04:
+                else:
-        #
+                    y = []
-        # `--progress`: added on Git 2.10:
+                if self.env['apt']:
-        #
+                    self.sh.run_cmd(
-        # * https://stackoverflow.com/questions/32944468/how-to-show-progress-for-submodule-fetching
+                        sudo + ['apt-get', 'update', LF]
-        # * https://stackoverflow.com/questions/4640020/progress-indicator-for-git-clone
+                    )
-        #
+                    if apt_get_pkgs:
-        # `--jobs"`: https://stackoverflow.com/questions/26957237/how-to-make-git-clone-faster-with-multiple-threads/52327638#52327638
+                        self.sh.run_cmd(
-        common.run_cmd(
+                            sudo + ['apt-get', 'install'] + y + [LF] +
-            git_cmd_common + ['--', common.Newline] +
+                            self.sh.add_newlines(sorted(apt_get_pkgs))
-            common.add_newlines([os.path.join(common.submodules_dir, x) for x in sorted(submodules)])
+                        )
-        )
+                    if apt_build_deps:
-    if submodules_shallow:
+                        self.sh.run_cmd(
-        # == Shallow cloning.
+                            sudo +
-        #
+                            ['apt-get', 'build-dep'] + y + [LF] +
-        # TODO Ideally we should shallow clone --depth 1 all of them.
+                            self.sh.add_newlines(sorted(apt_build_deps))
-        #
+                        )
-        # However, most git servers out there are crap or craply configured
+            if python2_pkgs:
-        # and don't allow shallow cloning except for branches.
+                self.sh.run_cmd(
-        #
+                    ['python', '-m', 'pip', 'install', '--user', LF] +
-        # So for now, let's shallow clone only the Linux kernel, which has by far
+                    self.sh.add_newlines(sorted(python2_pkgs))
-        # the largest .git repo history, and full clone the others.
+                )
-        #
+            if python3_pkgs:
-        # Then we will maintain a GitHub Linux kernel mirror / fork that always has a
+                # Not with pip executable directly:
-        # lkmc branch, and point to it, so that it will always succeed.
+                # https://stackoverflow.com/questions/49836676/error-after-upgrading-pip-cannot-import-name-main/51846054#51846054
-        #
+                self.sh.run_cmd(
-        # See also:
+                    ['python3', '-m', 'pip', 'install', '--user', LF] +
-        #
+                    self.sh.add_newlines(sorted(python3_pkgs))
-        # * https://stackoverflow.com/questions/3489173/how-to-clone-git-repository-with-specific-revision-changeset
+                )
-        # * https://stackoverflow.com/questions/2144406/git-shallow-submodules/47374702#47374702
+            git_cmd_common = ['git', 'submodule', 'update', '--init', '--recursive']
-        # * https://unix.stackexchange.com/questions/338578/why-is-the-git-clone-of-the-linux-kernel-source-code-much-larger-than-the-extrac
+            if submodules:
-        #
+                # == Other nice git options for when distros move to newer Git
-        common.run_cmd(
+                #
-            git_cmd_common + ['--depth', '1', '--', common.Newline] +
+                # Currently not on Ubuntu 16.04:
-            common.add_newlines([os.path.join(common.submodules_dir, x) for x in sorted(submodules_shallow)])
+                #
-        )
+                # `--progress`: added on Git 2.10:
+                #
+                # * https://stackoverflow.com/questions/32944468/how-to-show-progress-for-submodule-fetching
+                # * https://stackoverflow.com/questions/4640020/progress-indicator-for-git-clone
+                #
+                # `--jobs"`: https://stackoverflow.com/questions/26957237/how-to-make-git-clone-faster-with-multiple-threads/52327638#52327638
+                self.sh.run_cmd(
+                    git_cmd_common + ['--', LF] +
+                    self.sh.add_newlines([os.path.join(common.consts['submodules_dir'], x) for x in sorted(submodules)])
+                )
+            if submodules_shallow:
+                # == Shallow cloning.
+                #
+                # TODO Ideally we should shallow clone --depth 1 all of them.
+                #
+                # However, most git servers out there are crap or craply configured
+                # and don't allow shallow cloning except for branches.
+                #
+                # So for now, let's shallow clone only the Linux kernel, which has by far
+                # the largest .git repo history, and full clone the others.
+                #
+                # Then we will maintain a GitHub Linux kernel mirror / fork that always has a
+                # lkmc branch, and point to it, so that it will always succeed.
+                #
+                # See also:
+                #
+                # * https://stackoverflow.com/questions/3489173/how-to-clone-git-repository-with-specific-revision-changeset
+                # * https://stackoverflow.com/questions/2144406/git-shallow-submodules/47374702#47374702
+                # * https://unix.stackexchange.com/questions/338578/why-is-the-git-clone-of-the-linux-kernel-source-code-much-larger-than-the-extrac
+                #
+                self.sh.run_cmd(
+                    git_cmd_common + ['--depth', '1', '--', LF] +
+                    self.sh.add_newlines([os.path.join(common.consts['submodules_dir'], x) for x in sorted(submodules_shallow)])
+                )
 
-# Do the build.
+        # Do the build.
-for arch in archs:
+        for component in selected_components:
-    for component in selected_components:
+            if self.env['print_components']:
-        component.build(arch)
+                print(self.component_to_name_map[component])
+            else:
+                component.build(self.env['arch'])
+
+if __name__ == '__main__':
+    Main().cli()

build-baremetal

@@ -1,73 +1,80 @@
 #!/usr/bin/env python3
-
 import os
 
 import common
+from shell_helpers import LF
 
-class BaremetalComponent(common.Component):
+class Main(common.BuildCliFunction):
-    def do_build(self, args):
+    def __init__(self):
-        common.assert_crosstool_ng_supports_arch(args.arch)
+        super().__init__(
-        build_dir = self.get_build_dir(args)
+            description='''\
-        bootloader_obj = os.path.join(common.baremetal_build_lib_dir, 'bootloader{}'.format(common.obj_ext))
+Build the baremetal examples with crosstool-NG.
+''',
+            supported_archs=common.consts['crosstool_ng_supported_archs']
+        )
+
+    def build(self):
+        build_dir = self.get_build_dir()
+        bootloader_obj = os.path.join(self.env['baremetal_build_lib_dir'], 'bootloader{}'.format(self.env['obj_ext']))
         common_basename_noext = 'common'
-        common_src = os.path.join(common.root_dir, common_basename_noext + common.c_ext)
+        common_src = os.path.join(self.env['root_dir'], common_basename_noext + self.env['c_ext'])
-        common_obj = os.path.join(common.baremetal_build_lib_dir, common_basename_noext + common.obj_ext)
+        common_obj = os.path.join(self.env['baremetal_build_lib_dir'], common_basename_noext + self.env['obj_ext'])
         syscalls_basename_noext = 'syscalls'
-        syscalls_src = os.path.join(common.baremetal_src_lib_dir, syscalls_basename_noext + common.c_ext)
+        syscalls_src = os.path.join(self.env['baremetal_source_lib_dir'], syscalls_basename_noext + self.env['c_ext'])
-        syscalls_obj = os.path.join(common.baremetal_build_lib_dir, syscalls_basename_noext + common.obj_ext)
+        syscalls_obj = os.path.join(self.env['baremetal_build_lib_dir'], syscalls_basename_noext + self.env['obj_ext'])
         common_objs = [common_obj, syscalls_obj]
         cflags = [
-            '-I', common.baremetal_src_lib_dir, common.Newline,
+            '-I', self.env['baremetal_source_lib_dir'], LF,
-            '-I', common.root_dir, common.Newline,
+            '-I', self.env['root_dir'], LF,
-            '-O0', common.Newline,
+            '-O0', LF,
-            '-ggdb3', common.Newline,
+            '-ggdb3', LF,
-            '-mcpu={}'.format(common.mcpu), common.Newline,
+            '-mcpu={}'.format(self.env['mcpu']), LF,
-            '-nostartfiles', common.Newline,
+            '-nostartfiles', LF,
         ]
-        if args.prebuilt:
+        if self.env['prebuilt']:
             gcc = 'arm-none-eabi-gcc'
         else:
-            os.environ['PATH'] = common.crosstool_ng_bin_dir + os.environ['PATH']
+            os.environ['PATH'] = self.env['crosstool_ng_bin_dir'] + os.environ['PATH']
-            gcc = common.get_toolchain_tool('gcc', allowed_toolchains=['crosstool-ng'])
+            gcc = self.get_toolchain_tool('gcc', allowed_toolchains=['crosstool-ng'])
-        if common.emulator == 'gem5':
+        if self.env['emulator'] == 'gem5':
-            if common.machine == 'VExpress_GEM5_V1':
+            if self.env['machine'] == 'VExpress_GEM5_V1':
                 entry_address = 0x80000000
                 uart_address = 0x1c090000
-            elif common.machine == 'RealViewPBX':
+            elif self.env['machine'] == 'RealViewPBX':
                 entry_address = 0x10000
                 uart_address = 0x10009000
             else:
-                raise Exception('unknown machine: ' + common.machine)
+                raise Exception('unknown machine: ' + self.env['machine'])
-            cflags.extend(['-D', 'GEM5'.format(uart_address), common.Newline])
+            cflags.extend(['-D', 'GEM5'.format(uart_address), LF])
         else:
             entry_address = 0x40000000
             uart_address = 0x09000000
         os.makedirs(build_dir, exist_ok=True)
-        os.makedirs(common.baremetal_build_lib_dir, exist_ok=True)
+        os.makedirs(self.env['baremetal_build_lib_dir'], exist_ok=True)
-        src = os.path.join(common.baremetal_src_lib_dir, '{}{}'.format(args.arch, common.asm_ext))
+        src = os.path.join(self.env['baremetal_source_lib_dir'], '{}{}'.format(self.env['arch'], self.env['asm_ext']))
-        if common.need_rebuild([src], bootloader_obj):
+        if self.need_rebuild([src], bootloader_obj):
-            common.run_cmd(
+            self.sh.run_cmd(
-                [gcc,  common.Newline] +
+                [gcc,  LF] +
                 cflags +
                 [
-                    '-c', common.Newline,
+                    '-c', LF,
-                    '-o', bootloader_obj, common.Newline,
+                    '-o', bootloader_obj, LF,
-                    src, common.Newline,
+                    src, LF,
                 ]
             )
         for src, obj in [
             (common_src, common_obj),
             (syscalls_src, syscalls_obj),
         ]:
-            if common.need_rebuild([src], obj):
+            if self.need_rebuild([src], obj):
-                common.run_cmd(
+                self.sh.run_cmd(
-                    [gcc,  common.Newline] +
+                    [gcc,  LF] +
                     cflags +
                     [
-                        '-c', common.Newline,
+                        '-c', LF,
-                        '-D', 'UART0_ADDR={:#x}'.format(uart_address), common.Newline,
+                        '-D', 'UART0_ADDR={:#x}'.format(uart_address), LF,
-                        '-o', obj, common.Newline,
+                        '-o', obj, LF,
-                        src, common.Newline,
+                        src, LF,
                     ]
                 )
         self._build_dir(
@@ -86,18 +93,17 @@ class BaremetalComponent(common.Component):
             bootloader_obj=bootloader_obj,
             common_objs=common_objs,
         )
-        arch_dir = os.path.join('arch', args.arch)
+        if os.path.isdir(os.path.join(self.env['baremetal_source_arch_dir'])):
-        if os.path.isdir(os.path.join(common.baremetal_src_dir, arch_dir)):
             self._build_dir(
-                arch_dir,
+                self.env['baremetal_source_arch_subpath'],
                 gcc=gcc,
                 cflags=cflags,
                 entry_address=entry_address,
                 bootloader_obj=bootloader_obj,
                 common_objs=common_objs,
             )
-        arch_dir = os.path.join('arch', args.arch, 'no_bootloader')
+        arch_dir = os.path.join('arch', self.env['arch'], 'no_bootloader')
-        if os.path.isdir(os.path.join(common.baremetal_src_dir, arch_dir)):
+        if os.path.isdir(os.path.join(self.env['baremetal_source_dir'], arch_dir)):
             self._build_dir(
                 arch_dir,
                 gcc=gcc,
@@ -108,18 +114,8 @@ class BaremetalComponent(common.Component):
                 bootloader=False,
             )
 
-    def get_argparse_args(self):
+    def get_build_dir(self):
-        return {
+        return self.env['baremetal_build_dir']
-            'description': '''\
-Build the baremetal examples with crosstool-NG.
-'''
-        }
-
-    def get_build_dir(self, args):
-        return common.baremetal_build_dir
-
-    def get_default_args(self):
-        return {'baremetal': 'all'}
 
     def _build_dir(
             self,
@@ -131,48 +127,48 @@ Build the baremetal examples with crosstool-NG.
             common_objs,
             bootloader=True
         ):
-        """
+        '''
         Build all .c and .S files in a given subpath of the baremetal source
         directory non recursively.
 
         Place outputs on the same subpath or the output directory.
-        """
+        '''
-        in_dir = os.path.join(common.baremetal_src_dir, subpath)
+        in_dir = os.path.join(self.env['baremetal_source_dir'], subpath)
-        out_dir = os.path.join(common.baremetal_build_dir, subpath)
+        out_dir = os.path.join(self.env['baremetal_build_dir'], subpath)
         os.makedirs(out_dir, exist_ok=True)
         common_objs = common_objs.copy()
         if bootloader:
             common_objs.append(bootloader_obj)
         for in_basename in os.listdir(in_dir):
             in_path = os.path.join(in_dir, in_basename)
-            if os.path.isfile(in_path) and os.path.splitext(in_basename)[1] in (common.c_ext, common.asm_ext):
+            if os.path.isfile(in_path) and os.path.splitext(in_basename)[1] in (self.env['c_ext'], self.env['asm_ext']):
                 in_name = os.path.splitext(in_basename)[0]
-                main_obj = os.path.join(common.baremetal_build_dir, subpath, '{}{}'.format(in_name, common.obj_ext))
+                main_obj = os.path.join(self.env['baremetal_build_dir'], subpath, '{}{}'.format(in_name, self.env['obj_ext']))
-                src = os.path.join(common.baremetal_src_dir, in_path)
+                src = os.path.join(self.env['baremetal_source_dir'], in_path)
-                if common.need_rebuild([src], main_obj):
+                if self.need_rebuild([src], main_obj):
-                    common.run_cmd(
+                    self.sh.run_cmd(
-                        [gcc,  common.Newline] +
+                        [gcc,  LF] +
                         cflags +
                         [
-                            '-c', common.Newline,
+                            '-c', LF,
-                            '-o', main_obj, common.Newline,
+                            '-o', main_obj, LF,
-                            src, common.Newline,
+                            src, LF,
                         ]
                     )
                 objs = common_objs + [main_obj]
-                out = os.path.join(common.baremetal_build_dir, subpath, in_name + common.baremetal_build_ext)
+                out = os.path.join(self.env['baremetal_build_dir'], subpath, in_name + self.env['baremetal_build_ext'])
-                link_script = os.path.join(common.baremetal_src_dir, 'link.ld')
+                link_script = os.path.join(self.env['baremetal_source_dir'], 'link.ld')
-                if common.need_rebuild(objs + [link_script], out):
+                if self.need_rebuild(objs + [link_script], out):
-                    common.run_cmd(
+                    self.sh.run_cmd(
-                        [gcc,  common.Newline] +
+                        [gcc,  LF] +
                         cflags +
                         [
-                            '-Wl,--section-start=.text={:#x}'.format(entry_address), common.Newline,
+                            '-Wl,--section-start=.text={:#x}'.format(entry_address), LF,
-                            '-o', out, common.Newline,
+                            '-o', out, LF,
-                            '-T', link_script, common.Newline,
+                            '-T', link_script, LF,
                         ] +
-                        common.add_newlines(objs)
+                        self.sh.add_newlines(objs)
                     )
 
 if __name__ == '__main__':
-    BaremetalComponent().build()
+    Main().cli()

build-bench-boot

@@ -1,15 +0,0 @@
-#!/usr/bin/env bash
-set -eu
-test_size=1
-while [ $# -gt 0 ]; do
-  case "$1" in
-    --size)
-      test_size="$2"
-      shift 2
-      ;;
-  esac
-done
-./build --all-archs qemu-gem5-buildroot
-if [ "$test_size" -ge 3 ]; then
-  ./build --arch aarch64 all-linux
-fi

build-buildroot

@@ -9,11 +9,17 @@ import time
 import re
 
 import common
+from shell_helpers import LF
 
-class BuildrootComponent(common.Component):
+class Main(common.BuildCliFunction):
-    def add_parser_arguments(self, parser):
+    def __init__(self):
-        parser.add_argument(
+        super().__init__(
-            '--build-linux', default=self._defaults['build_linux'], action='store_true',
+            description='''\
+Build Buildroot. This includes, notably: the userland GCC cross-toolchain,
+and the root filesystem.
+''')
+        self.add_argument(
+            '--build-linux', default=False,
             help='''\
 Enable building the Linux kernel with Buildroot. This is done mostly
 to extract Buildroot's default kernel configurations when updating Buildroot.
@@ -21,35 +27,35 @@ This kernel will not be use by our other scripts. Configuring this kernel is
 not currently supported, juse use ./build-linux script if you want to do that.
 '''
         )
-        parser.add_argument(
+        self.add_argument(
-            '--baseline', default=self._defaults['baseline'], action='store_true',
+            '--baseline', default=False,
             help='''Do a default-ish Buildroot defconfig build, without any of our extra options.
 Mostly to track how much slower we are than a basic build.
 '''
         )
-        parser.add_argument(
+        self.add_argument(
-            '--config', default=self._defaults['config'], action='append',
+            '--config', default=[], action='append',
             help='''Add a single Buildroot config to the current build.
 Example value: 'BR2_TARGET_ROOTFS_EXT2_SIZE="512M"'.
 Can be used multiple times to add multiple configs.
 Takes precedence over any Buildroot config files.
 '''
         )
-        parser.add_argument(
+        self.add_argument(
-            '--config-fragment', default=self._defaults['config_fragment'], action='append',
+            '--config-fragment', default=[], action='append',
             help='''Also use the given Buildroot configuration fragment file.
 Pass multiple times to use multiple fragment files.
 '''
         )
-        parser.add_argument(
+        self.add_argument(
-            '--no-all', default=self._defaults['no_all'], action='store_true',
+            '--no-all', default=False,
             help='''\
 Don't build the all target which normally gets build by default.
 That target builds the root filesystem and all its dependencies.
 '''
         )
-        parser.add_argument(
+        self.add_argument(
-            '--no-overlay', default=self._defaults['no_all'], action='store_true',
+            '--no-overlay', default=False,
             help='''\
 Don't add our overlay which contains all files we build without going through Buildroot.
 This prevents us from overwriting certain Buildroot files. Remember however that you must
@@ -57,132 +63,116 @@ still rebuild the Buildroot package that provides those files to actually put th
 files on the root filesystem.
 '''
         )
-        parser.add_argument(
+        self.add_argument(
-            'extra_make_args', default=self._defaults['extra_make_args'], metavar='extra-make-args', nargs='*',
+            'extra-make-args', default=[], nargs='*',
             help='''\
 Extra arguments to be passed to the Buildroot make,
 usually extra Buildroot targets.
 '''
         )
 
-    def do_build(self, args):
+    def build(self):
-        build_dir = self.get_build_dir(args)
+        build_dir = self.get_build_dir()
-        os.makedirs(common.out_dir, exist_ok=True)
+        os.makedirs(self.env['out_dir'], exist_ok=True)
-        extra_make_args = common.add_newlines(args.extra_make_args)
+        extra_make_args = self.sh.add_newlines(self.env['extra_make_args'])
-        if args.build_linux:
+        if self.env['build_linux']:
-            extra_make_args.extend(['linux-reconfigure', common.Newline])
+            extra_make_args.extend(['linux-reconfigure', LF])
-        if common.emulator == 'gem5':
+        if self.env['arch'] == 'x86_64':
-            extra_make_args.extend(['gem5-reconfigure', common.Newline])
-        if args.arch == 'x86_64':
             defconfig = 'qemu_x86_64_defconfig'
-        elif args.arch == 'arm':
+        elif self.env['arch'] == 'arm':
             defconfig = 'qemu_arm_vexpress_defconfig'
-        elif args.arch == 'aarch64':
+        elif self.env['arch'] == 'aarch64':
             defconfig = 'qemu_aarch64_virt_defconfig'
         br2_external_dirs = []
-        for package_dir in os.listdir(common.packages_dir):
+        for package_dir in os.listdir(self.env['packages_dir']):
-            package_dir_abs = os.path.join(common.packages_dir, package_dir)
+            package_dir_abs = os.path.join(self.env['packages_dir'], package_dir)
             if os.path.isdir(package_dir_abs):
                 br2_external_dirs.append(self._path_relative_to_buildroot(package_dir_abs))
         br2_external_str = ':'.join(br2_external_dirs)
-        common.run_cmd(
+        self.sh.run_cmd(
             [
-                'make', common.Newline,
+                'make', LF,
-                'O={}'.format(common.buildroot_build_dir), common.Newline,
+                'O={}'.format(self.env['buildroot_build_dir']), LF,
-                'BR2_EXTERNAL={}'.format(br2_external_str), common.Newline,
+                'BR2_EXTERNAL={}'.format(br2_external_str), LF,
-                defconfig, common.Newline,
+                defconfig, LF,
             ],
-            cwd=common.buildroot_src_dir,
+            cwd=self.env['buildroot_source_dir'],
         )
-        configs = args.config
+        configs = self.env['config']
         configs.extend([
-            'BR2_JLEVEL={}'.format(args.nproc),
+            'BR2_JLEVEL={}'.format(self.env['nproc']),
-            'BR2_DL_DIR="{}"'.format(common.buildroot_download_dir),
+            'BR2_DL_DIR="{}"'.format(self.env['buildroot_download_dir']),
         ])
-        if not args.build_linux:
+        if not self.env['build_linux']:
             configs.extend([
                 '# BR2_LINUX_KERNEL is not set',
             ])
         config_fragments = []
-        if not args.baseline:
+        if not self.env['baseline']:
             configs.extend([
                 'BR2_GLOBAL_PATCH_DIR="{}"'.format(
-                    self._path_relative_to_buildroot(os.path.join(common.root_dir, 'patches', 'global'))
+                    self._path_relative_to_buildroot(os.path.join(self.env['root_dir'], 'patches', 'global'))
                 ),
                 'BR2_PACKAGE_BUSYBOX_CONFIG_FRAGMENT_FILES="{}"'.format(
-                    self._path_relative_to_buildroot(os.path.join(common.root_dir, 'busybox_config_fragment'))
+                    self._path_relative_to_buildroot(os.path.join(self.env['root_dir'], 'busybox_config_fragment'))
                 ),
                 'BR2_PACKAGE_OVERRIDE_FILE="{}"'.format(
-                    self._path_relative_to_buildroot(os.path.join(common.root_dir, 'buildroot_override'))
+                    self._path_relative_to_buildroot(os.path.join(self.env['root_dir'], 'buildroot_override'))
                 ),
                 'BR2_ROOTFS_POST_BUILD_SCRIPT="{}"'.format(
-                    self._path_relative_to_buildroot(os.path.join(common.root_dir, 'rootfs-post-build-script'))
+                    self._path_relative_to_buildroot(os.path.join(self.env['root_dir'], 'rootfs-post-build-script'))
                 ),
                 'BR2_ROOTFS_USERS_TABLES="{}"'.format(
-                    self._path_relative_to_buildroot(os.path.join(common.root_dir, 'user_table'))
+                    self._path_relative_to_buildroot(os.path.join(self.env['root_dir'], 'user_table'))
                 ),
             ])
-            if not args.no_overlay:
+            if not self.env['no_overlay']:
                 configs.append('BR2_ROOTFS_OVERLAY="{}"'.format(
-                    self._path_relative_to_buildroot(common.out_rootfs_overlay_dir)
+                    self._path_relative_to_buildroot(self.env['out_rootfs_overlay_dir'])
                 ))
             config_fragments = [
-                os.path.join(common.root_dir, 'buildroot_config', 'default')
+                os.path.join(self.env['root_dir'], 'buildroot_config', 'default')
-            ] + args.config_fragment
+            ] + self.env['config_fragment']
-        common.write_configs(common.buildroot_config_file, configs, config_fragments)
+        if self.env['initrd'] or self.env['initramfs']:
-        common.run_cmd(
+            configs.append('BR2_TARGET_ROOTFS_CPIO=y')
+        # TODO Can't get rid of these for now with nice fragments on Buildroot:
+        # http://stackoverflow.com/questions/44078245/is-it-possible-to-use-config-fragments-with-buildroots-config
+        self.sh.write_configs(self.env['buildroot_config_file'], configs, config_fragments)
+        self.sh.run_cmd(
             [
-                'make', common.Newline,
+                'make', LF,
-                'O={}'.format(common.buildroot_build_dir), common.Newline,
+                'O={}'.format(self.env['buildroot_build_dir']), LF,
-                'olddefconfig', common.Newline,
+                'olddefconfig', LF,
             ],
-            cwd=common.buildroot_src_dir,
+            cwd=self.env['buildroot_source_dir'],
         )
-        common.make_build_dirs()
+        self.make_build_dirs()
-        if not args.no_all:
+        if not self.env['no_all']:
-            extra_make_args.extend(['all', common.Newline])
+            extra_make_args.extend(['all', LF])
-        common.run_cmd(
+        self.sh.run_cmd(
             [
-                'make', common.Newline,
+                'make', LF,
-                'LKMC_GEM5_SRCDIR="{}"'.format(common.gem5_src_dir), common.Newline,
+                'LKMC_PARSEC_BENCHMARK_SRCDIR="{}"'.format(self.env['parsec_benchmark_source_dir']), LF,
-                'LKMC_PARSEC_BENCHMARK_SRCDIR="{}"'.format(common.parsec_benchmark_src_dir), common.Newline,
+                'O={}'.format(self.env['buildroot_build_dir']), LF,
-                'O={}'.format(common.buildroot_build_dir), common.Newline,
+                'V={}'.format(int(self.env['verbose'])), LF,
-                'V={}'.format(int(args.verbose)), common.Newline,
             ] +
             extra_make_args
             ,
-            out_file=os.path.join(common.buildroot_build_dir, 'lkmc.log'),
+            out_file=os.path.join(self.env['buildroot_build_dir'], 'lkmc.log'),
             delete_env=['LD_LIBRARY_PATH'],
-            cwd=common.buildroot_src_dir,
+            cwd=self.env['buildroot_source_dir'],
         )
         # Create the qcow2 from ext2.
         # Skip if qemu is not present, because gem5 does not need the qcow2.
         # so we don't force a QEMU build for gem5.
-        if not args.no_all and os.path.exists(common.qemu_img_executable):
+        if not self.env['no_all'] and os.path.exists(self.env['qemu_img_executable']):
-            common.raw_to_qcow2()
+            self.raw_to_qcow2()
 
-    def get_argparse_args(self):
+    def get_build_dir(self):
-        return {
+        return self.env['buildroot_build_dir']
-            'description': '''\
-Run Linux on an emulator
-'''
-        }
-
-    def get_build_dir(self, args):
-        return common.buildroot_build_dir
-
-    _defaults = {
-        'baseline': False,
-        'build_linux': False,
-        'config': [],
-        'config_fragment': [],
-        'extra_make_args': [],
-        'no_all': False,
-        'skip_configure': False,
-    }
 
     def _path_relative_to_buildroot(self, abspath):
-        return os.path.relpath(abspath, common.buildroot_src_dir)
+        return os.path.relpath(abspath, self.env['buildroot_source_dir'])
 
 if __name__ == '__main__':
-    BuildrootComponent().build()
+    Main().cli()

build-crosstool-ng

@@ -3,75 +3,71 @@
 import os
 
 import common
+from shell_helpers import LF
 
-class CrosstoolNgComponent(common.Component):
+class Main(common.BuildCliFunction):
-    def do_build(self, args):
+    def __init__(self):
-        common.assert_crosstool_ng_supports_arch(args.arch)
+        super().__init__(
-        build_dir = self.get_build_dir(args)
+            description='''\
-        defconfig_dest = os.path.join(common.crosstool_ng_util_dir, 'defconfig')
+Build crosstool-NG with Newlib for bare metal compilation
-        os.makedirs(common.crosstool_ng_util_dir, exist_ok=True)
+''',
-        os.makedirs(common.crosstool_ng_download_dir, exist_ok=True)
+            supported_archs=common.consts['crosstool_ng_supported_archs']
+        )
+
+    def build(self):
+        build_dir = self.get_build_dir()
+        defconfig_dest = os.path.join(self.env['crosstool_ng_util_dir'], 'defconfig')
+        os.makedirs(self.env['crosstool_ng_util_dir'], exist_ok=True)
+        os.makedirs(self.env['crosstool_ng_download_dir'], exist_ok=True)
 
         # Bootstrap out-ot-tree WONTFIX. I've tried.
         # https://github.com/crosstool-ng/crosstool-ng/issues/1021
-        os.chdir(common.crosstool_ng_src_dir)
+        os.chdir(self.env['crosstool_ng_source_dir'])
-        common.run_cmd(
+        self.sh.run_cmd(
-            [os.path.join(common.crosstool_ng_src_dir, 'bootstrap'), common.Newline],
+            [os.path.join(self.env['crosstool_ng_source_dir'], 'bootstrap'), LF],
         )
-        os.chdir(common.crosstool_ng_util_dir)
+        os.chdir(self.env['crosstool_ng_util_dir'])
-        common.run_cmd(
+        self.sh.run_cmd(
             [
-                os.path.join(common.crosstool_ng_src_dir, 'configure'), common.Newline,
+                os.path.join(self.env['crosstool_ng_source_dir'], 'configure'), LF,
-                '--enable-local', common.Newline,
+                '--enable-local', LF,
-            ],
-        )
-        common.run_cmd(
-            [
-                'make', common.Newline,
-                '-j', str(args.nproc), common.Newline,
             ],
         )
+        self.sh.run_cmd(['make', '-j', str(self.env['nproc']), LF])
 
         # Build the toolchain.
-        common.cp(
+        self.sh.cp(
-            os.path.join(common.root_dir, 'crosstool_ng_config', args.arch),
+            os.path.join(self.env['root_dir'], 'crosstool_ng_config', self.env['arch']),
             defconfig_dest
         )
-        common.write_configs(
+        self.sh.write_configs(
-            common.crosstool_ng_defconfig,
+            self.env['crosstool_ng_defconfig'],
             [
-                'CT_PREFIX_DIR="{}"'.format(common.crosstool_ng_install_dir),
+                'CT_PREFIX_DIR="{}"'.format(self.env['crosstool_ng_install_dir']),
                 'CT_WORK_DIR="{}"'.format(build_dir),
-                'CT_LOCAL_TARBALLS_DIR="{}"'.format(common.crosstool_ng_download_dir),
+                'CT_LOCAL_TARBALLS_DIR="{}"'.format(self.env['crosstool_ng_download_dir']),
             ]
         )
-        common.run_cmd(
+        self.sh.run_cmd(
             [
-                common.crosstool_ng_executable, common.Newline,
+                self.env['crosstool_ng_executable'], LF,
-                'defconfig', common.Newline,
+                'defconfig', LF,
             ],
         )
-        os.unlink(defconfig_dest)
+        self.sh.rmrf(defconfig_dest)
-        common.run_cmd(
+        self.sh.run_cmd(
             [
-                common.crosstool_ng_executable, common.Newline,
+                self.env['crosstool_ng_executable'], LF,
-                'build', common.Newline,
+                'build', LF,
-                'CT_JOBS={}'.format(str(args.nproc)), common.Newline,
+                'CT_JOBS={}'.format(str(self.env['nproc'])), LF,
             ],
             out_file=os.path.join(build_dir, 'lkmc.log'),
             delete_env=['LD_LIBRARY_PATH'],
-            extra_paths=[common.ccache_dir],
+            extra_paths=[self.env['ccache_dir']],
         )
 
-    def get_argparse_args(self):
+    def get_build_dir(self):
-        return {
+        return self.env['crosstool_ng_build_dir']
-            'description': '''\
-Build crosstool-NG with Newlib for bare metal compilation'
-'''
-        }
-
-    def get_build_dir(self, args):
-        return common.crosstool_ng_build_dir
 
 if __name__ == '__main__':
-    CrosstoolNgComponent().build()
+    Main().cli()

build-doc

@@ -1,2 +1,38 @@
-#!/usr/bin/env bash
+#!/usr/bin/env python3
-asciidoctor -o out/README.html -v README.adoc
+
+import re
+
+import common
+from shell_helpers import LF
+
+class Main(common.LkmcCliFunction):
+    def __init__(self):
+        super().__init__(
+            defaults = {
+                'print_time': False,
+            },
+            description='''\
+https://github.com/cirosantilli/linux-kernel-module-cheat#build-the-documentation
+''',
+        )
+
+    def timed_main(self):
+        self.sh.run_cmd(
+            [
+                'asciidoctor', LF,
+                '-o', self.env['readme_out'], LF,
+                '-v', self.env['readme'], LF,
+            ],
+            out_file=self.env['build_doc_log'],
+        )
+        error_re = re.compile('^asciidoctor: WARNING: ')
+        exit_status = 0
+        with open(self.env['build_doc_log']) as f:
+            for line in f:
+                if error_re.search(line):
+                    exit_status = 1
+                    break
+        return exit_status
+
+if __name__ == '__main__':
+    Main().cli()

build-docker

@@ -5,9 +5,10 @@ import subprocess
 import tarfile
 
 import common
+from shell_helpers import LF
 
 
-class DockerComponent(common.Component):
+class DockerComponent(self.Component):
     def get_argparse_args(self):
         return {
             'description': '''\
@@ -17,8 +18,8 @@ See also:https://github.com/cirosantilli/linux-kernel-module-cheat#ubuntu-guest-
 '''
         }
 
-    def do_build(self, args):
+    def build(self):
-        build_dir = self.get_build_dir(args)
+        build_dir = self.get_build_dir()
         container_name = 'lkmc-guest'
         target_dir = os.path.join('/root', 'linux-kernel-module-cheat')
         os.makedirs(build_dir, exist_ok=True)
@@ -29,12 +30,12 @@ See also:https://github.com/cirosantilli/linux-kernel-module-cheat#ubuntu-guest-
                 '--format', '{{.Names}}',
         ]).decode()
         if container_name in containers.split():
-            common.run_cmd([
+            self.sh.run_cmd([
                 'docker',
                 'rm',
                 container_name,
             ])
-        common.run_cmd([
+        self.sh.run_cmd([
             'docker',
             'create',
             '--name', container_name,
@@ -44,36 +45,36 @@ See also:https://github.com/cirosantilli/linux-kernel-module-cheat#ubuntu-guest-
             '--privileged',
             '-t',
             '-w', target_dir,
-            '-v', '{}:{}'.format(common.root_dir, target_dir),
+            '-v', '{}:{}'.format(kwargs['root_dir'], target_dir),
             'ubuntu:18.04',
             'bash',
         ])
-        common.run_cmd([
+        self.sh.run_cmd([
             'docker',
             'export',
             '-o',
-            common.docker_tar_file,
+            kwargs['docker_tar_file'],
             container_name,
         ])
-        tar = tarfile.open(common.docker_tar_file)
+        tar = tarfile.open(kwargs['docker_tar_file'])
-        tar.extractall(common.docker_tar_dir)
+        tar.extractall(kwargs['docker_tar_dir'])
         tar.close()
         # sudo not required in theory
         # https://askubuntu.com/questions/1046828/how-to-run-libguestfs-tools-tools-such-as-virt-make-fs-without-sudo
-        common.run_cmd([
+        self.sh.run_cmd([
             'virt-make-fs',
             '--format', 'raw',
             '--size', '+1G',
             '--type', 'ext2',
-            common.docker_tar_dir,
+            kwargs['docker_tar_dir'],
-            common.docker_rootfs_raw_file,
+            kwargs['docker_rootfs_raw_file'],
         ])
-        common.raw_to_qcow2(prebuilt=True)
+        self.raw_to_qcow2(prebuilt=True)
 
-    def get_build_dir(self, args):
+    def get_build_dir(self):
-        return common.docker_build_dir
+        return kwargs['docker_build_dir']
 
     def get_default_args(self):
         return {'docker': True}
 
-DockerComponent().build()
+Main().cli()

build-gem5

@@ -5,110 +5,105 @@ import pathlib
 import subprocess
 
 import common
+from shell_helpers import LF
 
-class Gem5Component(common.Component):
+class Main(common.BuildCliFunction):
-    def add_parser_arguments(self, parser):
+    def __init__(self):
-        parser.add_argument(
+        super().__init__()
+        self.add_argument(
             'extra_scons_args',
-            default=[],
             metavar='extra-scons-args',
-            nargs='*'
+            nargs='*',
         )
 
-    def do_build(self, args):
+    def build(self):
-        build_dir = self.get_build_dir(args)
+        build_dir = self.get_build_dir()
-        binaries_dir = os.path.join(common.gem5_system_dir, 'binaries')
+        binaries_dir = os.path.join(self.env['gem5_system_dir'], 'binaries')
-        disks_dir = os.path.join(common.gem5_system_dir, 'disks')
+        disks_dir = os.path.join(self.env['gem5_system_dir'], 'disks')
         os.makedirs(binaries_dir, exist_ok=True)
         os.makedirs(disks_dir, exist_ok=True)
-        if args.gem5_source_dir is None:
+        if self.env['gem5_source_dir'] is None:
-            if not os.path.exists(os.path.join(common.gem5_src_dir, '.git')):
+            if not os.path.exists(os.path.join(self.env['gem5_source_dir'], '.git')):
-                if common.gem5_src_dir == common.gem5_default_src_dir:
+                if self.env['gem5_source_dir'] == self.env['gem5_default_source_dir']:
                     raise Exception('gem5 submodule not checked out')
-                common.run_cmd([
+                self.sh.run_cmd([
-                    'git', common.Newline,
+                    'git', LF,
-                    '-C', common.gem5_default_src_dir, common.Newline,
+                    '-C', self.env['gem5_default_source_dir'], LF,
-                    'worktree', 'add', common.Newline,
+                    'worktree', 'add', LF,
-                    '-b', os.path.join('wt', args.gem5_build_id), common.Newline,
+                    '-b', os.path.join('wt', self.env['gem5_build_id']), LF,
-                    common.gem5_src_dir, common.Newline,
+                    self.env['gem5_source_dir'], LF,
                 ])
-        if args.verbose:
+        if self.env['verbose']:
-            verbose = ['--verbose', common.Newline]
+            verbose = ['--verbose', LF]
         else:
             verbose = []
-        if args.arch == 'x86_64':
+        if self.env['arch'] == 'x86_64':
             dummy_img_path = os.path.join(disks_dir, 'linux-bigswap2.img')
             with open(dummy_img_path, 'wb') as dummy_img_file:
                 zeroes = b'\x00' * (2 ** 16)
                 for i in range(2 ** 10):
                     dummy_img_file.write(zeroes)
-            common.run_cmd(['mkswap', dummy_img_path])
+            self.sh.run_cmd(['mkswap', dummy_img_path, LF])
             with open(os.path.join(binaries_dir, 'x86_64-vmlinux-2.6.22.9'), 'w'):
                 # This file must always be present, despite --kernel overriding that default and selecting the kernel.
                 # I'm not even joking. No one has ever built x86 gem5 without the magic dist dir present.
                 pass
-        elif args.arch == 'arm' or args.arch == 'aarch64':
+        elif self.env['arch'] == 'arm' or self.env['arch'] == 'aarch64':
-            gem5_system_src_dir = os.path.join(common.gem5_src_dir, 'system')
+            gem5_system_source_dir = os.path.join(self.env['gem5_source_dir'], 'system')
 
             # dtb
-            dt_src_dir = os.path.join(gem5_system_src_dir, 'arm', 'dt')
+            dt_source_dir = os.path.join(gem5_system_source_dir, 'arm', 'dt')
-            dt_build_dir = os.path.join(common.gem5_system_dir, 'arm', 'dt')
+            dt_build_dir = os.path.join(self.env['gem5_system_dir'], 'arm', 'dt')
-            common.run_cmd([
+            self.sh.run_cmd(['make', '-C', dt_source_dir, LF])
-                'make', common.Newline,
+            self.sh.copy_dir_if_update_non_recursive(
-                '-C', dt_src_dir, common.Newline,
+                srcdir=dt_source_dir,
-            ])
-            common.copy_dir_if_update_non_recursive(
-                srcdir=dt_src_dir,
                 destdir=dt_build_dir,
                 filter_ext='.dtb',
             )
 
             # Bootloader 32.
-            bootloader32_dir = os.path.join(gem5_system_src_dir, 'arm', 'simple_bootloader')
+            bootloader32_dir = os.path.join(gem5_system_source_dir, 'arm', 'simple_bootloader')
             # TODO use the buildroot cross compiler here, and remove the dependencies from configure.
-            common.run_cmd([
+            self.sh.run_cmd([
-                'make', common.Newline,
+                'make', LF,
-                '-C', bootloader32_dir, common.Newline,
+                '-C', bootloader32_dir, LF,
-                'CROSS_COMPILE=arm-linux-gnueabihf-', common.Newline,
+                'CROSS_COMPILE=arm-linux-gnueabihf-', LF,
             ])
             # bootloader
-            common.cp(os.path.join(bootloader32_dir, 'boot_emm.arm'), binaries_dir)
+            self.sh.cp(os.path.join(bootloader32_dir, 'boot_emm.arm'), binaries_dir)
 
             # Bootloader 64.
-            bootloader64_dir = os.path.join(gem5_system_src_dir, 'arm', 'aarch64_bootloader')
+            bootloader64_dir = os.path.join(gem5_system_source_dir, 'arm', 'aarch64_bootloader')
             # TODO cross_compile is ignored because the make does not use CC...
-            common.run_cmd([
+            self.sh.run_cmd(['make', '-C', bootloader64_dir, LF])
-                'make', common.Newline,
+            self.sh.cp(os.path.join(bootloader64_dir, 'boot_emm.arm64'), binaries_dir)
-                '-C', bootloader64_dir, common.Newline
+        self.sh.run_cmd(
-            ])
-            common.cp(os.path.join(bootloader64_dir, 'boot_emm.arm64'), binaries_dir)
-        common.run_cmd(
             (
                 [
-                    'scons', common.Newline,
+                    'scons', LF,
-                    '-j', str(args.nproc), common.Newline,
+                    '-j', str(self.env['nproc']), LF,
-                    '--gold-linker', common.Newline,
+                    '--gold-linker', LF,
-                    '--ignore-style', common.Newline,
+                    '--ignore-style', LF,
-                    common.gem5_executable, common.Newline,
+                    self.env['gem5_executable'], LF,
                 ] +
                 verbose +
-                common.add_newlines(args.extra_scons_args)
+                self.sh.add_newlines(self.env['extra_scons_args'])
             ),
-            cwd=common.gem5_src_dir,
+            cwd=self.env['gem5_source_dir'],
-            extra_paths=[common.ccache_dir],
+            extra_paths=[self.env['ccache_dir']],
         )
-        term_src_dir = os.path.join(common.gem5_src_dir, 'util/term')
+        term_source_dir = os.path.join(self.env['gem5_source_dir'], 'util/term')
-        m5term_build = os.path.join(term_src_dir, 'm5term')
+        m5term_build = os.path.join(term_source_dir, 'm5term')
-        common.run_cmd(['make', '-C', term_src_dir])
+        self.sh.run_cmd(['make', '-C', term_source_dir, LF])
-        if os.path.exists(common.gem5_m5term):
+        if os.path.exists(self.env['gem5_m5term']):
-            # Otherwise common.cp would fail with "Text file busy" if you
+            # Otherwise self.sh.cp would fail with "Text file busy" if you
             # tried to rebuild while running m5term:
             # https://stackoverflow.com/questions/16764946/what-generates-the-text-file-busy-message-in-unix/52427512#52427512
-            os.unlink(common.gem5_m5term)
+            self.sh.rmrf(self.env['gem5_m5term'])
-        common.cp(m5term_build, common.gem5_m5term)
+        self.sh.cp(m5term_build, self.env['gem5_m5term'])
 
-    def get_build_dir(self, args):
+    def get_build_dir(self):
-        return common.gem5_build_dir
+        return self.env['gem5_build_dir']
 
 if __name__ == '__main__':
-    Gem5Component().build()
+    Main().cli()

build-linux

@@ -4,10 +4,16 @@ import os
 import shutil
 
 import common
+from shell_helpers import LF
 
-class LinuxComponent(common.Component):
+class Main(common.BuildCliFunction):
-    def add_parser_arguments(self, parser):
+    def __init__(self):
-        parser.add_argument(
+        super().__init__(
+            description='''\
+Build the Linux kernel.
+'''
+        )
+        self.add_argument(
             '--config', default=[], action='append',
             help='''\
 Add a single kernel config configs to the current build. Sample value:
@@ -15,14 +21,20 @@ Add a single kernel config configs to the current build. Sample value:
 configs. Takes precedence over any config files.
 '''
         )
-        parser.add_argument(
+        self.add_argument(
             '--config-fragment', default=[], action='append',
             help='''\
 Also use the given kernel configuration fragment file.
 Pass multiple times to use multiple fragment files.
 '''
         )
-        parser.add_argument(
+        self.add_argument(
+            '--config-only', default=False,
+            help='''\
+Configure the kernel, but don't build it.
+'''
+        )
+        self.add_argument(
             '--custom-config-file',
             help='''\
 Ignore all default kernel configurations and use this file instead.
@@ -30,137 +42,126 @@ Still uses options explicitly passed with `--config` and
 `--config-fragment` on top of it.
 '''
         )
-        parser.add_argument(
+        self.add_argument(
-            '--custom-config-file-gem5', default=False, action='store_true',
+            '--custom-config-file-gem5', default=False,
             help='''\
 Use the gem5 Linux kernel fork config as the custom config file.
 Ignore --custom-config-file.
 '''
         )
-        parser.add_argument(
+        self.add_argument(
-            '--config-only', default=False, action='store_true',
+            '--modules-install', default=True,
             help='''\
-Configure the kernel, but don't build it.
+Run `make modules_install` after `make`.
 '''
         )
-        parser.add_argument(
+        self.add_argument(
-            '--initramfs', default=False, action='store_true',
-        )
-        parser.add_argument(
-            '--initrd', default=False, action='store_true',
-        )
-        parser.add_argument(
             'extra_make_args',
             default=[],
             metavar='extra-make-args',
             nargs='*'
         )
 
-    def do_build(self, args):
+    def build(self):
-        build_dir = self.get_build_dir(args)
+        build_dir = self.get_build_dir()
-        if args.initrd or args.initramfs:
-            raise Exception('just trolling, --initrd and --initramfs are broken for now')
         os.makedirs(build_dir, exist_ok=True)
         tool = 'gcc'
-        gcc = common.get_toolchain_tool(tool)
+        gcc = self.get_toolchain_tool(tool)
         prefix = gcc[:-len(tool)]
         common_args = {
-            'cwd': common.linux_source_dir,
+            'cwd': self.env['linux_source_dir'],
         }
         ccache = shutil.which('ccache')
         if ccache is not None:
             cc = '{} {}'.format(ccache, gcc)
         else:
             cc = gcc
-        if args.verbose:
+        if self.env['verbose']:
             verbose = ['V=1']
         else:
             verbose = []
         common_make_args = [
-            'make', common.Newline,
+            'make', LF,
-            '-j', str(args.nproc), common.Newline,
+            '-j', str(self.env['nproc']), LF,
-            'ARCH={}'.format(common.linux_arch), common.Newline,
+            'ARCH={}'.format(self.env['linux_arch']), LF,
-            'CROSS_COMPILE={}'.format(prefix), common.Newline,
+            'CROSS_COMPILE={}'.format(prefix), LF,
-            'CC={}'.format(cc), common.Newline,
+            'CC={}'.format(cc), LF,
-            'O={}'.format(build_dir), common.Newline,
+            'O={}'.format(build_dir), LF,
         ] + verbose
-        if args.custom_config_file_gem5:
+        if self.env['custom_config_file_gem5']:
-            custom_config_file = os.path.join(common.linux_source_dir, 'arch', common.linux_arch, 'configs', 'gem5_defconfig')
+            custom_config_file = os.path.join(self.env['linux_source_dir'], 'arch', self.env['linux_arch'], 'configs', 'gem5_defconfig')
         else:
-            custom_config_file = args.custom_config_file
+            custom_config_file = self.env['custom_config_file']
         if custom_config_file is not None:
             if not os.path.exists(custom_config_file):
-                raise Exception('config fragment file does not exist: {}'.format(args.custom_config_file))
+                raise Exception('config fragment file does not exist: {}'.format(custom_config_file))
             base_config_file = custom_config_file
             config_fragments = []
         else:
-            base_config_file = os.path.join(common.linux_config_dir, 'buildroot-{}'.format(args.arch))
+            base_config_file = os.path.join(self.env['linux_config_dir'], 'buildroot-{}'.format(self.env['arch']))
             config_fragments = ['min', 'default']
             for i, config_fragment in enumerate(config_fragments):
-                config_fragments[i] = os.path.join(common.linux_config_dir, config_fragment)
+                config_fragments[i] = os.path.join(self.env['linux_config_dir'], config_fragment)
-        config_fragments.extend(args.config_fragment)
+        config_fragments.extend(self.env['config_fragment'])
-        if args.config != []:
+        cli_configs = self.env['config']
+        if self.env['initramfs']:
+            cli_configs.append('CONFIG_INITRAMFS_SOURCE="{}"'.format(self.env['buildroot_cpio']))
+        if cli_configs:
             cli_config_fragment_path = os.path.join(build_dir, 'lkmc_cli_config_fragment')
-            cli_config_str = '\n'.join(args.config)
+            self.sh.write_configs(cli_config_fragment_path, cli_configs, mode='w')
-            common.write_string_to_file(cli_config_fragment_path, cli_config_str)
             config_fragments.append(cli_config_fragment_path)
-        common.cp(
+        self.sh.cp(
             base_config_file,
             os.path.join(build_dir, '.config'),
         )
-        common.run_cmd(
+        self.sh.run_cmd(
             [
-                os.path.join(common.linux_source_dir, 'scripts', 'kconfig', 'merge_config.sh'), common.Newline,
+                os.path.join(self.env['linux_source_dir'], 'scripts', 'kconfig', 'merge_config.sh'), LF,
-                '-m', common.Newline,
+                '-m', LF,
-                '-O', build_dir, common.Newline,
+                '-O', build_dir, LF,
-                os.path.join(build_dir, '.config'), common.Newline,
+                os.path.join(build_dir, '.config'), LF,
             ] +
-            common.add_newlines(config_fragments)
+            self.sh.add_newlines(config_fragments)
         )
-        common.run_cmd(
+        self.sh.run_cmd(
             (
                 common_make_args +
-                ['olddefconfig', common.Newline]
+                ['olddefconfig', LF]
             ),
             **common_args
         )
-        if not args.config_only:
+        if not self.env['config_only']:
-            common.run_cmd(
+            self.sh.run_cmd(
                 (
                     common_make_args +
-                    common.add_newlines(args.extra_make_args)
+                    self.sh.add_newlines(self.env['extra_make_args'])
                 ),
+                # https://github.com/cirosantilli/linux-kernel-module-cheat#proc-version
                 extra_env={
                     'KBUILD_BUILD_VERSION': '1',
                     'KBUILD_BUILD_TIMESTAMP': 'Thu Jan  1 00:00:00 UTC 1970',
                     'KBUILD_BUILD_USER': 'lkmc',
-                    'KBUILD_BUILD_HOST': common.git_sha(common.linux_source_dir),
+                    'KBUILD_BUILD_HOST': common.git_sha(self.env['linux_source_dir']),
                 },
                 **common_args
             )
-            common.run_cmd(
+            if self.env['modules_install']:
-                (
+                self.sh.run_cmd(
-                    common_make_args +
+                    (
-                    [
+                        common_make_args +
-                        'INSTALL_MOD_PATH={}'.format(common.out_rootfs_overlay_dir), common.Newline,
+                        [
-                        'modules_install', common.Newline,
+                            'INSTALL_MOD_PATH={}'.format(self.env['out_rootfs_overlay_dir']), LF,
-                    ]
+                            'modules_install', LF,
-                ),
+                        ]
-                **common_args
+                    ),
-            )
+                    **common_args
-            # TODO: remove build and source https://stackoverflow.com/questions/13578618/what-does-build-and-source-link-do-in-lib-modules-kernel-version
+                )
-            # TODO Basically all kernel modules also basically leak full host paths. Just terrible. Buildroot deals with that stuff nicely for us.
+                # TODO: remove build and source https://stackoverflow.com/questions/13578618/what-does-build-and-source-link-do-in-lib-modules-kernel-version
-            # common.rmrf()
+                # TODO Basically all kernel modules also basically leak full host paths. Just terrible. Buildroot deals with that stuff nicely for us.
+                # self.rmrf()
 
-    def get_argparse_args(self):
+    def get_build_dir(self):
-        return {
+        return self.env['linux_build_dir']
-            'description': '''\
-Build the Linux kernel.
-'''
-        }
-
-    def get_build_dir(self, args):
-        return common.linux_build_dir
 
 if __name__ == '__main__':
-    LinuxComponent().build()
+    Main().cli()

build-m5

@@ -3,42 +3,47 @@
 import os
 
 import common
+from shell_helpers import LF
 
-class M5Component(common.Component):
+class Main(common.BuildCliFunction):
-    def get_make_cmd(self, args):
+    def __init__(self):
+        super().__init__()
+
+    def _get_make_cmd(self):
         allowed_toolchains = ['buildroot']
-        cc = common.get_toolchain_tool('gcc', allowed_toolchains=allowed_toolchains)
+        cc = self.get_toolchain_tool('gcc', allowed_toolchains=allowed_toolchains)
-        ld = common.get_toolchain_tool('ld', allowed_toolchains=allowed_toolchains)
+        ld = self.get_toolchain_tool('ld', allowed_toolchains=allowed_toolchains)
-        if args.arch == 'x86_64':
+        if self.env['arch'] == 'x86_64':
             arch = 'x86'
         else:
-            arch = args.arch
+            arch = self.env['arch']
         return [
-            'make', common.Newline,
+            'make', LF,
-            '-j', str(args.nproc), common.Newline,
+            '-j', str(self.env['nproc']), LF,
-            '-f', 'Makefile.{}'.format(arch), common.Newline,
+            '-f', 'Makefile.{}'.format(arch), LF,
-            'CC={}'.format(cc), common.Newline,
+            'CC={}'.format(cc), LF,
-            'LD={}'.format(ld), common.Newline,
+            'LD={}'.format(ld), LF,
-            'PWD={}'.format(common.gem5_m5_src_dir), common.Newline,
+            'PWD={}'.format(self.env['gem5_m5_source_dir']), LF,
         ]
 
-    def do_build(self, args):
+    def build(self):
-        os.makedirs(common.gem5_m5_build_dir, exist_ok=True)
+        os.makedirs(self.env['gem5_m5_build_dir'], exist_ok=True)
         # We must clean first or else the build outputs of one arch can conflict with the other.
         # I should stop being lazy and go actually patch gem5 to support out of tree m5 build...
-        self.clean(args)
+        self.clean()
-        common.run_cmd(
+        self.sh.run_cmd(
-            self.get_make_cmd(args),
+            self._get_make_cmd(),
-            cwd=common.gem5_m5_src_dir,
+            cwd=self.env['gem5_m5_source_dir'],
         )
-        os.makedirs(common.out_rootfs_overlay_bin_dir, exist_ok=True)
+        os.makedirs(self.env['out_rootfs_overlay_bin_dir'], exist_ok=True)
-        common.cp(os.path.join(common.gem5_m5_src_dir, 'm5'), common.out_rootfs_overlay_bin_dir)
+        self.sh.cp(os.path.join(self.env['gem5_m5_source_dir'], 'm5'), self.env['out_rootfs_overlay_bin_dir'])
 
-    def clean(self, args):
+    def clean(self):
-        common.run_cmd(
+        self.sh.run_cmd(
-            self.get_make_cmd(args) + ['clean', common.Newline],
+            self._get_make_cmd() + ['clean', LF],
-            cwd=common.gem5_m5_src_dir,
+            cwd=self.env['gem5_m5_source_dir'],
         )
+        return None
 
 if __name__ == '__main__':
-    M5Component().build()
+    Main().cli()

build-modules

@@ -6,32 +6,37 @@ import platform
 import shutil
 
 import common
+from shell_helpers import LF
 
-class ModulesComponent(common.Component):
+class Main(common.BuildCliFunction):
-    def add_parser_arguments(self, parser):
+    def __init__(self):
-        parser.add_argument(
+        super().__init__(
+            description='''\
+Build our Linux kernel modules without using Buildroot.
+
+See also: https://github.com/cirosantilli/linux-kernel-module-cheat#host
+''')
+        self.add_argument(
             '--make-args',
             default='',
         )
-        parser.add_argument(
+        self.add_argument(
             '--host',
-            action='store_true',
             default=False,
             help='''\
 Build the Linux kernel modules for the host instead of guest.
 Use the host packaged cross toolchain.
 ''',
         )
-        parser.add_argument(
+        self.add_argument(
-            'kernel_modules',
+            'kernel-modules',
             default=[],
             help='Which kernel modules to build. Default: build all',
-            metavar='kernel-modules',
             nargs='*',
         )
 
-    def do_build(self, args):
+    def build(self):
-        build_dir = self.get_build_dir(args)
+        build_dir = self.get_build_dir()
         os.makedirs(build_dir, exist_ok=True)
         # I kid you not, out-of-tree build is not possible, O= does not work as for the kernel build:
         #
@@ -42,87 +47,78 @@ Use the host packaged cross toolchain.
         # This copies only modified files as per:
         # https://stackoverflow.com/questions/5718899/building-an-out-of-tree-linux-kernel-module-in-a-separate-object-directory
         distutils.dir_util.copy_tree(
-            common.kernel_modules_src_dir,
+            self.env['kernel_modules_source_dir'],
-            os.path.join(build_dir, common.kernel_modules_subdir),
+            os.path.join(build_dir, self.env['kernel_modules_subdir']),
             update=1,
         )
         distutils.dir_util.copy_tree(
-            common.include_src_dir,
+            self.env['include_source_dir'],
-            os.path.join(build_dir, common.include_subdir),
+            os.path.join(build_dir, self.env['include_subdir']),
             update=1,
         )
         all_kernel_modules = []
-        for basename in os.listdir(common.kernel_modules_src_dir):
+        for basename in os.listdir(self.env['kernel_modules_source_dir']):
-            src = os.path.join(common.kernel_modules_src_dir, basename)
+            src = os.path.join(self.env['kernel_modules_source_dir'], basename)
             if os.path.isfile(src):
                 noext, ext = os.path.splitext(basename)
-                if ext == common.c_ext:
+                if ext == self.env['c_ext']:
                     all_kernel_modules.append(noext)
-        if args.kernel_modules == []:
+        if self.env['kernel_modules'] == []:
             kernel_modules = all_kernel_modules
         else:
-            kernel_modules = map(lambda x: os.path.splitext(os.path.split(x)[1])[0], args.kernel_modules)
+            kernel_modules = map(lambda x: os.path.splitext(os.path.split(x)[1])[0], self.env['kernel_modules'])
-        object_files = map(lambda x: x + common.obj_ext, kernel_modules)
+        object_files = map(lambda x: x + self.env['obj_ext'], kernel_modules)
         tool = 'gcc'
-        if args.host:
+        if self.env['host']:
             allowed_toolchains = ['host']
-            build_subdir = common.kernel_modules_build_host_subdir
+            build_subdir = self.env['kernel_modules_build_host_subdir']
         else:
             allowed_toolchains = None
-            build_subdir = common.kernel_modules_build_subdir
+            build_subdir = self.env['kernel_modules_build_subdir']
-        gcc = common.get_toolchain_tool(tool, allowed_toolchains=allowed_toolchains)
+        gcc = self.get_toolchain_tool(tool, allowed_toolchains=allowed_toolchains)
         prefix = gcc[:-len(tool)]
         ccache = shutil.which('ccache')
         if ccache is not None:
             cc = '{} {}'.format(ccache, gcc)
         else:
             cc = gcc
-        if args.verbose:
+        if self.env['verbose']:
             verbose = ['V=1']
         else:
             verbose = []
-        if args.host:
+        if self.env['host']:
             linux_dir = os.path.join('/lib', 'modules', platform.uname().release, 'build')
         else:
-            linux_dir = common.linux_build_dir
+            linux_dir = self.env['linux_build_dir']
-        common.run_cmd(
+        self.sh.run_cmd(
             (
                 [
-                    'make', common.Newline,
+                    'make', LF,
-                    '-j', str(args.nproc), common.Newline,
+                    '-j', str(self.env['nproc']), LF,
-                    'ARCH={}'.format(common.linux_arch), common.Newline,
+                    'ARCH={}'.format(self.env['linux_arch']), LF,
-                    'CC={}'.format(cc), common.Newline,
+                    'CC={}'.format(cc), LF,
-                    'CROSS_COMPILE={}'.format(prefix), common.Newline,
+                    'CROSS_COMPILE={}'.format(prefix), LF,
-                    'LINUX_DIR={}'.format(linux_dir), common.Newline,
+                    'LINUX_DIR={}'.format(linux_dir), LF,
-                    'M={}'.format(build_subdir), common.Newline,
+                    'M={}'.format(build_subdir), LF,
-                    'OBJECT_FILES={}'.format(' '.join(object_files)), common.Newline,
+                    'OBJECT_FILES={}'.format(' '.join(object_files)), LF,
                 ] +
-                common.shlex_split(args.make_args) +
+                self.sh.shlex_split(self.env['make_args']) +
                 verbose
             ),
-            cwd=os.path.join(common.kernel_modules_build_subdir),
+            cwd=os.path.join(self.env['kernel_modules_build_subdir']),
         )
-        if not args.host:
+        if not self.env['host']:
-            common.copy_dir_if_update_non_recursive(
+            self.sh.copy_dir_if_update_non_recursive(
-                srcdir=common.kernel_modules_build_subdir,
+                srcdir=self.env['kernel_modules_build_subdir'],
-                destdir=common.out_rootfs_overlay_dir,
+                destdir=self.env['out_rootfs_overlay_dir'],
-                filter_ext=common.kernel_module_ext,
+                filter_ext=self.env['kernel_module_ext'],
             )
 
-    def get_argparse_args(self):
+    def get_build_dir(self):
-        return {
+        if self.env['host']:
-                'description': '''\
+            return self.env['kernel_modules_build_host_dir']
-Build our Linux kernel modules without using Buildroot.
-
-See also: https://github.com/cirosantilli/linux-kernel-module-cheat#host
-'''
-        }
-
-    def get_build_dir(self, args):
-        if args.host:
-            return os.path.join(common.kernel_modules_build_host_dir)
         else:
-            return os.path.join(common.kernel_modules_build_dir)
+            return self.env['kernel_modules_build_dir']
 
 if __name__ == '__main__':
-    ModulesComponent().build()
+    Main().cli()

build-qemu

@@ -3,61 +3,62 @@
 import os
 
 import common
+from shell_helpers import LF
 
-class QemuComponent(common.Component):
+class Main(common.BuildCliFunction):
-    def add_parser_arguments(self, parser):
+    def __init__(self):
-        parser.add_argument(
+        super().__init__()
-            '--userland',
+        self.add_argument(
+            '--user-mode',
             default=False,
-            action='store_true',
             help='Build QEMU user mode instead of system.',
         )
-        parser.add_argument(
+        self.add_argument(
             'extra_config_args',
             default=[],
             metavar='extra-config-args',
             nargs='*'
         )
 
-    def do_build(self, args):
+    def build(self):
-        build_dir = self.get_build_dir(args)
+        build_dir = self.get_build_dir()
         os.makedirs(build_dir, exist_ok=True)
-        if args.verbose:
+        if self.env['verbose']:
             verbose = ['V=1']
         else:
             verbose = []
-        if args.userland:
+        if self.env['user_mode']:
-            target_list = '{}-linux-user'.format(args.arch)
+            target_list = '{}-linux-user'.format(self.env['arch'])
         else:
-            target_list = '{}-softmmu'.format(args.arch)
+            target_list = '{}-softmmu'.format(self.env['arch'])
-        common.run_cmd(
+        self.sh.run_cmd(
             [
-                os.path.join(common.qemu_src_dir, 'configure'), common.Newline,
+                os.path.join(self.env['qemu_source_dir'], 'configure'), LF,
-                '--enable-debug', common.Newline,
+                '--enable-debug', LF,
-                '--enable-trace-backends=simple', common.Newline,
+                '--enable-trace-backends=simple', LF,
-                '--target-list={}'.format(target_list), common.Newline,
+                '--target-list={}'.format(target_list), LF,
-                '--enable-sdl', common.Newline,
+                '--enable-sdl', LF,
-                '--with-sdlabi=2.0', common.Newline,
+                '--with-sdlabi=2.0', LF,
             ] +
-            common.add_newlines(args.extra_config_args),
+            self.sh.add_newlines(self.env['extra_config_args']),
-            extra_paths=[common.ccache_dir],
+            extra_paths=[self.env['ccache_dir']],
             cwd=build_dir
         )
-        common.run_cmd(
+        self.sh.run_cmd(
             (
                 [
-                    'make', common.Newline,
+                    'make', LF,
-                    '-j', str(args.nproc), common.Newline,
+                    '-j', str(self.env['nproc']), LF,
 
                 ] +
                 verbose
             ),
             cwd=build_dir,
-            extra_paths=[common.ccache_dir],
+            extra_paths=[self.env['ccache_dir']],
         )
 
-    def get_build_dir(self, args):
+    def get_build_dir(self):
-        return common.qemu_build_dir
+        return self.env['qemu_build_dir']
 
 if __name__ == '__main__':
-    QemuComponent().build()
+    Main().cli()

build-test

@@ -1,4 +1,6 @@
 #!/usr/bin/env bash
+# Build just enough to run ./test:
+# https://github.com/cirosantilli/linux-kernel-module-cheat#automated-tests
 set -eu
 test_size=1
 while [ $# -gt 0 ]; do
@@ -9,5 +11,5 @@ while [ $# -gt 0 ]; do
       ;;
   esac
 done
-./build-bench-boot --size "$test_size"
+./build-test-boot --size "$test_size"
-./build-test-gdb
+./build --all-archs test-gdb test-user-mode

build-test-boot

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# We want to move this into ./build. The only reason we haven't is that
+# what to build depends on --size, which ./build does not support right now.
+# The best way to solve this is to move the dependency checking into the run
+# scripts, which will take a while to refactor.
+set -eu
+test_size=1
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --size)
+      test_size="$2"
+      shift 2
+      ;;
+  esac
+done
+./build --all-archs qemu-gem5-buildroot
+if [ "$test_size" -ge 3 ]; then
+  ./build-gem5 --arch aarch64 --gem5-build-type debug
+  ./build-gem5 --arch aarch64 --gem5-build-type fast
+fi

build-test-gdb

@@ -1,2 +0,0 @@
-#!/usr/bin/env bash
-./build --all-archs all-baremetal

build-userland

@@ -1,16 +1,19 @@
 #!/usr/bin/env python3
 
 import os
-import platform
 import shlex
-import shutil
-import subprocess
 
 import common
+from shell_helpers import LF
 
-class UserlandComponent(common.Component):
+class Main(common.BuildCliFunction):
-    def add_parser_arguments(self, parser):
+    def __init__(self):
-        parser.add_argument(
+        super().__init__(
+            description='''\
+Build our compiled userland examples.
+'''
+        )
+        self.add_argument(
             '--has-package',
             action='append',
             default=[],
@@ -19,20 +22,27 @@ Indicate that a given package is present in the root filesystem, which
 allows us to build examples that rely on it.
 ''',
         )
-        parser.add_argument(
+        self.add_argument(
             '--host',
-            action='store_true',
             default=False,
             help='''\
 Build the userland programs for the host instead of guest.
 Use the host packaged cross toolchain.
 ''',
         )
-        parser.add_argument(
+        self.add_argument(
             '--make-args',
             default='',
         )
-        parser.add_argument(
+        self.add_argument(
+            '--static',
+            default=False,
+            help='''\
+Build the executables statically. TODO not implemented: Set the build id to 'static'
+if one was not given explicitly.
+''',
+        )
+        self.add_argument(
             'targets',
             default=[],
             help='''\
@@ -44,49 +54,47 @@ has the OpenBLAS libraries and headers installed.
             nargs='*',
         )
 
-    def do_build(self, args):
+    def build(self):
-        build_dir = self.get_build_dir(args)
+        build_dir = self.get_build_dir()
         os.makedirs(build_dir, exist_ok=True)
-        if args.host:
+        if self.env['host']:
             allowed_toolchains = ['host']
         else:
             allowed_toolchains = ['buildroot']
-        cc = common.get_toolchain_tool('gcc', allowed_toolchains=allowed_toolchains)
+        cc = self.get_toolchain_tool('gcc', allowed_toolchains=allowed_toolchains)
-        cxx = common.get_toolchain_tool('g++', allowed_toolchains=allowed_toolchains)
+        cxx = self.get_toolchain_tool('g++', allowed_toolchains=allowed_toolchains)
-        common.run_cmd(
+        make_args = shlex.split(self.env['make_args'])
+        if self.env['static']:
+            make_args.extend(['CCFLAGS_EXTRA=-static', LF])
+        self.sh.run_cmd(
             (
                 [
-                    'make', common.Newline,
+                    'make', LF,
-                    '-j', str(args.nproc), common.Newline,
+                    '-j', str(self.env['nproc']), LF,
-                    'ARCH={}'.format(args.arch), common.Newline,
+                    'ARCH={}'.format(self.env['arch']), LF,
-                    'CCFLAGS_SCRIPT={} {}'.format('-I', common.userland_src_dir), common.Newline,
+                    'CCFLAGS_SCRIPT={} {}'.format('-I', self.env['userland_source_dir']), LF,
-                    'COMMON_DIR={}'.format(common.root_dir), common.Newline,
+                    'COMMON_DIR={}'.format(self.env['root_dir']), LF,
-                    'CC={}'.format(cc), common.Newline,
+                    'CC={}'.format(cc), LF,
-                    'CXX={}'.format(cxx), common.Newline,
+                    'CXX={}'.format(cxx), LF,
-                    'PKG_CONFIG={}'.format(common.buildroot_pkg_config), common.Newline,
+                    'PKG_CONFIG={}'.format(self.env['buildroot_pkg_config']), LF,
-                    'STAGING_DIR={}'.format(common.buildroot_staging_dir), common.Newline,
+                    'STAGING_DIR={}'.format(self.env['buildroot_staging_dir']), LF,
-                    'OUT_DIR={}'.format(build_dir), common.Newline,
+                    'OUT_DIR={}'.format(build_dir), LF,
                 ] +
-                common.add_newlines(['HAS_{}=y'.format(package.upper()) for package in args.has_package]) +
+                self.sh.add_newlines(['HAS_{}=y'.format(package.upper()) for package in self.env['has_package']]) +
-                shlex.split(args.make_args) +
+                make_args +
-                common.add_newlines([os.path.join(build_dir, os.path.splitext(os.path.split(target)[1])[0]) + common.userland_build_ext for target in args.targets])
+                self.sh.add_newlines([os.path.join(build_dir, os.path.splitext(os.path.split(target)[1])[0]) + self.env['userland_build_ext'] for target in self.env['targets']])
             ),
-            cwd=common.userland_src_dir,
+            cwd=self.env['userland_source_dir'],
-            extra_paths=[common.ccache_dir],
+            extra_paths=[self.env['ccache_dir']],
         )
-        common.copy_dir_if_update_non_recursive(
+        self.sh.copy_dir_if_update_non_recursive(
             srcdir=build_dir,
-            destdir=common.out_rootfs_overlay_dir,
+            destdir=self.env['out_rootfs_overlay_dir'],
-            filter_ext=common.userland_build_ext,
+            filter_ext=self.env['userland_build_ext'],
         )
 
-    def get_argparse_args(self):
+    def get_build_dir(self):
-        return {
+        return self.env['userland_build_dir']
-            'description': 'Build our compiled userland examples',
-        }
-
-    def get_build_dir(self, args):
-        return common.userland_build_dir
 
 if __name__ == '__main__':
-    UserlandComponent().build()
+    Main().cli()

build-xen

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+# TODO get working, aarch64 Xen integration attempt.
+# Current state: prints to Boot-wrapper v0.2 to screen and hangs.
+# Bibliography:
+# https://wiki.xenproject.org/wiki/Xen_ARM_with_Virtualization_Extensions/qemu-system-aarch64
+# https://blog.xenproject.org/2014/04/01/virtualization-on-arm-with-xen/
+cd submodules/xen
+make \
+  -j`nproc` \
+  dist-xen \
+  CONFIG_DEBUG=y \
+  CONFIG_EARLY_PRINTK=vexpress \
+  CROSS_COMPILE=aarch64-linux-gnu- \
+  XEN_TARGET_ARCH=arm64 \
+;
+cd ../boot-wraper-aarch64
+autoreconf -i
+# DTB dumped from QEMU with: -machine dumpdtb=dtb.dtb
+./configure \
+  --enable-gicv3 \
+  --enable-psci \
+  --host=aarch64-linux-gnu \
+  --with-cmdline="console=hvc0 root=/dev/vda rw mem=1G" \
+  --with-dtb=dtb.dtb \
+  --with-kernel-dir=../../out/linux/default/aarch64 \
+  --with-xen-cmdline="dtuart=/uart@1c090000 console=dtuart no-bootscrub dom0_mem=1G loglvl=all guest_loglvl=all" \
+  --with-xen=../xen/xen/xen \
+;
+dtb.dtb -j`nproc`
+../../out/qemu/default/aarch64-softmmu/qemu-system-aarch64 \
+  -M virt \
+  -M virtualization=on \
+  -cpu cortex-a57 \
+  -kernel xen-system.axf \
+  -serial mon:stdio \
+  -nographic \
+;

buildroot_config/default

@@ -23,10 +23,6 @@ BR2_TARGET_ROOTFS_INITRAMFS=n
 BR2_TARGET_ROOTFS_TAR=n
 
 # Host GDB
-BR2_GDB_VERSION="7.11.1"
-BR2_GDB_VERSION_7_10=n
-BR2_GDB_VERSION_7_11=y
-BR2_GDB_VERSION_7_12=n
 BR2_PACKAGE_HOST_GDB=y
 BR2_PACKAGE_HOST_GDB_PYTHON=y
 BR2_PACKAGE_HOST_GDB_SIM=y

cli_function.py

@@ -0,0 +1,450 @@
+#!/usr/bin/env python3
+
+import argparse
+import bisect
+import collections
+import imp
+import os
+import sys
+
+class _Argument:
+    def __init__(
+            self,
+            long_or_short_1,
+            long_or_short_2=None,
+            default=None,
+            dest=None,
+            help=None,
+            nargs=None,
+            **kwargs
+        ):
+            self.args = []
+            # argparse is crappy and cannot tell us if arguments were given or not.
+            # We need that information to decide if the config file should override argparse or not.
+            # So we just use None as a sentinel.
+            self.kwargs = {'default': None}
+            shortname, longname, key, is_option = self.get_key(
+                long_or_short_1,
+                long_or_short_2,
+                dest
+            )
+            if shortname is not None:
+                self.args.append(shortname)
+            if is_option:
+                self.args.append(longname)
+            else:
+                self.args.append(key)
+                self.kwargs['metavar'] = longname
+                if default is not None and nargs is None:
+                    self.kwargs['nargs'] = '?'
+            if dest is not None:
+                self.kwargs['dest'] = dest
+            if nargs is not None:
+                self.kwargs['nargs'] = nargs
+            if default is True or default is False:
+                bool_action = 'store_true'
+                self.is_bool = True
+            else:
+                self.is_bool = False
+                if default is None and (
+                    nargs in ('*', '+')
+                    or ('action' in kwargs and kwargs['action'] == 'append')
+                ):
+                    default = []
+            if self.is_bool and not 'action' in kwargs:
+                self.kwargs['action'] = bool_action
+            if help is not None:
+                if default is not None:
+                    if help[-1] == '\n':
+                        if '\n\n' in help[:-1]:
+                            help += '\n'
+                    elif help[-1] == ' ':
+                        pass
+                    else:
+                        help += ' '
+                    help += 'Default: {}'.format(default)
+                self.kwargs['help'] = help
+            self.optional = (
+                default is not None or
+                self.is_bool or
+                is_option or
+                nargs in ('?', '*', '+')
+            )
+            self.kwargs.update(kwargs)
+            self.default = default
+            self.longname = longname
+            self.key = key
+            self.is_option = is_option
+            self.nargs = nargs
+
+    def __str__(self):
+        return str(self.args) + ' ' + str(self.kwargs)
+
+    @staticmethod
+    def get_key(
+        long_or_short_1,
+        long_or_short_2=None,
+        dest=None,
+        **kwargs
+    ):
+        if long_or_short_2 is None:
+            shortname = None
+            longname = long_or_short_1
+        else:
+            shortname = long_or_short_1
+            longname = long_or_short_2
+        if longname[0] == '-':
+            key = longname.lstrip('-').replace('-', '_')
+            is_option = True
+        else:
+            key = longname.replace('-', '_')
+            is_option = False
+        if dest is not None:
+            key = dest
+        return shortname, longname, key, is_option
+
+class CliFunction:
+    '''
+    Represent a function that can be called either from Python code, or
+    from the command line.
+
+    Features:
+
+    * single argument description in format very similar to argparse
+    * handle default arguments transparently in both cases
+    * expose a configuration file mechanism to get default parameters from a file
+    * fix some argparse.ArgumentParser() annoyances:
+    ** allow dashes in positional arguments:
+       https://stackoverflow.com/questions/12834785/having-options-in-argparse-with-a-dash
+    ** boolean defaults automatically use store_true or store_false, and add a --no-* CLI
+       option to invert them if set from the config
+    * from a Python call, get the corresponding CLI string list. See get_cli.
+    * easily determine if arguments were given on the command line
+      https://stackoverflow.com/questions/30487767/check-if-argparse-optional-argument-is-set-or-not/30491369
+
+    This somewhat duplicates: https://click.palletsprojects.com but:
+
+    * that decorator API is insane
+    * CLI + Python for single functions was wontfixed: https://github.com/pallets/click/issues/40
+    '''
+    def __call__(self, **kwargs):
+        '''
+        Python version of the function call. Not called by cli() indirectly,
+        so can be overridden to distinguish between Python and CLI calls.
+
+        :type arguments: Dict
+        '''
+        return self._do_main(kwargs)
+
+    def _do_main(self, kwargs):
+        return self.main(**self._get_args(kwargs))
+
+    def __init__(self, config_file=None, description=None, extra_config_params=None):
+        self._arguments = collections.OrderedDict()
+        self._config_file = config_file
+        self._description = description
+        self.extra_config_params = extra_config_params
+        if self._config_file is not None:
+            self.add_argument(
+                '--config-file',
+                default=self._config_file,
+                help='Path to the configuration file to use'
+            )
+
+    def __str__(self):
+        return '\n'.join(str(arg[key]) for key in self._arguments)
+
+    def _get_args(self, kwargs):
+        '''
+        Resolve default arguments from the config file and CLI param defaults.
+
+        Add an extra _args_given argument which determines if an argument was given or not.
+        Args set from the config file count as given.
+        '''
+        args_with_defaults = kwargs.copy()
+        # Add missing args from config file.
+        config_file = None
+        if 'config_file' in args_with_defaults and args_with_defaults['config_file'] is not None:
+            config_file = args_with_defaults['config_file']
+        else:
+            config_file = self._config_file
+        args_given = {}
+        for key in self._arguments:
+            args_given[key] = not (
+                not key in args_with_defaults or
+                args_with_defaults[key] is None or
+                self._arguments[key].nargs == '*' and args_with_defaults[key] == []
+            )
+        if config_file is not None and os.path.exists(config_file):
+            config_configs = {}
+            config = imp.load_source('config', config_file)
+            if self.extra_config_params is None:
+                config.set_args(config_configs)
+            else:
+                config.set_args(config_configs, self.extra_config_params)
+            for key in config_configs:
+                if key not in self._arguments:
+                    raise Exception('Unknown key in config file: ' + key)
+                if not args_given[key]:
+                    args_with_defaults[key] = config_configs[key]
+                    args_given[key] = True
+        # Add missing args from hard-coded defaults.
+        for key in self._arguments:
+            argument = self._arguments[key]
+            if (not key in args_with_defaults) or args_with_defaults[key] is None:
+                if argument.optional:
+                    args_with_defaults[key] = argument.default
+                else:
+                    raise Exception('Value not given for mandatory argument: ' + key)
+        args_with_defaults['_args_given'] = args_given
+        if 'config_file' in args_with_defaults:
+            del args_with_defaults['config_file']
+        return args_with_defaults
+
+    def add_argument(
+            self,
+            *args,
+            **kwargs
+        ):
+            argument = _Argument(*args, **kwargs)
+            self._arguments[argument.key] = argument
+
+    def cli_noexit(self, cli_args=None):
+        '''
+        Call the function from the CLI. Parse command line arguments
+        to get all arguments.
+
+        :return: the return of main
+        '''
+        parser = argparse.ArgumentParser(
+            description=self._description,
+            formatter_class=argparse.RawTextHelpFormatter,
+        )
+        for key in self._arguments:
+            argument = self._arguments[key]
+            parser.add_argument(*argument.args, **argument.kwargs)
+            if argument.is_bool:
+                new_longname = '--no' + argument.longname[1:]
+                kwargs = argument.kwargs.copy()
+                kwargs['default'] = not argument.default
+                if kwargs['action'] in ('store_true', 'store_false'):
+                    kwargs['action'] = 'store_false'
+                if 'help' in kwargs:
+                    del kwargs['help']
+                parser.add_argument(new_longname, dest=argument.key, **kwargs)
+        args = parser.parse_args(args=cli_args)
+        return self._do_main(vars(args))
+
+    def cli(self, *args, **kwargs):
+        '''
+        Same as cli, but also exit the program with status equal to the return value of main.
+        main must return an integer for this to be used.
+
+        None is considered 0.
+        '''
+        exit_status = self.cli_noexit(*args, **kwargs)
+        if exit_status is None:
+            exit_status = 0
+        sys.exit(exit_status)
+
+    def get_cli(self, **kwargs):
+        '''
+        :rtype: List[Type(str)]
+        :return: the canonical command line arguments arguments that would
+                 generate this Python function call.
+
+                 (--key, value) option pairs are grouped into tuples, and all
+                 other values are grouped in their own tuple (positional_arg,)
+                 or (--bool-arg,).
+
+                 Arguments with default values are not added, but arguments
+                 that are set by the config are also given.
+
+                 The optional arguments are sorted alphabetically, followed by
+                 positional arguments.
+
+                 The long option name is used if both long and short versions
+                 are given.
+        '''
+        options = []
+        positional_dict = {}
+        kwargs = self._get_args(kwargs)
+        for key in kwargs:
+            if not key in ('_args_given',):
+                argument = self._arguments[key]
+                default = argument.default
+                value = kwargs[key]
+                if value != default:
+                    if argument.is_option:
+                        if argument.is_bool:
+                            vals = [(argument.longname,)]
+                        elif 'action' in argument.kwargs and argument.kwargs['action'] == 'append':
+                            vals = [(argument.longname, str(val)) for val in value]
+                        else:
+                            vals = [(argument.longname, str(value))]
+                        for val in vals:
+                            bisect.insort(options, val)
+                    else:
+                        if type(value) is list:
+                            positional_dict[key] = [tuple([v]) for v in value]
+                        else:
+                            positional_dict[key] = [(str(value),)]
+        # Python built-in data structures suck.
+        # https://stackoverflow.com/questions/27726245/getting-the-key-index-in-a-python-ordereddict/27726534#27726534
+        positional = []
+        for key in self._arguments.keys():
+            if key in positional_dict:
+                positional.extend(positional_dict[key])
+        return options + positional
+
+    @staticmethod
+    def get_key(*args, **kwargs):
+        return _Argument.get_key(*args, **kwargs)
+
+    def main(self, **kwargs):
+        '''
+        Do the main function call work.
+
+        :type arguments: Dict
+        '''
+        raise NotImplementedError
+
+if __name__ == '__main__':
+    class OneCliFunction(CliFunction):
+        def __init__(self):
+            super().__init__(
+                config_file='cli_function_test_config.py',
+                description = '''\
+Description of this
+amazing function!
+''',
+            )
+            self.add_argument('-a', '--asdf', default='A', help='Help for asdf'),
+            self.add_argument('-q', '--qwer', default='Q', help='Help for qwer'),
+            self.add_argument('-b', '--bool-true', default=True, help='Help for bool-true'),
+            self.add_argument('--bool-false', default=False, help='Help for bool-false'),
+            self.add_argument('--dest', dest='custom_dest', help='Help for dest'),
+            self.add_argument('--bool-cli', default=False, help='Help for bool'),
+            self.add_argument('--bool-nargs', default=False, nargs='?', action='store', const='')
+            self.add_argument('--no-default', help='Help for no-bool'),
+            self.add_argument('--append', action='append')
+            self.add_argument('pos-mandatory', help='Help for pos-mandatory', type=int),
+            self.add_argument('pos-optional', default=0, help='Help for pos-optional', type=int),
+            self.add_argument('args-star', help='Help for args-star', nargs='*'),
+
+        def main(self, **kwargs):
+            del kwargs['_args_given']
+            return kwargs
+
+    one_cli_function = OneCliFunction()
+
+    # Default code call.
+    default = one_cli_function(pos_mandatory=1)
+    assert default == {
+        'asdf': 'A',
+        'qwer': 'Q',
+        'bool_true': True,
+        'bool_false': False,
+        'bool_nargs': False,
+        'bool_cli': True,
+        'custom_dest': None,
+        'no_default': None,
+        'append': [],
+        'pos_mandatory': 1,
+        'pos_optional': 0,
+        'args_star': []
+    }
+
+    # Default CLI call with programmatic CLI arguments.
+    out = one_cli_function.cli_noexit(['1'])
+    assert out == default
+
+    # asdf
+    out = one_cli_function(pos_mandatory=1, asdf='B')
+    assert out['asdf'] == 'B'
+    out['asdf'] = default['asdf']
+    assert out == default
+
+    # asdf and qwer
+    out = one_cli_function(pos_mandatory=1, asdf='B', qwer='R')
+    assert out['asdf'] == 'B'
+    assert out['qwer'] == 'R'
+    out['asdf'] = default['asdf']
+    out['qwer'] = default['qwer']
+    assert out == default
+
+    if '--bool-true':
+        out = one_cli_function(pos_mandatory=1, bool_true=False)
+        cli_out = one_cli_function.cli_noexit(['--no-bool-true', '1'])
+        assert out == cli_out
+        assert out['bool_true'] == False
+        out['bool_true'] = default['bool_true']
+        assert out == default
+
+    if '--bool-false':
+        out = one_cli_function(pos_mandatory=1, bool_false=True)
+        cli_out = one_cli_function.cli_noexit(['--bool-false', '1'])
+        assert out == cli_out
+        assert out['bool_false'] == True
+        out['bool_false'] = default['bool_false']
+        assert out == default
+
+    if '--bool-nargs':
+        out = one_cli_function(pos_mandatory=1, bool_nargs=True)
+        assert out['bool_nargs'] == True
+        out['bool_nargs'] = default['bool_nargs']
+        assert out == default
+
+        out = one_cli_function(pos_mandatory=1, bool_nargs='asdf')
+        assert out['bool_nargs'] == 'asdf'
+        out['bool_nargs'] = default['bool_nargs']
+        assert out == default
+
+    # --dest
+    out = one_cli_function(pos_mandatory=1, custom_dest='a')
+    cli_out = one_cli_function.cli_noexit(['--dest', 'a', '1'])
+    assert out == cli_out
+    assert out['custom_dest'] == 'a'
+    out['custom_dest'] = default['custom_dest']
+    assert out == default
+
+    # Positional
+    out = one_cli_function(pos_mandatory=1, pos_optional=2, args_star=['3', '4'])
+    assert out['pos_mandatory'] == 1
+    assert out['pos_optional'] == 2
+    assert out['args_star'] == ['3', '4']
+    cli_out = one_cli_function.cli_noexit(['1', '2', '3', '4'])
+    assert out == cli_out
+    out['pos_mandatory'] = default['pos_mandatory']
+    out['pos_optional'] = default['pos_optional']
+    out['args_star'] = default['args_star']
+    assert out == default
+
+    # Star
+    out = one_cli_function(append=['1', '2'], pos_mandatory=1)
+    cli_out = one_cli_function.cli_noexit(['--append', '1', '--append', '2', '1'])
+    assert out == cli_out
+    assert out['append'] == ['1', '2']
+    out['append'] = default['append']
+    assert out == default
+
+    # Force a boolean value set on the config to be False on CLI.
+    assert one_cli_function.cli_noexit(['--no-bool-cli', '1'])['bool_cli'] is False
+
+    # Pick another config file.
+    assert one_cli_function.cli_noexit(['--config-file', 'cli_function_test_config_2.py', '1'])['bool_cli'] is False
+
+    # Extra config file for '*'.
+    assert one_cli_function.cli_noexit(['--config-file', 'cli_function_test_config_2.py', '1', '2', '3', '4'])['args_star'] == ['3', '4']
+    assert one_cli_function.cli_noexit(['--config-file', 'cli_function_test_config_2.py', '1', '2'])['args_star'] == ['asdf', 'qwer']
+
+    # get_cli
+    assert one_cli_function.get_cli(pos_mandatory=1, asdf='B') == [('--asdf', 'B'), ('--bool-cli',), ('1',)]
+    assert one_cli_function.get_cli(pos_mandatory=1, asdf='B', qwer='R') == [('--asdf', 'B'), ('--bool-cli',), ('--qwer', 'R'), ('1',)]
+    assert one_cli_function.get_cli(pos_mandatory=1, bool_true=False) == [('--bool-cli',), ('--bool-true',), ('1',)]
+    assert one_cli_function.get_cli(pos_mandatory=1, pos_optional=2, args_star=['asdf', 'qwer']) == [('--bool-cli',), ('1',), ('2',), ('asdf',), ('qwer',)]
+    assert one_cli_function.get_cli(pos_mandatory=1, append=['2', '3']) == [('--append', '2'), ('--append', '3',), ('--bool-cli',), ('1',)]
+
+    if len(sys.argv) > 1:
+        # CLI call with argv command line arguments.
+        print(one_cli_function.cli())

cli_function_test_config.py

@@ -0,0 +1,5 @@
+def set_args(args):
+    '''
+    :type args: Dict[str, Any]
+    '''
+    args['bool_cli'] = True

cli_function_test_config_2.py

@@ -0,0 +1,6 @@
+def set_args(args):
+    '''
+    :type args: Dict[str, Any]
+    '''
+    args['bool_cli'] = False
+    args['args_star'] = ['asdf', 'qwer']

config.example

@@ -1,4 +0,0 @@
-#!/usr/bin/env python
-arch = 'aarch64'
-gem5 = True
-run_id = 'asdf'

config.py

@@ -0,0 +1,10 @@
+'''
+https://github.com/cirosantilli/linux-kernel-module-cheat#default-command-line-arguments
+'''
+
+def set_args(args, script_name):
+    args['arch'] = 'aarch64'
+    args['emulators'] = ['gem5']
+    if script_name == 'build-gem5':
+        # This argument is defined only for ./build-gem5.
+        args['extra_scons_args'] = ['ADSF=qwer']

copy-overlay

@@ -5,25 +5,21 @@ import os
 import shutil
 
 import common
+from shell_helpers import LF
 
-class CopyOverlayComponent(common.Component):
+class Main(common.BuildCliFunction):
-    def do_build(self, args):
+    def __init__(self):
+        super().__init__(
+            description='''\
+https://github.com/cirosantilli/linux-kernel-module-cheat#rootfs_overlay
+''')
+    def build(self):
+        # TODO: print rsync equivalent, move into shell_helpers.
         distutils.dir_util.copy_tree(
-            common.rootfs_overlay_dir,
+            self.env['rootfs_overlay_dir'],
-            common.out_rootfs_overlay_dir,
+            self.env['out_rootfs_overlay_dir'],
             update=1,
         )
 
-    def get_argparse_args(self):
-        return {
-            'description': '''\
-Copy our git tracked rootfs_overlay to the final generated rootfs_overlay
-that also contains generated build outputs. This has the following advantages
-over just adding that to BR2_ROOTFS_OVERLAY:
-- also works for non Buildroot root filesystesms
-- places everything in one place for a nice 9P mount
-''',
-        }
-
 if __name__ == '__main__':
-    CopyOverlayComponent().build()
+    Main().cli()

gem5-bench-cache

@@ -13,7 +13,7 @@ while getopts "C" OPT; do
   esac
 done
 shift "$(($OPTIND - 1))"
-common_opts="--gem5 $@"
+common_opts="--emulator gem5 $@"
 
 # Vars
 cmd="./run ${common_opts}"

gem5-bench-dhrystone

@@ -6,11 +6,11 @@ set -eu
 root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
 outfile="${root_dir}/out/gem5-bench-dhrystone.txt"
 arch=aarch64
-cmd="./run -a '$arch' --gem5 --eval-busybox '/gem5.sh'"
+cmd="./run --arch '$arch' --emulator gem5 --eval-busybox '/gem5.sh'"
 
 # These cache sizes roughly match the ARM Cortex A75
 # https://en.wikipedia.org/wiki/ARM_Cortex-A75
-restore='-l 1 -- --cpu-type=HPI --restore-with-cpu=HPI --caches --l2cache --l1d_size=64kB --l1i_size=64kB --l2_size=256kB'
+restore='--gem5-restore 1 -- --cpu-type=HPI --restore-with-cpu=HPI --caches --l2cache --l1d_size=64kB --l1i_size=64kB --l2_size=256kB'
 
 # Generate a checkpoint after Linux boots, using the faster and less detailed CPU.
 # The boot takes a while, be patient young Padawan.

gem5-shell

@@ -1,16 +1,23 @@
 #!/usr/bin/env python3
 
-import sys
-
 import common
+from shell_helpers import LF
 
-parser = common.get_argparse(
+class Main(common.LkmcCliFunction):
-    default_args={'gem5':True},
+    def __init__(self):
-    argparse_args={'description':'Connect a terminal to a running gem5 instance'}
+        super().__init__(
-)
+            defaults={
-args = common.setup(parser)
+                'emulators': ['gem5'],
-sys.exit(common.run_cmd([
+            },
-    common.gem5_m5term, common.Newline,
+            description='Connect a terminal to a running gem5 instance',
-    'localhost', common.Newline,
+        )
-    str(common.gem5_telnet_port), common.Newline,
+    def timed_main(self):
-]))
+        return self.sh.run_cmd([
+            self.env['gem5_m5term'],
+            'localhost',
+            str(self.env['gem5_telnet_port']),
+            LF,
+        ])
+
+if __name__ == '__main__':
+    Main().cli()

gem5-stat

@@ -1,14 +1,26 @@
 #!/usr/bin/env python3
+
 import common
-parser = common.get_argparse(
+
-    argparse_args={'description':'Get the value of a gem5 stat from the stats.txt file.'}
+class Main(common.LkmcCliFunction):
-)
+    def __init__(self):
-parser.add_argument(
+        super().__init__(
-    'stat',
+            defaults={
-    default=None,
+                'print_time': False,
-    help='Python regexp matching the full stat name of interest',
+            },
-    nargs='?',
+            description='''\
-)
+Get the value of a gem5 stat from the stats.txt file.
-args = common.setup(parser)
+''',
-stats = common.get_stats(args.stat)
+        )
-print('\n'.join(stats))
+        self.add_argument(
+            'stat',
+            help='Python regexp matching the full stat name of interest',
+            nargs='?',
+        )
+
+    def timed_main(self):
+        stats = self.get_stats(self.env['stat'])
+        print('\n'.join(stats))
+
+if __name__ == '__main__':
+    Main().cli()

getvar

@@ -1,16 +1,20 @@
 #!/usr/bin/env python3
 
-import types
-
 import common
 
-parser = common.get_argparse(argparse_args={
+class Main(common.LkmcCliFunction):
-    'description': '''Print the value of a common.py variable.
+    def __init__(self):
+        super().__init__(
+            defaults = {
+                'print_time': False,
+            },
+            description='''\
+Print the value of a self.env['py'] variable.
 
 This is useful to:
 
 * give dry commands on the README that don't change when we refactor directory structure
-* create simple bash scripts that call use common.py variables
+* create simple bash scripts that call use self.env['py'] variables
 
 For example, to get the Buildroot output directory for an ARM build, use:
 
@@ -23,16 +27,22 @@ List all available variables:
 ....
 ./%(prog)s
 ....
-....
+''',
-'''
+        )
-})
+        self.add_argument('--type', choices=['input', 'all'], default='all')
-parser.add_argument('variable', nargs='?')
+        self.add_argument('variable', nargs='?')
-args = common.setup(parser)
+
-if args.variable:
+    def timed_main(self):
-    print(getattr(common, args.variable))
+        variable = self.env['variable']
-else:
+        if variable:
-    for attr in dir(common):
+            print(self.env[variable])
-        if not attr.startswith('__'):
+        else:
-            val = getattr(common, attr)
+            if self.env['type'] == 'input':
-            if not callable(val) and not type(val) is types.ModuleType:
+                to_print = self.input_args
-                print('{} {}'.format(attr, val))
+            elif self.env['type'] == 'all':
+                to_print = self.env
+            for key in sorted(to_print):
+                print('{}={}'.format(key, self.env[key]))
+
+if __name__ == '__main__':
+    Main().cli()

qemu-monitor

@@ -5,10 +5,11 @@ import sys
 import telnetlib
 
 import common
+from shell_helpers import LF
 
 prompt = b'\n(qemu) '
 
-parser = common.get_argparse({
+parser = self.get_argparse({
     'description': '''\
 Run a command on the QEMU monitor of a running QEMU instance
 
@@ -21,24 +22,24 @@ parser.add_argument(
     help='If given, run this command and quit',
     nargs='*',
 )
-args = common.setup(parser)
+args = self.setup(parser)
 
 def write_and_read(tn, cmd, prompt):
     tn.write(cmd.encode('utf-8'))
     return '\n'.join(tn.read_until(prompt).decode('utf-8').splitlines()[1:])[:-len(prompt)]
 
-with telnetlib.Telnet('localhost', common.qemu_monitor_port) as tn:
+with telnetlib.Telnet('localhost', kwargs['qemu_monitor_port']) as tn:
     # Couldn't disable server echo, so just removing the write for now.
     # https://stackoverflow.com/questions/12421799/how-to-disable-telnet-echo-in-python-telnetlib
     # sock = tn.get_socket()
     # sock.send(telnetlib.IAC + telnetlib.WILL + telnetlib.ECHO)
     if os.isatty(sys.stdin.fileno()):
-        if args.command == []:
+        if kwargs['command'] == []:
             print(tn.read_until(prompt).decode('utf-8'), end='')
             tn.interact()
         else:
             tn.read_until(prompt)
-            print(write_and_read(tn, ' '.join(args.command) + '\n', prompt))
+            print(write_and_read(tn, ' '.join(kwargs['command']) + '\n', prompt))
     else:
         tn.read_until(prompt)
         print(write_and_read(tn, sys.stdin.read() + '\n', prompt))

qemu-trace2txt

@@ -1,26 +1,29 @@
 #!/usr/bin/env python3
 
 import os
-import subprocess
-import sys
 
 import common
+from shell_helpers import LF
 
-def main():
+class Main(common.LkmcCliFunction):
-    return common.run_cmd(
+    def __init__(self):
-        [
+        super().__init__(
-            os.path.join(common.qemu_src_dir, 'scripts/simpletrace.py'), common.Newline,
+            description='''\
-            os.path.join(common.qemu_build_dir, 'trace-events-all'), common.Newline,
+Convert a QEMU `-trace exec_tb` to text form.
-            os.path.join(common.qemu_trace_file), common.Newline,
+'''
-        ],
+        )
-        cmd_file=os.path.join(common.run_dir, 'qemu-trace2txt'),
+
-        out_file=common.qemu_trace_txt_file,
+    def timed_main(self):
-        show_stdout=False,
+        return self.sh.run_cmd(
-    )
+            [
+                os.path.join(self.env['qemu_source_dir'], 'scripts/simpletrace.py'), LF,
+                os.path.join(self.env['qemu_build_dir'], 'trace-events-all'), LF,
+                os.path.join(self.env['qemu_trace_file']), LF,
+            ],
+            cmd_file=os.path.join(self.env['run_dir'], 'qemu-trace2txt'),
+            out_file=self.env['qemu_trace_txt_file'],
+            show_stdout=False,
+        )
 
 if __name__ == '__main__':
-    parser = common.get_argparse(argparse_args={
+    Main().cli()
-        'description': 'Convert a QEMU `-trace exec_tb` to text form.'
-    })
-    args = common.setup(parser)
-    sys.exit(main())

release

@@ -1,32 +0,0 @@
-#!/usr/bin/env python3
-
-'''
-https://upload.com/cirosantilli/linux-kernel-module-cheat#release
-'''
-
-import imp
-import os
-import subprocess
-import time
-
-import common
-release_zip = imp.load_source('release_zip', os.path.join(common.root_dir, 'release-zip'))
-release_upload = imp.load_source('release_upload', os.path.join(common.root_dir, 'release-upload'))
-
-start_time = time.time()
-# TODO factor those out so we don't redo the same thing multiple times.
-# subprocess.check_call([os.path.join(common.root_dir, 'test')])
-# subprocess.check_call([os.path.join(common.root_dir, ''bench-all', '-A', '-u'])
-
-# A clean release requires a full rebuild unless we hack it :-(
-# We can't just use our current build as it contains packages we've
-# installed in random experiments. And with EXT2: we can't easily
-# know what the smallest root filesystem size is and use it either...
-# https://stackoverflow.com/questions/47320800/how-to-clean-only-target-in-buildroot
-subprocess.check_call([os.path.join(common.root_dir, 'configure'), '--all'])
-subprocess.check_call([os.path.join(common.root_dir, 'build'), '--all-archs', 'release'])
-release_zip.main()
-subprocess.check_call(['git', 'push'])
-release_upload.main()
-end_time = time.time()
-common.print_time(end_time - start_time)

release-download-latest

@@ -1,16 +1,26 @@
 #!/usr/bin/env python3
 
-'''
-Usage: https://github.com/cirosantilli/linux-kernel-module-cheat#prebuilt
-
-Implementation:
-https://stackoverflow.com/questions/24987542/is-there-a-link-to-github-for-downloading-a-file-in-the-latest-release-of-a-repo/50540591#50540591
-'''
-
 import urllib.request
 
 import common
+from shell_helpers import LF
 
-_json = common.github_make_request(path='/releases')
+class Main(common.LkmcCliFunction):
-asset = _json[0]['assets'][0]
+    def __init__(self):
-urllib.request.urlretrieve(asset['browser_download_url'], asset['name'])
+        super().__init__(
+            description='''\
+Usage: https://github.com/cirosantilli/linux-kernel-module-cheat#prebuilt
+
+Implementation:
+https://stackoverflow.com/questions/24987542/is-there-a-link-to-github-for-downloading-a-file-in-the-latest-release-of-a-repo/50540591#50540591
+''',
+        )
+
+    def timed_main(self):
+        self.log_info('Downloading the release, this may take several seconds / a few minutes.')
+        _json = self.github_make_request(path='/releases')
+        asset = _json[0]['assets'][0]
+        urllib.request.urlretrieve(asset['browser_download_url'], asset['name'])
+
+if __name__ == '__main__':
+    Main().cli()

release-upload

@@ -1,78 +1,84 @@
 #!/usr/bin/env python3
 
-'''
-Usage: https://github.com/cirosantilli/linux-kernel-module-cheat#release-zip
-
-Implementation:
-
-* https://stackoverflow.com/questions/5207269/how-to-release-a-build-artifact-asset-on-github-with-a-script/52354732#52354732
-* https://stackoverflow.com/questions/38153418/can-someone-give-a-python-requests-example-of-uploading-a-release-asset-in-githu/52354681#52354681
-'''
-
 import json
 import os
+import subprocess
 import sys
-
 import urllib.error
 
 import common
+from shell_helpers import LF
 
-def main():
+class Main(common.LkmcCliFunction):
-    repo = common.github_repo_id
+    def __init__(self):
-    tag = 'sha-{}'.format(common.sha)
+        super().__init__(
-    upload_path = common.release_zip_file
+            description='''\
-
+https://github.com/cirosantilli/linux-kernel-module-cheat#release-upload
-    # Check the release already exists.
+''',
-    try:
-        _json = common.github_make_request(path='/releases/tags/' + tag)
-    except urllib.error.HTTPError as e:
-        if e.code == 404:
-            release_exists = False
-        else:
-            raise e
-    else:
-        release_exists = True
-        release_id = _json['id']
-
-    # Create release if not yet created.
-    if not release_exists:
-        _json = common.github_make_request(
-            authenticate=True,
-            data=json.dumps({
-                'tag_name': tag,
-                'name': tag,
-                'prerelease': True,
-            }).encode(),
-            path='/releases'
         )
-        release_id = _json['id']
 
-    asset_name = os.path.split(upload_path)[1]
+    def timed_main(self):
+        # https://stackoverflow.com/questions/3404936/show-which-git-tag-you-are-on
+        tag = subprocess.check_output([
+            'git',
+            'describe',
+            '--exact-match',
+            '--tags'
+        ]).decode().rstrip()
+        upload_path = self.env['release_zip_file']
 
-    # Clear the prebuilts for a upload.
+        # Check the release already exists.
-    _json = common.github_make_request(
+        try:
-        path=('/releases/' + str(release_id) + '/assets'),
+            _json = self.github_make_request(path='/releases/tags/' + tag)
-    )
+        except urllib.error.HTTPError as e:
-    for asset in _json:
+            if e.code == 404:
-        if asset['name'] == asset_name:
+                release_exists = False
-            _json = common.github_make_request(
+            else:
+                raise e
+        else:
+            release_exists = True
+            release_id = _json['id']
+
+        # Create release if not yet created.
+        if not release_exists:
+            _json = self.github_make_request(
                 authenticate=True,
-                path=('/releases/assets/' + str(asset['id'])),
+                data=json.dumps({
-                method='DELETE',
+                    'tag_name': tag,
+                    'name': tag,
+                    'prerelease': True,
+                }).encode(),
+                path='/releases'
             )
-            break
+            release_id = _json['id']
 
-    # Upload the prebuilt.
+        asset_name = os.path.split(upload_path)[1]
-    with open(upload_path, 'br') as myfile:
+
-        content = myfile.read()
+        # Clear the prebuilts for a upload.
-    _json = common.github_make_request(
+        _json = self.github_make_request(
-        authenticate=True,
+            path=('/releases/' + str(release_id) + '/assets'),
-        data=content,
+        )
-        extra_headers={'Content-Type': 'application/zip'},
+        for asset in _json:
-        path=('/releases/' + str(release_id) + '/assets'),
+            if asset['name'] == asset_name:
-        subdomain='uploads',
+                _json = self.github_make_request(
-        url_params={'name': asset_name},
+                    authenticate=True,
-    )
+                    path=('/releases/assets/' + str(asset['id'])),
+                    method='DELETE',
+                )
+                break
+
+        # Upload the prebuilt.
+        self.log_info('Uploading the release, this may take several seconds / a few minutes.')
+        with open(upload_path, 'br') as myfile:
+            content = myfile.read()
+        _json = self.github_make_request(
+            authenticate=True,
+            data=content,
+            extra_headers={'Content-Type': 'application/zip'},
+            path=('/releases/' + str(release_id) + '/assets'),
+            subdomain='uploads',
+            url_params={'name': asset_name},
+        )
 
 if __name__ == '__main__':
-    main()
+    Main().cli()

release-zip

@@ -1,25 +1,39 @@
 #!/usr/bin/env python3
 
-'''
-https://github.com/cirosantilli/linux-kernel-module-cheat#release-zip
-'''
-
 import os
-import subprocess
 import zipfile
 
 import common
 
-def main():
+class Main(common.LkmcCliFunction):
-    os.makedirs(common.release_dir, exist_ok=True)
+    def __init__(self):
-    if os.path.exists(common.release_zip_file):
+        super().__init__(
-        os.unlink(common.release_zip_file)
+            description='''\
-    zipf = zipfile.ZipFile(common.release_zip_file, 'w', zipfile.ZIP_DEFLATED)
+https://github.com/cirosantilli/linux-kernel-module-cheat#release-zip
-    for arch in common.all_archs:
+''',
-        common.setup(common.get_argparse(default_args={'arch': arch}))
+            defaults = {
-        zipf.write(common.qcow2_file, arcname=os.path.relpath(common.qcow2_file, common.root_dir))
+                'print_time': False,
-        zipf.write(common.linux_image, arcname=os.path.relpath(common.linux_image, common.root_dir))
+            }
-    zipf.close()
+        )
+        self.zip_files = []
+
+    def timed_main(self):
+        self.zip_files.append(self.env['qcow2_file'])
+        self.zip_files.append(self.env['linux_image'])
+        for root, dirs, files in os.walk(self.env['baremetal_build_dir']):
+            for file in files:
+                path = os.path.join(root, file)
+                if os.path.splitext(path)[1] == self.env['baremetal_build_ext']:
+                    self.zip_files.append(path)
+
+    def teardown(self):
+        os.makedirs(self.env['release_dir'], exist_ok=True)
+        self.sh.rmrf(self.env['release_zip_file'])
+        self.log_info('Creating zip: ' + self.env['release_zip_file'])
+        with zipfile.ZipFile(self.env['release_zip_file'], 'w', zipfile.ZIP_DEFLATED) as zipf:
+            for zip_file in self.zip_files:
+                self.log_info('Adding file: ' + zip_file)
+                zipf.write(zip_file, arcname=os.path.relpath(zip_file, self.env['root_dir']))
 
 if __name__ == '__main__':
-    main()
+    Main().cli()

rootfs_overlay/gem5_exit.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+# To be able to do init=/gem5_exit.sh, since kernel CLI argument passing is too messy:
+# https://github.com/cirosantilli/linux-kernel-module-cheat#init-arguments
+m5 exit

rootfs_overlay/test_all.sh

@@ -1,25 +1,27 @@
 #!/bin/sh
+test_dir="${1:-.}"
 for test in \
-  /anonymous_inode.sh \
+  anonymous_inode.sh \
-  /character_device.sh \
+  character_device.sh \
-  /character_device_create.sh \
+  character_device_create.sh \
-  /debugfs.sh \
+  debugfs.sh \
-  /dep.sh \
+  dep.sh \
-  /fops.sh \
+  fops.sh \
-  /init_module.sh \
+  init_module.sh \
-  /ioctl.sh \
+  ioctl.sh \
-  /kstrto.sh \
+  kstrto.sh \
-  /mmap.sh \
+  mmap.sh \
-  /netlink.sh \
+  netlink.sh \
-  /params.sh \
+  params.sh \
-  /procfs.sh \
+  procfs.sh \
-  /seq_file.sh \
+  seq_file.sh \
-  /seq_file_single_open.sh \
+  seq_file_single_open.sh \
-  /sysfs.sh \
+  sysfs.sh \
 ; do
-  if ! "$test"; then
+  if ! "${test_dir}/${test}"; then
-    echo "lkmc_test_fail: ${test}"
+    echo "Test failed: ${test}"
+    test_fail.sh
     exit 1
   fi
 done
-echo lkmc_test_pass
+echo 'All tests passed.'

rootfs_overlay/test_fail.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+# https://github.com/cirosantilli/linux-kernel-module-cheat#magic-failure-string
+echo lkmc_test_fail

run-docker

@@ -4,61 +4,63 @@ import argparse
 import os
 
 import common
+import shell_helpers
+from shell_helpers import LF
 
-container_name = common.repo_short_id
+container_name = common.consts['repo_short_id']
-container_hostname = common.repo_short_id
+container_hostname = common.consts['repo_short_id']
-image_name = common.repo_short_id
+image_name = common.consts['repo_short_id']
-target_dir = '/root/{}'.format(common.repo_short_id)
+target_dir = '/root/{}'.format(common.consts['repo_short_id'])
 docker = ['sudo', 'docker']
 def create(args):
-    common.run_cmd(docker + ['build', '-t', image_name, '.', common.Newline])
+    sh.run_cmd(docker + ['build', '-t', image_name, '.', LF])
     # --privileged for KVM:
     # https://stackoverflow.com/questions/48422001/launching-qemu-kvm-from-inside-docker-container
-    common.run_cmd(
+    sh.run_cmd(
         docker +
         [
-            'create', common.Newline,
+            'create', LF,
-            '--hostname', container_hostname, common.Newline,
+            '--hostname', container_hostname, LF,
-            '-i', common.Newline,
+            '-i', LF,
-            '--name', container_name, common.Newline,
+            '--name', container_name, LF,
-            '--net', 'host', common.Newline,
+            '--net', 'host', LF,
-            '--privileged', common.Newline,
+            '--privileged', LF,
-            '-t', common.Newline,
+            '-t', LF,
-            '-w', target_dir, common.Newline,
+            '-w', target_dir, LF,
-            '-v', '{}:{}'.format(os.getcwd(), target_dir), common.Newline,
+            '-v', '{}:{}'.format(os.getcwd(), target_dir), LF,
             image_name,
         ]
     )
 def destroy(args):
     stop(args)
-    common.run_cmd(docker + ['rm', container_name, common.Newline])
+    sh.run_cmd(docker + ['rm', container_name, LF])
-    common.run_cmd(docker + ['rmi', image_name, common.Newline])
+    sh.run_cmd(docker + ['rmi', image_name, LF])
-def sh(args):
+def sh_func(args):
     start(args)
     if args:
         sh_args = args
     else:
         sh_args = ['bash']
-    common.run_cmd(
+    sh.run_cmd(
         docker + ['exec', '-i', '-t', container_name] +
         sh_args +
-        [common.Newline],
+        [LF],
     )
 def start(args):
-    common.run_cmd(docker + ['start', container_name, common.Newline])
+    sh.run_cmd(docker + ['start', container_name, LF])
 def stop(args):
-    common.run_cmd(docker + ['stop', container_name, common.Newline])
+    sh.run_cmd(docker + ['stop', container_name, LF])
 cmd_action_map = {
     'create': lambda args: create(args),
     'DESTROY': lambda args: destroy(args),
-    'sh': lambda args: sh(args),
+    'sh': lambda args: sh_func(args),
     'start': lambda args: start(args),
     'stop': lambda args: stop(args),
 }
 parser = argparse.ArgumentParser()
+parser.add_argument('--dry-run', default=False, action='store_true')
 parser.add_argument('cmd', choices=cmd_action_map)
 parser.add_argument('args', nargs='*')
-common.add_dry_run_argument(parser)
 args = parser.parse_args()
-common.setup_dry_run_arguments(args)
+sh = shell_helpers.ShellHelpers(dry_run=args.dry_run)
 cmd_action_map[args.cmd](args.args)

run-gdb

@@ -7,18 +7,7 @@ import subprocess
 import sys
 
 import common
-
+from shell_helpers import LF
-defaults = {
-    'after': '',
-    'before': '',
-    'break_at': None,
-    'kgdb': False,
-    'no_continue': False,
-    'no_lxsymbols': False,
-    'test': False,
-    'sim': False,
-    'userland': None,
-}
 
 class GdbTestcase:
     def __init__(
@@ -34,8 +23,6 @@ class GdbTestcase:
         '''
         self.prompt = '\(gdb\) '
         self.source_path = source_path
-        common.print_cmd(cmd)
-        cmd = common.strip_newlines(cmd)
         import pexpect
         self.child = pexpect.spawn(
             cmd[0],
@@ -48,9 +35,15 @@ class GdbTestcase:
         self.child.waitnoecho()
         self.child.expect(self.prompt)
         test = imp.load_source('test', test_script_path)
-        test.test(self)
+        exception = None
+        try:
+            test.test(self)
+        except AssertionError as e:
+            exception = e
         self.child.sendcontrol('d')
         self.child.close()
+        if exception is not None:
+            raise exception
 
     def before(self):
         return self.child.before.rstrip()
@@ -84,155 +77,147 @@ class GdbTestcase:
         self.child.sendline(line)
         self.child.expect(self.prompt)
 
-def main(args, extra_args=None):
+class Main(common.LkmcCliFunction):
-    '''
+    def __init__(self):
-    :param args: argparse parse_argument() output. Must contain all the common options,
+        super().__init__(description='''\
-                 but does not need GDB specific ones.
+Connect with GDB to an emulator to debug Linux itself
-    :type args: argparse.Namespace
+''')
-
+        self.add_argument(
-    :param extra_args: extra arguments to be added to args
+            '--after', default='',
-    :type extra_args: Dict[str,Any]
+            help='Pass extra arguments to GDB, to be appended after all other arguments'
-
-    :return: GDB exit status
-    :rtype: int
-    '''
-    global defaults
-    args = common.resolve_args(defaults, args, extra_args)
-    after = common.shlex_split(args.after)
-    before = common.shlex_split(args.before)
-    no_continue = args.no_continue
-    if args.test:
-        no_continue = True
-        before.extend([
-            '-q', common.Newline,
-            '-nh', common.Newline,
-            '-ex', 'set confirm off', common.Newline
-        ])
-    elif args.verbose:
-        # The output of this would affect the tests.
-        # https://stackoverflow.com/questions/13496389/gdb-remote-protocol-how-to-analyse-packets
-        # Also be opinionated and set remotetimeout to allow you to step debug the emulator at the same time.
-        before.extend([
-            '-ex', 'set debug remote 1', common.Newline,
-            '-ex', 'set remotetimeout 99999', common.Newline,
-        ])
-    if args.break_at is not None:
-        break_at = ['-ex', 'break {}'.format(args.break_at), common.Newline]
-    else:
-        break_at = []
-    linux_full_system = (common.baremetal is None and args.userland is None)
-    if args.userland:
-        image = common.resolve_userland(args.userland)
-    elif common.baremetal:
-        image = common.image
-        test_script_path = os.path.splitext(common.source_path)[0] + '.py'
-    else:
-        image = common.vmlinux
-    if common.baremetal:
-        allowed_toolchains = ['crosstool-ng', 'buildroot', 'host']
-    else:
-        allowed_toolchains = ['buildroot', 'crosstool-ng', 'host']
-    cmd = (
-        [common.get_toolchain_tool('gdb', allowed_toolchains=allowed_toolchains), common.Newline] +
-        before +
-        ['-q', common.Newline]
-    )
-    if linux_full_system:
-        cmd.extend(['-ex', 'add-auto-load-safe-path {}'.format(common.linux_build_dir), common.Newline])
-    if args.sim:
-        target = 'sim'
-    else:
-        if args.kgdb:
-            port = common.extra_serial_port
-        else:
-            port = common.gdb_port
-        target = 'remote localhost:{}'.format(port)
-    cmd.extend([
-        '-ex', 'file {}'.format(image), common.Newline,
-        '-ex', 'target {}'.format(target), common.Newline,
-    ])
-    if not args.kgdb:
-        cmd.extend(break_at)
-    if not no_continue:
-        # ## lx-symbols
-        #
-        # ### lx-symbols after continue
-        #
-        # lx symbols must be run after continue.
-        #
-        # running it immediately after the connect on the bootloader leads to failure,
-        # likely because kernel structure on which it depends are not yet available.
-        #
-        # With this setup, continue runs, and lx-symbols only runs when a break happens,
-        # either by hitting the breakpoint, or by entering Ctrl + C.
-        #
-        # Sure, if the user sets a break on a raw address of the bootloader,
-        # problems will still arise, but let's think about that some other time.
-        #
-        # ### lx-symbols autoload
-        #
-        # The lx-symbols commands gets loaded through the file vmlinux-gdb.py
-        # which gets put on the kernel build root when python debugging scripts are enabled.
-        cmd.extend(['-ex', 'continue', common.Newline])
-        if not args.no_lxsymbols and linux_full_system:
-            cmd.extend(['-ex', 'lx-symbols {}'.format(common.kernel_modules_build_subdir), common.Newline])
-    cmd.extend(after)
-    if args.test:
-        GdbTestcase(
-            common.source_path,
-            test_script_path,
-            cmd,
-            verbose=args.verbose,
         )
-    else:
+        self.add_argument(
-        # I would rather have cwd be out_rootfs_overlay_dir,
+            '--before', default='',
-        # but then lx-symbols cannot fine the vmlinux and fails with:
+            help='Pass extra arguments to GDB to be prepended before any of the arguments passed by this script'
-        # vmlinux: No such file or directory.
-        return common.run_cmd(
-            cmd,
-            cmd_file=os.path.join(common.run_dir, 'run-gdb.sh'),
-            cwd=common.linux_build_dir
         )
-
+        self.add_argument(
-if __name__ == '__main__':
+            'break_at', nargs='?',
-    parser = common.get_argparse(argparse_args={'description': 'Connect with GDB to an emulator to debug Linux itself'})
+            help='Extra options to append at the end of the emulator command line'
-    parser.add_argument(
+        )
-        '-A', '--after', default=defaults['after'],
+        self.add_argument(
-        help='Pass extra arguments to GDB, to be appended after all other arguments'
+            '-k', '--kgdb', default=False,
-    )
+        )
-    parser.add_argument(
+        self.add_argument(
-        '--before', default=defaults['before'],
+            '-C', '--no-continue', default=False,
-        help='Pass extra arguments to GDB to be prepended before any of the arguments passed by this script'
+            help="Don't run continue after connecting"
-    )
+        )
-    parser.add_argument(
+        self.add_argument(
-        '-C', '--no-continue', default=defaults['no_continue'], action='store_true',
+            '-X', '--no-lxsymbols', default=False,
-        help="Don't run continue after connecting"
+        )
-    )
+        self.add_argument(
-    parser.add_argument(
+            '--test', default=False,
-        '-k', '--kgdb', default=defaults['kgdb'], action='store_true'
+            help='''\
-    )
-    parser.add_argument(
-        '--sim', default=defaults['sim'], action='store_true',
-        help='''Use the built-in GDB CPU simulator
-See: https://github.com/cirosantilli/linux-kernel-module-cheat#gdb-builtin-cpu-simulator
-'''
-    )
-    parser.add_argument(
-        '-X', '--no-lxsymbols', default=defaults['no_lxsymbols'], action='store_true'
-    )
-    parser.add_argument(
-        '--test', default=defaults['test'], action='store_true',
-        help='''\
 Run an expect test case instead of interactive usage. For baremetal and userland,
 the script is a .py file next to the source code.
 '''
-    )
+        )
-    parser.add_argument(
+        self.add_argument(
-        '-u', '--userland', default=defaults['userland'],
+            '--sim', default=False,
-    )
+            help='''Use the built-in GDB CPU simulator
-    parser.add_argument(
+See: https://github.com/cirosantilli/linux-kernel-module-cheat#gdb-builtin-cpu-simulator
-        'break_at', nargs='?',
+'''
-        help='Extra options to append at the end of the emulator command line'
+        )
-    )
+        self.add_argument(
-    args = common.setup(parser)
+            '-u', '--userland',
-    sys.exit(main(args))
+        )
+
+    def timed_main(self):
+        after = self.sh.shlex_split(self.env['after'])
+        before = self.sh.shlex_split(self.env['before'])
+        no_continue = self.env['no_continue']
+        if self.env['test']:
+            no_continue = True
+            before.extend([
+                '-q', LF,
+                '-nh', LF,
+                '-ex', 'set confirm off', LF
+            ])
+        elif self.env['verbose']:
+            # The output of this would affect the tests.
+            # https://stackoverflow.com/questions/13496389/gdb-remote-protocol-how-to-analyse-packets
+            # Also be opinionated and set remotetimeout to allow you to step debug the emulator at the same time.
+            before.extend([
+                '-ex', 'set debug remote 1', LF,
+                '-ex', 'set remotetimeout 99999', LF,
+            ])
+        if self.env['break_at'] is not None:
+            break_at = ['-ex', 'break {}'.format(self.env['break_at']), LF]
+        else:
+            break_at = []
+        linux_full_system = (self.env['baremetal'] is None and self.env['userland'] is None)
+        if self.env['userland']:
+            image = self.resolve_userland(self.env['userland'])
+        elif self.env['baremetal']:
+            image = self.env['image']
+            test_script_path = os.path.splitext(self.env['source_path'])[0] + '.py'
+        else:
+            image = self.env['vmlinux']
+        if self.env['baremetal']:
+            allowed_toolchains = ['crosstool-ng', 'buildroot', 'host']
+        else:
+            allowed_toolchains = ['buildroot', 'crosstool-ng', 'host']
+        cmd = (
+            [self.get_toolchain_tool('gdb', allowed_toolchains=allowed_toolchains), LF] +
+            before
+        )
+        if linux_full_system:
+            cmd.extend(['-ex', 'add-auto-load-safe-path {}'.format(self.env['linux_build_dir']), LF])
+        if self.env['sim']:
+            target = 'sim'
+        else:
+            if self.env['kgdb']:
+                port = self.env['extra_serial_port']
+            else:
+                port = self.env['gdb_port']
+            target = 'remote localhost:{}'.format(port)
+        cmd.extend([
+            '-ex', 'file {}'.format(image), LF,
+            '-ex', 'target {}'.format(target), LF,
+        ])
+        if not self.env['kgdb']:
+            cmd.extend(break_at)
+        if not no_continue:
+            # ## lx-symbols
+            #
+            # ### lx-symbols after continue
+            #
+            # lx symbols must be run after continue.
+            #
+            # running it immediately after the connect on the bootloader leads to failure,
+            # likely because kernel structure on which it depends are not yet available.
+            #
+            # With this setup, continue runs, and lx-symbols only runs when a break happens,
+            # either by hitting the breakpoint, or by entering Ctrl + C.
+            #
+            # Sure, if the user sets a break on a raw address of the bootloader,
+            # problems will still arise, but let's think about that some other time.
+            #
+            # ### lx-symbols autoload
+            #
+            # The lx-symbols commands gets loaded through the file vmlinux-gdb.py
+            # which gets put on the kernel build root when python debugging scripts are enabled.
+            cmd.extend(['-ex', 'continue', LF])
+            if not self.env['no_lxsymbols'] and linux_full_system:
+                cmd.extend(['-ex', 'lx-symbols {}'.format(self.env['kernel_modules_build_subdir']), LF])
+        cmd.extend(after)
+        if self.env['test']:
+            self.sh.print_cmd(cmd)
+            if not self.env['dry_run']:
+                GdbTestcase(
+                    self.env['source_path'],
+                    test_script_path,
+                    self.sh.strip_newlines(cmd),
+                    verbose=self.env['verbose'],
+                )
+        else:
+            # I would rather have cwd be out_rootfs_overlay_dir,
+            # but then lx-symbols cannot fine the vmlinux and fails with:
+            # vmlinux: No such file or directory.
+            return self.sh.run_cmd(
+                cmd,
+                cmd_file=os.path.join(self.env['run_dir'], 'run-gdb.sh'),
+                cwd=self.env['linux_build_dir']
+            )
+
+if __name__ == '__main__':
+    Main().cli()

run-gdb-user

@@ -5,9 +5,9 @@ import os
 import sys
 
 import common
-rungdb = imp.load_source('rungdb', os.path.join(common.root_dir, 'run-gdb'))
+rungdb = imp.load_source('run_gdb', os.path.join(kwargs['root_dir'], 'run-gdb'))
 
-parser = common.get_argparse(argparse_args={
+parser = self.get_argparse(argparse_args={
     'description': '''GDB step debug guest userland processes without gdbserver.
 
 More information at: https://github.com/cirosantilli/linux-kernel-module-cheat#gdb-step-debug-userland-processes
@@ -23,9 +23,9 @@ parser.add_argument(
     help='Break at this point, e.g. main.',
     nargs='?'
 )
-args = common.setup(parser)
+args = self.setup(parser)
-executable = common.resolve_userland(args.executable)
+executable = self.resolve_userland(kwargs['executable'])
-addr = common.get_elf_entry(os.path.join(common.buildroot_build_build_dir, executable))
+addr = self.get_elf_entry(os.path.join(kwargs['buildroot_build_build_dir'], executable))
 extra_args = {}
 extra_args['before'] = '-ex \"add-symbol-file {} {}\"'.format(executable, hex(addr))
 # Or else lx-symbols throws for arm:
@@ -33,5 +33,5 @@ extra_args['before'] = '-ex \"add-symbol-file {} {}\"'.format(executable, hex(ad
 # TODO understand better.
 # Also, lx-symbols overrides the add-symbol-file commands.
 extra_args['no_lxsymbols'] = True
-extra_args['break_at'] = args.break_at
+extra_args['break_at'] = kwargs['break_at']
 sys.exit(rungdb.main(args, extra_args))

run-gdbserver

@@ -5,8 +5,9 @@ import subprocess
 import sys
 
 import common
+from shell_helpers import LF
 
-parser = common.get_argparse(argparse_args={
+parser = self.get_argparse(argparse_args={
     'description':'Connect to gdbserver running on the guest.'
 })
 parser.add_argument(
@@ -16,13 +17,13 @@ parser.add_argument(
 parser.add_argument(
     'break_at', default='main', nargs='?'
 )
-args = common.setup(parser)
+args = self.setup(parser)
 sys.exit(subprocess.Popen([
-  common.get_toolchain_tool('gdb'),
+  self.get_toolchain_tool('gdb'),
   '-q',
-  '-ex', 'set sysroot {}'.format(common.buildroot_staging_dir),
+  '-ex', 'set sysroot {}'.format(kwargs['buildroot_staging_dir']),
-  '-ex', 'target remote localhost:{}'.format(common.qemu_hostfwd_generic_port),
+  '-ex', 'target remote localhost:{}'.format(kwargs['qemu_hostfwd_generic_port']),
-  '-ex', 'tbreak {}'.format(args.break_at),
+  '-ex', 'tbreak {}'.format(kwargs['break_at']),
   '-ex', 'continue',
-  os.path.join(common.buildroot_build_build_dir, common.resolve_userland(args.executable)),
+  os.path.join(kwargs['buildroot_build_build_dir'], self.resolve_userland(kwargs['executable'])),
 ]).wait())

run-toolchain

@@ -4,8 +4,9 @@ import os
 import sys
 
 import common
+from shell_helpers import LF
 
-parser = common.get_argparse(argparse_args={
+parser = self.get_argparse(argparse_args={
     'description': '''Run a Buildroot ToolChain tool like readelf or objdump.
 
 For example, to get some information about the arm vmlinux:
@@ -24,7 +25,6 @@ ls "$(./getvar -a arm host_bin_dir)"
 parser.add_argument(
     '--dry',
     help='Just output the tool path to stdout but actually run it',
-    action='store_true',
 )
 parser.add_argument('tool', help='Which tool to run.')
 parser.add_argument(
@@ -34,17 +34,17 @@ parser.add_argument(
     metavar='extra-args',
     nargs='*'
 )
-args = common.setup(parser)
+args = self.setup(parser)
-if common.baremetal is None:
+if kwargs['baremetal'] is None:
-    image = common.vmlinux
+    image = kwargs['vmlinux']
 else:
-    image = common.image
+    image = kwargs['image']
-tool= common.get_toolchain_tool(args.tool)
+tool= self.get_toolchain_tool(kwargs['tool'])
-if args.dry:
+if kwargs['dry']:
     print(tool)
 else:
-    sys.exit(common.run_cmd(
+    sys.exit(self.sh.run_cmd(
-        [tool, common.Newline]
+        [tool, LF]
-        + common.add_newlines(args.extra_args),
+        + self.sh.add_newlines(kwargs['extra_args']),
-        cmd_file=os.path.join(common.run_dir, 'run-toolchain.sh'),
+        cmd_file=os.path.join(kwargs['run_dir'], 'run-toolchain.sh'),
     ))

shell_helpers.py

@@ -0,0 +1,296 @@
+#!/usr/bin/env python3
+
+import distutils.file_util
+import itertools
+import os
+import shlex
+import shutil
+import signal
+import stat
+import subprocess
+import sys
+import threading
+
+class LF:
+    '''
+    LineFeed (AKA newline).
+
+    Singleton class. Can be used in print_cmd to print out nicer command lines
+    with --key on the same line as "--key value".
+    '''
+    pass
+
+class ShellHelpers:
+    '''
+    Helpers to do things which are easy from the shell,
+    usually filesystem, process or pipe operations.
+
+    Attempt to print shell equivalents of all commands to make things
+    easy to debug and understand what is going on.
+    '''
+
+    _print_lock = threading.Lock()
+
+    def __init__(self, dry_run=False, quiet=False):
+        '''
+        :param dry_run: don't run the commands, just potentially print them. Debug aid.
+        :type dry_run: Bool
+
+        :param quiet: don't print the commands
+        :type dry_run: Bool
+        '''
+        self.dry_run = dry_run
+        self.quiet = quiet
+
+    @classmethod
+    def _print_thread_safe(cls, string):
+        '''
+        Python sucks: a naive print adds a bunch of random spaces to stdout,
+        and then copy pasting the command fails.
+        https://stackoverflow.com/questions/3029816/how-do-i-get-a-thread-safe-print-in-python-2-6
+        The initial use case was test-gdb which must create a thread for GDB to run the program in parallel.
+        '''
+        cls._print_lock.acquire()
+        sys.stdout.write(string + '\n')
+        sys.stdout.flush()
+        cls._print_lock.release()
+
+    def add_newlines(self, cmd):
+        out = []
+        for arg in cmd:
+            out.extend([arg, LF])
+        return out
+
+    def cp(self, src, dest, **kwargs):
+        self.print_cmd(['cp', src, dest])
+        if not self.dry_run:
+            shutil.copy2(src, dest)
+
+    @staticmethod
+    def cmd_to_string(cmd, cwd=None, extra_env=None, extra_paths=None):
+        '''
+        Format a command given as a list of strings so that it can
+        be viewed nicely and executed by bash directly and print it to stdout.
+        '''
+        last_newline = ' \\\n'
+        newline_separator = last_newline + '  '
+        out = []
+        if extra_env is None:
+            extra_env = {}
+        if cwd is not None:
+            out.append('cd {} &&'.format(shlex.quote(cwd)))
+        if extra_paths is not None:
+            out.append('PATH="{}:${{PATH}}"'.format(':'.join(extra_paths)))
+        for key in extra_env:
+            out.append('{}={}'.format(shlex.quote(key), shlex.quote(extra_env[key])))
+        cmd_quote = []
+        newline_count = 0
+        for arg in cmd:
+            if arg == LF:
+                cmd_quote.append(arg)
+                newline_count += 1
+            else:
+                cmd_quote.append(shlex.quote(arg))
+        if newline_count > 0:
+            cmd_quote = [' '.join(list(y)) for x, y in itertools.groupby(cmd_quote, lambda z: z == LF) if not x]
+        out.extend(cmd_quote)
+        if newline_count == 1 and cmd[-1] == LF:
+            ending = ''
+        else:
+            ending = last_newline + ';'
+        return newline_separator.join(out) + ending
+
+    def copy_dir_if_update_non_recursive(self, srcdir, destdir, filter_ext=None):
+        # TODO print rsync equivalent.
+        os.makedirs(destdir, exist_ok=True)
+        for basename in os.listdir(srcdir):
+            src = os.path.join(srcdir, basename)
+            if os.path.isfile(src):
+                noext, ext = os.path.splitext(basename)
+                if filter_ext is not None and ext == filter_ext:
+                    distutils.file_util.copy_file(
+                        src,
+                        os.path.join(destdir, basename),
+                        update=1,
+                    )
+
+    def print_cmd(self, cmd, cwd=None, cmd_file=None, extra_env=None, extra_paths=None):
+        '''
+        Print cmd_to_string to stdout.
+
+        Optionally save the command to cmd_file file, and add extra_env
+        environment variables to the command generated.
+
+        If cmd contains at least one LF, newlines are only added on LF.
+        Otherwise, newlines are added automatically after every word.
+        '''
+        if type(cmd) is str:
+            cmd_string = cmd
+        else:
+            cmd_string = self.cmd_to_string(cmd, cwd=cwd, extra_env=extra_env, extra_paths=extra_paths)
+        if not self.quiet:
+            self._print_thread_safe('+ ' + cmd_string)
+        if cmd_file is not None:
+            with open(cmd_file, 'w') as f:
+                f.write('#!/usr/bin/env bash\n')
+                f.write(cmd_string)
+            st = os.stat(cmd_file)
+            os.chmod(cmd_file, st.st_mode | stat.S_IXUSR)
+
+    def run_cmd(
+            self,
+            cmd,
+            cmd_file=None,
+            out_file=None,
+            show_stdout=True,
+            show_cmd=True,
+            extra_env=None,
+            extra_paths=None,
+            delete_env=None,
+            raise_on_failure=True,
+            **kwargs
+        ):
+        '''
+        Run a command. Write the command to stdout before running it.
+
+        Wait until the command finishes execution.
+
+        :param cmd: command to run. LF entries are magic get skipped.
+        :type cmd: List[str]
+
+        :param cmd_file: if not None, write the command to be run to that file
+        :type cmd_file: str
+
+        :param out_file: if not None, write the stdout and stderr of the command the file
+        :type out_file: str
+
+        :param show_stdout: wether to show stdout and stderr on the terminal or not
+        :type show_stdout: bool
+
+        :param extra_env: extra environment variables to add when running the command
+        :type extra_env: Dict[str,str]
+
+        :return: exit status of the command
+        :rtype: int
+        '''
+        if out_file is not None:
+            stdout = subprocess.PIPE
+            stderr = subprocess.STDOUT
+        else:
+            if show_stdout:
+                stdout = None
+                stderr = None
+            else:
+                stdout = subprocess.DEVNULL
+                stderr = subprocess.DEVNULL
+        if extra_env is None:
+            extra_env = {}
+        if delete_env is None:
+            delete_env = []
+        if 'cwd' in kwargs:
+            cwd = kwargs['cwd']
+        else:
+            cwd = None
+        env = os.environ.copy()
+        env.update(extra_env)
+        if extra_paths is not None:
+            path = ':'.join(extra_paths)
+            if 'PATH' in os.environ:
+                path += ':' + os.environ['PATH']
+            env['PATH'] = path
+        for key in delete_env:
+            if key in env:
+                del env[key]
+        if show_cmd:
+            self.print_cmd(cmd, cwd=cwd, cmd_file=cmd_file, extra_env=extra_env, extra_paths=extra_paths)
+
+        # Otherwise, if called from a non-main thread:
+        # ValueError: signal only works in main thread
+        if threading.current_thread() == threading.main_thread():
+            # Otherwise Ctrl + C gives:
+            # - ugly Python stack trace for gem5 (QEMU takes over terminal and is fine).
+            # - kills Python, and that then kills GDB: https://stackoverflow.com/questions/19807134/does-python-always-raise-an-exception-if-you-do-ctrlc-when-a-subprocess-is-exec
+            sigint_old = signal.getsignal(signal.SIGINT)
+            signal.signal(signal.SIGINT, signal.SIG_IGN)
+
+            # Otherwise BrokenPipeError when piping through | grep
+            # But if I do this_module, my terminal gets broken at the end. Why, why, why.
+            # https://stackoverflow.com/questions/14207708/ioerror-errno-32-broken-pipe-python
+            # Ignoring the exception is not enough as it prints a warning anyways.
+            #sigpipe_old = signal.getsignal(signal.SIGPIPE)
+            #signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+
+        cmd = self.strip_newlines(cmd)
+        if not self.dry_run:
+            # https://stackoverflow.com/questions/15535240/python-popen-write-to-stdout-and-log-file-simultaneously/52090802#52090802
+            with subprocess.Popen(cmd, stdout=stdout, stderr=stderr, env=env, **kwargs) as proc:
+                if out_file is not None:
+                    os.makedirs(os.path.split(os.path.abspath(out_file))[0], exist_ok=True)
+                    with open(out_file, 'bw') as logfile:
+                        while True:
+                            byte = proc.stdout.read(1)
+                            if byte:
+                                if show_stdout:
+                                    sys.stdout.buffer.write(byte)
+                                    try:
+                                        sys.stdout.flush()
+                                    except BlockingIOError:
+                                        # TODO understand. Why, Python, why.
+                                        pass
+                                logfile.write(byte)
+                            else:
+                                break
+            if threading.current_thread() == threading.main_thread():
+                signal.signal(signal.SIGINT, sigint_old)
+                #signal.signal(signal.SIGPIPE, sigpipe_old)
+            returncode = proc.returncode
+            if returncode != 0 and raise_on_failure:
+                raise Exception('Command exited with status: {}'.format(returncode))
+            return returncode
+        else:
+            return 0
+
+    def shlex_split(self, string):
+        '''
+        shlex_split, but also add Newline after every word.
+
+        Not perfect since it does not group arguments, but I don't see a solution.
+        '''
+        return self.add_newlines(shlex.split(string))
+
+    def strip_newlines(self, cmd):
+        return [x for x in cmd if x != LF]
+
+    def rmrf(self, path):
+        self.print_cmd(['rm', '-r', '-f', path, LF])
+        if not self.dry_run and os.path.exists(path):
+            if os.path.isdir(path):
+                shutil.rmtree(path)
+            else:
+                os.unlink(path)
+
+    def write_configs(self, config_path, configs, config_fragments=None, mode='a'):
+        '''
+        Append extra KEY=val configs into the given config file.
+        '''
+        if config_fragments is None:
+            config_fragments = []
+        for config_fragment in config_fragments:
+            self.print_cmd(['cat', config_fragment, '>>', config_path])
+        if not self.dry_run:
+            with open(config_path, 'a') as config_file:
+                for config_fragment in config_fragments:
+                    with open(config_fragment, 'r') as config_fragment_file:
+                        for line in config_fragment_file:
+                            config_file.write(line)
+        self.write_string_to_file(config_path, '\n'.join(configs), mode=mode)
+
+    def write_string_to_file(self, path, string, mode='w'):
+        if mode == 'a':
+            redirect = '>>'
+        else:
+            redirect = '>'
+        self.print_cmd("cat << 'EOF' {} {}\n{}\nEOF".format(redirect, path, string))
+        if not self.dry_run:
+            with open(path, mode) as f:
+                f.write(string)

test

@@ -1,14 +1,39 @@
-#!/usr/bin/env bash
+#!/usr/bin/env python3
-set -eu
+
-test_size=1
+import common
-while [ $# -gt 0 ]; do
+import shell_helpers
-  case "$1" in
+from shell_helpers import LF
-    --size)
+
-      test_size="$2"
+class Main(common.TestCliFunction):
-      shift 2
+    def __init__(self):
-      ;;
+        super().__init__(
-  esac
+            description='''\
-done
+https://github.com/cirosantilli/linux-kernel-module-cheat#automated-tests
-./bench-boot --size "$test_size"
+'''
-./test-modules
+        )
-./test-gdb
+        self.add_argument(
+            '--size',
+            default=1,
+            type=int,
+            help='''\
+Size of the tests to run. Scale:
+
+* 1: a few seconds and important
+* 2: < 5 minutes and important or a few seconds and not too important
+* 3: all
+'''
+        )
+
+    def timed_main(self):
+        run_args = self.get_common_args()
+        test_boot_args = run_args.copy()
+        test_boot_args['size'] = self.env['size']
+        self.run_test(self.import_path_main('test-boot'), test_boot_args, 'test-boot')
+        self.run_test(self.import_path_main('test-userland-full-system'), run_args, 'test-userland')
+        self.run_test(self.import_path_main('test-baremetal'), run_args, 'test-baremetal')
+        self.run_test(self.import_path_main('test-user-mode'), run_args, 'test-user-mode')
+        self.run_test(self.import_path_main('test-gdb'), run_args, 'test-gdb')
+
+if __name__ == '__main__':
+    Main().cli()
+

test-baremetal

@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+
+import common
+
+class Main(common.TestCliFunction):
+    def __init__(self):
+        super().__init__(
+            supported_archs=common.consts['crosstool_ng_supported_archs'],
+        )
+        self.add_argument(
+            'tests',
+            nargs='*',
+            help='''\
+If given, run only the given tests. Otherwise, run all tests.
+'''
+        )
+
+    def timed_main(self):
+        run = self.import_path_main('run')
+        run_args = self.get_common_args()
+        if self.env['emulator'] == 'gem5':
+            run_args['userland_build_id'] = 'static'
+        if self.env['tests'] == []:
+            baremetal_source_exts = (self.env['c_ext'], self.env['asm_ext'])
+            paths = []
+            for f in os.listdir(self.env['baremetal_source_dir']):
+                path = os.path.join(self.env['baremetal_source_dir'], f)
+                if os.path.isfile(path) and os.path.splitext(path)[1] in baremetal_source_exts:
+                    paths.append(path)
+            for root, dirs, files in os.walk(self.env['baremetal_source_arch_dir'], topdown=True):
+                dirs[:] = [d for d in dirs if d != 'interactive']
+                for file in files:
+                    path = os.path.join(root, file)
+                    if os.path.splitext(path)[1] in baremetal_source_exts:
+                        paths.append(path)
+            sources = []
+            for path in paths:
+                if not (
+                        self.env['emulator'] == 'gem5' and os.path.basename(path).startswith('semihost_') or
+                        self.env['emulator'] == 'qemu' and os.path.basename(path).startswith('gem5_')
+                    ):
+                        sources.append(os.path.relpath(path, self.env['baremetal_source_dir']))
+        else:
+            sources = self.env['tests']
+        for source in sources:
+            run_args['baremetal'] = source
+            run_args['ctrl_c_host'] = True
+            if os.path.splitext(os.path.basename(source))[0] == 'multicore':
+                run_args['cpus'] = 2
+            self.run_test(run, run_args, source)
+
+if __name__ == '__main__':
+    Main().cli()

test-boot

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+
+import common
+import shell_helpers
+from shell_helpers import LF
+
+class Main(common.TestCliFunction):
+    def __init__(self):
+        super().__init__(
+            description='''\
+Test and benchmark the Linux kernel boot. Use inits that exit immediately.
+'''
+        )
+        self.add_argument(
+            '--size',
+            default=1,
+            type=int,
+            help='''\
+See ./test --help for --size.
+'''
+        )
+
+    def _bench(self, **kwargs):
+        words = []
+        for line in self.run.get_cli(**kwargs):
+            words.extend(line)
+        extra_params = shell_helpers.ShellHelpers().cmd_to_string(words + [LF])
+        run_args = kwargs.copy()
+        run_args.update(self.common_args)
+        self.run_test(self.run, run_args, extra_params)
+
+    def timed_main(self):
+        # TODO bring this benchmark code back to life. Likely should go inside run with an option
+        #gem5_insts() (
+        #  printf "instructions $(./gem5-stat --arch "$1" sim_insts)\n" >> "$self.env['test_boot_benchmark_file']"
+        #  newline
+        #)
+        #
+        #qemu_insts() (
+        #  common_arch="$1"
+        #  ./qemu-trace2txt --arch "$common_arch"
+        #  common_qemu_trace_txt_file="$("$getvar" --arch "$common_arch" qemu_trace_txt_file)"
+        #  printf "instructions $(wc -l "${common_qemu_trace_txt_file}" | cut -d' ' -f1)\n" >> "$self.env['test_boot_benchmark_file']"
+        #  newline
+        #)
+        #
+        #rm -f "${self.env['test_boot_benchmark_file']}"
+        self.run = self.import_path_main('run')
+        self.common_args = self.get_common_args()
+        self.common_args['ctrl_c_host'] = True
+        self.common_args['quit_after_boot'] = True
+        if (self.env['emulator'] == 'qemu' or
+                (self.env['emulator'] == 'gem5' and self.env['size'] >= 2)):
+            self._bench()
+        if self.env['host_arch'] == self.env['arch']:
+            # TODO: find out why it fails.
+            if self.env['emulator'] != 'gem5':
+                self._bench(kvm=True)
+        if self.env['emulator'] == 'qemu' and self.env['size'] >= 2:
+            self._bench(trace='exec_tb')
+        if self.env['emulator'] == 'gem5' and self.env['size'] >= 3:
+            if self.env['arch'] == 'x86_64':
+                cpu_types = [
+                    # TODO segfault
+                    #'DerivO3CPU'
+                ]
+            elif self.env['is_arm']:
+                cpu_types = [
+                    'DerivO3CPU',
+                    'HPI',
+                ]
+            for cpu_type in cpu_types:
+                self._bench(
+                    extra_emulator_args=[
+                        '--cpu-type', cpu_type,
+                        '--caches',
+                        '--l2cache',
+                        '--l1d_size', '1024kB',
+                        '--l1i_size', '1024kB',
+                        '--l2_size', '1024kB',
+                        '--l3_size', '1024kB',
+                    ],
+                )
+            if self.env['arch'] == 'aarch64':
+                # Do a fuller testing for aarch64.
+                for build_type in ['debug', 'fast']:
+                    self._bench(gem5_build_type=build_type)
+                # Requires patching the executable.
+                # self._bench(gem5_script='biglittle')
+
+if __name__ == '__main__':
+    Main().cli()

test-gdb

@@ -1,35 +1,59 @@
-#!/usr/bin/env bash
+#!/usr/bin/env python3
-set -eux
-for emulator in --qemu --gem5; do
-  # Userland.
-  # TODO make work.
-  #./run --arch x86_64 --background --userland add "$emulator" --wait-gdb &
-  #./run-gdb --arch x86_64 --userland add "$emulator" --test "$@"
-  #wait
 
-  # Baremetal.
+import threading
-  ./run --arch arm --background --baremetal add "$emulator" --wait-gdb &
+import os
-  ./run-gdb --arch arm --baremetal add "$emulator" --test "$@"
+
-  wait
+import common
-  ./run --arch arm --background --baremetal arch/arm/add "$emulator" --wait-gdb &
+
-  ./run-gdb --arch arm --baremetal arch/arm/add "$emulator" --test "$@"
+class Main(common.TestCliFunction):
-  wait
+    def __init__(self):
-  ./run --arch arm --background --baremetal arch/arm/regs "$emulator" --wait-gdb &
+        super().__init__(
-  ./run-gdb --arch arm --baremetal arch/arm/regs "$emulator" --test "$@"
+            description='''\
-  wait
+https://github.com/cirosantilli/linux-kernel-module-cheat#test-gdb
-  ./run --arch aarch64 --background --baremetal add "$emulator" --wait-gdb &
+'''
-  ./run-gdb --arch aarch64 --baremetal add "$emulator" --test "$@"
+        )
-  wait
+        self.add_argument(
-  ./run --arch aarch64 --background --baremetal arch/aarch64/add "$emulator" --wait-gdb &
+            'tests',
-  ./run-gdb --arch aarch64 --baremetal arch/aarch64/add "$emulator" --test "$@"
+            nargs='*',
-  wait
+            help='''\
-  ./run --arch aarch64 --background --baremetal arch/aarch64/regs "$emulator" --wait-gdb &
+If given, run only the given tests. Otherwise, run all tests,
-  ./run-gdb --arch aarch64 --baremetal arch/aarch64/regs "$emulator" --test "$@"
+found by searching for the Python test files.
-  wait
+'''
-  ./run --arch aarch64 --background --baremetal arch/aarch64/fadd "$emulator" --wait-gdb &
+        )
-  ./run-gdb --arch aarch64 --baremetal arch/aarch64/fadd "$emulator" --test "$@"
+
-  wait
+    def timed_main(self):
-  ./run --arch aarch64 --background --baremetal arch/aarch64/regs "$emulator" --wait-gdb &
+        run = self.import_path_main('run')
-  ./run-gdb --arch aarch64 --baremetal arch/aarch64/regs "$emulator" --test "$@"
+        run_gdb = self.import_path_main('run-gdb')
-  wait
+        if self.env['arch'] in self.env['crosstool_ng_supported_archs']:
-done
+            if self.env['tests'] == []:
+                test_scripts_noext = []
+                for f in os.listdir(self.env['baremetal_source_dir']):
+                    base, ext = os.path.splitext(f)
+                    if ext == '.py':
+                        test_scripts_noext.append(base)
+                for root, dirs, files in os.walk(os.path.join(self.env['baremetal_source_dir'], 'arch', self.env['arch'])):
+                    for f in files:
+                        base, ext = os.path.splitext(f)
+                        if ext == '.py':
+                            full_path = os.path.join(root, base)
+                            relpath = os.path.relpath(full_path, self.env['baremetal_source_dir'])
+                            test_scripts_noext.append(relpath)
+            else:
+                test_scripts_noext = self.env['tests']
+            for test_script_noext in test_scripts_noext:
+                common_args = self.get_common_args()
+                common_args['baremetal'] = test_script_noext
+                test_id_string = self.test_setup(test_script_noext)
+                run_args = common_args.copy()
+                run_args['wait_gdb'] = True
+                run_args['background'] = True
+                run_thread = threading.Thread(target=lambda: run(**run_args))
+                run_thread.start()
+                gdb_args = common_args.copy()
+                gdb_args['test'] = True
+                run_gdb(**gdb_args)
+                run_thread.join()
+                self.test_teardown(run, 0, test_id_string)
+
+if __name__ == '__main__':
+    Main().cli()

test-modules

@@ -1,7 +0,0 @@
-#!/usr/bin/env bash
-set -eu
-root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
-getvar="${root_dir}/getvar"
-termout_file="$("$getvar" termout_file)"
-./run --eval-busybox '/test_all.sh;/poweroff.out' --kvm
-grep -q lkmc_test_pass "$termout_file"

test-user-mode

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+
+import common
+
+class Main(common.TestCliFunction):
+    def __init__(self):
+        super().__init__(
+            description='''\
+https://github.com/cirosantilli/linux-kernel-module-cheat#user-mode-tests
+'''
+            ,
+        )
+        self.add_argument(
+            'tests',
+            nargs='*',
+            help='''\
+If given, run only the given tests. Otherwise, run all tests.
+'''
+        )
+
+    def timed_main(self):
+        run = self.import_path_main('run')
+        run_args = self.get_common_args()
+        run_args['ctrl_c_host'] = True
+        if self.env['emulator'] == 'gem5':
+            run_args['userland_build_id'] = 'static'
+        if self.env['tests'] == []:
+            sources = [
+                'add.c',
+                'hello.c',
+                'hello_cpp.cpp',
+                'print_argv.c',
+            ]
+            if self.env['arch'] == 'x86_64':
+                arch_sources = [
+                    'asm_hello'
+                ]
+            elif self.env['arch'] == 'aarch64':
+                arch_sources = [
+                    'asm_hello'
+                ]
+            else:
+                arch_sources = []
+            arch_sources[:] = [
+                os.path.join('arch', self.env['arch'], arch_source)
+                for arch_source
+                in arch_sources
+            ]
+            sources.extend(arch_sources)
+        else:
+            sources = self.env['tests']
+        for source in sources:
+            run_args['userland'] = source
+            self.run_test(run, run_args, source)
+
+if __name__ == '__main__':
+    Main().cli()

test-userland-full-system

@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+
+import common
+
+class Main(common.TestCliFunction):
+    def __init__(self):
+        super().__init__(
+            description='''\
+https://github.com/cirosantilli/linux-kernel-module-cheat#test-userland-in-full-system
+'''
+        )
+    def timed_main(self):
+        run = self.import_path_main('run')
+        run_args = self.get_common_args()
+        run_args['eval_after'] = '/test_all.sh;{};'.format(self.env['userland_quit_cmd'])
+        self.run_test(run, run_args)
+
+if __name__ == '__main__':
+    Main().cli()

trace-boot

@@ -1,63 +1,57 @@
 #!/usr/bin/env python3
 
-import imp
-import os
-import subprocess
-import re
-
 import common
-run = imp.load_source('run', os.path.join(common.root_dir, 'run'))
+from shell_helpers import LF
-qemu_trace2txt = imp.load_source('qemu_trace2txt', os.path.join(common.root_dir, 'qemu-trace2txt'))
 
-parser = common.get_argparse(argparse_args={
+class Main(common.LkmcCliFunction):
-    'description': '''Trace the PIC addresses executed on a Linux kernel boot.
+    def __init__(self):
+        super().__init__(
+            description='''Trace the PIC addresses executed on a Linux kernel boot.
 
 More information at: https://github.com/cirosantilli/linux-kernel-module-cheat#tracing
 '''
-})
+        )
-parser.add_argument(
+
-    'extra_emulator_args', nargs='*',
+    def timed_main(self):
-    help='Extra options to append at the end of the emulator command line'
+        args = self.get_common_args()
-)
+        run = self.import_path_main('run')
-args = common.setup(parser)
+        if self.env['emulator'] == 'gem5':
-extra_args = {
+            args['trace'] = 'Exec,-ExecSymbol,-ExecMicro'
-    'extra_emulator_args': args.extra_emulator_args,
+            run.main(**args)
-}
+        elif self.env['emulator'] == 'qemu':
-if common.emulator == 'gem5':
+            run_args = args.copy()
-    extra_args.update({
+            run_args['trace'] = 'exec_tb'
-        'eval': 'm5 exit',
+            run_args['quit_after_boot'] = True
-        'trace': 'Exec,-ExecSymbol,-ExecMicro',
+            run.main(**run_args)
-    })
+            qemu_trace2txt = self.import_path_main('qemu-trace2txt')
-    run.main(args, extra_args)
+            qemu_trace2txt.main(**args)
-else:
+            # Instruction count.
-    extra_args.update({
+            # We could put this on a separate script, but it just adds more arch boilerplate to a new script.
-        'kernel_cli': 'init=/poweroff.out',
+            # So let's just leave it here for now since it did not add a significant processing time.
-        'trace': 'exec_tb',
+            kernel_entry_addr = hex(self.get_elf_entry(self.env['vmlinux']))
-    })
+            nlines = 0
-    run.main(args, extra_args)
+            nlines_firmware = 0
-    qemu_trace2txt.main()
+            with open(self.env['qemu_trace_txt_file'], 'r') as trace_file:
-    # Instruction count.
+                in_firmware = True
-    # We could put this on a separate script, but it just adds more arch boilerplate to a new script.
+                for line in trace_file:
-    # So let's just leave it here for now since it did not add a significant processing time.
+                    line = line.rstrip()
-    kernel_entry_addr = hex(common.get_elf_entry(common.vmlinux))
+                    nlines += 1
-    nlines = 0
+                    pc = line.split('=')[-1]
-    nlines_firmware = 0
+                    if pc == kernel_entry_addr:
-    with open(common.qemu_trace_txt_file, 'r') as trace_file:
+                        in_firmware = False
-        in_firmware = True
+                    if in_firmware:
-        for line in trace_file:
+                        nlines_firmware += 1
-            line = line.rstrip()
+            print('''\
-            nlines += 1
-            pc = line.split('=')[-1]
-            if pc == kernel_entry_addr:
-                in_firmware = False
-            if in_firmware:
-                nlines_firmware += 1
-    print('''\
 instructions {}
 entry_address {}
-instructions_firmware {}\
+instructions_firmware {}
 '''.format(
-        nlines,
+                    nlines,
-        kernel_entry_addr,
+                    kernel_entry_addr,
-        nlines_firmware
+                    nlines_firmware
-    ))
+                ),
+                end=''
+            )
+
+if __name__ == '__main__':
+    Main().cli()

trace2line

@@ -13,23 +13,24 @@ import subprocess
 import sys
 
 import common
+from shell_helpers import LF
 
-parser = common.get_argparse(argparse_args={
+parser = self.get_argparse(argparse_args={
     'description': 'Convert an execution trace containing PC values into the Linux kernel linex executed'
 })
-args = common.setup(parser)
+args = self.setup(parser)
 sys.exit(subprocess.Popen([
-    os.path.join(common.root_dir, 'trace2line.sh'),
+    os.path.join(kwargs['root_dir'], 'trace2line.sh'),
-    'true' if common.emulator == 'gem5' else 'false',
+    'true' if kwargs['emulator'] == 'gem5' else 'false',
-    common.trace_txt_file,
+    kwargs['trace_txt_file'],
-    common.get_toolchain_tool('addr2line'),
+    self.get_toolchain_tool('addr2line'),
-    common.vmlinux,
+    kwargs['vmlinux'],
-    common.run_dir,
+    kwargs['run_dir'],
 ]).wait())
 
 # This was the full conversion attempt.
 
-# if common.emulator == 'gem5':
+# if kwargs['emulator'] == 'gem5':
 #     def get_pc(line):
 #         # TODO
 #         # stdin = sed -r 's/^.* (0x[^. ]*)[. ].*/\1/' "$common_trace_txt_file")
@@ -40,17 +41,17 @@ sys.exit(subprocess.Popen([
 # with \
 #         subprocess.Popen(
 #             [
-#                 common.get_toolchain_tool('addr2line'),
+#                 self.get_toolchain_tool('addr2line'),
 #                 '-e',
-#                 common.vmlinux,
+#                 kwargs['vmlinux'],
 #                 '-f',
 #                 '-p',
 #             ],
 #             stdout=subprocess.PIPE,
 #             stdin=subprocess.PIPE,
 #         ) as proc, \
-#         open(common.trace_txt_file, 'r') as infile, \
+#         open(kwargs['trace_txt_file'], 'r') as infile, \
-#         open(os.path.join(common.run_dir, 'trace-lines.txt'), 'w') as outfile \
+#         open(os.path.join(kwargs['run_dir'], 'trace-lines.txt'), 'w') as outfile \
 #     :
 #     for in_line in infile:
 #         proc.stdin.write(get_pc(in_line).encode())
@@ -58,5 +59,5 @@ sys.exit(subprocess.Popen([
 #         stdout = proc.stdout.read()
 #         outfile.write(stdout.decode())
 #     # TODO
-#     # sed -E "s|at ${common.linux_build_dir}/(\./\|)||"
+#     # sed -E "s|at ${kwargs['linux_build_dir']}/(\./\|)||"
 #     # uniq -c

userland/Makefile

@@ -46,7 +46,9 @@ OUTS := $(addprefix $(OUT_DIR)/,$(OUTS))
 
 all: mkdir $(OUTS)
 	for subdir in $(SUBDIRS); do \
-	  $(MAKE) -C $${subdir} OUT_DIR="$(OUT_DIR)/$$subdir"; \
+	  if [ -d "$${subdir}" ]; then \
+	    $(MAKE) -C "$${subdir}" OUT_DIR="$(OUT_DIR)/$$subdir"; \
+	  fi \
 	done
 
 $(COMMON_OBJ): $(COMMON_DIR)/$(COMMON_BASENAME)$(IN_EXT_C)
@@ -64,7 +66,9 @@ $(OUT_DIR)/%$(OUT_EXT): %$(IN_EXT_CXX) $(COMMON_OBJ)
 clean:
 	rm -f *'$(OBJ_EXT)' *'$(OUT_EXT)'
 	for subdir in $(SUBDIRS); do \
-	  $(MAKE) -C $${subdir} clean; \
+	  if [ -d "$${subdir}" ]; then \
+	    $(MAKE) -C $${subdir} clean; \
+	  fi \
 	done
 
 mkdir:

userland/count.c

@@ -1,11 +1,18 @@
+#include <limits.h>
 #include <stdio.h>
+#include <stdlib.h>
 #include <unistd.h>
 
-int main(void) {
+int main(int argc, char **argv) {
-	int i = 0;
+    unsigned long i = 0, max;
-	while (1) {
+    if (argc > 1) {
-		printf("%d\n", i);
+        max = strtoul(argv[1], NULL, 10);
-		i++;
+    } else {
-		sleep(1);
+        max = ULONG_MAX;
-	}
+    }
+    while (i < max) {
+        printf("%lu\n", i);
+        i++;
+        sleep(1);
+    }
 }

userland/external.desc

@@ -1 +0,0 @@
-name: USERLAND

userland/false.c

@@ -0,0 +1,13 @@
+/* Test that emulators forward the exit status properly. */
+
+#include <stdlib.h>
+
+int main(int argc, char **argv) {
+    int ret;
+    if (argc == 1) {
+        ret = 1;
+    } else {
+        ret = strtoull(argv[1], NULL, 0);
+    }
+	return ret;
+}

userland/print_argv.c

@@ -1,3 +1,5 @@
+/* Print each command line argument received, one per line. */
+
 #include <stdio.h>
 
 int main(int argc, char **argv) {