gem5: update to 7bfb7f3a43f382eb49853f47b140bfd6caad0fb8

The update is required to include 3c3ca64b5f0dd9eef7b1ce1c65cc6e8e9147dd38
otherwise baremetal does not on VExpress.

baremetal: create a baremetal setup with crosstool-ng

buildroot: improve directory location: move out/dl inside
out/buildroot/download, and add a new out/buildroot/build level

tagline: generalize, deliver more value than howto, since now howtos
are starting to multiply

rename all top scripts to separate words with hyphen more consistently,
e.g. run-gdb instead of rungdb

getvar: list all variables

gem5: make m5out section to focus all releated information at

Prevent m5term Text file busy when rebuilding gem5 while it is running.

.gitmodules

@@ -2,6 +2,9 @@
 	path = submodules/buildroot
 	url = https://github.com/cirosantilli/buildroot
 	ignore = dirty
+[submodule "submodules/crosstool-ng"]
+	path = submodules/crosstool-ng
+	url = https://github.com/crosstool-ng/crosstool-ng
 [submodule "submodules/gem5"]
 	path = submodules/gem5
 	url = https://gem5.googlesource.com/public/gem5

baremetal/arch/arm/gem5_assert.S

@@ -0,0 +1,19 @@
+/* assert 0x12345678 + 1 == 0x12345679 */
+.global main
+main:
+    movw r0, #:lower16:myvar
+    movt r0, #:upper16:myvar
+    ldr r1, [r0]
+    add r1, r1, #1
+    str r1, [r0]
+    movw r2, #0x5679
+    movt r2, #0x1234
+    cmp r1, r2
+    beq ok
+    # m5 fail 1
+    mov r0, #0; mov r1, #0; mov r2, #1; mov r3, #0; .inst 0xEE000110 | (0x22 << 16);
+ok:
+    # m5 exit
+    mov r0, #0; mov r1, #0; .inst 0xEE000110 | (0x21 << 16);
+myvar:
+    .word 0x12345678

baremetal/arch/arm/no_bootloader/semihost_exit.S

@@ -0,0 +1,5 @@
+.global mystart
+mystart:
+    mov r0, #0x18
+    ldr r1, =#0x20026
+    svc 0x00123456

baremetal/arch/arm/semihost_exit.S

@@ -0,0 +1,5 @@
+.global main
+main:
+    mov r0, #0x18
+    ldr r1, =#0x20026
+    svc 0x00123456

baremetal/exit.c

@@ -0,0 +1,7 @@
+#include <stdio.h>
+#include <stdlib.h>
+
+void main(void) {
+	exit(0);
+}
+

baremetal/lib/aarch64.S

@@ -0,0 +1,11 @@
+.global mystart
+mystart:
+    /* = NEON setup */
+    mov x1, #(0x3 << 20)
+    msr cpacr_el1, x1
+    isb
+
+    ldr x0, =stack_top
+    mov sp, x0
+    bl main
+    b .

baremetal/lib/arm.S

@@ -0,0 +1,5 @@
+.global mystart
+mystart:
+    ldr sp, =stack_top
+    bl main
+    b .

baremetal/lib/common.c

@@ -0,0 +1,65 @@
+#include <sys/stat.h>
+
+enum {
+    UART_FR_RXFE = 0x10,
+};
+
+#define UART_DR(baseaddr) (*(unsigned int *)(baseaddr))
+#define UART_FR(baseaddr) (*(((unsigned int *)(baseaddr))+6))
+
+int _close(int file) { return -1; }
+
+int _fstat(int file, struct stat *st) {
+    st->st_mode = S_IFCHR;
+    return 0;
+}
+
+int _isatty(int file) { return 1; }
+int _lseek(int file, int ptr, int dir) { return 0; }
+int _open(const char *name, int flags, int mode) { return -1; }
+
+int _read(int file, char *ptr, int len) {
+    int todo;
+    if (len == 0)
+        return 0;
+    while (UART_FR(UART0_ADDR) & UART_FR_RXFE);
+    *ptr++ = UART_DR(UART0_ADDR);
+    for (todo = 1; todo < len; todo++) {
+        if (UART_FR(UART0_ADDR) & UART_FR_RXFE) {
+            break;
+        }
+        *ptr++ = UART_DR(UART0_ADDR);
+    }
+    return todo;
+}
+
+char *heap_end = 0;
+caddr_t _sbrk(int incr) {
+    extern char heap_low;
+    extern char heap_top;
+    char *prev_heap_end;
+    if (heap_end == 0) {
+        heap_end = &heap_low;
+    }
+    prev_heap_end = heap_end;
+    if (heap_end + incr > &heap_top) {
+        /* Heap and stack collision */
+        return (caddr_t)0;
+    }
+    heap_end += incr;
+    return (caddr_t)prev_heap_end;
+}
+
+int _write(int file, char *ptr, int len) {
+    int todo;
+    for (todo = 0; todo < len; todo++) {
+        UART_DR(UART0_ADDR) = *ptr++;
+    }
+    return len;
+}
+
+void _exit(int status) {
+#if defined(__arm__)
+    __asm__ __volatile__ ("mov r0, #0x18; ldr r1, =#0x20026; svc 0x00123456");
+#endif
+}

baremetal/link.ld

@@ -0,0 +1,19 @@
+ENTRY(mystart)
+SECTIONS
+{
+  .text : {
+    */bootloader.o(.text)
+    *(.text)
+    *(.rodata)
+    *(.data)
+    *(COMMON)
+  }
+  /* gem5 uses the bss as a measure of the kernel size. */
+  .bss : { *(.bss) }
+  heap_low = .;
+  . = . + 0x1000000;
+  heap_top = .;
+  . = . + 0x1000000;
+  stack_top = .;
+}
+

baremetal/prompt.c

@@ -0,0 +1,22 @@
+#include <stdio.h>
+#include <stdlib.h>
+
+void main(void) {
+    char c;
+    char *ptr = NULL;
+    size_t alloc_size = 1;
+    while (1) {
+        printf("enter a character\n");
+        c = getchar();
+        printf("got: %c\n", c);
+        ptr = realloc(ptr, alloc_size);
+        if (ptr == NULL) {
+            puts("out of memory");
+            break;
+        } else {
+            printf("new alloc of %d bytes at address 0x%p\n", alloc_size, ptr);
+            alloc_size <<= 1;
+        }
+    }
+}
+

bench-all

@@ -79,7 +79,7 @@ do_bench_buildroot_build() (
     baseline_suffix=
   fi
   common_build_dir="$("$getvar" --arch "$arch" --buildroot-build-id "$build_id" build_dir)"
-  common_images_dir="$("$getvar" --arch "$arch" --buildroot-build-id "$build_id" images_dir)"
+  common_images_dir="$("$getvar" --arch "$arch" --buildroot-build-id "$build_id" buildroot_images_dir)"
   "${root_dir}/build-buildroot" --arch "$arch" $baseline --buildroot-build-id "$build_id" --clean
   "${root_dir}/build-buildroot" --arch "$arch" $baseline --buildroot-build-id "$build_id" --no-all -- source
   "${root_dir}/build-buildroot" --arch "$arch" $baseline --buildroot-build-id "$build_id"

bench-boot

@@ -30,8 +30,13 @@ bench() (
   "${root_dir}/bench-cmd" "./run --arch ${1}${extra_args}" "$common_bench_boot"
 )
 
+newline() (
+  echo >> "$common_bench_boot"
+)
+
 gem5_insts() (
   printf "instructions $(./gem5-stat --arch "$1" sim_insts)\n" >> "$common_bench_boot"
+  newline
 )
 
 qemu_insts() (
@@ -39,10 +44,7 @@ qemu_insts() (
   ./qemu-trace2txt --arch "$common_arch"
   common_qemu_trace_txt_file="$("$getvar" --arch "$common_arch" qemu_trace_txt_file)"
   printf "instructions $(wc -l "${common_qemu_trace_txt_file}" | cut -d' ' -f1)\n" >> "$common_bench_boot"
-)
+  newline
-
-newline() (
-  echo >> "$common_bench_boot"
 )
 
 rm -f "${common_bench_boot}"
@@ -55,31 +57,26 @@ newline
 if [ "$test_size" -ge 2 ]; then
   bench "${arch} --eval '/poweroff.out' --trace exec_tb"
   qemu_insts "$arch"
-  newline
   bench "$arch --eval 'm5 exit' --gem5"
   gem5_insts "$arch"
-  newline
 fi
 #bench "$arch --eval 'm5 exit' --gem5 -- --cpu-type=DerivO3CPU ${caches}"
 #gem5_insts "$arch"
-#newline
 
 arch=arm
 bench "$arch --eval '/poweroff.out'"
-newline
 if [ "$test_size" -ge 2 ]; then
   bench "$arch --eval '/poweroff.out' --trace exec_tb"
   qemu_insts "$arch"
-  newline
   #bench "$arch --eval 'm5 exit' --gem5"
   #gem5_insts "$arch"
-  #newline
 fi
-#if [ "$test_size" -ge 3 ]; then
+if [ "$test_size" -ge 3 ]; then
   #bench "$arch --eval 'm5 exit' --gem5 -- --cpu-type=HPI ${caches}"
   #gem5_insts "$arch"
-#  newline
+  bench "$arch --eval 'm5 exit' --gem5 --gem5-biglittle"
-#fi
+  gem5_insts "$arch"
+fi
 
 arch=aarch64
 bench "$arch --eval '/poweroff.out'"
@@ -87,13 +84,10 @@ newline
 if [ "$test_size" -ge 2 ]; then
   bench "$arch --eval '/poweroff.out' --trace exec_tb"
   qemu_insts "$arch"
-  newline
   bench "$arch --eval 'm5 exit' --gem5"
   gem5_insts "$arch"
-  newline
 fi
 if [ "$test_size" -ge 3 ]; then
   bench "$arch --eval 'm5 exit' --gem5 -- --cpu-type=HPI ${caches}"
   gem5_insts "$arch"
-  newline
 fi

build-all

@@ -15,8 +15,14 @@ done
 shift "$(($OPTIND - 1))"
 for arch in $archs; do
   ./build-qemu --arch "$arch"
-  ./build-buildroot --arch "$arch" --gem5 --kernel-modules -l "$@"
   if "$gem5"; then
     ./build-gem5 --arch "$arch"
   fi
+  ./build-buildroot --arch "$arch" --gem5 --kernel-modules -l "$@"
+  if [ ! "$arch" = x86_64 ]; then
+    ./build-crosstool-ng --arch "$arch"
+    ./build-baremetal --arch "$arch"
+    ./build-baremetal --arch "$arch" -g
+    ./build-baremetal --arch "$arch" -g --machine RealViewPBX
+  fi
 done

build-baremetal

@@ -0,0 +1,151 @@
+#!/usr/bin/env python3
+
+import glob
+import multiprocessing
+import os
+import shutil
+import sys
+import time
+
+import common
+
+def build_dir(subpath, gcc, cflags, entry_address, bootloader_obj, common_obj, bootloader=True):
+    """
+    Build all .c and .S files in a given subpath of the baremetal source
+    directory non recursively.
+
+    Place outputs on the same subpath or the output directory.
+    """
+    in_dir = os.path.join(common.baremetal_src_dir, subpath)
+    out_dir = os.path.join(common.baremetal_out_dir, subpath)
+    os.makedirs(out_dir, exist_ok=True)
+    if bootloader:
+        bootloader_cmd = [bootloader_obj]
+    else:
+        bootloader_cmd = []
+    for in_basename in os.listdir(in_dir):
+        in_path = os.path.join(in_dir, in_basename)
+        if os.path.isfile(in_path) and os.path.splitext(in_basename)[1] in (common.c_ext, common.asm_ext):
+            in_name = os.path.splitext(in_basename)[0]
+            main_obj = os.path.join(common.baremetal_out_dir, subpath, '{}{}'.format(in_name, common.obj_ext))
+            assert common.run_cmd(
+                [gcc] +
+                cflags +
+                [
+                    '-c',
+                    '-o', main_obj,
+                    os.path.join(common.baremetal_src_dir, in_path),
+                ]
+            ) == 0
+            assert common.run_cmd(
+                [gcc] +
+                cflags +
+                [
+                    '-Wl,--section-start=.text={:#x}'.format(entry_address),
+                    '-o', os.path.join(common.baremetal_out_dir, subpath, in_name + common.baremetal_out_ext),
+                    '-T', os.path.join(common.baremetal_src_dir, 'link.ld'),
+                ] +
+                bootloader_cmd +
+                [
+                    common_obj,
+                    main_obj,
+                ]
+            ) == 0
+
+def get_argparse():
+        parser = common.get_argparse(argparse_args={
+            'description': 'Build the baremetal examples with crosstool-NG'
+        })
+        common.add_build_arguments(parser)
+        return parser
+
+def main(args, extra_args=None):
+    if args.clean:
+        common.rmrf(common.baremetal_out_dir)
+    else:
+        common.raise_no_x86(args.arch)
+        bootloader_obj = os.path.join(common.baremetal_out_lib_dir, 'bootloader{}'.format(common.obj_ext))
+        common_obj = os.path.join(common.baremetal_out_lib_dir, 'common{}'.format(common.obj_ext))
+        cflags = [
+            '-ggdb3',
+            '-mcpu={}'.format(common.mcpu),
+            '-nostartfiles',
+            '-O0',
+        ]
+        if args.prebuilt:
+            gcc = 'arm-none-eabi-gcc'
+        else:
+            os.environ['PATH'] = common.crosstool_ng_bin_dir + os.environ['PATH']
+            gcc = common.get_toolchain_tool('gcc')
+        if args.gem5:
+            if common.machine == 'VExpress_GEM5_V1':
+                entry_address = 0x80000000
+                uart_address = 0x1c090000
+            elif common.machine == 'RealViewPBX':
+                entry_address = 0x10000
+                uart_address = 0x10009000
+            else:
+                raise Exception('unknown machine: ' + common.machine)
+        else:
+            entry_address = 0x40000000
+            uart_address = 0x09000000
+        os.makedirs(common.baremetal_out_dir, exist_ok=True)
+        os.makedirs(common.baremetal_out_lib_dir, exist_ok=True)
+        assert common.run_cmd(
+            [gcc] +
+            cflags +
+            [
+                '-c',
+                '-o', bootloader_obj,
+                os.path.join(common.baremetal_src_lib_dir, '{}{}'.format(args.arch, common.asm_ext)),
+            ]
+        ) == 0
+        assert common.run_cmd(
+            [gcc] +
+            cflags +
+            [
+                '-c',
+                '-D',
+                'UART0_ADDR={:#x}'.format(uart_address),
+                '-o', common_obj,
+                os.path.join(common.baremetal_src_lib_dir, 'common' + common.c_ext),
+            ]
+        ) == 0
+        build_dir(
+            '',
+            gcc=gcc,
+            cflags=cflags,
+            entry_address=entry_address,
+            bootloader_obj=bootloader_obj,
+            common_obj=common_obj,
+        )
+        build_dir(
+            os.path.join('arch', args.arch),
+            gcc=gcc,
+            cflags=cflags,
+            entry_address=entry_address,
+            bootloader_obj=bootloader_obj,
+            common_obj=common_obj,
+        )
+        build_dir(
+            os.path.join('arch', args.arch, 'no_bootloader'),
+            gcc=gcc,
+            cflags=cflags,
+            entry_address=entry_address,
+            bootloader_obj=bootloader_obj,
+            common_obj=common_obj,
+            bootloader=False,
+        )
+    return 0
+
+if __name__ == '__main__':
+    parser = common.get_argparse(
+        default_args={'baremetal': 'all'},
+    )
+    common.add_build_arguments(parser)
+    args = common.setup(parser)
+    start_time = time.time()
+    exit_status = main(args)
+    end_time = time.time()
+    common.print_time(end_time - start_time)
+    sys.exit(exit_status)

build-buildroot

@@ -168,9 +168,9 @@ def main(args, extra_args=None):
             buildroot_configs = args.buildroot_config
             buildroot_configs.extend([
                 'BR2_JLEVEL={}'.format(nproc),
-                'BR2_DL_DIR="{}"'.format(common.dl_dir),
+                'BR2_DL_DIR="{}"'.format(common.buildroot_download_dir),
             ])
-            write_configs(buildroot_configs)
+            common.write_configs(common.buildroot_config_file, buildroot_configs)
             if not args.baseline:
                 buildroot_configs.extend([
                     'BR2_GLOBAL_PATCH_DIR="{}"'.format(
@@ -182,7 +182,7 @@ def main(args, extra_args=None):
                     'BR2_ROOTFS_OVERLAY="{}"'.format(
                         path_relative_to_buildroot(os.path.join(common.root_dir, 'rootfs_overlay'))),
                     'BR2_ROOTFS_POST_BUILD_SCRIPT="{}"'.format(
-                        path_relative_to_buildroot(os.path.join(common.root_dir, 'rootfs_post_build_script'))),
+                        path_relative_to_buildroot(os.path.join(common.root_dir, 'rootfs-post-build-script'))),
                     'BR2_ROOTFS_USERS_TABLES="{}"'.format(
                         path_relative_to_buildroot(os.path.join(common.root_dir, 'user_table'))),
                 ])
@@ -253,7 +253,7 @@ def main(args, extra_args=None):
                     kernel_config_fragments[i] = path_relative_to_buildroot(frag)
                 buildroot_kernel_config_fragment_str = 'BR2_LINUX_KERNEL_CONFIG_FRAGMENT_FILES="{}"'.format(' '.join(kernel_config_fragments))
                 buildroot_configs.append(buildroot_kernel_config_fragment_str)
-                write_configs(buildroot_configs, buildroot_config_fragments)
+                common.write_configs(common.buildroot_config_file, buildroot_configs, buildroot_config_fragments)
                 subprocess.check_call(
                     [
                         'make',
@@ -304,22 +304,6 @@ def main(args, extra_args=None):
 def path_relative_to_buildroot(abspath):
     return os.path.relpath(abspath, common.buildroot_src_dir)
 
-def write_configs(buildroot_configs, buildroot_config_fragments=None):
-    """
-    Write extra configs into the Buildroot config file.
-    TODO Can't get rid of these for now with nice fragments:
-    http://stackoverflow.com/questions/44078245/is-it-possible-to-use-config-fragments-with-buildroots-config
-    """
-    if buildroot_config_fragments is None:
-        buildroot_config_fragments = []
-    with open(common.buildroot_config_file, 'a') as br2_config_file:
-        for buildroot_config_fragment in buildroot_config_fragments:
-            with open(buildroot_config_fragment, 'r') as br2_config_fragment:
-                for line in br2_config_fragment:
-                    br2_config_file.write(line)
-        for buildroot_config in buildroot_configs:
-            br2_config_file.write(buildroot_config + '\n')
-
 if __name__ == '__main__':
     parser = get_argparse()
     args = common.setup(parser)

build-crosstool-ng

@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+
+import multiprocessing
+import os
+import shutil
+import sys
+import time
+
+import common
+
+def get_argparse():
+    parser = common.get_argparse(argparse_args={
+        'description': 'Build crosstool-NG with Newlib for bare metal compilation'
+    })
+    common.add_build_arguments(parser)
+    return parser
+
+def main(args, extra_args=None):
+    common.raise_no_x86(args.arch)
+    nproc = multiprocessing.cpu_count()
+    defconfig_dest = os.path.join(common.crosstool_ng_util_dir, 'defconfig')
+    os.makedirs(common.crosstool_ng_util_dir, exist_ok=True)
+    os.makedirs(common.crosstool_ng_download_dir, exist_ok=True)
+
+    # Bootstrap out-ot-tree WONTFIX. I've tried.
+    # https://github.com/crosstool-ng/crosstool-ng/issues/1021
+    os.chdir(common.crosstool_ng_src_dir)
+    assert common.run_cmd(
+        [os.path.join(common.crosstool_ng_src_dir, 'bootstrap')],
+    ) == 0
+    os.chdir(common.crosstool_ng_util_dir)
+    assert common.run_cmd(
+        [
+            os.path.join(common.crosstool_ng_src_dir, 'configure'),
+            '--enable-local',
+        ],
+    ) == 0
+    assert common.run_cmd(
+        [
+            'make',
+            '-j', str(nproc)
+        ],
+    ) == 0
+
+    # Build the toolchain.
+    shutil.copy2(
+        os.path.join(common.root_dir, 'crosstool_ng_config', args.arch),
+        defconfig_dest
+    )
+    common.write_configs(
+        common.crosstool_ng_defconfig,
+        [
+            'CT_PREFIX_DIR="{}"'.format(common.crosstool_ng_install_dir),
+            'CT_WORK_DIR="{}"'.format(common.crosstool_ng_build_dir),
+            'CT_LOCAL_TARBALLS_DIR="{}"'.format(common.crosstool_ng_download_dir),
+        ]
+    )
+    assert common.run_cmd(
+        [
+            common.crosstool_ng_executable,
+            'defconfig',
+        ],
+    ) == 0
+    os.unlink(defconfig_dest)
+    assert common.run_cmd(
+        [
+            common.crosstool_ng_executable,
+            'build',
+            '-j', str(nproc)
+        ],
+        out_file=os.path.join(common.crosstool_ng_build_dir, 'lkmc.log'),
+        delete_env=['LD_LIBRARY_PATH'],
+        extra_env={'PATH': common.ccache_dir + ':' + os.environ['PATH']},
+    ) == 0
+
+if __name__ == '__main__':
+    parser = get_argparse()
+    args = common.setup(parser)
+    start_time = time.time()
+    exit_status = main(args)
+    end_time = time.time()
+    common.print_time(end_time - start_time)
+    sys.exit(exit_status)

build-gem5

@@ -84,7 +84,13 @@ else:
         extra_env={'PATH': common.ccache_dir + ':' + os.environ['PATH']},
     ) == 0
     term_src_dir = os.path.join(common.gem5_src_dir, 'util/term')
+    m5term_build = os.path.join(term_src_dir, 'm5term')
     subprocess.check_call(['make', '-C', term_src_dir])
-    shutil.copy2(os.path.join(term_src_dir, 'm5term'), common.gem5_m5term)
+    if os.path.exists(common.gem5_m5term):
+        # Otherwise shutil.copy2 would fail with "Text file busy" if you
+        # tried to rebuild while running m5term:
+        # https://stackoverflow.com/questions/16764946/what-generates-the-text-file-busy-message-in-unix/52427512#52427512
+        os.unlink(common.gem5_m5term)
+    shutil.copy2(m5term_build, common.gem5_m5term)
     end_time = time.time()
     common.print_time(end_time - start_time)

common.py

@@ -26,9 +26,11 @@ p9_dir = os.path.join(data_dir, '9p')
 gem5_non_default_src_root_dir = os.path.join(data_dir, 'gem5')
 out_dir = os.path.join(root_dir, 'out')
 bench_boot = os.path.join(out_dir, 'bench-boot.txt')
-dl_dir = os.path.join(out_dir, 'dl')
+packages_dir = os.path.join(root_dir, 'packages')
+kernel_modules_src_dir = os.path.join(this.packages_dir, 'kernel_modules')
 submodules_dir = os.path.join(root_dir, 'submodules')
 buildroot_src_dir = os.path.join(submodules_dir, 'buildroot')
+crosstool_ng_src_dir = os.path.join(submodules_dir, 'crosstool-ng')
 gem5_default_src_dir = os.path.join(submodules_dir, 'gem5')
 linux_src_dir = os.path.join(submodules_dir, 'linux')
 extract_vmlinux = os.path.join(linux_src_dir, 'scripts', 'extract-vmlinux')
@@ -97,6 +99,14 @@ def get_argparse(default_args=None, argparse_args=None):
         '-a', '--arch', choices=arch_choices, default='x86_64',
         help='CPU architecture. Default: %(default)s'
     )
+    parser.add_argument(
+        '--baremetal',
+        help='Use Baremetal examples instead of Linux kernel ones'
+    )
+    parser.add_argument(
+        '--crosstool-ng-build-id', default=default_build_id,
+        help='Crosstool-NG build ID. Allows you to keep multiple separate crosstool-NG builds. Default: %(default)s'
+    )
     parser.add_argument(
         '-g', '--gem5', default=False, action='store_true',
         help='Use gem5 instead of QEMU'
@@ -105,6 +115,14 @@ def get_argparse(default_args=None, argparse_args=None):
         '-L', '--linux-build-id', default=default_build_id,
         help='Linux build ID. Allows you to keep multiple separate Linux builds. Default: %(default)s'
     )
+    parser.add_argument(
+        '--machine',
+        help='''Machine type.
+QEMU default: virt
+gem5 default: VExpress_GEM5_V1
+See the documentation for other values known to work.
+'''
+    )
     parser.add_argument(
         '-M', '--gem5-build-id', default=default_build_id,
         help='gem5 build ID. Allows you to keep multiple separate gem5 builds. Default: %(default)s'
@@ -123,6 +141,14 @@ given, just use the submodule source.
 ID for run outputs such as gem5's m5out. Allows you to do multiple runs,
 and then inspect separate outputs later in different output directories.
 Default: %(default)s
+'''
+    )
+    parser.add_argument(
+        '-P', '--prebuilt', default=False, action='store_true',
+        help='''\
+Use prebuilt packaged host utilities as much as possible instead
+of the ones we built ourselves. Saves build time, but decreases
+the likelihood of compatibility.
 '''
     )
     parser.add_argument(
@@ -190,7 +216,10 @@ def get_stats(stat_re=None, stats_file=None):
 
 def get_toolchain_tool(tool):
     global this
+    if this.baremetal is None:
         return glob.glob(os.path.join(this.host_bin_dir, '*-buildroot-*-{}'.format(tool)))[0]
+    else:
+        return os.path.join(this.crosstool_ng_bin_dir, '{}-{}'.format(this.crosstool_ng_prefix, tool))
 
 def github_make_request(
         authenticate=False,
@@ -228,6 +257,14 @@ def github_make_request(
 def log_error(msg):
     print('error: {}'.format(msg), file=sys.stderr)
 
+def mkdir():
+    global this
+    os.makedirs(this.build_dir, exist_ok=True)
+    os.makedirs(this.gem5_build_dir, exist_ok=True)
+    os.makedirs(this.gem5_run_dir, exist_ok=True)
+    os.makedirs(this.qemu_run_dir, exist_ok=True)
+    os.makedirs(this.p9_dir, exist_ok=True)
+
 def print_cmd(cmd, cmd_file=None, extra_env=None):
     '''
     Format a command given as a list of strings so that it can
@@ -284,6 +321,10 @@ def raw_to_qcow2(prebuilt=False, reverse=False):
         outfile,
     ]) == 0
 
+def raise_no_x86(arch):
+    if (arch == 'x86_64'):
+        raise Exception('x86_64 not yet supported')
+
 def resolve_args(defaults, args, extra_args):
     if extra_args is None:
         extra_args = {}
@@ -362,6 +403,7 @@ def run_cmd(
     # https://stackoverflow.com/questions/15535240/python-popen-write-to-stdout-and-log-file-simultaneously/52090802#52090802
     with subprocess.Popen(cmd, stdout=stdout, stderr=stderr, env=env, **kwargs) as proc:
         if out_file is not None:
+            os.makedirs(os.path.split(os.path.abspath(out_file))[0], exist_ok=True)
             with open(out_file, 'bw') as logfile:
                 while True:
                     byte = proc.stdout.read(1)
@@ -385,15 +427,41 @@ def setup(parser):
     args = parser.parse_args()
     if args.arch in this.arch_map:
         args.arch = this.arch_map[args.arch]
+    this.machine = args.machine
     if args.arch == 'arm':
         this.armv = 7
         this.gem5_arch = 'ARM'
+        this.mcpu = 'cortex-a15'
+        this.crosstool_ng_prefix = 'arm-unknown-eabi'
+        if args.gem5:
+            if this.machine is None:
+                this.machine = 'VExpress_GEM5_V1'
+        else:
+            if this.machine is None:
+                this.machine = 'virt'
     elif args.arch == 'aarch64':
         this.armv = 8
         this.gem5_arch = 'ARM'
+        this.mcpu = 'cortex-a57'
+        this.crosstool_ng_prefix = 'aarch64-unknown-elf'
+        if args.gem5:
+            if this.machine is None:
+                this.machine = 'VExpress_GEM5_V1'
+        else:
+            if this.machine is None:
+                this.machine = 'virt'
     elif args.arch == 'x86_64':
+        this.crosstool_ng_prefix = 'TODO'
         this.gem5_arch = 'X86'
-    this.buildroot_build_dir = os.path.join(this.out_dir, 'buildroot', args.arch, args.buildroot_build_id)
+        if args.gem5:
+            if this.machine is None:
+                this.machine = 'TODO'
+        else:
+            if this.machine is None:
+                this.machine = 'pc'
+    this.buildroot_out_dir = os.path.join(this.out_dir, 'buildroot')
+    this.buildroot_build_dir = os.path.join(this.buildroot_out_dir, 'build', args.buildroot_build_id, args.arch)
+    this.buildroot_download_dir = os.path.join(this.buildroot_out_dir, 'download')
     this.buildroot_config_file = os.path.join(this.buildroot_build_dir, '.config')
     this.build_dir = os.path.join(this.buildroot_build_dir, 'build')
     this.linux_build_dir = os.path.join(this.build_dir, 'linux-custom')
@@ -407,8 +475,8 @@ def setup(parser):
     this.qemu_guest_build_dir = os.path.join(this.build_dir, 'qemu-custom')
     this.host_dir = os.path.join(this.buildroot_build_dir, 'host')
     this.host_bin_dir = os.path.join(this.host_dir, 'usr', 'bin')
-    this.images_dir = os.path.join(this.buildroot_build_dir, 'images')
+    this.buildroot_images_dir = os.path.join(this.buildroot_build_dir, 'images')
-    this.rootfs_raw_file = os.path.join(this.images_dir, 'rootfs.ext2')
+    this.rootfs_raw_file = os.path.join(this.buildroot_images_dir, 'rootfs.ext2')
     this.qcow2_file = this.rootfs_raw_file + '.qcow2'
     this.staging_dir = os.path.join(this.buildroot_build_dir, 'staging')
     this.target_dir = os.path.join(this.buildroot_build_dir, 'target')
@@ -425,11 +493,23 @@ def setup(parser):
     this.qemu_trace_txt_file = os.path.join(this.qemu_run_dir, 'trace.txt')
     this.qemu_termout_file = os.path.join(this.qemu_run_dir, 'termout.txt')
     this.qemu_rrfile = os.path.join(this.qemu_run_dir, 'rrfile')
-    this.gem5_build_dir = os.path.join(this.out_dir, 'gem5', args.gem5_build_id)
+    this.gem5_out_dir = os.path.join(this.out_dir, 'gem5')
+    this.gem5_build_dir = os.path.join(this.gem5_out_dir, args.gem5_build_id)
+    this.gem5_fake_iso = os.path.join(this.gem5_out_dir, 'fake.iso')
     this.gem5_m5term = os.path.join(this.gem5_build_dir, 'm5term')
     this.gem5_build_build_dir = os.path.join(this.gem5_build_dir, 'build')
     this.gem5_executable = os.path.join(this.gem5_build_build_dir, gem5_arch, 'gem5.{}'.format(args.gem5_build_type))
     this.gem5_system_dir = os.path.join(this.gem5_build_dir, 'system')
+    this.crosstool_ng_out_dir = os.path.join(this.out_dir, 'crosstool-ng')
+    this.crosstool_ng_buildid_dir = os.path.join(this.crosstool_ng_out_dir, 'build', args.crosstool_ng_build_id)
+    this.crosstool_ng_install_dir = os.path.join(this.crosstool_ng_buildid_dir, 'install', args.arch)
+    this.crosstool_ng_bin_dir = os.path.join(this.crosstool_ng_install_dir, 'bin')
+    this.crosstool_ng_util_dir = os.path.join(this.crosstool_ng_buildid_dir, 'util')
+    this.crosstool_ng_config = os.path.join(this.crosstool_ng_util_dir, '.config')
+    this.crosstool_ng_defconfig = os.path.join(this.crosstool_ng_util_dir, 'defconfig')
+    this.crosstool_ng_executable = os.path.join(this.crosstool_ng_util_dir, 'ct-ng')
+    this.crosstool_ng_build_dir = os.path.join(this.crosstool_ng_buildid_dir, 'build')
+    this.crosstool_ng_download_dir = os.path.join(this.crosstool_ng_out_dir, 'download')
     if args.gem5_worktree is not None:
         this.gem5_src_dir = os.path.join(this.gem5_non_default_src_root_dir, args.gem5_worktree)
     else:
@@ -470,12 +550,63 @@ def setup(parser):
         this.qemu_hostfwd_ssh_port = this.qemu_base_port + 2
         this.qemu_gdb_port = this.qemu_base_port + 3
         this.gdb_port = this.qemu_gdb_port
+
+    # Baremetal.
+    this.baremetal = args.baremetal
+    this.baremetal_lib_basename = 'lib'
+    this.baremetal_src_dir = os.path.join(this.root_dir, 'baremetal')
+    this.baremetal_src_lib_dir = os.path.join(this.baremetal_src_dir, this.baremetal_lib_basename)
+    this.c_ext = '.c'
+    this.asm_ext = '.S'
+    this.obj_ext = '.o'
+    if args.gem5:
+        this.simulator_name = 'gem5'
+    else:
+        this.simulator_name = 'qemu'
+    this.baremetal_out_dir = os.path.join(out_dir, 'baremetal', args.arch, this.simulator_name, this.machine)
+    this.baremetal_out_lib_dir = os.path.join(this.baremetal_out_dir, this.baremetal_lib_basename)
+    this.baremetal_out_ext = '.elf'
+
+    # Image.
+    if args.baremetal is None:
+        if args.gem5:
+            this.image = this.vmlinux
+            this.disk_image = this.rootfs_raw_file
+        else:
+            this.image = this.linux_image
+            this.disk_image = this.qcow2_file
+    else:
+        this.disk_image = this.gem5_fake_iso
+        paths = [
+            os.path.join(this.baremetal_out_dir, this.baremetal),
+            os.path.join(
+                this.baremetal_out_dir,
+                os.path.relpath(this.baremetal, this.baremetal_src_dir),
+            )
+        ]
+        paths[:] = [os.path.splitext(path)[0] + this.baremetal_out_ext for path in paths]
+        found = False
+        for path in paths:
+            if os.path.exists(path):
+                found = True
+                break
+        if not found and this.baremetal != 'all':
+            raise Exception('Baremetal ELF file not found. Tried:\n' + '\n'.join(paths))
+        this.image = path
     return args
 
-def mkdir():
+def write_configs(config_path, configs, config_fragments=None):
-    global this
+    """
-    os.makedirs(this.build_dir, exist_ok=True)
+    Write extra configs into the Buildroot config file.
-    os.makedirs(this.gem5_build_dir, exist_ok=True)
+    TODO Can't get rid of these for now with nice fragments:
-    os.makedirs(this.gem5_run_dir, exist_ok=True)
+    http://stackoverflow.com/questions/44078245/is-it-possible-to-use-config-fragments-with-buildroots-config
-    os.makedirs(this.qemu_run_dir, exist_ok=True)
+    """
-    os.makedirs(this.p9_dir, exist_ok=True)
+    if config_fragments is None:
+        config_fragments = []
+    with open(config_path, 'a') as config_file:
+        for config_fragment in config_fragments:
+            with open(config_fragment, 'r') as config_fragment:
+                for line in config_fragment:
+                    config_file.write(line)
+        for config in configs:
+            config_file.write(config + '\n')

configure

@@ -1,29 +1,51 @@
 #!/usr/bin/env bash
 set -eu
+apt_get=true
+baremetal=false
+baremetal_given=false
+buildroot=true
+buildroot_given=false
+linux=true
+linux_given=false
 interactive_pkgs=libsdl2-dev
 gem5=false
 gem5_given=false
 qemu=true
 qemu_given=false
 submodules_dir=submodules
-submodules=buildroot
+submodules=
 y=
 while [ $# -gt 0 ]; do
   case "$1" in
+    --baremetal)
+      baremetal=true
+      baremetal_given=true
+      shift
+    ;;
+    --buildroot)
+      buildroot_given=true
+      shift
+    ;;
     --gem5)
       gem5_given=true
       shift
     ;;
     --parsec-benchmark)
       submodules="${submodules} parsec-benchmark"
+      shift
     ;;
     --qemu)
       qemu_given=true
       shift
     ;;
+    --no-apt-get)
+      apt_get=false
+      shift
+    ;;
     --travis)
       interactive_pkgs=
       y=-y
+      shift
     ;;
     *)
       echo 'unknown option' 1>&2
@@ -37,9 +59,11 @@ fi
 if "$gem5_given"; then
   gem5=true
 fi
+if "$baremetal_given" && ! "$buildroot_given"; then
+  buildroot=false
+fi
 
-## apt-get
+if "$apt_get"; then
-
   pkgs="\
   automake \
   bc \
@@ -112,9 +136,16 @@ if "$gem5"; then
     # https://stackoverflow.com/questions/49836676/error-after-upgrading-pip-cannot-import-name-main/51846054#51846054
     python -m pip install --user pydot
   fi
+fi
 
 ## Submodules
 
+if "$baremetal"; then
+  submodules="${submodules} crosstool-ng"
+fi
+if "$buildroot"; then
+  submodules="${submodules} buildroot"
+fi
 if "$qemu"; then
   submodules="${submodules} qemu"
 fi
@@ -154,4 +185,6 @@ submodules="$(for submodule in ${submodules}; do printf "${submodules_dir}/${sub
 # `--jobs"`: https://stackoverflow.com/questions/26957237/how-to-make-git-clone-faster-with-multiple-threads/52327638#52327638
 #
 git submodule update --init --recursive -- ${submodules}
+if "$linux"; then
   git submodule update --depth 1 --init --recursive -- "${submodules_dir}/linux"
+fi

crosstool_ng_config/aarch64

@@ -0,0 +1,9 @@
+CT_ARCH_64=y
+CT_ARCH_ARM=y
+CT_CC_LANG_CXX=y
+CT_DEBUG_GDB=y
+CT_GDB_CROSS_SIM=y
+CT_LIBC_NEWLIB_DISABLE_SUPPLIED_SYSCALLS=y
+CT_LIBC_NEWLIB_IO_C99FMT=y
+CT_LIBC_NEWLIB_IO_FLOAT=y
+CT_LIBC_NEWLIB_IO_LL=y

crosstool_ng_config/arm

@@ -0,0 +1,12 @@
+# ./ct-ng arm-unknown-eabi
+CT_ARCH_ARM=y
+CT_ARCH_FLOAT_SW=y
+CT_LIBC_NEWLIB_IO_C99FMT=y
+CT_LIBC_NEWLIB_IO_LL=y
+CT_LIBC_NEWLIB_IO_FLOAT=y
+CT_LIBC_NEWLIB_DISABLE_SUPPLIED_SYSCALLS=y
+CT_CC_LANG_CXX=y
+
+# Modifications.
+CT_DEBUG_GDB=y
+CT_GDB_CROSS_SIM=y

gem5-bench-dhrystone

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# https://github.com/cirosantilli/linux-kernel-module-cheat/tree/58de3f7243016c052ad080f82dd757d61878219b#gem5-run-benchmark
+# https://github.com/cirosantilli/linux-kernel-module-cheat#gem5-run-benchmark
 
 set -eu
 root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
@@ -10,7 +10,7 @@ cmd="./run -a '$arch' --gem5 --eval-busybox '/gem5.sh'"
 
 # These cache sizes roughly match the ARM Cortex A75
 # https://en.wikipedia.org/wiki/ARM_Cortex-A75
-restore='-l 1 -- --cpu-type=HPI --restore-with-cpu=HPI --caches --l2cache --l1d_size=128kB --l1i_size=1024kB --l2_size=256kB'
+restore='-l 1 -- --cpu-type=HPI --restore-with-cpu=HPI --caches --l2cache --l1d_size=64kB --l1i_size=64kB --l2_size=256kB'
 
 # Generate a checkpoint after Linux boots, using the faster and less detailed CPU.
 # The boot takes a while, be patient young Padawan.

getvar

@@ -1,5 +1,9 @@
 #!/usr/bin/env python3
+
+import types
+
 import common
+
 parser = common.get_argparse(argparse_args={
     'description': '''Print the value of a common.py variable.
 
@@ -14,8 +18,21 @@ For example, to get the Buildroot output directory for an ARM build, use:
 ./%(prog)s -a arm buildroot_build_dir
 ....
 
+List all available variables:
+
+....
+./%(prog)s
+....
+....
 '''
 })
-parser.add_argument('variable')
+parser.add_argument('variable', nargs='?')
 args = common.setup(parser)
+if args.variable:
     print(getattr(common, args.variable))
+else:
+    for attr in dir(common):
+        if not attr.startswith('__'):
+            val = getattr(common, attr)
+            if not callable(val) and not type(val) is types.ModuleType:
+                print('{} {}'.format(attr, val))

patches/manual/gem5-aarch64-baremetal.patch

@@ -0,0 +1,12 @@
+diff --git a/configs/example/fs.py b/configs/example/fs.py
+index 3997ed76c..c3259825d 100644
+--- a/configs/example/fs.py
++++ b/configs/example/fs.py
+@@ -376,5 +376,7 @@ if buildEnv['TARGET_ISA'] == "arm" and options.generate_dtb:
+             sys = getattr(root, sysname)
+             sys.dtb_filename = create_dtb_for_system(sys, '%s.dtb' % sysname)
+ 
++test_sys.highest_el_is_64 = True
++test_sys.auto_reset_addr_64 = True
+ Simulation.setWorkCountOptions(test_sys, options)
+ Simulation.run(options, root, test_sys, FutureClass)

patches/manual/gem5-wait-gdb.patch

@@ -0,0 +1,11 @@
+diff --git a/configs/example/fs.py b/configs/example/fs.py
+index 3997ed76c..b4267ebc0 100644
+--- a/configs/example/fs.py
++++ b/configs/example/fs.py
+@@ -376,5 +376,6 @@ if buildEnv['TARGET_ISA'] == "arm" and options.generate_dtb:
+             sys = getattr(root, sysname)
+             sys.dtb_filename = create_dtb_for_system(sys, '%s.dtb' % sysname)
+ 
++test_sys.cpu[0].wait_for_remote_gdb = True
+ Simulation.setWorkCountOptions(test_sys, options)
+ Simulation.run(options, root, test_sys, FutureClass)

run

@@ -29,7 +29,6 @@ defaults = {
     'kgdb': False,
     'kvm': False,
     'memory': '256M',
-    'prebuilt': False,
     'record': False,
     'replay': False,
     'terminal': False,
@@ -98,17 +97,27 @@ def main(args, extra_args=None):
 
     def raise_rootfs_not_found():
         raise Exception('Root filesystem not found. Did you build it?\n' \
-                        'Tried to use: ' + common.rootfs_raw_file)
+                        'Tried to use: ' + common.disk_image)
+    def raise_image_not_found():
+        raise Exception('Executable image not found. Did you build it?\n' \
+                        'Tried to use: ' + common.image)
+    if common.image is None:
+        raise Exception('Baremetal ELF file not found. Tried:\n' + '\n'.join(paths))
     if args.gem5:
+        if args.baremetal is None:
             if not os.path.exists(common.rootfs_raw_file):
                 if not os.path.exists(common.qcow2_file):
                     raise_rootfs_not_found()
                 common.raw_to_qcow2(prebuilt=args.prebuilt, reverse=True)
-        if not os.path.exists(common.vmlinux):
+        else:
+            if not os.path.exists(common.gem5_fake_iso):
+                os.makedirs(os.path.dirname(common.gem5_fake_iso), exist_ok=True)
+                with open(common.gem5_fake_iso, 'w') as f:
+                    f.write('a' * 512)
+        if not os.path.exists(common.image):
             # This is to run gem5 from a prebuilt download.
-            if not os.path.exists(common.linux_image):
+            if (not args.baremetal is None) or (not os.path.exists(common.linux_image)):
-                raise Exception('Linux kernel image not found. Did you compile it?\n' \
+                raise_image_not_found()
-                                'Tried: ' + common.vmlinux)
             assert common.run_cmd([os.path.join(common.extract_vmlinux, common.linux_image)]) == 0
         os.makedirs(os.path.dirname(common.gem5_readfile), exist_ok=True)
         with open(common.gem5_readfile, 'w') as readfile:
@@ -139,9 +148,9 @@ def main(args, extra_args=None):
                 os.path.join(common.gem5_src_dir, 'configs', 'example', 'arm', 'fs_bigLITTLE.py'),
                 '--big-cpus', '2',
                 '--cpu-type', 'atomic',
-                '--disk', common.rootfs_raw_file,
+                '--disk', common.disk_image,
                 '--dtb', os.path.join(common.gem5_system_dir, 'arm', 'dt', 'armv8_gem5_v1_big_little_2_2.dtb'),
-                '--kernel', common.vmlinux,
+                '--kernel', common.image,
                 '--little-cpus', '2'
             ]
         else:
@@ -152,8 +161,8 @@ def main(args, extra_args=None):
                 extra_emulator_args.extend(['-r', str(sorted(cpt_dirs).index(cpt_dir) + 1)])
             cmd += [
                 common.gem5_fs_file,
-                '--disk-image', common.rootfs_raw_file,
+                '--disk-image', common.disk_image,
-                '--kernel', common.vmlinux,
+                '--kernel', common.image,
                 '--mem-size', memory,
                 '--num-cpus', str(args.cpus),
                 '--script', common.gem5_readfile,
@@ -168,9 +177,13 @@ def main(args, extra_args=None):
                 cmd += [
                     '--command-line', 'earlyprintk=pl011,0x1c090000 console=ttyAMA0 lpj=19988480 rw loglevel=8 mem={} root=/dev/sda {}'.format(memory, kernel_cli),
                     '--dtb-file', os.path.join(common.gem5_system_dir, 'arm', 'dt', 'armv{}_gem5_v1_{}cpu.dtb'.format(common.armv, args.cpus)),
-                    '--machine-type', 'VExpress_GEM5_V1',
+                    '--machine-type', common.machine,
                 ]
+                if not args.baremetal is None:
+                    cmd.append('--bare-metal')
     else:
+        if not os.path.exists(common.image):
+            raise_image_not_found()
         extra_emulator_args.extend(extra_qemu_args)
         os.makedirs(common.run_dir, exist_ok=True)
         if args.prebuilt:
@@ -183,10 +196,6 @@ def main(args, extra_args=None):
         if not qemu_found:
             raise Exception('QEMU executable not found, did you forget to build or install it?\n' \
                             'Tried to use: ' + qemu_executable)
-        if not os.path.exists(common.qcow2_file):
-            if not os.path.exists(common.rootfs_raw_file):
-                raise_rootfs_not_found()
-            common.raw_to_qcow2(prebuilt=args.prebuilt)
         if args.debug_vm:
             serial_monitor = []
         else:
@@ -201,7 +210,7 @@ def main(args, extra_args=None):
                 qemu_executable,
                 '-device', 'rtl8139,netdev=net0',
                 '-gdb', 'tcp::{}'.format(common.gdb_port),
-                '-kernel', common.linux_image,
+                '-kernel', common.image,
                 '-m', args.memory,
                 '-monitor', 'telnet::{},server,nowait'.format(common.qemu_monitor_port),
                 '-netdev', 'user,hostfwd=tcp::{}-:{},hostfwd=tcp::{}-:22,id=net0'.format(common.qemu_hostfwd_generic_port, common.qemu_hostfwd_generic_port, common.qemu_hostfwd_ssh_port),
@@ -215,7 +224,7 @@ def main(args, extra_args=None):
             vnc
         )
         if args.initrd:
-            extra_emulator_args.extend(['-initrd', os.path.join(common.images_dir, 'rootfs.cpio')])
+            extra_emulator_args.extend(['-initrd', os.path.join(common.buildroot_images_dir, 'rootfs.cpio')])
         rr = args.record or args.replay
         if ramfs:
             # TODO why is this needed, and why any string works.
@@ -231,9 +240,14 @@ def main(args, extra_args=None):
                 root = 'root=/dev/vda'
                 rrid = ''
                 snapshot = ',snapshot'
+            if args.baremetal is None:
+                if not os.path.exists(common.qcow2_file):
+                    if not os.path.exists(common.rootfs_raw_file):
+                        raise_rootfs_not_found()
+                    common.raw_to_qcow2(prebuilt=args.prebuilt)
                 extra_emulator_args.extend([
                     '-drive',
-                'file={},format=qcow2,if={}{}{}'.format(common.qcow2_file, driveif, snapshot, rrid)
+                    'file={},format=qcow2,if={}{}{}'.format(common.disk_image, driveif, snapshot, rrid)
                 ])
                 if rr:
                     extra_emulator_args.extend([
@@ -251,29 +265,32 @@ def main(args, extra_args=None):
         if args.arch == 'x86_64':
             if args.kgdb:
                 kernel_cli += ' kgdboc=ttyS0,115200'
+            append = ['-append', '{} nopat {}'.format(root, kernel_cli)]
             cmd.extend([
-                '-M', 'pc',
+                '-M', common.machine,
-                '-append', '{} nopat {}'.format(root, kernel_cli),
                 '-device', 'edu',
             ])
         elif args.arch == 'arm' or args.arch == 'aarch64':
+            extra_qemu_args.append('-semihosting')
             if args.kgdb:
                 kernel_cli += ' kgdboc=ttyAMA0,115200'
             if args.arch == 'arm':
                 cpu = 'cortex-a15'
             else:
                 cpu = 'cortex-a57'
-            # highmem=off needed since v3.0.0 due to:
+            append = ['-append', '{} {}'.format(root, kernel_cli)]
-            # http://lists.nongnu.org/archive/html/qemu-discuss/2018-08/msg00034.html
             cmd = (
                 cmd +
                 [
-                    '-M', 'virt,highmem=off',
+                    # highmem=off needed since v3.0.0 due to:
-                    '-append', '{} {}'.format(root, kernel_cli),
+                    # http://lists.nongnu.org/archive/html/qemu-discuss/2018-08/msg00034.html
+                    '-M', '{},highmem=off'.format(common.machine),
                     '-cpu', cpu,
                 ] +
                 virtio_gpu_pci
             )
+        if args.baremetal is None:
+            cmd.extend(append)
     if args.tmux:
         if args.gem5:
             subprocess.Popen([os.path.join(common.root_dir, 'tmu'),
@@ -286,7 +303,7 @@ def main(args, extra_args=None):
             # but it cannot be used as a library properly it seems, and it is
             # slower than tmux.
             subprocess.Popen([os.path.join(common.root_dir, 'tmu'),
-                "sleep 2;./rungdb -a '{}' -L '{}' -n '{}' {}" \
+                "sleep 2;./run-gdb -a '{}' -L '{}' -n '{}' {}" \
                 .format(args.arch, args.linux_build_id, args.run_id, args.tmux_args)
             ])
     cmd.extend(extra_emulator_args)
@@ -414,10 +431,6 @@ some arch to fail to boot.
 Default: %(default)s
 '''
     )
-    parser.add_argument(
-        '-P', '--prebuilt', default=defaults['prebuilt'], action='store_true',
-        help='Run the downloaded prebuilt images with pre-packaged host tools.'
-    )
     group = parser.add_mutually_exclusive_group()
     group.add_argument(
         '-R', '--replay', default=defaults['replay'], action='store_true',

run-gdb

@@ -11,6 +11,7 @@ import common
 defaults = {
     'after': '',
     'before': '',
+    'sim': False,
     'no_continue': False,
     'kgdb': False,
     'no_lxsymbols': False,
@@ -33,7 +34,7 @@ def main(args, extra_args=None):
     args = common.resolve_args(defaults, args, extra_args)
     after = shlex.split(args.after)
     before = shlex.split(args.before)
-    if args.no_lxsymbols:
+    if args.no_lxsymbols or args.baremetal is not None:
         lx_symbols = []
     else:
         lx_symbols = ['-ex', 'lx-symbols ../kernel_modules-1.0/']
@@ -41,19 +42,25 @@ def main(args, extra_args=None):
         break_at = ['-ex', 'break {}'.format(args.break_at)]
     else:
         break_at = []
+    if args.baremetal is None:
+        image = common.vmlinux
+    else:
+        image = common.image
     cmd = (
-        [
+        [common.get_toolchain_tool('gdb')] +
-            os.path.join(common.host_bin_dir,
-            '{}-linux-gdb'.format(args.arch))
-        ] +
         before +
-        [
+        ['-q']
-            '-q',
-            '-ex', 'add-auto-load-safe-path {}'.format(common.linux_variant_dir),
-            '-ex', 'file {}'.format(common.vmlinux),
-            '-ex', 'target remote localhost:{}'.format(common.gdb_port),
-        ]
     )
+    if args.baremetal is None:
+        cmd.extend(['-ex', 'add-auto-load-safe-path {}'.format(common.linux_variant_dir)])
+    if args.sim:
+        target = 'sim'
+    else:
+        target = 'remote localhost:{}'.format(common.gdb_port)
+    cmd.extend([
+        '-ex', 'file {}'.format(image),
+        '-ex', 'target {}'.format(target),
+    ])
     if not args.kgdb:
         cmd.extend(break_at)
     if not args.no_continue:
@@ -78,7 +85,7 @@ def main(args, extra_args=None):
         # which gets put on the kernel build root when python debugging scripts are enabled.
         cmd.extend(['-ex', 'continue'] + lx_symbols)
     cmd.extend(after)
-    return common.run_cmd(cmd, cmd_file=os.path.join(common.run_dir, 'rungdb.sh'), cwd=common.linux_variant_dir)
+    return common.run_cmd(cmd, cmd_file=os.path.join(common.run_dir, 'run-gdb.sh'), cwd=common.linux_variant_dir)
 
 if __name__ == '__main__':
     parser = common.get_argparse(argparse_args={'description': 'Connect with GDB to an emulator to debug Linux itself'})
@@ -97,6 +104,12 @@ if __name__ == '__main__':
     parser.add_argument(
         '-k', '--kgdb', default=defaults['kgdb'], action='store_true'
     )
+    parser.add_argument(
+        '--sim', default=defaults['sim'], action='store_true',
+        help='''Use the built-in GDB CPU simulator
+See: https://github.com/cirosantilli/linux-kernel-module-cheat#gdb-builtin-cpu-simulator
+'''
+    )
     parser.add_argument(
         '-X', '--no-lxsymbols', default=defaults['no_lxsymbols'], action='store_true'
     )

run-gdb-user

@@ -5,7 +5,7 @@ import os
 import sys
 
 import common
-rungdb = imp.load_source('rungdb', os.path.join(common.root_dir, 'rungdb'))
+rungdb = imp.load_source('rungdb', os.path.join(common.root_dir, 'run-gdb'))
 
 parser = common.get_argparse(argparse_args={
     'description': '''GDB step debug guest userland processes without gdbserver.

run-toolchain

@@ -42,6 +42,4 @@ else:
     sys.exit(common.run_cmd(
         [common.get_toolchain_tool(args.tool)] + args.extra_args,
         cmd_file=os.path.join(common.run_dir, 'run-toolchain.sh'),
-        cwd=common.linux_variant_dir,
-        show_cmd=False,
     ))