build: make baremetal parts more flexible and powerful

Document test-gdb on readme

README.adoc

@@ -736,9 +736,7 @@ QEMU:
 
 ....
 ./download-dependencies --baremetal --qemu && \
-./build-qemu --arch arm && \
+./build --arch arm qemu-baremetal
-./build-crosstool-ng --arch arm && \
-./build-baremetal --arch arm && \
 ./run --arch arm --baremetal interactive/prompt
 ....
 
@@ -761,7 +759,19 @@ got: c
 new alloc of 4 bytes at address 0x0x4000a2c8
 ....
 
-`./build-baremetal` is the command that actually builds the baremetal system for us. It uses crosstool-NG, so that command must be preceded by `./build-crosstool-ng`.
+To modify that program, edit:
+
+....
+vim baremetal/interactive/prompt.c
+....
+
+and run:
+
+....
+./build-baremetal --arch arm
+....
+
+`./build qemu-baremetal` had called link:build-baremetal[] for us previously, in addition to its requirements. `./build-baremetal` uses crosstool-NG, and so it must be preceded by link:build-crosstool-ng[], which `./build qemu-baremetal` also calls.
 
 Every `.c` file inside link:baremetal/[] and `.S` file inside `baremetal/arch/<arch>/` generates a separate baremetal image. You can run a different image with commands such as:
 
@@ -791,10 +801,8 @@ Absolute paths however are used as is and must point to the actual executable:
 To use gem5 instead of QEMU do:
 
 ....
-./download-dependencies --baremetal --gem5
+./download-dependencies --baremetal --gem5 && \
-./build-gem5 --arch arm
+./build gem5-baremetal
-./build-crosstool-ng --arch arm
-./build-baremetal --arch arm --gem5
 ./run --arch arm --baremetal interactive/prompt --gem5
 ....
 
@@ -11410,7 +11418,8 @@ It takes too much time to be feasible for every patch, but it should be done for
 ==== Automated tests
 
 ....
-./build-bench-boot --size 3 && ./test --size 3
+./build-test --size 3 && \
+./test --size 3
 echo $?
 ....
 
@@ -11418,7 +11427,7 @@ should output 0.
 
 Sources:
 
-* link:build[]
+* link:build-test[]
 * link:test[]
 
 Test just the kernel modules:
@@ -11440,6 +11449,20 @@ Source: link:rootfs_overlay/test_all.sh[].
 
 ===== Test GDB
 
+We have some link:https://github.com/pexpect/pexpect[pexpect] automated tests for the baremetal programs!
+
+....
+./build-test-gdb && \
+./test-gdb
+....
+
+Sources:
+
+* link:build-test-gdb[]
+* link:test-gdb[]
+
+For the Linux kenel, do the following manual tests for now.
+
 Shell 1:
 
 ....

build

@@ -16,22 +16,25 @@ class Component:
     and buildroot optionally depends on qemu to build the qcow2 version
     of the image.
     '''
-    def __init__(self, build_callback=None, dependencies=None):
+    def __init__(
+        self,
+        build_callback=None,
+        dependencies=None,
+        supported_archs=None,
+    ):
         self.build_callback = build_callback
+        self.supported_archs = supported_archs
         if dependencies is None:
             self.dependencies = []
         else:
             self.dependencies = dependencies
     def build(self, arch):
-        if self.build_callback is not None:
+        if (
+            (self.build_callback is not None) and
+            (self.supported_archs is None or arch in self.supported_archs)
+        ):
             self.build_callback(arch)
 
-def build_baremetal(arch):
-    run_cmd(['build-crosstool-ng'], arch)
-    run_cmd(['build-baremetal'], arch)
-    run_cmd(['build-baremetal', '--gem5'], arch)
-    run_cmd(['build-baremetal', '--gem5', '--machine', 'RealViewPBX'], arch)
-
 def run_cmd(cmd, arch):
     global args
     cmd_abs = cmd.copy()
@@ -43,8 +46,17 @@ def run_cmd(cmd, arch):
 
 name_to_component_map = {
     # Leaves without dependencies.
-    'baremetal': Component(
+    'baremetal-qemu': Component(
-        lambda arch: build_baremetal(arch),
+        lambda arch: run_cmd(['build-baremetal'], arch),
+        supported_archs=common.crosstool_ng_supported_archs,
+    ),
+    'baremetal-gem5': Component(
+        lambda arch: run_cmd(['build-baremetal', '--gem5'], arch),
+        supported_archs=common.crosstool_ng_supported_archs,
+    ),
+    'baremetal-gem5-pbx': Component(
+        lambda arch: run_cmd(['build-baremetal', '--gem5', '--machine', 'RealViewPBX'], arch),
+        supported_archs=common.crosstool_ng_supported_archs,
     ),
     'buildroot': Component(
         lambda arch: run_cmd(['build-buildroot'], arch),
@@ -55,6 +67,10 @@ name_to_component_map = {
     'copy-overlay': Component(
         lambda arch: run_cmd(['copy-overlay'], arch),
     ),
+    'crosstool-ng': Component(
+        lambda arch: run_cmd(['build-crosstool-ng'], arch),
+        supported_archs=common.crosstool_ng_supported_archs,
+    ),
     'gem5': Component(
         lambda arch: run_cmd(['build-gem5'], arch),
     ),
@@ -84,12 +100,27 @@ name_to_component_map = {
     ),
 
     # Dependency only nodes.
+    'all': Component(dependencies=[
+        'all-linux',
+        'all-baremetal',
+    ]),
+    'all-baremetal': Component(dependencies=[
+            'qemu-baremetal',
+            'gem5-baremetal',
+            'baremetal-gem5-pbx',
+        ],
+        supported_archs=common.crosstool_ng_supported_archs,
+    ),
     'all-linux': Component(dependencies=[
         'qemu-gem5-buildroot',
         'gem5-debug',
         'gem5-fast',
         'qemu-user',
     ]),
+    'baremetal': Component(dependencies=[
+        'baremetal-gem5',
+        'baremetal-qemu',
+    ]),
     'gem5-buildroot': Component(dependencies=[
         'buildroot-gcc',
         'linux',
@@ -97,12 +128,22 @@ name_to_component_map = {
         'overlay',
         'gem5',
     ]),
+    'gem5-baremetal': Component(dependencies=[
+        'gem5',
+        'crosstool-ng',
+        'baremetal-gem5',
+    ]),
     'overlay': Component(dependencies=[
         'copy-overlay',
         'modules',
         'userland',
         'buildroot',
     ]),
+    'qemu-baremetal': Component(dependencies=[
+        'qemu',
+        'crosstool-ng',
+        'baremetal-qemu',
+    ]),
     'qemu-buildroot': Component(dependencies=[
         'qemu',
         'buildroot-gcc',
@@ -116,10 +157,6 @@ name_to_component_map = {
     'release': Component(dependencies=[
         'qemu-buildroot',
     ]),
-    'all': Component(dependencies=[
-        'all-linux',
-        'baremetal',
-    ])
 }
 parser = argparse.ArgumentParser(
     description= '''\
@@ -168,11 +205,11 @@ parser.add_argument('--all', default=False, action='store_true', help='''\
 Build absolutely everything for all archs.
 ''')
 group = parser.add_mutually_exclusive_group(required=False)
-group.add_argument('--all-archs', default=False, action='store_true', help='''\
+group.add_argument('-A', '--all-archs', default=False, action='store_true', help='''\
 Build the selected components for all archs.
 ''')
-group.add_argument('--arch', choices=common.arch_choices, default=[], action='append', help='''\
+group.add_argument('-a', '--arch', choices=common.arch_choices, default=[], action='append', help='''\
-Build the selected components for this arch. Select multiple arches by
+Build the selected components for this arch. Select multiple archs by
 passing this option multiple times. Default: [{}]
 '''.format(common.default_arch))
 parser.add_argument('--extra-args', default='', help='''\

build-baremetal

@@ -6,7 +6,7 @@ import common
 
 class BaremetalComponent(common.Component):
     def do_build(self, args):
-        common.raise_no_x86(args.arch)
+        common.assert_crosstool_ng_supports_arch(args.arch)
         build_dir = self.get_build_dir(args)
         bootloader_obj = os.path.join(common.baremetal_build_lib_dir, 'bootloader{}'.format(common.obj_ext))
         common_obj = os.path.join(common.baremetal_build_lib_dir, 'common{}'.format(common.obj_ext))

build-crosstool-ng

@@ -6,7 +6,7 @@ import common
 
 class CrosstoolNgComponent(common.Component):
     def do_build(self, args):
-        common.raise_no_x86(args.arch)
+        common.assert_crosstool_ng_supports_arch(args.arch)
         build_dir = self.get_build_dir(args)
         defconfig_dest = os.path.join(common.crosstool_ng_util_dir, 'defconfig')
         os.makedirs(common.crosstool_ng_util_dir, exist_ok=True)

build-test

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -eu
+test_size=1
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --size)
+      test_size="$2"
+      shift 2
+      ;;
+  esac
+done
+./build-bench-boot --size "$test_size"
+./build-test-gdb

build-test-gdb

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+./build --all-archs all-baremetal

common.py

@@ -41,6 +41,7 @@ include_src_dir = os.path.join(this_module.root_dir, this_module.include_subdir)
 submodules_dir = os.path.join(root_dir, 'submodules')
 buildroot_src_dir = os.path.join(submodules_dir, 'buildroot')
 crosstool_ng_src_dir = os.path.join(submodules_dir, 'crosstool-ng')
+crosstool_ng_supported_archs = set(['arm', 'aarch64'])
 linux_src_dir = os.path.join(submodules_dir, 'linux')
 linux_config_dir = os.path.join(this_module.root_dir, 'linux_config')
 rootfs_overlay_dir = os.path.join(this_module.root_dir, 'rootfs_overlay')
@@ -169,6 +170,10 @@ def add_newlines(cmd):
         out.extend([arg, this_module.Newline])
     return out
 
+def assert_crosstool_ng_supports_arch(arch):
+    if arch not in this_module.crosstool_ng_supported_archs:
+        raise Exception('arch not yet supported: ' + arch)
+
 def base64_encode(string):
     return base64.b64encode(string.encode()).decode()
 
@@ -555,10 +560,6 @@ def raw_to_qcow2(prebuilt=False, reverse=False):
         outfile, this_module.Newline,
     ])
 
-def raise_no_x86(arch):
-    if (arch == 'x86_64'):
-        raise Exception('x86_64 not yet supported')
-
 def resolve_args(defaults, args, extra_args):
     if extra_args is None:
         extra_args = {}

download-dependencies

@@ -99,11 +99,18 @@ g++-arm-linux-gnueabihf \
 git \
 libguestfs-tools \
 moreutils \
+python3-pip \
 rsync \
 tmux \
 unzip \
 vinagre \
 wget \
+"
+  # gem5 users Python 2.
+  pip2_pkgs="\
+"
+  pip3_pkgs="\
+pexpect==4.6.0 \
 "
   if "$gem5"; then
     pkgs="${pkgs}\
@@ -114,6 +121,9 @@ protobuf-compiler \
 python-dev \
 python-pip \
 scons \
+"
+    pip2_pkgs="${pip2_pkgs}\
+pydot \
 "
   fi
   if "$baremetal"; then
@@ -171,12 +181,13 @@ EOF
   if "$qemu"; then
     $mysudo apt-get build-dep $y qemu
   fi
-  if "$gem5"; then
   # Generate graphs of config.ini under m5out.
   # Not with pip directly:
   # https://stackoverflow.com/questions/49836676/error-after-upgrading-pip-cannot-import-name-main/51846054#51846054
-    python -m pip install --user pydot
+  if "$gem5"; then
+    python -m pip install --user $pip2_pkgs
   fi
+  python3 -m pip install --user $pip3_pkgs
 fi
 
 ## Submodules

test

@@ -1,4 +1,14 @@
 #!/usr/bin/env bash
 set -eu
-./bench-boot --size "${1:-1}"
+test_size=1
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --size)
+      test_size="$2"
+      shift 2
+      ;;
+  esac
+done
+./bench-boot --size "$test_size"
 ./test-modules
+./test-gdb