From 5d649377fc5160d4d525091d77441dd136b0bca7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ciro=20Santilli=20=E5=85=AD=E5=9B=9B=E4=BA=8B=E4=BB=B6=20?= =?UTF-8?q?=E6=B3=95=E8=BD=AE=E5=8A=9F?= Date: Fri, 9 Nov 2018 00:00:00 +0000 Subject: [PATCH] build: make baremetal parts more flexible and powerful Document test-gdb on readme --- README.adoc | 43 ++++++++++++++++++++------ build | 71 ++++++++++++++++++++++++++++++++----------- build-baremetal | 2 +- build-crosstool-ng | 2 +- build-test | 13 ++++++++ build-test-gdb | 2 ++ common.py | 9 +++--- download-dependencies | 21 ++++++++++--- test | 12 +++++++- 9 files changed, 136 insertions(+), 39 deletions(-) create mode 100755 build-test create mode 100755 build-test-gdb diff --git a/README.adoc b/README.adoc index eea2a3e..496f45b 100644 --- a/README.adoc +++ b/README.adoc @@ -736,9 +736,7 @@ QEMU: .... ./download-dependencies --baremetal --qemu && \ -./build-qemu --arch arm && \ -./build-crosstool-ng --arch arm && \ -./build-baremetal --arch arm && \ +./build --arch arm qemu-baremetal ./run --arch arm --baremetal interactive/prompt .... @@ -761,7 +759,19 @@ got: c new alloc of 4 bytes at address 0x0x4000a2c8 .... -`./build-baremetal` is the command that actually builds the baremetal system for us. It uses crosstool-NG, so that command must be preceded by `./build-crosstool-ng`. +To modify that program, edit: + +.... +vim baremetal/interactive/prompt.c +.... + +and run: + +.... +./build-baremetal --arch arm +.... + +`./build qemu-baremetal` had called link:build-baremetal[] for us previously, in addition to its requirements. `./build-baremetal` uses crosstool-NG, and so it must be preceded by link:build-crosstool-ng[], which `./build qemu-baremetal` also calls. Every `.c` file inside link:baremetal/[] and `.S` file inside `baremetal/arch//` generates a separate baremetal image. You can run a different image with commands such as: @@ -791,10 +801,8 @@ Absolute paths however are used as is and must point to the actual executable: To use gem5 instead of QEMU do: .... -./download-dependencies --baremetal --gem5 -./build-gem5 --arch arm -./build-crosstool-ng --arch arm -./build-baremetal --arch arm --gem5 +./download-dependencies --baremetal --gem5 && \ +./build gem5-baremetal ./run --arch arm --baremetal interactive/prompt --gem5 .... @@ -11410,7 +11418,8 @@ It takes too much time to be feasible for every patch, but it should be done for ==== Automated tests .... -./build-bench-boot --size 3 && ./test --size 3 +./build-test --size 3 && \ +./test --size 3 echo $? .... @@ -11418,7 +11427,7 @@ should output 0. Sources: -* link:build[] +* link:build-test[] * link:test[] Test just the kernel modules: @@ -11440,6 +11449,20 @@ Source: link:rootfs_overlay/test_all.sh[]. ===== Test GDB +We have some link:https://github.com/pexpect/pexpect[pexpect] automated tests for the baremetal programs! + +.... +./build-test-gdb && \ +./test-gdb +.... + +Sources: + +* link:build-test-gdb[] +* link:test-gdb[] + +For the Linux kenel, do the following manual tests for now. + Shell 1: .... diff --git a/build b/build index 0fe933d..a4fd7b2 100755 --- a/build +++ b/build @@ -16,22 +16,25 @@ class Component: and buildroot optionally depends on qemu to build the qcow2 version of the image. ''' - def __init__(self, build_callback=None, dependencies=None): + def __init__( + self, + build_callback=None, + dependencies=None, + supported_archs=None, + ): self.build_callback = build_callback + self.supported_archs = supported_archs if dependencies is None: self.dependencies = [] else: self.dependencies = dependencies def build(self, arch): - if self.build_callback is not None: + if ( + (self.build_callback is not None) and + (self.supported_archs is None or arch in self.supported_archs) + ): self.build_callback(arch) -def build_baremetal(arch): - run_cmd(['build-crosstool-ng'], arch) - run_cmd(['build-baremetal'], arch) - run_cmd(['build-baremetal', '--gem5'], arch) - run_cmd(['build-baremetal', '--gem5', '--machine', 'RealViewPBX'], arch) - def run_cmd(cmd, arch): global args cmd_abs = cmd.copy() @@ -43,8 +46,17 @@ def run_cmd(cmd, arch): name_to_component_map = { # Leaves without dependencies. - 'baremetal': Component( - lambda arch: build_baremetal(arch), + 'baremetal-qemu': Component( + lambda arch: run_cmd(['build-baremetal'], arch), + supported_archs=common.crosstool_ng_supported_archs, + ), + 'baremetal-gem5': Component( + lambda arch: run_cmd(['build-baremetal', '--gem5'], arch), + supported_archs=common.crosstool_ng_supported_archs, + ), + 'baremetal-gem5-pbx': Component( + lambda arch: run_cmd(['build-baremetal', '--gem5', '--machine', 'RealViewPBX'], arch), + supported_archs=common.crosstool_ng_supported_archs, ), 'buildroot': Component( lambda arch: run_cmd(['build-buildroot'], arch), @@ -55,6 +67,10 @@ name_to_component_map = { 'copy-overlay': Component( lambda arch: run_cmd(['copy-overlay'], arch), ), + 'crosstool-ng': Component( + lambda arch: run_cmd(['build-crosstool-ng'], arch), + supported_archs=common.crosstool_ng_supported_archs, + ), 'gem5': Component( lambda arch: run_cmd(['build-gem5'], arch), ), @@ -84,12 +100,27 @@ name_to_component_map = { ), # Dependency only nodes. + 'all': Component(dependencies=[ + 'all-linux', + 'all-baremetal', + ]), + 'all-baremetal': Component(dependencies=[ + 'qemu-baremetal', + 'gem5-baremetal', + 'baremetal-gem5-pbx', + ], + supported_archs=common.crosstool_ng_supported_archs, + ), 'all-linux': Component(dependencies=[ 'qemu-gem5-buildroot', 'gem5-debug', 'gem5-fast', 'qemu-user', ]), + 'baremetal': Component(dependencies=[ + 'baremetal-gem5', + 'baremetal-qemu', + ]), 'gem5-buildroot': Component(dependencies=[ 'buildroot-gcc', 'linux', @@ -97,12 +128,22 @@ name_to_component_map = { 'overlay', 'gem5', ]), + 'gem5-baremetal': Component(dependencies=[ + 'gem5', + 'crosstool-ng', + 'baremetal-gem5', + ]), 'overlay': Component(dependencies=[ 'copy-overlay', 'modules', 'userland', 'buildroot', ]), + 'qemu-baremetal': Component(dependencies=[ + 'qemu', + 'crosstool-ng', + 'baremetal-qemu', + ]), 'qemu-buildroot': Component(dependencies=[ 'qemu', 'buildroot-gcc', @@ -116,10 +157,6 @@ name_to_component_map = { 'release': Component(dependencies=[ 'qemu-buildroot', ]), - 'all': Component(dependencies=[ - 'all-linux', - 'baremetal', - ]) } parser = argparse.ArgumentParser( description= '''\ @@ -168,11 +205,11 @@ parser.add_argument('--all', default=False, action='store_true', help='''\ Build absolutely everything for all archs. ''') group = parser.add_mutually_exclusive_group(required=False) -group.add_argument('--all-archs', default=False, action='store_true', help='''\ +group.add_argument('-A', '--all-archs', default=False, action='store_true', help='''\ Build the selected components for all archs. ''') -group.add_argument('--arch', choices=common.arch_choices, default=[], action='append', help='''\ -Build the selected components for this arch. Select multiple arches by +group.add_argument('-a', '--arch', choices=common.arch_choices, default=[], action='append', help='''\ +Build the selected components for this arch. Select multiple archs by passing this option multiple times. Default: [{}] '''.format(common.default_arch)) parser.add_argument('--extra-args', default='', help='''\ diff --git a/build-baremetal b/build-baremetal index 69700e8..3904fd2 100755 --- a/build-baremetal +++ b/build-baremetal @@ -6,7 +6,7 @@ import common class BaremetalComponent(common.Component): def do_build(self, args): - common.raise_no_x86(args.arch) + common.assert_crosstool_ng_supports_arch(args.arch) build_dir = self.get_build_dir(args) bootloader_obj = os.path.join(common.baremetal_build_lib_dir, 'bootloader{}'.format(common.obj_ext)) common_obj = os.path.join(common.baremetal_build_lib_dir, 'common{}'.format(common.obj_ext)) diff --git a/build-crosstool-ng b/build-crosstool-ng index 485ed0b..3386ad6 100755 --- a/build-crosstool-ng +++ b/build-crosstool-ng @@ -6,7 +6,7 @@ import common class CrosstoolNgComponent(common.Component): def do_build(self, args): - common.raise_no_x86(args.arch) + common.assert_crosstool_ng_supports_arch(args.arch) build_dir = self.get_build_dir(args) defconfig_dest = os.path.join(common.crosstool_ng_util_dir, 'defconfig') os.makedirs(common.crosstool_ng_util_dir, exist_ok=True) diff --git a/build-test b/build-test new file mode 100755 index 0000000..552770a --- /dev/null +++ b/build-test @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -eu +test_size=1 +while [ $# -gt 0 ]; do + case "$1" in + --size) + test_size="$2" + shift 2 + ;; + esac +done +./build-bench-boot --size "$test_size" +./build-test-gdb diff --git a/build-test-gdb b/build-test-gdb new file mode 100755 index 0000000..1fcb056 --- /dev/null +++ b/build-test-gdb @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +./build --all-archs all-baremetal diff --git a/common.py b/common.py index 01729ce..c8fd152 100644 --- a/common.py +++ b/common.py @@ -41,6 +41,7 @@ include_src_dir = os.path.join(this_module.root_dir, this_module.include_subdir) submodules_dir = os.path.join(root_dir, 'submodules') buildroot_src_dir = os.path.join(submodules_dir, 'buildroot') crosstool_ng_src_dir = os.path.join(submodules_dir, 'crosstool-ng') +crosstool_ng_supported_archs = set(['arm', 'aarch64']) linux_src_dir = os.path.join(submodules_dir, 'linux') linux_config_dir = os.path.join(this_module.root_dir, 'linux_config') rootfs_overlay_dir = os.path.join(this_module.root_dir, 'rootfs_overlay') @@ -169,6 +170,10 @@ def add_newlines(cmd): out.extend([arg, this_module.Newline]) return out +def assert_crosstool_ng_supports_arch(arch): + if arch not in this_module.crosstool_ng_supported_archs: + raise Exception('arch not yet supported: ' + arch) + def base64_encode(string): return base64.b64encode(string.encode()).decode() @@ -555,10 +560,6 @@ def raw_to_qcow2(prebuilt=False, reverse=False): outfile, this_module.Newline, ]) -def raise_no_x86(arch): - if (arch == 'x86_64'): - raise Exception('x86_64 not yet supported') - def resolve_args(defaults, args, extra_args): if extra_args is None: extra_args = {} diff --git a/download-dependencies b/download-dependencies index ed96450..defe44b 100755 --- a/download-dependencies +++ b/download-dependencies @@ -99,14 +99,21 @@ g++-arm-linux-gnueabihf \ git \ libguestfs-tools \ moreutils \ +python3-pip \ rsync \ tmux \ unzip \ vinagre \ wget \ +" + # gem5 users Python 2. + pip2_pkgs="\ +" + pip3_pkgs="\ +pexpect==4.6.0 \ " if "$gem5"; then - pkgs="${pkgs} \ + pkgs="${pkgs}\ ccache \ diod \ libgoogle-perftools-dev \ @@ -114,6 +121,9 @@ protobuf-compiler \ python-dev \ python-pip \ scons \ +" + pip2_pkgs="${pip2_pkgs}\ +pydot \ " fi if "$baremetal"; then @@ -171,12 +181,13 @@ EOF if "$qemu"; then $mysudo apt-get build-dep $y qemu fi + # Generate graphs of config.ini under m5out. + # Not with pip directly: + # https://stackoverflow.com/questions/49836676/error-after-upgrading-pip-cannot-import-name-main/51846054#51846054 if "$gem5"; then - # Generate graphs of config.ini under m5out. - # Not with pip directly: - # https://stackoverflow.com/questions/49836676/error-after-upgrading-pip-cannot-import-name-main/51846054#51846054 - python -m pip install --user pydot + python -m pip install --user $pip2_pkgs fi + python3 -m pip install --user $pip3_pkgs fi ## Submodules diff --git a/test b/test index 809b30d..e0f144d 100755 --- a/test +++ b/test @@ -1,4 +1,14 @@ #!/usr/bin/env bash set -eu -./bench-boot --size "${1:-1}" +test_size=1 +while [ $# -gt 0 ]; do + case "$1" in + --size) + test_size="$2" + shift 2 + ;; + esac +done +./bench-boot --size "$test_size" ./test-modules +./test-gdb