From 8f1c662e3d7171fb0dbda978d9fc7a8ff9563431 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ciro=20Santilli=20=E5=85=AD=E5=9B=9B=E4=BA=8B=E4=BB=B6=20?= =?UTF-8?q?=E6=B3=95=E8=BD=AE=E5=8A=9F?= Date: Fri, 26 Oct 2018 00:00:00 +0000 Subject: [PATCH] build kind of works --- README.adoc | 6 +- build | 212 ++++++++++++++++++++++++++-------------------------- common.py | 5 +- release | 2 +- 4 files changed, 114 insertions(+), 111 deletions(-) diff --git a/README.adoc b/README.adoc index 2fcffc5..7385246 100644 --- a/README.adoc +++ b/README.adoc @@ -10413,7 +10413,7 @@ We tried to automate it on Travis with link:.travis.yml[] but it hits the curren Benchmark all: .... -./build --all-linux-components +./build all-linux ./bench-boot cat "$(./getvar bench_boot)" .... @@ -11158,7 +11158,7 @@ It takes too much time to be feasible for every patch, but it should be done for ==== Automated tests .... -./build --all-linux-components +./build all-linux ./test --size 3 echo $? .... @@ -11341,7 +11341,7 @@ TODO also run tests and only release if they pass. Create a zip containing all files required for <>: .... -./build --all-linux-components +./build all-linux ./release-zip .... diff --git a/build b/build index 319ad87..dc6198e 100755 --- a/build +++ b/build @@ -6,26 +6,31 @@ import os import common -class Component(): - def __init__(self, default_selected, build_callback): - self.default_selected = default_selected +class Component: + ''' + Yes, we are re-inventing a crappy dependency resolution system. + I can't believe it. + + The hard part is that we have optional dependencies as well... + e.g. buildroot optionally depends on m5 to put m5 in the root filesystem, + and buildroot optionally depends on qemu to build the qcow2 version + of the image. + ''' + def __init__(self, build_callback=None, dependencies=None): self.build_callback = build_callback + if dependencies is None: + self.dependencies = [] + else: + self.dependencies = dependencies def build(self, arch, dry_run): - self.build_callback(arch, dry_run) + if self.build_callback is not None: + self.build_callback(arch, dry_run) -class BaremetalComponent(Component): - def __init__(self, default_selected): - self.default_selected = default_selected - def build(self, arch, dry_run): - common.run_cmd(['build-crosstool-ng', '--arch', arch], dry_run=dry_run) - common.run_cmd(['build-baremetal', '--arch', arch], dry_run=dry_run) - common.run_cmd(['build-baremetal', '--arch', arch, '--gem5'], dry_run=dry_run) - common.run_cmd(['build-baremetal', '--arch', arch, '--gem5', '--machine', 'RealViewPBX'], dry_run=dry_run) - -def add_bool_arg(parser, name, default=False): - group = parser.add_mutually_exclusive_group(required=False) - group.add_argument('--' + name, default=False, action='store_true', dest=name) - group.add_argument('--no-' + name, default=False, action='store_true', dest=name) +def build_baremetal(arch, dry_run): + common.run_cmd(['build-crosstool-ng', '--arch', arch], dry_run=dry_run) + common.run_cmd(['build-baremetal', '--arch', arch], dry_run=dry_run) + common.run_cmd(['build-baremetal', '--arch', arch, '--gem5'], dry_run=dry_run) + common.run_cmd(['build-baremetal', '--arch', arch, '--gem5', '--machine', 'RealViewPBX'], dry_run=dry_run) def run_cmd(cmd, dry_run): cmd_abs = cmd.copy() @@ -33,136 +38,126 @@ def run_cmd(cmd, dry_run): common.run_cmd(cmd_abs, dry_run=dry_run) name_to_component_map = { - 'baremetal': BaremetalComponent(False), + # Leaves without dependencies. + 'baremetal': Component( + lambda arch, dry_run: build_baremetal(arch, dry_run), + ), 'buildroot': Component( - True, + lambda arch, dry_run: run_cmd(['build-buildroot', '--arch', arch], dry_run=dry_run), + ), + 'buildroot-gcc': Component( lambda arch, dry_run: run_cmd(['build-buildroot', '--arch', arch], dry_run=dry_run), ), 'copy-overlay': Component( - True, - lambda arch, dry_run: run_cmd(['copy-overlay', '--arch', arch], dry_run=dry_run) + lambda arch, dry_run: run_cmd(['copy-overlay', '--arch', arch], dry_run=dry_run), ), 'gem5': Component( - False, - lambda arch, dry_run: run_cmd(['build-gem5', '--arch', arch], dry_run=dry_run) + lambda arch, dry_run: run_cmd(['build-gem5', '--arch', arch], dry_run=dry_run), ), 'linux': Component( - True, - lambda arch, dry_run: run_cmd(['build-linux', '--arch', arch], dry_run=dry_run) + lambda arch, dry_run: run_cmd(['build-linux', '--arch', arch], dry_run=dry_run), ), 'modules': Component( - True, - lambda arch, dry_run: run_cmd(['build-modules', '--arch', arch], dry_run=dry_run) + lambda arch, dry_run: run_cmd(['build-modules', '--arch', arch], dry_run=dry_run), ), 'm5': Component( - False, - lambda arch, dry_run: run_cmd(['build-m5', '--arch', arch], dry_run=dry_run) + lambda arch, dry_run: run_cmd(['build-m5', '--arch', arch], dry_run=dry_run), ), 'qemu': Component( - True, - lambda arch, dry_run: run_cmd(['build-qemu', '--arch', arch], dry_run=dry_run) + lambda arch, dry_run: run_cmd(['build-qemu', '--arch', arch], dry_run=dry_run), ), 'userland': Component( - True, lambda arch, dry_run: run_cmd(['build-userland', '--arch', arch], dry_run=dry_run), ), -} -# Topological sorted on build order. -component_order = [ - 'baremetal', - 'gem5', - 'qemu', - # Need one extra one here to build the toolchain. - 'buildroot', - 'linux', - 'copy-overlay', - 'modules', - 'userland', - 'm5', - 'buildroot', -] -component_names = name_to_component_map.keys() -linux_component_names = { - 'gem5', - 'qemu', - 'linux', - 'copy-overlay', - 'modules', - 'userland', - 'm5', - 'buildroot', -} + # Dependency only nodes. + 'all-linux': Component(dependencies=[ + 'qemu', + 'gem5-buildroot', + ]), + 'gem5-buildroot': Component(dependencies=[ + 'buildroot-gcc', + 'linux', + 'm5', + 'overlay', + 'gem5', + ]), + 'overlay': Component(dependencies=[ + 'copy-overlay', + 'modules', + 'userland', + 'buildroot', + ]), + 'qemu-buildroot': Component(dependencies=[ + 'qemu', + 'buildroot-gcc', + 'overlay', + 'linux', + ]), + 'all': Component(dependencies=[ + 'all-linux', + 'baremetal', + ]) +} parser = argparse.ArgumentParser( - description= ''' + description= '''\ Shallow helper to build everything, or a subset of everything conveniently. -Our build-* scripts don't build any dependencies. +Our build-* scripts don't build any dependencies to make iterative +development fast and more predictable. -While developing something however, you will likely want to just run the -required sub-build commands manually to speed things up and better understand -what is going on. +While modifying a specific component however, you will likely want to just run the +individual build-* commands which: + +* build no dependencies, and so are fast and predictable +* can take multiple options to custumize the build Without any args, build only what is necessary for https://github.com/cirosantilli/linux-kernel-module-cheat#qemu-buildroot-setup +for x86_64: .... ./%(prog)s .... -This includes: - -* QEMU -* Linux kernel -* kernel modules and userland tools -* Buildroot - -just for x86_64. - -To build EVERYTHING: - -This will build QEMU, gem5, Buildroot, Linux, etc. -for x86_64, arm and aarch64. - -With --archs, build everything for just the given archs: +This is equivalent to: .... -./%(prog)s --archs 'arm aarch64' +./%(prog)s --arch x86_64 qemu-buildroot .... -Other options make this script build only the given components. E.g., to build -just Linux and QEMU for all archs, but not gem5, Buildroot, etc.: +If `--arch` is given, build just for the given archs: .... -./%(prog)s --linux --qemu +./%(prog)s --arch arm --arch aarch64 .... -this is useful to while developing those components to prepare to quickly +This will build `qemu-buildroot` for arm and aarch64 only, but not `x86_64`. ''', formatter_class=argparse.RawTextHelpFormatter, ) + parser.add_argument('--all', default=False, action='store_true', help='''\ -Build absolutely everything. -''') -parser.add_argument('--all-components', default=False, action='store_true', help='''\ -Build all components within the selected archs. -''') -parser.add_argument('--all-linux-components', default=False, action='store_true', help='''\ -Build all Linux-releated components within the selected archs. -Excludes for example baremetal examples. +Build absolutely everything for all archs. ''') group = parser.add_mutually_exclusive_group(required=False) + group.add_argument('--all-archs', default=False, action='store_true', help='''\ Build the selected components for all archs. ''') + group.add_argument('--arch', choices=common.arch_choices, default=[], action='append', help='''\ Build the selected components for this arch. Select multiple arches by passing this option multiple times. Default: [{}] '''.format(common.default_arch)) -for component in component_names: - add_bool_arg(parser, component) + +parser.add_argument('components', choices=list(name_to_component_map.keys()) + [[]], default=[], nargs='*', help='''\ +Which components to build. +'''.format(common.default_arch)) + common.add_dry_run_argument(parser) args = parser.parse_args() +common.setup_dry_run_arguments(args) # Decide archs. if args.arch == []: @@ -178,19 +173,24 @@ else: archs.add(arch) # Decide components. -selected_component_names = [] -for name in component_order: - component = name_to_component_map[name] - if ( - args.all or - args.all_components or - (args.all_linux_components and name in linux_component_names) or - getattr(args, name) or - component.default_selected - ): - selected_component_names.append(name) +components = args.components +if args.all: + components = ['all'] +elif components == []: + components = ['qemu-buildroot'] +selected_components = [] +selected_component_name_set = set() +for component_name in components: + todo = [component_name] + while todo != []: + current_name = todo.pop(0) + if current_name not in selected_component_name_set: + selected_component_name_set.add(current_name) + component = name_to_component_map[current_name] + selected_components.append(component) + todo.extend(component.dependencies) # Do the build. for arch in archs: - for name in selected_component_names: - name_to_component_map[name].build(arch, args.dry_run) + for component in selected_components: + component.build(arch, args.dry_run) diff --git a/common.py b/common.py index c586efc..eca178d 100644 --- a/common.py +++ b/common.py @@ -658,7 +658,7 @@ def setup(parser): if args.gem5_worktree is not None and not gem5_build_id_given: args.gem5_build_id = args.gem5_worktree this_module.machine = args.machine - this_module.dry_run = args.dry_run + this_module.setup_dry_run_arguments(args) if args.arch == 'arm': this_module.armv = 7 this_module.gem5_arch = 'ARM' @@ -867,6 +867,9 @@ def setup(parser): this_module.image = path return args +def setup_dry_run_arguments(args): + this_module.dry_run = args.dry_run + def write_configs(config_path, configs, config_fragments=None): """ Write extra configs into the Buildroot config file. diff --git a/release b/release index 0f73e44..014cdaf 100755 --- a/release +++ b/release @@ -24,7 +24,7 @@ start_time = time.time() # know what the smallest root filesystem size is and use it either... # https://stackoverflow.com/questions/47320800/how-to-clean-only-target-in-buildroot subprocess.check_call([os.path.join(common.root_dir, 'configure'), '--all']) -subprocess.check_call([os.path.join(common.root_dir, 'build'), '--all-linux-components']) +subprocess.check_call([os.path.join(common.root_dir, 'build'), 'all-linux']) release_zip.main() subprocess.check_call(['git', 'push']) release_upload.main()