build kind of works

README.adoc

@@ -10413,7 +10413,7 @@ We tried to automate it on Travis with link:.travis.yml[] but it hits the curren
 Benchmark all:
 
 ....
-./build --all-linux-components
+./build all-linux
 ./bench-boot
 cat "$(./getvar bench_boot)"
 ....
@@ -11158,7 +11158,7 @@ It takes too much time to be feasible for every patch, but it should be done for
 ==== Automated tests
 
 ....
-./build --all-linux-components
+./build all-linux
 ./test --size 3
 echo $?
 ....
@@ -11341,7 +11341,7 @@ TODO also run tests and only release if they pass.
 Create a zip containing all files required for <<prebuilt>>:
 
 ....
-./build --all-linux-components
+./build all-linux
 ./release-zip
 ....
 

build

@@ -6,26 +6,31 @@ import os
 
 import common
 
-class Component():
+class Component:
-    def __init__(self, default_selected, build_callback):
+    '''
-        self.default_selected = default_selected
+    Yes, we are re-inventing a crappy dependency resolution system.
+    I can't believe it.
+
+    The hard part is that we have optional dependencies as well...
+    e.g. buildroot optionally depends on m5 to put m5 in the root filesystem,
+    and buildroot optionally depends on qemu to build the qcow2 version
+    of the image.
+    '''
+    def __init__(self, build_callback=None, dependencies=None):
         self.build_callback = build_callback
+        if dependencies is None:
+            self.dependencies = []
+        else:
+            self.dependencies = dependencies
     def build(self, arch, dry_run):
-        self.build_callback(arch, dry_run)
+        if self.build_callback is not None:
+            self.build_callback(arch, dry_run)
 
-class BaremetalComponent(Component):
+def build_baremetal(arch, dry_run):
-    def __init__(self, default_selected):
+    common.run_cmd(['build-crosstool-ng', '--arch', arch], dry_run=dry_run)
-        self.default_selected = default_selected
+    common.run_cmd(['build-baremetal', '--arch', arch], dry_run=dry_run)
-    def build(self, arch, dry_run):
+    common.run_cmd(['build-baremetal', '--arch', arch, '--gem5'], dry_run=dry_run)
-        common.run_cmd(['build-crosstool-ng', '--arch', arch], dry_run=dry_run)
+    common.run_cmd(['build-baremetal', '--arch', arch, '--gem5', '--machine', 'RealViewPBX'], dry_run=dry_run)
-        common.run_cmd(['build-baremetal', '--arch', arch], dry_run=dry_run)
-        common.run_cmd(['build-baremetal', '--arch', arch, '--gem5'], dry_run=dry_run)
-        common.run_cmd(['build-baremetal', '--arch', arch, '--gem5', '--machine', 'RealViewPBX'], dry_run=dry_run)
-
-def add_bool_arg(parser, name, default=False):
-    group = parser.add_mutually_exclusive_group(required=False)
-    group.add_argument('--' + name, default=False, action='store_true', dest=name)
-    group.add_argument('--no-' + name, default=False, action='store_true', dest=name)
 
 def run_cmd(cmd, dry_run):
     cmd_abs = cmd.copy()
@@ -33,136 +38,126 @@ def run_cmd(cmd, dry_run):
     common.run_cmd(cmd_abs, dry_run=dry_run)
 
 name_to_component_map = {
-    'baremetal': BaremetalComponent(False),
+    # Leaves without dependencies.
+    'baremetal': Component(
+        lambda arch, dry_run: build_baremetal(arch, dry_run),
+    ),
     'buildroot': Component(
-        True,
+        lambda arch, dry_run: run_cmd(['build-buildroot', '--arch', arch], dry_run=dry_run),
+    ),
+    'buildroot-gcc': Component(
         lambda arch, dry_run: run_cmd(['build-buildroot', '--arch', arch], dry_run=dry_run),
     ),
     'copy-overlay': Component(
-        True,
+        lambda arch, dry_run: run_cmd(['copy-overlay', '--arch', arch], dry_run=dry_run),
-        lambda arch, dry_run: run_cmd(['copy-overlay', '--arch', arch], dry_run=dry_run)
     ),
     'gem5': Component(
-        False,
+        lambda arch, dry_run: run_cmd(['build-gem5', '--arch', arch], dry_run=dry_run),
-        lambda arch, dry_run: run_cmd(['build-gem5', '--arch', arch], dry_run=dry_run)
     ),
     'linux': Component(
-        True,
+        lambda arch, dry_run: run_cmd(['build-linux', '--arch', arch], dry_run=dry_run),
-        lambda arch, dry_run: run_cmd(['build-linux', '--arch', arch], dry_run=dry_run)
     ),
     'modules': Component(
-        True,
+        lambda arch, dry_run: run_cmd(['build-modules', '--arch', arch], dry_run=dry_run),
-        lambda arch, dry_run: run_cmd(['build-modules', '--arch', arch], dry_run=dry_run)
     ),
     'm5': Component(
-        False,
+        lambda arch, dry_run: run_cmd(['build-m5', '--arch', arch], dry_run=dry_run),
-        lambda arch, dry_run: run_cmd(['build-m5', '--arch', arch], dry_run=dry_run)
     ),
     'qemu': Component(
-        True,
+        lambda arch, dry_run: run_cmd(['build-qemu', '--arch', arch], dry_run=dry_run),
-        lambda arch, dry_run: run_cmd(['build-qemu', '--arch', arch], dry_run=dry_run)
     ),
     'userland': Component(
-        True,
         lambda arch, dry_run: run_cmd(['build-userland', '--arch', arch], dry_run=dry_run),
     ),
-}
-# Topological sorted on build order.
-component_order = [
-    'baremetal',
-    'gem5',
-    'qemu',
-    # Need one extra one here to build the toolchain.
-    'buildroot',
-    'linux',
-    'copy-overlay',
-    'modules',
-    'userland',
-    'm5',
-    'buildroot',
-]
-component_names = name_to_component_map.keys()
-linux_component_names = {
-    'gem5',
-    'qemu',
-    'linux',
-    'copy-overlay',
-    'modules',
-    'userland',
-    'm5',
-    'buildroot',
-}
 
+    # Dependency only nodes.
+    'all-linux': Component(dependencies=[
+        'qemu',
+        'gem5-buildroot',
+    ]),
+    'gem5-buildroot': Component(dependencies=[
+        'buildroot-gcc',
+        'linux',
+        'm5',
+        'overlay',
+        'gem5',
+    ]),
+    'overlay': Component(dependencies=[
+        'copy-overlay',
+        'modules',
+        'userland',
+        'buildroot',
+    ]),
+    'qemu-buildroot': Component(dependencies=[
+        'qemu',
+        'buildroot-gcc',
+        'overlay',
+        'linux',
+    ]),
+    'all': Component(dependencies=[
+        'all-linux',
+        'baremetal',
+    ])
+}
 parser = argparse.ArgumentParser(
-    description= '''
+    description= '''\
 Shallow helper to build everything, or a subset of everything conveniently.
 
-Our build-* scripts don't build any dependencies.
+Our build-* scripts don't build any dependencies to make iterative
+development fast and more predictable.
 
-While developing something however, you will likely want to just run the
+While modifying a specific component however, you will likely want to just run the
-required sub-build commands manually to speed things up and better understand
+individual build-* commands which:
-what is going on.
+
+* build no dependencies, and so are fast and predictable
+* can take multiple options to custumize the build
 
 Without any args, build only what is necessary for
 https://github.com/cirosantilli/linux-kernel-module-cheat#qemu-buildroot-setup
+for x86_64:
 
 ....
 ./%(prog)s
 ....
 
-This includes:
+This is equivalent to:
-
-* QEMU
-* Linux kernel
-* kernel modules and userland tools
-* Buildroot
-
-just for x86_64.
-
-To build EVERYTHING:
-
-This will build QEMU, gem5, Buildroot, Linux, etc.
-for x86_64, arm and aarch64.
-
-With --archs, build everything for just the given archs:
 
 ....
-./%(prog)s --archs 'arm aarch64'
+./%(prog)s --arch x86_64 qemu-buildroot
 ....
 
-Other options make this script build only the given components. E.g., to build
+If `--arch` is given, build just for the given archs:
-just Linux and QEMU for all archs, but not gem5, Buildroot, etc.:
 
 ....
-./%(prog)s --linux --qemu
+./%(prog)s --arch arm --arch aarch64
 ....
 
-this is useful to while developing those components to prepare to quickly
+This will build `qemu-buildroot` for arm and aarch64 only, but not `x86_64`.
 ''',
     formatter_class=argparse.RawTextHelpFormatter,
 )
+
 parser.add_argument('--all', default=False, action='store_true', help='''\
-Build absolutely everything.
+Build absolutely everything for all archs.
-''')
-parser.add_argument('--all-components', default=False, action='store_true', help='''\
-Build all components within the selected archs.
-''')
-parser.add_argument('--all-linux-components', default=False, action='store_true', help='''\
-Build all Linux-releated components within the selected archs.
-Excludes for example baremetal examples.
 ''')
 group = parser.add_mutually_exclusive_group(required=False)
+
 group.add_argument('--all-archs', default=False, action='store_true', help='''\
 Build the selected components for all archs.
 ''')
+
 group.add_argument('--arch', choices=common.arch_choices, default=[], action='append', help='''\
 Build the selected components for this arch. Select multiple arches by
 passing this option multiple times. Default: [{}]
 '''.format(common.default_arch))
-for component in component_names:
+
-    add_bool_arg(parser, component)
+parser.add_argument('components', choices=list(name_to_component_map.keys()) + [[]], default=[], nargs='*', help='''\
+Which components to build.
+'''.format(common.default_arch))
+
 common.add_dry_run_argument(parser)
 args = parser.parse_args()
+common.setup_dry_run_arguments(args)
 
 # Decide archs.
 if args.arch == []:
@@ -178,19 +173,24 @@ else:
         archs.add(arch)
 
 # Decide components.
-selected_component_names = []
+components = args.components
-for name in component_order:
+if args.all:
-    component = name_to_component_map[name]
+    components = ['all']
-    if (
+elif components == []:
-        args.all or
+    components = ['qemu-buildroot']
-        args.all_components or
+selected_components = []
-        (args.all_linux_components and name in linux_component_names) or
+selected_component_name_set = set()
-        getattr(args, name) or
+for component_name in components:
-        component.default_selected
+    todo = [component_name]
-    ):
+    while todo != []:
-        selected_component_names.append(name)
+        current_name = todo.pop(0)
+        if current_name not in selected_component_name_set:
+            selected_component_name_set.add(current_name)
+            component = name_to_component_map[current_name]
+            selected_components.append(component)
+            todo.extend(component.dependencies)
 
 # Do the build.
 for arch in archs:
-    for name in selected_component_names:
+    for component in selected_components:
-        name_to_component_map[name].build(arch, args.dry_run)
+        component.build(arch, args.dry_run)

common.py

@@ -658,7 +658,7 @@ def setup(parser):
     if args.gem5_worktree is not None and not gem5_build_id_given:
         args.gem5_build_id = args.gem5_worktree
     this_module.machine = args.machine
-    this_module.dry_run = args.dry_run
+    this_module.setup_dry_run_arguments(args)
     if args.arch == 'arm':
         this_module.armv = 7
         this_module.gem5_arch = 'ARM'
@@ -867,6 +867,9 @@ def setup(parser):
         this_module.image = path
     return args
 
+def setup_dry_run_arguments(args):
+    this_module.dry_run = args.dry_run
+
 def write_configs(config_path, configs, config_fragments=None):
     """
     Write extra configs into the Buildroot config file.

release

@@ -24,7 +24,7 @@ start_time = time.time()
 # know what the smallest root filesystem size is and use it either...
 # https://stackoverflow.com/questions/47320800/how-to-clean-only-target-in-buildroot
 subprocess.check_call([os.path.join(common.root_dir, 'configure'), '--all'])
-subprocess.check_call([os.path.join(common.root_dir, 'build'), '--all-linux-components'])
+subprocess.check_call([os.path.join(common.root_dir, 'build'), 'all-linux'])
 release_zip.main()
 subprocess.check_call(['git', 'push'])
 release_upload.main()