build-buildroot twice, and split build-m5

README.adoc

@@ -396,19 +396,19 @@ If you haven't built Buildroot yet for <<qemu-buildroot-setup>>, you can build f
 ....
 ./download-dependencies --gem5
 ./build --gem5 --no-qemu
-./build-buildroot --gem5
 ./run --gem5
 ....
 
 `--no-qemu` is optional, but it makes the build slightly faster TODO: after first build:
 ....
 ./download-dependencies --gem5
-./build-gem5 --gem5
+./build-gem5
-./build-buildroot --gem5
+./build-m5
+./build-buildroot
 ./run --gem5
 ....
 
-If you have already built  previously, don't be afraid: gem5 and QEMU use almost the same root filesystem and kernel, so `./build-buildroot --gem` will be fast. It is currently only needed for the <<m5>> tool.
+If you have already built previously, don't be afraid: gem5 and QEMU use almost the same root filesystem and kernel, so `./build-buildroot --gem` will be fast. It is currently only needed for the <<m5>> tool.
 
 To get a terminal, either open a new shell and run:
 
@@ -601,7 +601,7 @@ Maybe we could work around this by just downloading the kernel source somehow, a
 +
 ** there is no Debian package for it, so you have to compile your own, so you might as well just build the image itself
 ** it does not handle <<gem5-qcow2,qcow2>>, and we haven't gotten <<squashfs>> to work yet, therefore we would have to either distribute large ext2 images, or constantly fight with <<br2_target_rootfs_ext2_size>>
-** QEMU uses  `bzImage` and gem5 the raw `vmlinux`, and we don't want to distribute the same thing twice...
+** QEMU uses `bzImage` and gem5 the raw `vmlinux`, and we don't want to distribute the same thing twice...
 +
 And our attempt at using link:https://github.com/torvalds/linux/blob/master/scripts/extract-vmlinux[`extract-vmlinux`] failed for `aarch64` with:
 +
@@ -2913,7 +2913,7 @@ man init_module
 documents that:
 
 ____
-The finit_module() system call is like init_module(), but reads the module to be loaded from the file descriptor fd.  It is useful when the authenticity of a kernel module can be determined from its location in the filesystem; in cases where that is possible, the overhead of using cryptographically signed modules to determine the authenticity of a module can be avoided.  The param_values  argument is as for init_module().
+The finit_module() system call is like init_module(), but reads the module to be loaded from the file descriptor fd. It is useful when the authenticity of a kernel module can be determined from its location in the filesystem; in cases where that is possible, the overhead of using cryptographically signed modules to determine the authenticity of a module can be avoided. The param_values argument is as for init_module().
 ____
 
 `finit` is newer and was added only in v3.8. More rationale: https://lwn.net/Articles/519010/

build

@@ -32,34 +32,50 @@ def run_cmd(cmd, dry_run):
     cmd_abs[0] = os.path.join(common.root_dir, cmd[0])
     common.run_cmd(cmd_abs, dry_run=dry_run)
 
-# Topological sorted on build dependencies.
+name_to_component_map = {
-name_to_component_map = collections.OrderedDict([
+    'baremetal': BaremetalComponent(False),
-    ('baremetal', BaremetalComponent(False)),
+    'buildroot': Component(
-    ('gem5', Component(
-        False,
-        lambda arch, dry_run: run_cmd(['build-gem5', '--arch', arch], dry_run)
-    )),
-    ('qemu', Component(
-        True,
-        lambda arch, dry_run: run_cmd(['build-qemu', '--arch', arch], dry_run)
-    )),
-    ('linux', Component(
-        True,
-        lambda arch, dry_run: run_cmd(['build-linux', '--arch', arch], dry_run=dry_run)
-    )),
-    ('modules', Component(
-        True,
-        lambda arch, dry_run: run_cmd(['build-modules', '--arch', arch], dry_run=dry_run)
-    )),
-    ('userland', Component(
-        True,
-        lambda arch, dry_run: run_cmd(['build-userland', '--arch', arch], dry_run=dry_run),
-    )),
-    ('buildroot', Component(
         True,
         lambda arch, dry_run: run_cmd(['build-buildroot', '--arch', arch, '--gem5'], dry_run=dry_run),
-    )),
+    ),
-])
+    'gem5': Component(
+        False,
+        lambda arch, dry_run: run_cmd(['build-gem5', '--arch', arch], dry_run)
+    ),
+    'linux': Component(
+        True,
+        lambda arch, dry_run: run_cmd(['build-linux', '--arch', arch], dry_run=dry_run)
+    ),
+    'modules': Component(
+        True,
+        lambda arch, dry_run: run_cmd(['build-modules', '--arch', arch], dry_run=dry_run)
+    ),
+    'm5': Component(
+        True,
+        lambda arch, dry_run: run_cmd(['build-m5', '--arch', arch], dry_run=dry_run)
+    ),
+    'qemu': Component(
+        True,
+        lambda arch, dry_run: run_cmd(['build-qemu', '--arch', arch], dry_run)
+    ),
+    'userland': Component(
+        True,
+        lambda arch, dry_run: run_cmd(['build-userland', '--arch', arch], dry_run=dry_run),
+    ),
+}
+# Topological sorted on build order.
+component_order = [
+    'baremetal',
+    'gem5',
+    'qemu',
+    # Need one extra one here to build the toolchain.
+    'buildroot',
+    'linux',
+    'modules',
+    'userland',
+    'm5',
+    'buildroot',
+]
 component_names = name_to_component_map.keys()
 linux_component_names = {
     'gem5',
@@ -67,6 +83,7 @@ linux_component_names = {
     'linux',
     'modules',
     'userland',
+    'm5',
     'buildroot',
 }
 
@@ -156,7 +173,7 @@ else:
 
 # Decide components.
 selected_component_names = []
-for name in component_names:
+for name in component_order:
     component = name_to_component_map[name]
     if (
         args.all or
@@ -167,6 +184,7 @@ for name in component_names:
     ):
         selected_component_names.append(name)
 
+# Do the build.
 for arch in archs:
     for name in selected_component_names:
         name_to_component_map[name].build(arch, args.dry_run)

build-m5

@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+
+import multiprocessing
+import os
+import platform
+import shutil
+import subprocess
+import time
+
+import common
+
+parser = common.get_argparse(argparse_args={
+    'description': 'Build the m5 executable',
+})
+common.add_build_arguments(parser)
+args = common.setup(parser)
+start_time = time.time()
+os.makedirs(common.gem5_m5_build_dir, exist_ok=True)
+allowed_toolchains = ['buildroot']
+cc = common.get_toolchain_tool('gcc', allowed_toolchains=allowed_toolchains)
+ld = common.get_toolchain_tool('ld', allowed_toolchains=allowed_toolchains)
+if args.arch == 'x86_64':
+    arch = 'x86'
+else:
+    arch = args.arch
+assert common.run_cmd(
+    (
+        [
+            'make',
+            '-j', str(multiprocessing.cpu_count()),
+            '-f', 'Makefile.{}'.format(arch),
+            'CC={}'.format(cc),
+            'LD={}'.format(ld),
+            'PWD={}'.format(common.gem5_m5_src_dir),
+        ]
+    ),
+    cwd=common.gem5_m5_src_dir,
+) == 0
+print(common.out_rootfs_overlay_bin_dir)
+os.makedirs(common.out_rootfs_overlay_bin_dir, exist_ok=True)
+shutil.copy2(os.path.join(common.gem5_m5_src_dir, 'm5'), common.out_rootfs_overlay_bin_dir)
+end_time = time.time()
+common.print_time(end_time - start_time)

build-userland

@@ -1,6 +1,5 @@
 #!/usr/bin/env python3
 
-import distutils.file_util
 import multiprocessing
 import os
 import platform

common.py

@@ -649,6 +649,8 @@ def setup(parser):
             this.gem5_src_dir = os.path.join(this.gem5_non_default_src_root_dir, args.gem5_worktree)
         else:
             this.gem5_src_dir = this.gem5_default_src_dir
+    this.gem5_m5_src_dir = os.path.join(this.gem5_src_dir, 'util', 'm5')
+    this.gem5_m5_build_dir = os.path.join(this.out_dir, 'util', 'm5')
     if args.gem5:
         this.executable = this.gem5_executable
         this.run_dir = this.gem5_run_dir
@@ -683,6 +685,7 @@ def setup(parser):
     this.kernel_modules_build_host_dir = os.path.join(this.kernel_modules_build_base_dir, 'host')
     this.userland_build_dir = os.path.join(this.out_dir, 'userland', args.arch)
     this.out_rootfs_overlay_dir = os.path.join(this.out_dir, 'rootfs_overlay', args.arch)
+    this.out_rootfs_overlay_bin_dir = os.path.join(this.out_rootfs_overlay_dir, 'bin')
 
     # Ports
     if args.port_offset is None: