From 23a9d767ba05b9cfd915337a47b6af4423a1cbf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ciro=20Santilli=20=E5=85=AD=E5=9B=9B=E4=BA=8B=E4=BB=B6=20?= =?UTF-8?q?=E6=B3=95=E8=BD=AE=E5=8A=9F?= Date: Thu, 18 Oct 2018 00:00:00 +0000 Subject: [PATCH] build-buildroot twice, and split build-m5 --- README.adoc | 12 ++++----- build | 72 +++++++++++++++++++++++++++++++------------------- build-m5 | 43 ++++++++++++++++++++++++++++++ build-userland | 1 - common.py | 3 +++ 5 files changed, 97 insertions(+), 34 deletions(-) create mode 100755 build-m5 diff --git a/README.adoc b/README.adoc index 644833f..a197d2e 100644 --- a/README.adoc +++ b/README.adoc @@ -396,19 +396,19 @@ If you haven't built Buildroot yet for <>, you can build f .... ./download-dependencies --gem5 ./build --gem5 --no-qemu -./build-buildroot --gem5 ./run --gem5 .... `--no-qemu` is optional, but it makes the build slightly faster TODO: after first build: .... ./download-dependencies --gem5 -./build-gem5 --gem5 -./build-buildroot --gem5 +./build-gem5 +./build-m5 +./build-buildroot ./run --gem5 .... -If you have already built previously, don't be afraid: gem5 and QEMU use almost the same root filesystem and kernel, so `./build-buildroot --gem` will be fast. It is currently only needed for the <> tool. +If you have already built previously, don't be afraid: gem5 and QEMU use almost the same root filesystem and kernel, so `./build-buildroot --gem` will be fast. It is currently only needed for the <> tool. To get a terminal, either open a new shell and run: @@ -601,7 +601,7 @@ Maybe we could work around this by just downloading the kernel source somehow, a + ** there is no Debian package for it, so you have to compile your own, so you might as well just build the image itself ** it does not handle <>, and we haven't gotten <> to work yet, therefore we would have to either distribute large ext2 images, or constantly fight with <> -** QEMU uses `bzImage` and gem5 the raw `vmlinux`, and we don't want to distribute the same thing twice... +** QEMU uses `bzImage` and gem5 the raw `vmlinux`, and we don't want to distribute the same thing twice... + And our attempt at using link:https://github.com/torvalds/linux/blob/master/scripts/extract-vmlinux[`extract-vmlinux`] failed for `aarch64` with: + @@ -2913,7 +2913,7 @@ man init_module documents that: ____ -The finit_module() system call is like init_module(), but reads the module to be loaded from the file descriptor fd. It is useful when the authenticity of a kernel module can be determined from its location in the filesystem; in cases where that is possible, the overhead of using cryptographically signed modules to determine the authenticity of a module can be avoided. The param_values argument is as for init_module(). +The finit_module() system call is like init_module(), but reads the module to be loaded from the file descriptor fd. It is useful when the authenticity of a kernel module can be determined from its location in the filesystem; in cases where that is possible, the overhead of using cryptographically signed modules to determine the authenticity of a module can be avoided. The param_values argument is as for init_module(). ____ `finit` is newer and was added only in v3.8. More rationale: https://lwn.net/Articles/519010/ diff --git a/build b/build index 2cf13d1..30f82d9 100755 --- a/build +++ b/build @@ -32,34 +32,50 @@ def run_cmd(cmd, dry_run): cmd_abs[0] = os.path.join(common.root_dir, cmd[0]) common.run_cmd(cmd_abs, dry_run=dry_run) -# Topological sorted on build dependencies. -name_to_component_map = collections.OrderedDict([ - ('baremetal', BaremetalComponent(False)), - ('gem5', Component( - False, - lambda arch, dry_run: run_cmd(['build-gem5', '--arch', arch], dry_run) - )), - ('qemu', Component( - True, - lambda arch, dry_run: run_cmd(['build-qemu', '--arch', arch], dry_run) - )), - ('linux', Component( - True, - lambda arch, dry_run: run_cmd(['build-linux', '--arch', arch], dry_run=dry_run) - )), - ('modules', Component( - True, - lambda arch, dry_run: run_cmd(['build-modules', '--arch', arch], dry_run=dry_run) - )), - ('userland', Component( - True, - lambda arch, dry_run: run_cmd(['build-userland', '--arch', arch], dry_run=dry_run), - )), - ('buildroot', Component( +name_to_component_map = { + 'baremetal': BaremetalComponent(False), + 'buildroot': Component( True, lambda arch, dry_run: run_cmd(['build-buildroot', '--arch', arch, '--gem5'], dry_run=dry_run), - )), -]) + ), + 'gem5': Component( + False, + lambda arch, dry_run: run_cmd(['build-gem5', '--arch', arch], dry_run) + ), + 'linux': Component( + True, + lambda arch, dry_run: run_cmd(['build-linux', '--arch', arch], dry_run=dry_run) + ), + 'modules': Component( + True, + lambda arch, dry_run: run_cmd(['build-modules', '--arch', arch], dry_run=dry_run) + ), + 'm5': Component( + True, + lambda arch, dry_run: run_cmd(['build-m5', '--arch', arch], dry_run=dry_run) + ), + 'qemu': Component( + True, + lambda arch, dry_run: run_cmd(['build-qemu', '--arch', arch], dry_run) + ), + 'userland': Component( + True, + lambda arch, dry_run: run_cmd(['build-userland', '--arch', arch], dry_run=dry_run), + ), +} +# Topological sorted on build order. +component_order = [ + 'baremetal', + 'gem5', + 'qemu', + # Need one extra one here to build the toolchain. + 'buildroot', + 'linux', + 'modules', + 'userland', + 'm5', + 'buildroot', +] component_names = name_to_component_map.keys() linux_component_names = { 'gem5', @@ -67,6 +83,7 @@ linux_component_names = { 'linux', 'modules', 'userland', + 'm5', 'buildroot', } @@ -156,7 +173,7 @@ else: # Decide components. selected_component_names = [] -for name in component_names: +for name in component_order: component = name_to_component_map[name] if ( args.all or @@ -167,6 +184,7 @@ for name in component_names: ): selected_component_names.append(name) +# Do the build. for arch in archs: for name in selected_component_names: name_to_component_map[name].build(arch, args.dry_run) diff --git a/build-m5 b/build-m5 new file mode 100755 index 0000000..fa25af1 --- /dev/null +++ b/build-m5 @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 + +import multiprocessing +import os +import platform +import shutil +import subprocess +import time + +import common + +parser = common.get_argparse(argparse_args={ + 'description': 'Build the m5 executable', +}) +common.add_build_arguments(parser) +args = common.setup(parser) +start_time = time.time() +os.makedirs(common.gem5_m5_build_dir, exist_ok=True) +allowed_toolchains = ['buildroot'] +cc = common.get_toolchain_tool('gcc', allowed_toolchains=allowed_toolchains) +ld = common.get_toolchain_tool('ld', allowed_toolchains=allowed_toolchains) +if args.arch == 'x86_64': + arch = 'x86' +else: + arch = args.arch +assert common.run_cmd( + ( + [ + 'make', + '-j', str(multiprocessing.cpu_count()), + '-f', 'Makefile.{}'.format(arch), + 'CC={}'.format(cc), + 'LD={}'.format(ld), + 'PWD={}'.format(common.gem5_m5_src_dir), + ] + ), + cwd=common.gem5_m5_src_dir, +) == 0 +print(common.out_rootfs_overlay_bin_dir) +os.makedirs(common.out_rootfs_overlay_bin_dir, exist_ok=True) +shutil.copy2(os.path.join(common.gem5_m5_src_dir, 'm5'), common.out_rootfs_overlay_bin_dir) +end_time = time.time() +common.print_time(end_time - start_time) diff --git a/build-userland b/build-userland index 75d3972..797fed7 100755 --- a/build-userland +++ b/build-userland @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -import distutils.file_util import multiprocessing import os import platform diff --git a/common.py b/common.py index a250c92..063f075 100644 --- a/common.py +++ b/common.py @@ -649,6 +649,8 @@ def setup(parser): this.gem5_src_dir = os.path.join(this.gem5_non_default_src_root_dir, args.gem5_worktree) else: this.gem5_src_dir = this.gem5_default_src_dir + this.gem5_m5_src_dir = os.path.join(this.gem5_src_dir, 'util', 'm5') + this.gem5_m5_build_dir = os.path.join(this.out_dir, 'util', 'm5') if args.gem5: this.executable = this.gem5_executable this.run_dir = this.gem5_run_dir @@ -683,6 +685,7 @@ def setup(parser): this.kernel_modules_build_host_dir = os.path.join(this.kernel_modules_build_base_dir, 'host') this.userland_build_dir = os.path.join(this.out_dir, 'userland', args.arch) this.out_rootfs_overlay_dir = os.path.join(this.out_dir, 'rootfs_overlay', args.arch) + this.out_rootfs_overlay_bin_dir = os.path.join(this.out_rootfs_overlay_dir, 'bin') # Ports if args.port_offset is None: