move all builds to components

This commit is contained in:
Ciro Santilli 六四事件 法轮功
2018-10-23 00:00:00 +00:00
parent b6ccceb344
commit 4cd9c533b8
11 changed files with 281 additions and 269 deletions

View File

@@ -6,7 +6,7 @@ import shutil
import sys
import common
build = imp.load_source('build', os.path.join(common.root_dir, 'build'))
build_linux = imp.load_source('build-linux', os.path.join(common.root_dir, 'build_linux'))
run = imp.load_source('run', os.path.join(common.root_dir, 'run'))
parser = common.get_argparse(
@@ -25,7 +25,7 @@ args = common.setup(parser)
# - may fail
# - may not actually rebuild all files, e.g. on header changes
common.rmrf(common.linux_build_dir)
assert build.main(args) == 0
build_linux.LinuxComponent().do_build(args)
status = run.main(args, {
'eval': 'm5 exit',
})

View File

@@ -1,71 +1,16 @@
#!/usr/bin/env python3
import glob
import multiprocessing
import os
import shutil
import sys
import time
import common
def build_dir(subpath, gcc, cflags, entry_address, bootloader_obj, common_obj, bootloader=True):
"""
Build all .c and .S files in a given subpath of the baremetal source
directory non recursively.
Place outputs on the same subpath or the output directory.
"""
in_dir = os.path.join(common.baremetal_src_dir, subpath)
out_dir = os.path.join(common.baremetal_out_dir, subpath)
os.makedirs(out_dir, exist_ok=True)
if bootloader:
bootloader_cmd = [bootloader_obj]
else:
bootloader_cmd = []
for in_basename in os.listdir(in_dir):
in_path = os.path.join(in_dir, in_basename)
if os.path.isfile(in_path) and os.path.splitext(in_basename)[1] in (common.c_ext, common.asm_ext):
in_name = os.path.splitext(in_basename)[0]
main_obj = os.path.join(common.baremetal_out_dir, subpath, '{}{}'.format(in_name, common.obj_ext))
assert common.run_cmd(
[gcc] +
cflags +
[
'-c',
'-o', main_obj,
os.path.join(common.baremetal_src_dir, in_path),
]
) == 0
assert common.run_cmd(
[gcc] +
cflags +
[
'-Wl,--section-start=.text={:#x}'.format(entry_address),
'-o', os.path.join(common.baremetal_out_dir, subpath, in_name + common.baremetal_out_ext),
'-T', os.path.join(common.baremetal_src_dir, 'link.ld'),
] +
bootloader_cmd +
[
common_obj,
main_obj,
]
) == 0
def get_argparse():
parser = common.get_argparse(argparse_args={
'description': 'Build the baremetal examples with crosstool-NG'
})
common.add_build_arguments(parser)
return parser
def main(args, extra_args=None):
if args.clean:
common.rmrf(common.baremetal_out_dir)
else:
class BaremetalComponent(common.Component):
def do_build(self, args):
common.raise_no_x86(args.arch)
bootloader_obj = os.path.join(common.baremetal_out_lib_dir, 'bootloader{}'.format(common.obj_ext))
common_obj = os.path.join(common.baremetal_out_lib_dir, 'common{}'.format(common.obj_ext))
build_dir = self.get_build_dir(args)
bootloader_obj = os.path.join(common.baremetal_build_lib_dir, 'bootloader{}'.format(common.obj_ext))
common_obj = os.path.join(common.baremetal_build_lib_dir, 'common{}'.format(common.obj_ext))
cflags = [
'-ggdb3',
'-mcpu={}'.format(common.mcpu),
@@ -89,8 +34,8 @@ def main(args, extra_args=None):
else:
entry_address = 0x40000000
uart_address = 0x09000000
os.makedirs(common.baremetal_out_dir, exist_ok=True)
os.makedirs(common.baremetal_out_lib_dir, exist_ok=True)
os.makedirs(build_dir, exist_ok=True)
os.makedirs(common.baremetal_build_lib_dir, exist_ok=True)
assert common.run_cmd(
[gcc] +
cflags +
@@ -111,7 +56,7 @@ def main(args, extra_args=None):
os.path.join(common.baremetal_src_lib_dir, 'common' + common.c_ext),
]
) == 0
build_dir(
self._build_dir(
'',
gcc=gcc,
cflags=cflags,
@@ -119,7 +64,7 @@ def main(args, extra_args=None):
bootloader_obj=bootloader_obj,
common_obj=common_obj,
)
build_dir(
self._build_dir(
'interactive',
gcc=gcc,
cflags=cflags,
@@ -129,7 +74,7 @@ def main(args, extra_args=None):
)
arch_dir = os.path.join('arch', args.arch)
if os.path.isdir(os.path.join(common.baremetal_src_dir, arch_dir)):
build_dir(
self._build_dir(
arch_dir,
gcc=gcc,
cflags=cflags,
@@ -139,7 +84,7 @@ def main(args, extra_args=None):
)
arch_dir = os.path.join('arch', args.arch, 'no_bootloader')
if os.path.isdir(os.path.join(common.baremetal_src_dir, arch_dir)):
build_dir(
self._build_dir(
arch_dir,
gcc=gcc,
cflags=cflags,
@@ -148,16 +93,62 @@ def main(args, extra_args=None):
common_obj=common_obj,
bootloader=False,
)
return 0
def get_argparse_args(self):
return {
'description': '''\
Build the baremetal examples with crosstool-NG.
'''
}
def get_build_dir(self, args):
return common.baremetal_build_dir
def get_default_args(self):
return {'baremetal': 'all'}
def _build_dir(self, subpath, gcc, cflags, entry_address, bootloader_obj, common_obj, bootloader=True):
"""
Build all .c and .S files in a given subpath of the baremetal source
directory non recursively.
Place outputs on the same subpath or the output directory.
"""
in_dir = os.path.join(common.baremetal_src_dir, subpath)
out_dir = os.path.join(common.baremetal_build_dir, subpath)
os.makedirs(out_dir, exist_ok=True)
if bootloader:
bootloader_cmd = [bootloader_obj]
else:
bootloader_cmd = []
for in_basename in os.listdir(in_dir):
in_path = os.path.join(in_dir, in_basename)
if os.path.isfile(in_path) and os.path.splitext(in_basename)[1] in (common.c_ext, common.asm_ext):
in_name = os.path.splitext(in_basename)[0]
main_obj = os.path.join(common.baremetal_build_dir, subpath, '{}{}'.format(in_name, common.obj_ext))
assert common.run_cmd(
[gcc] +
cflags +
[
'-c',
'-o', main_obj,
os.path.join(common.baremetal_src_dir, in_path),
]
) == 0
assert common.run_cmd(
[gcc] +
cflags +
[
'-Wl,--section-start=.text={:#x}'.format(entry_address),
'-o', os.path.join(common.baremetal_build_dir, subpath, in_name + common.baremetal_build_ext),
'-T', os.path.join(common.baremetal_src_dir, 'link.ld'),
] +
bootloader_cmd +
[
common_obj,
main_obj,
]
) == 0
if __name__ == '__main__':
parser = common.get_argparse(
default_args={'baremetal': 'all'},
)
common.add_build_arguments(parser)
args = common.setup(parser)
start_time = time.time()
exit_status = main(args)
end_time = time.time()
common.print_time(end_time - start_time)
sys.exit(exit_status)
BaremetalComponent().build()

View File

@@ -11,29 +11,10 @@ import re
import common
defaults = {
'baseline': False,
'build_linux': False,
'buildroot_config': [],
'buildroot_config_fragment': [],
'initramfs': False,
'initrd': False,
'kernel_config': [],
'kernel_config_fragment': [],
'kernel_custom_config_file': None,
'kernel_modules': False,
'linux_reconfigure': False,
'no_all': False,
'nproc': None,
'skip_configure': False,
'extra_make_args': [],
}
def get_argparse():
parser = common.get_argparse(argparse_args={'description':'Run Linux on an emulator'})
common.add_build_arguments(parser)
class BuildrootComponent(common.Component):
def add_parser_arguments(self, parser):
parser.add_argument(
'-B', '--buildroot-config', default=defaults['buildroot_config'], action='append',
'-B', '--buildroot-config', default=self._defaults['buildroot_config'], action='append',
help='''Add a single Buildroot config to the current build.
Example value: 'BR2_TARGET_ROOTFS_EXT2_SIZE="512M"'.
Can be used multiple times to add multiple configs.
@@ -41,12 +22,13 @@ Takes precedence over any Buildroot config files.
'''
)
parser.add_argument(
'-b', '--buildroot-config-fragment', default=defaults['buildroot_config_fragment'], action='append',
'-b', '--buildroot-config-fragment', default=self._defaults['buildroot_config_fragment'], action='append',
help='''Also use the given Buildroot configuration fragment file.
Pass multiple times to use multiple fragment files.'''
Pass multiple times to use multiple fragment files.
'''
)
parser.add_argument(
'--build-linux', default=defaults['build_linux'], action='store_true',
'--build-linux', default=self._defaults['build_linux'], action='store_true',
help='''\
Enable building the Linux kernel with Buildroot. This is done mostly
to extract Buildroot's default kernel configurations when updating Buildroot.
@@ -54,13 +36,13 @@ That kernel will not be use by our scripts.
'''
)
parser.add_argument(
'--baseline', default=defaults['baseline'], action='store_true',
'--baseline', default=self._defaults['baseline'], action='store_true',
help='''Do a default-ish Buildroot defconfig build, without any of our extra options.
Mostly to track how much slower we are than a basic build.
'''
)
parser.add_argument(
'-C', '--kernel-config', default=defaults['kernel_config'], action='append',
'-C', '--kernel-config', default=self._defaults['kernel_config'], action='append',
help='''\
Add a single kernel config configs to the current build.
Example value: 'CONFIG_FORTIFY_SOURCE=y'.
@@ -69,70 +51,65 @@ Takes precedence over any Buildroot config files.
'''
)
parser.add_argument(
'-c', '--kernel-config-fragment', default=defaults['kernel_config_fragment'], action='append',
'-c', '--kernel-config-fragment', default=self._defaults['kernel_config_fragment'], action='append',
help='''\
Also use the given kernel configuration fragment file.
Pass multiple times to use multiple fragment files.
'''
)
parser.add_argument(
'-I', '--initramfs', default=defaults['initramfs'], action='store_true',
'-I', '--initramfs', default=self._defaults['initramfs'], action='store_true',
)
parser.add_argument(
'-i', '--initrd', default=defaults['initrd'], action='store_true',
'-i', '--initrd', default=self._defaults['initrd'], action='store_true',
)
parser.add_argument(
'-j', '--nproc', default=defaults['nproc'], type=int,
'-j', '--nproc', default=self._defaults['nproc'], type=int,
help='Number of processors to use for the build. Default: all.'
)
parser.add_argument(
'-K', '--kernel-custom-config-file', default=defaults['kernel_custom_config_file'],
'-K', '--kernel-custom-config-file', default=self._defaults['kernel_custom_config_file'],
help='''\
Ignore all default kernel configurations and use this file instead.
Still uses options explicitly passed with `-C` and `-c` on top of it.
'''
)
parser.add_argument(
'-k', '--kernel-modules', default=defaults['kernel_modules'], action='store_true',
'-k', '--kernel-modules', default=self._defaults['kernel_modules'], action='store_true',
help='''Reconfigure and rebuild the kernel modules package.
Force --build-linux to true, since building the modules requires building Linux.
'''
)
parser.add_argument(
'--no-all', default=defaults['no_all'], action='store_true',
'--no-all', default=self._defaults['no_all'], action='store_true',
help='''\
Don't build the all target which normally gets build by default.
That target builds the root filesystem and all its dependencies.
'''
)
parser.add_argument(
'--skip-configure', default=defaults['skip_configure'], action='store_true',
'--skip-configure', default=self._defaults['skip_configure'], action='store_true',
help='''\
Skip the Buildroot configuration. Saves a few seconds,
but requires you to know what you are doing :-)
'''
)
parser.add_argument(
'extra_make_args', default=defaults['extra_make_args'], metavar='extra-make-args', nargs='*',
'extra_make_args', default=self._defaults['extra_make_args'], metavar='extra-make-args', nargs='*',
help='''\
Extra arguments to be passed to the Buildroot make,
usually extra Buildroot targets.
'''
)
return parser
def main(args, extra_args=None):
global defaults
args = common.resolve_args(defaults, args, extra_args)
if args.clean:
common.rmrf(common.buildroot_build_dir)
else:
def do_build(self, args):
build_dir = self.get_build_dir(args)
os.makedirs(common.out_dir, exist_ok=True)
extra_make_args = args.extra_make_args.copy()
if args.kernel_modules:
args.build_linux = True
extra_make_args.append('lkmc-reconfigure')
if args.linux_reconfigure:
if args.build_linux:
extra_make_args.append('linux-reconfigure')
if args.gem5:
extra_make_args.append('gem5-reconfigure')
@@ -154,7 +131,7 @@ def main(args, extra_args=None):
for package_dir in os.listdir(packages_dir):
package_dir_abs = os.path.join(packages_dir, package_dir)
if os.path.isdir(package_dir_abs):
br2_external_dirs.append(path_relative_to_buildroot(package_dir_abs))
br2_external_dirs.append(self._path_relative_to_buildroot(package_dir_abs))
br2_external_str = ':'.join(br2_external_dirs)
assert common.run_cmd(
[
@@ -178,23 +155,23 @@ def main(args, extra_args=None):
if not args.baseline:
buildroot_configs.extend([
'BR2_GLOBAL_PATCH_DIR="{}"'.format(
path_relative_to_buildroot(os.path.join(common.root_dir, 'patches', 'global'))
self._path_relative_to_buildroot(os.path.join(common.root_dir, 'patches', 'global'))
),
'BR2_PACKAGE_BUSYBOX_CONFIG_FRAGMENT_FILES="{}"'.format(
path_relative_to_buildroot(os.path.join(common.root_dir, 'busybox_config_fragment'))
self._path_relative_to_buildroot(os.path.join(common.root_dir, 'busybox_config_fragment'))
),
'BR2_PACKAGE_OVERRIDE_FILE="{}"'.format(
path_relative_to_buildroot(os.path.join(common.root_dir, 'buildroot_override'))
self._path_relative_to_buildroot(os.path.join(common.root_dir, 'buildroot_override'))
),
'BR2_ROOTFS_OVERLAY="{} {}"'.format(
path_relative_to_buildroot(common.rootfs_overlay_dir),
path_relative_to_buildroot(common.out_rootfs_overlay_dir),
self._path_relative_to_buildroot(common.rootfs_overlay_dir),
self._path_relative_to_buildroot(common.out_rootfs_overlay_dir),
),
'BR2_ROOTFS_POST_BUILD_SCRIPT="{}"'.format(
path_relative_to_buildroot(os.path.join(common.root_dir, 'rootfs-post-build-script'))
self._path_relative_to_buildroot(os.path.join(common.root_dir, 'rootfs-post-build-script'))
),
'BR2_ROOTFS_USERS_TABLES="{}"'.format(
path_relative_to_buildroot(os.path.join(common.root_dir, 'user_table'))
self._path_relative_to_buildroot(os.path.join(common.root_dir, 'user_table'))
),
])
if args.kernel_modules:
@@ -246,21 +223,21 @@ def main(args, extra_args=None):
'BR2_LINUX_KERNEL_USE_CUSTOM_CONFIG=y',
'BR2_LINUX_KERNEL_CUSTOM_CONFIG_FILE=\"{}\"'.format(args.kernel_custom_config_file),
])
if args.linux_reconfigure:
if args.build_linux:
pathlib.Path(args.kernel_custom_config_file).touch()
else:
raise Exception('Kernel config fragment file does not exist: {}'.format(args.kernel_custom_config_file))
default_kernel_config_fragments = []
else:
default_kernel_config_fragments = ['min', 'default']
if args.linux_reconfigure:
if args.build_linux:
# https://stackoverflow.com/questions/49260466/why-when-i-change-br2-linux-kernel-custom-config-file-and-run-make-linux-reconfi
pathlib.Path(os.path.join(common.linux_config_dir, 'min')).touch()
for i, default_kernel_config_fragment in enumerate(default_kernel_config_fragments):
default_kernel_config_fragments[i] = os.path.join(common.linux_config_dir, default_kernel_config_fragment)
kernel_config_fragments.extend(default_kernel_config_fragments)
for i, frag in enumerate(kernel_config_fragments):
kernel_config_fragments[i] = path_relative_to_buildroot(frag)
kernel_config_fragments[i] = self._path_relative_to_buildroot(frag)
buildroot_kernel_config_fragment_str = 'BR2_LINUX_KERNEL_CONFIG_FRAGMENT_FILES="{}"'.format(' '.join(kernel_config_fragments))
buildroot_configs.append(buildroot_kernel_config_fragment_str)
common.write_configs(common.buildroot_config_file, buildroot_configs, buildroot_config_fragments)
@@ -298,16 +275,35 @@ def main(args, extra_args=None):
if not args.no_all and os.path.exists(common.qemu_img_executable):
common.raw_to_qcow2()
return 0
def get_argparse_args(self):
return {
'description': '''\
Run Linux on an emulator
'''
}
def path_relative_to_buildroot(abspath):
def get_build_dir(self, args):
return common.buildroot_build_dir
_defaults = {
'baseline': False,
'build_linux': False,
'buildroot_config': [],
'buildroot_config_fragment': [],
'initramfs': False,
'initrd': False,
'kernel_config': [],
'kernel_config_fragment': [],
'kernel_custom_config_file': None,
'kernel_modules': False,
'no_all': False,
'nproc': None,
'skip_configure': False,
'extra_make_args': [],
}
def _path_relative_to_buildroot(self, abspath):
return os.path.relpath(abspath, common.buildroot_src_dir)
if __name__ == '__main__':
parser = get_argparse()
args = common.setup(parser)
start_time = time.time()
exit_status = main(args)
end_time = time.time()
common.print_time(end_time - start_time)
sys.exit(exit_status)
BuildrootComponent().build()

View File

@@ -2,20 +2,13 @@
import multiprocessing
import os
import sys
import time
import common
def get_argparse():
parser = common.get_argparse(argparse_args={
'description': 'Build crosstool-NG with Newlib for bare metal compilation'
})
common.add_build_arguments(parser)
return parser
def main(args, extra_args=None):
class CrosstoolNgComponent(common.Component):
def do_build(self, args):
common.raise_no_x86(args.arch)
build_dir = self.get_build_dir(args)
nproc = multiprocessing.cpu_count()
defconfig_dest = os.path.join(common.crosstool_ng_util_dir, 'defconfig')
os.makedirs(common.crosstool_ng_util_dir, exist_ok=True)
@@ -50,7 +43,7 @@ def main(args, extra_args=None):
common.crosstool_ng_defconfig,
[
'CT_PREFIX_DIR="{}"'.format(common.crosstool_ng_install_dir),
'CT_WORK_DIR="{}"'.format(common.crosstool_ng_build_dir),
'CT_WORK_DIR="{}"'.format(build_dir),
'CT_LOCAL_TARBALLS_DIR="{}"'.format(common.crosstool_ng_download_dir),
]
)
@@ -67,16 +60,20 @@ def main(args, extra_args=None):
'build',
'CT_JOBS={}'.format(str(nproc)),
],
out_file=os.path.join(common.crosstool_ng_build_dir, 'lkmc.log'),
out_file=os.path.join(build_dir, 'lkmc.log'),
delete_env=['LD_LIBRARY_PATH'],
extra_paths=[common.ccache_dir],
) == 0
def get_argparse_args(self):
return {
'description': '''\
Build crosstool-NG with Newlib for bare metal compilation'
'''
}
def get_build_dir(self, args):
return common.crosstool_ng_build_dir
if __name__ == '__main__':
parser = get_argparse()
args = common.setup(parser)
start_time = time.time()
exit_status = main(args)
end_time = time.time()
common.print_time(end_time - start_time)
sys.exit(exit_status)
CrosstoolNgComponent().build()

View File

@@ -99,4 +99,5 @@ class Gem5Component(common.Component):
def get_build_dir(self, args):
return common.gem5_build_dir
Gem5Component().build()
if __name__ == '__main__':
Gem5Component().build()

View File

@@ -89,4 +89,5 @@ Build the Linux kernel.
def get_build_dir(self, args):
return common.linux_build_dir
LinuxComponent().build()
if __name__ == '__main__':
LinuxComponent().build()

View File

@@ -42,4 +42,5 @@ class M5Component(common.Component):
cwd=common.gem5_m5_src_dir,
) == 0
M5Component().build()
if __name__ == '__main__':
M5Component().build()

View File

@@ -116,4 +116,5 @@ See also: https://github.com/cirosantilli/linux-kernel-module-cheat#host
else:
return os.path.join(common.kernel_modules_build_dir)
ModulesComponent().build()
if __name__ == '__main__':
ModulesComponent().build()

View File

@@ -51,4 +51,5 @@ class QemuComponent(common.Component):
def get_build_dir(self, args):
return common.qemu_build_dir
QemuComponent().build()
if __name__ == '__main__':
QemuComponent().build()

View File

@@ -69,4 +69,5 @@ has the OpenBLAS libraries and headers installed.
def get_build_dir(self, args):
return common.userland_build_dir
UserlandComponent().build()
if __name__ == '__main__':
UserlandComponent().build()

View File

@@ -78,7 +78,15 @@ class Component:
pass
def build(self):
parser = this_module.get_argparse(argparse_args=self.get_argparse_args())
'''
Parse CLI, and to the build based on it.
The actual build work is done by do_build in implementing classes.
'''
parser = this_module.get_argparse(
argparse_args=self.get_argparse_args(),
default_args=self.get_default_args(),
)
self.add_parser_arguments(parser)
this_module.add_build_arguments(parser)
args = this_module.setup(parser)
@@ -99,14 +107,29 @@ class Component:
this_module.rmrf(self.get_build_dir(args))
def do_build(self, args):
'''
Do the actual main build work.
'''
raise NotImplementedError()
def get_argparse_args(self):
'''
Extra arguments for argparse.ArgumentParser.
'''
return {}
def get_build_dir(self, args):
'''
Build directory, gets cleaned by --clean.
'''
raise NotImplementedError()
def get_default_args(self):
'''
Default values for command line arguments.
'''
return {}
def add_build_arguments(parser):
parser.add_argument(
'--clean',
@@ -760,10 +783,9 @@ def setup(parser):
this_module.simulator_name = 'gem5'
else:
this_module.simulator_name = 'qemu'
this_module.baremetal_out_dir = os.path.join(out_dir, 'baremetal', args.arch, this_module.simulator_name, this_module.machine)
this_module.baremetal_out_lib_dir = os.path.join(this_module.baremetal_out_dir, this_module.baremetal_lib_basename)
this_module.baremetal_out_ext = '.elf'
this_module.baremetal_build_dir = os.path.join(out_dir, 'baremetal', args.arch, this_module.simulator_name, this_module.machine)
this_module.baremetal_build_lib_dir = os.path.join(this_module.baremetal_build_dir, this_module.baremetal_lib_basename)
this_module.baremetal_build_ext = '.elf'
# Docker
this_module.docker_build_dir = os.path.join(this_module.out_dir, 'docker', args.arch)
@@ -789,13 +811,13 @@ def setup(parser):
else:
this_module.disk_image = this_module.gem5_fake_iso
paths = [
os.path.join(this_module.baremetal_out_dir, this_module.baremetal),
os.path.join(this_module.baremetal_build_dir, this_module.baremetal),
os.path.join(
this_module.baremetal_out_dir,
this_module.baremetal_build_dir,
os.path.relpath(this_module.baremetal, this_module.baremetal_src_dir),
)
]
paths[:] = [os.path.splitext(path)[0] + this_module.baremetal_out_ext for path in paths]
paths[:] = [os.path.splitext(path)[0] + this_module.baremetal_build_ext for path in paths]
found = False
for path in paths:
if os.path.exists(path):