linux-kernel-module-cheat/build-buildroot

#!/usr/bin/env python3

import os
import pathlib
import shutil
import subprocess
import sys
import time
import re

import common
from shell_helpers import LF

class Main(common.BuildCliFunction):
    def __init__(self):
        super().__init__(
            description='''\
Build Buildroot. This includes, notably: the userland GCC cross-toolchain,
and the root filesystem.
''')
        self.add_argument(
            '--build-linux', default=False,
            help='''\
Enable building the Linux kernel with Buildroot. This is done mostly
to extract Buildroot's default kernel configurations when updating Buildroot.
This kernel will not be use by our other scripts. Configuring this kernel is
not currently supported, juse use ./build-linux script if you want to do that.
'''
        )
        self.add_argument(
            '--baseline', default=False,
            help='''Do a default-ish Buildroot defconfig build, without any of our extra options.
Mostly to track how much slower we are than a basic build.
'''
        )
        self.add_argument(
            '--config', default=[], action='append',
            help='''Add a single Buildroot config to the current build.
Example value: 'BR2_TARGET_ROOTFS_EXT2_SIZE="512M"'.
Can be used multiple times to add multiple configs.
Takes precedence over any Buildroot config files.
'''
        )
        self.add_argument(
            '--config-fragment', default=[], action='append',
            help='''Also use the given Buildroot configuration fragment file.
Pass multiple times to use multiple fragment files.
'''
        )
        self.add_argument(
            '--no-all', default=False,
            help='''\
Don't build the all target which normally gets build by default.
That target builds the root filesystem and all its dependencies.
'''
        )
        self.add_argument(
            '--no-overlay', default=False,
            help='''\
Don't add our overlay which contains all files we build without going through Buildroot.
This prevents us from overwriting certain Buildroot files. Remember however that you must
still rebuild the Buildroot package that provides those files to actually put the Buildroot
files on the root filesystem.
'''
        )
        self._add_argument('--force-rebuild')
        self._add_argument('extra_make_args')

    def build(self):
        build_dir = self.get_build_dir()
        os.makedirs(self.env['out_dir'], exist_ok=True)
        extra_make_args = self.sh.add_newlines(self.env['extra_make_args'])
        if self.env['build_linux']:
            extra_make_args.extend(['linux-reconfigure', LF])
        if self.env['arch'] == 'x86_64':
            defconfig = 'qemu_x86_64_defconfig'
        elif self.env['arch'] == 'arm':
            defconfig = 'qemu_arm_vexpress_defconfig'
        elif self.env['arch'] == 'aarch64':
            defconfig = 'qemu_aarch64_virt_defconfig'
        br2_external_dirs = []
        for package_dir in sorted(os.listdir(self.env['packages_dir'])):
            package_dir_abs = os.path.join(self.env['packages_dir'], package_dir)
            if os.path.isdir(package_dir_abs):
                br2_external_dirs.append(self._path_relative_to_buildroot(package_dir_abs))
        br2_external_str = ':'.join(br2_external_dirs)
        self.sh.run_cmd(
            [
                'make', LF,
                'O={}'.format(self.env['buildroot_build_dir']), LF,
                'BR2_EXTERNAL={}'.format(br2_external_str), LF,
                defconfig, LF,
            ],
            cwd=self.env['buildroot_source_dir'],
        )
        configs = self.env['config']
        configs.extend([
            'BR2_JLEVEL={}'.format(self.env['nproc']),
            'BR2_DL_DIR="{}"'.format(self.env['buildroot_download_dir']),
        ])
        if not self.env['build_linux']:
            configs.extend([
                '# BR2_LINUX_KERNEL is not set',
            ])
        config_fragments = []
        if not self.env['baseline']:
            configs.extend([
                'BR2_GLOBAL_PATCH_DIR="{}"'.format(
                    self._path_relative_to_buildroot(os.path.join(self.env['root_dir'], 'patches', 'global'))
                ),
                'BR2_PACKAGE_BUSYBOX_CONFIG_FRAGMENT_FILES="{}"'.format(
                    self._path_relative_to_buildroot(os.path.join(self.env['root_dir'], 'busybox_config_fragment'))
                ),
                'BR2_PACKAGE_OVERRIDE_FILE="{}"'.format(
                    self._path_relative_to_buildroot(os.path.join(self.env['root_dir'], 'buildroot_override'))
                ),
                'BR2_ROOTFS_POST_BUILD_SCRIPT="{}"'.format(
                    self._path_relative_to_buildroot(os.path.join(self.env['root_dir'], 'rootfs-post-build-script'))
                ),
                'BR2_ROOTFS_USERS_TABLES="{}"'.format(
                    self._path_relative_to_buildroot(os.path.join(self.env['root_dir'], 'user_table'))
                ),
            ])
            if not self.env['no_overlay']:
                configs.append('BR2_ROOTFS_OVERLAY="{}"'.format(
                    self._path_relative_to_buildroot(self.env['out_rootfs_overlay_dir'])
                ))
            config_fragments = [
                os.path.join(self.env['root_dir'], 'buildroot_config', 'default')
            ] + self.env['config_fragment']
        if self.env['initrd'] or self.env['initramfs']:
            configs.append('BR2_TARGET_ROOTFS_CPIO=y')
        # TODO Can't get rid of these for now with nice fragments on Buildroot:
        # http://stackoverflow.com/questions/44078245/is-it-possible-to-use-config-fragments-with-buildroots-config
        self.sh.write_configs(self.env['buildroot_config_file'], configs, config_fragments)
        self.sh.run_cmd(
            [
                'make', LF,
                'O={}'.format(self.env['buildroot_build_dir']), LF,
                'olddefconfig', LF,
            ],
            cwd=self.env['buildroot_source_dir'],
        )
        self.make_build_dirs()
        if self.env['force_rebuild']:
            extra_make_args.extend(['-B', LF])
        if not self.env['no_all']:
            extra_make_args.extend(['all', LF])
        self.sh.run_cmd(
            [
                'make', LF,
                'LKMC_PARSEC_BENCHMARK_SRCDIR="{}"'.format(self.env['parsec_benchmark_source_dir']), LF,
                'O={}'.format(self.env['buildroot_build_dir']), LF,
                'V={}'.format(int(self.env['verbose'])), LF,
            ] +
            extra_make_args
            ,
            out_file=os.path.join(self.env['buildroot_build_dir'], self.env['repo_short_id'] + '.log'),
            delete_env=['LD_LIBRARY_PATH'],
            cwd=self.env['buildroot_source_dir'],
        )
        # Create the qcow2 from ext2.
        # Skip if qemu is not present, because gem5 does not need the qcow2.
        # so we don't force a QEMU build for gem5.
        if not self.env['no_all'] and os.path.exists(self.env['qemu_img_executable']):
            self.raw_to_qcow2()

    def get_build_dir(self):
        return self.env['buildroot_build_dir']

    def _path_relative_to_buildroot(self, abspath):
        return os.path.relpath(abspath, self.env['buildroot_source_dir'])

if __name__ == '__main__':
    Main().cli()