From 21627ff9d84286cd69d5c5e22bbd0a791ff9aa94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ciro=20Santilli=20=E5=85=AD=E5=9B=9B=E4=BA=8B=E4=BB=B6=20?= =?UTF-8?q?=E6=B3=95=E8=BD=AE=E5=8A=9F?= Date: Wed, 24 Oct 2018 00:00:01 +0000 Subject: [PATCH] Move build-buildroot linux configs to build-linux. Needs a lot of testing, and need to cleanup readme, but seems to work. --- build-buildroot | 245 +++++------------- build-linux | 100 +++++-- common.py | 48 ++-- .../buildroot-aarch64 | 0 {kernel_config => linux_config}/buildroot-arm | 0 .../buildroot-x86_64 | 0 {kernel_config => linux_config}/default | 0 {kernel_config => linux_config}/display | 0 {kernel_config => linux_config}/gpio | 0 {kernel_config => linux_config}/leds | 0 {kernel_config => linux_config}/min | 0 11 files changed, 164 insertions(+), 229 deletions(-) rename {kernel_config => linux_config}/buildroot-aarch64 (100%) rename {kernel_config => linux_config}/buildroot-arm (100%) rename {kernel_config => linux_config}/buildroot-x86_64 (100%) rename {kernel_config => linux_config}/default (100%) rename {kernel_config => linux_config}/display (100%) rename {kernel_config => linux_config}/gpio (100%) rename {kernel_config => linux_config}/leds (100%) rename {kernel_config => linux_config}/min (100%) diff --git a/build-buildroot b/build-buildroot index 7ba9ebb..272b21d 100755 --- a/build-buildroot +++ b/build-buildroot @@ -12,26 +12,13 @@ import common class BuildrootComponent(common.Component): def add_parser_arguments(self, parser): - parser.add_argument( - '-B', '--buildroot-config', default=self._defaults['buildroot_config'], action='append', - help='''Add a single Buildroot config to the current build. -Example value: 'BR2_TARGET_ROOTFS_EXT2_SIZE="512M"'. -Can be used multiple times to add multiple configs. -Takes precedence over any Buildroot config files. -''' - ) - parser.add_argument( - '-b', '--buildroot-config-fragment', default=self._defaults['buildroot_config_fragment'], action='append', - help='''Also use the given Buildroot configuration fragment file. -Pass multiple times to use multiple fragment files. -''' - ) parser.add_argument( '--build-linux', default=self._defaults['build_linux'], action='store_true', help='''\ Enable building the Linux kernel with Buildroot. This is done mostly to extract Buildroot's default kernel configurations when updating Buildroot. -That kernel will not be use by our scripts. +This kernel will not be use by our other scripts. Configuring this kernel is +not currently supported, juse use ./build-linux script if you want to do that. ''' ) parser.add_argument( @@ -41,38 +28,17 @@ Mostly to track how much slower we are than a basic build. ''' ) parser.add_argument( - '-C', '--kernel-config', default=self._defaults['kernel_config'], action='append', - help='''\ -Add a single kernel config configs to the current build. -Example value: 'CONFIG_FORTIFY_SOURCE=y'. + '--config', default=self._defaults['config'], action='append', + help='''Add a single Buildroot config to the current build. +Example value: 'BR2_TARGET_ROOTFS_EXT2_SIZE="512M"'. Can be used multiple times to add multiple configs. Takes precedence over any Buildroot config files. ''' ) parser.add_argument( - '-c', '--kernel-config-fragment', default=self._defaults['kernel_config_fragment'], action='append', - help='''\ -Also use the given kernel configuration fragment file. + '--config-fragment', default=self._defaults['config_fragment'], action='append', + help='''Also use the given Buildroot configuration fragment file. Pass multiple times to use multiple fragment files. -''' - ) - parser.add_argument( - '-I', '--initramfs', default=self._defaults['initramfs'], action='store_true', - ) - parser.add_argument( - '-i', '--initrd', default=self._defaults['initrd'], action='store_true', - ) - parser.add_argument( - '-K', '--kernel-custom-config-file', default=self._defaults['kernel_custom_config_file'], - help='''\ -Ignore all default kernel configurations and use this file instead. -Still uses options explicitly passed with `-C` and `-c` on top of it. -''' - ) - parser.add_argument( - '-k', '--kernel-modules', default=self._defaults['kernel_modules'], action='store_true', - help='''Reconfigure and rebuild the kernel modules package. -Force --build-linux to true, since building the modules requires building Linux. ''' ) parser.add_argument( @@ -80,13 +46,6 @@ Force --build-linux to true, since building the modules requires building Linux. help='''\ Don't build the all target which normally gets build by default. That target builds the root filesystem and all its dependencies. -''' - ) - parser.add_argument( - '--skip-configure', default=self._defaults['skip_configure'], action='store_true', - help='''\ -Skip the Buildroot configuration. Saves a few seconds, -but requires you to know what you are doing :-) ''' ) parser.add_argument( @@ -101,9 +60,6 @@ usually extra Buildroot targets. build_dir = self.get_build_dir(args) os.makedirs(common.out_dir, exist_ok=True) extra_make_args = args.extra_make_args.copy() - if args.kernel_modules: - args.build_linux = True - extra_make_args.append('lkmc-reconfigure') if args.build_linux: extra_make_args.append('linux-reconfigure') if args.gem5: @@ -114,134 +70,66 @@ usually extra Buildroot targets. defconfig = 'qemu_arm_vexpress_defconfig' elif args.arch == 'aarch64': defconfig = 'qemu_aarch64_virt_defconfig' - - # Configure. - if not args.skip_configure: - br2_external_dirs = [] - packages_dir = os.path.join(common.root_dir, 'packages') - for package_dir in os.listdir(packages_dir): - package_dir_abs = os.path.join(packages_dir, package_dir) - if os.path.isdir(package_dir_abs): - br2_external_dirs.append(self._path_relative_to_buildroot(package_dir_abs)) - br2_external_str = ':'.join(br2_external_dirs) + br2_external_dirs = [] + packages_dir = os.path.join(common.root_dir, 'packages') + for package_dir in os.listdir(packages_dir): + package_dir_abs = os.path.join(packages_dir, package_dir) + if os.path.isdir(package_dir_abs): + br2_external_dirs.append(self._path_relative_to_buildroot(package_dir_abs)) + br2_external_str = ':'.join(br2_external_dirs) + common.run_cmd( + [ + 'make', + 'O={}'.format(common.buildroot_build_dir), + 'BR2_EXTERNAL={}'.format(br2_external_str), + defconfig, + ], + cwd=common.buildroot_src_dir, + ) + configs = args.config + configs.extend([ + 'BR2_JLEVEL={}'.format(args.nproc), + 'BR2_DL_DIR="{}"'.format(common.buildroot_download_dir), + ]) + common.write_configs(common.buildroot_config_file, configs) + if not args.build_linux: + configs.extend([ + '# BR2_LINUX_KERNEL is not set', + ]) + if not args.baseline: + configs.extend([ + 'BR2_GLOBAL_PATCH_DIR="{}"'.format( + self._path_relative_to_buildroot(os.path.join(common.root_dir, 'patches', 'global')) + ), + 'BR2_PACKAGE_BUSYBOX_CONFIG_FRAGMENT_FILES="{}"'.format( + self._path_relative_to_buildroot(os.path.join(common.root_dir, 'busybox_config_fragment')) + ), + 'BR2_PACKAGE_OVERRIDE_FILE="{}"'.format( + self._path_relative_to_buildroot(os.path.join(common.root_dir, 'buildroot_override')) + ), + 'BR2_ROOTFS_OVERLAY="{} {}"'.format( + self._path_relative_to_buildroot(common.rootfs_overlay_dir), + self._path_relative_to_buildroot(common.out_rootfs_overlay_dir), + ), + 'BR2_ROOTFS_POST_BUILD_SCRIPT="{}"'.format( + self._path_relative_to_buildroot(os.path.join(common.root_dir, 'rootfs-post-build-script')) + ), + 'BR2_ROOTFS_USERS_TABLES="{}"'.format( + self._path_relative_to_buildroot(os.path.join(common.root_dir, 'user_table')) + ), + ]) + config_fragments = [ + os.path.join(common.root_dir, 'buildroot_config', 'default') + ] + args.config_fragment + common.write_configs(common.buildroot_config_file, configs, config_fragments) common.run_cmd( [ 'make', 'O={}'.format(common.buildroot_build_dir), - 'BR2_EXTERNAL={}'.format(br2_external_str), - defconfig, + 'olddefconfig', ], cwd=common.buildroot_src_dir, ) - buildroot_configs = args.buildroot_config - buildroot_configs.extend([ - 'BR2_JLEVEL={}'.format(args.nproc), - 'BR2_DL_DIR="{}"'.format(common.buildroot_download_dir), - ]) - common.write_configs(common.buildroot_config_file, buildroot_configs) - if not args.build_linux: - buildroot_configs.extend([ - '# BR2_LINUX_KERNEL is not set', - ]) - if not args.baseline: - buildroot_configs.extend([ - 'BR2_GLOBAL_PATCH_DIR="{}"'.format( - self._path_relative_to_buildroot(os.path.join(common.root_dir, 'patches', 'global')) - ), - 'BR2_PACKAGE_BUSYBOX_CONFIG_FRAGMENT_FILES="{}"'.format( - self._path_relative_to_buildroot(os.path.join(common.root_dir, 'busybox_config_fragment')) - ), - 'BR2_PACKAGE_OVERRIDE_FILE="{}"'.format( - self._path_relative_to_buildroot(os.path.join(common.root_dir, 'buildroot_override')) - ), - 'BR2_ROOTFS_OVERLAY="{} {}"'.format( - self._path_relative_to_buildroot(common.rootfs_overlay_dir), - self._path_relative_to_buildroot(common.out_rootfs_overlay_dir), - ), - 'BR2_ROOTFS_POST_BUILD_SCRIPT="{}"'.format( - self._path_relative_to_buildroot(os.path.join(common.root_dir, 'rootfs-post-build-script')) - ), - 'BR2_ROOTFS_USERS_TABLES="{}"'.format( - self._path_relative_to_buildroot(os.path.join(common.root_dir, 'user_table')) - ), - ]) - if args.kernel_modules: - buildroot_configs.append('BR2_PACKAGE_LKMC=y') - if args.gem5: - buildroot_configs.append('BR2_PACKAGE_GEM5=y') - if args.initramfs: - buildroot_configs.extend([ - 'BR2_TARGET_ROOTFS_CPIO=n', - 'BR2_TARGET_ROOTFS_EXT2=n', - 'BR2_TARGET_ROOTFS_INITRAMFS=y', - ]) - if args.initrd: - buildroot_configs.extend([ - 'BR2_TARGET_ROOTFS_CPIO=y', - 'BR2_TARGET_ROOTFS_EXT2=n' - 'BR2_TARGET_ROOTFS_INITRAMFS=n', - ]) - buildroot_config_fragments = [ - os.path.join(common.root_dir, 'buildroot_config', 'default') - ] + args.buildroot_config_fragment - - # Decide kernel configuration. - kernel_config_fragments = [] - if True: - # CLI kernel configurations. - kernel_config_fragment_cli_path = os.path.join(common.buildroot_build_dir, 'lkmc_kernel_config_fragment_cli') - kernel_config_cli_str = '\n'.join(args.kernel_config) - do_write = False - if os.path.exists(kernel_config_fragment_cli_path): - with open(kernel_config_fragment_cli_path, 'r') as kernel_config_fragment_cli_file: - kernel_config_cli_str_old = kernel_config_fragment_cli_file.read() - if kernel_config_cli_str != kernel_config_cli_str_old: - do_write = True - else: - do_write = True - if do_write: - # Only update if modified, otherwise Buildroot tries to - # rebuilds the kernel every time, which takes a few seconds. - # even when the kernel has already been built. - with open(kernel_config_fragment_cli_path, 'w') as kernel_config_fragment_cli_file: - kernel_config_fragment_cli_file.write(kernel_config_cli_str) - kernel_config_fragments.append(os.path.join(kernel_config_fragment_cli_path)) - if True: - # Kernel configuration fragments. - if args.kernel_custom_config_file is not None: - if os.path.exists(args.kernel_custom_config_file): - buildroot_configs.extend([ - 'BR2_LINUX_KERNEL_USE_CUSTOM_CONFIG=y', - 'BR2_LINUX_KERNEL_CUSTOM_CONFIG_FILE=\"{}\"'.format(args.kernel_custom_config_file), - ]) - if args.build_linux: - pathlib.Path(args.kernel_custom_config_file).touch() - else: - raise Exception('Kernel config fragment file does not exist: {}'.format(args.kernel_custom_config_file)) - default_kernel_config_fragments = [] - else: - default_kernel_config_fragments = ['min', 'default'] - if args.build_linux: - # https://stackoverflow.com/questions/49260466/why-when-i-change-br2-linux-kernel-custom-config-file-and-run-make-linux-reconfi - pathlib.Path(os.path.join(common.linux_config_dir, 'min')).touch() - for i, default_kernel_config_fragment in enumerate(default_kernel_config_fragments): - default_kernel_config_fragments[i] = os.path.join(common.linux_config_dir, default_kernel_config_fragment) - kernel_config_fragments.extend(default_kernel_config_fragments) - for i, frag in enumerate(kernel_config_fragments): - kernel_config_fragments[i] = self._path_relative_to_buildroot(frag) - buildroot_kernel_config_fragment_str = 'BR2_LINUX_KERNEL_CONFIG_FRAGMENT_FILES="{}"'.format(' '.join(kernel_config_fragments)) - buildroot_configs.append(buildroot_kernel_config_fragment_str) - common.write_configs(common.buildroot_config_file, buildroot_configs, buildroot_config_fragments) - common.run_cmd( - [ - 'make', - 'O={}'.format(common.buildroot_build_dir), - 'olddefconfig', - ], - cwd=common.buildroot_src_dir, - ) - - # Do the actual build. common.make_build_dirs() if not args.no_all: extra_make_args.append('all') @@ -259,7 +147,6 @@ usually extra Buildroot targets. delete_env=['LD_LIBRARY_PATH'], cwd=common.buildroot_src_dir, ) - # Create the qcow2 from ext2. # Skip if qemu is not present, because gem5 does not need the qcow2. # so we don't force a QEMU build for gem5. @@ -279,17 +166,11 @@ Run Linux on an emulator _defaults = { 'baseline': False, 'build_linux': False, - 'buildroot_config': [], - 'buildroot_config_fragment': [], - 'initramfs': False, - 'initrd': False, - 'kernel_config': [], - 'kernel_config_fragment': [], - 'kernel_custom_config_file': None, - 'kernel_modules': False, + 'config': [], + 'config_fragment': [], + 'extra_make_args': [], 'no_all': False, 'skip_configure': False, - 'extra_make_args': [], } def _path_relative_to_buildroot(self, abspath): diff --git a/build-linux b/build-linux index 15f4db0..6db57da 100755 --- a/build-linux +++ b/build-linux @@ -7,6 +7,42 @@ import common class LinuxComponent(common.Component): def add_parser_arguments(self, parser): + parser.add_argument( + '--config', default=[], action='append', + help='''\ +Add a single kernel config configs to the current build. +The `CONFIG_` prefix is added automatically. Sample values: +'FORTIFY_SOURCE=y', `KGDB=n`. Can be used multiple times to add multiple +configs. Takes precedence over any config files. +''' + ) + parser.add_argument( + '--config-fragment', default=[], action='append', + help='''\ +Also use the given kernel configuration fragment file. +Pass multiple times to use multiple fragment files. +''' + ) + parser.add_argument( + '--custom-config-file', + help='''\ +Ignore all default kernel configurations and use this file instead. +Still uses options explicitly passed with `--config` and +`--config-fragment` on top of it. +''' + ) + parser.add_argument( + '--config-only', default=False, action='store_true', + help='''\ +Configure the kernel, but don't build it. +''' + ) + parser.add_argument( + '--initramfs', default=False, action='store_true', + ) + parser.add_argument( + '--initrd', default=False, action='store_true', + ) parser.add_argument( 'extra_make_args', default=[], @@ -16,11 +52,9 @@ class LinuxComponent(common.Component): def do_build(self, args): build_dir = self.get_build_dir(args) + if args.initrd or args.initramfs: + raise Exception('just trolling, --initrd and --initramfs are broken for now') os.makedirs(build_dir, exist_ok=True) - common.cp( - os.path.join(common.linux_config_dir, 'buildroot-{}'.format(args.arch)), - os.path.join(build_dir, '.config'), - ) tool = 'gcc' gcc = common.get_toolchain_tool(tool) prefix = gcc[:-len(tool)] @@ -33,6 +67,8 @@ class LinuxComponent(common.Component): else: cc = gcc common_make_args = [ + 'make', + '-j', str(args.nproc), 'ARCH={}'.format(common.linux_arch), 'CROSS_COMPILE={}'.format(prefix), 'CC={}'.format(cc), @@ -42,41 +78,51 @@ class LinuxComponent(common.Component): verbose = ['V=1'] else: verbose = [] + if args.custom_config_file is not None: + if not os.path.exists(args.custom_config_file): + raise Exception('config fragment file does not exist: {}'.format(args.custom_config_file)) + base_config_file = args.custom_config_file + config_fragments = [] + else: + base_config_file = os.path.join(common.linux_config_dir, 'buildroot-{}'.format(args.arch)) + config_fragments = ['min', 'default'] + for i, config_fragment in enumerate(config_fragments): + config_fragments[i] = os.path.join(common.linux_config_dir, config_fragment) + config_fragments.extend(args.config_fragment) + if args.config != []: + cli_config_fragment_path = os.path.join(build_dir, 'lkmc_cli_config_fragment') + cli_config_str = '\n'.join(map(lambda x: 'CONFIG_' + x, args.config)) + with open(cli_config_fragment_path, 'w') as cli_config_fragment_cli: + cli_config_fragment_cli.write(cli_config_str) + config_fragments.append(cli_config_fragment_path) + common.cp( + base_config_file, + os.path.join(build_dir, '.config'), + ) common.run_cmd( [ os.path.join(common.linux_src_dir, 'scripts', 'kconfig', 'merge_config.sh'), '-m', '-O', build_dir, os.path.join(build_dir, '.config'), - os.path.join(common.linux_config_dir, 'min'), - os.path.join(common.linux_config_dir, 'default'), - ], + ] + + config_fragments ) common.run_cmd( ( - [ - 'make', - '-j', str(args.nproc), - ] + common_make_args + - [ - 'olddefconfig', - ] + ['olddefconfig'] ), - **common_args, - ) - common.run_cmd( - ( - [ - 'make', - '-j', str(args.nproc), - ] + - common_make_args + - verbose + - args.extra_make_args - ), - **common_args, + **common_args ) + if not args.config_only: + common.run_cmd( + ( + common_make_args + + args.extra_make_args + ), + **common_args + ) def get_argparse_args(self): return { diff --git a/common.py b/common.py index 030e293..c803ee0 100644 --- a/common.py +++ b/common.py @@ -39,7 +39,7 @@ submodules_dir = os.path.join(root_dir, 'submodules') buildroot_src_dir = os.path.join(submodules_dir, 'buildroot') crosstool_ng_src_dir = os.path.join(submodules_dir, 'crosstool-ng') linux_src_dir = os.path.join(submodules_dir, 'linux') -linux_config_dir = os.path.join(this_module.root_dir, 'kernel_config') +linux_config_dir = os.path.join(this_module.root_dir, 'linux_config') rootfs_overlay_dir = os.path.join(this_module.root_dir, 'rootfs_overlay') extract_vmlinux = os.path.join(linux_src_dir, 'scripts', 'extract-vmlinux') qemu_src_dir = os.path.join(submodules_dir, 'qemu') @@ -89,7 +89,17 @@ class Component: default_args=self.get_default_args(), ) self.add_parser_arguments(parser) - this_module.add_build_arguments(parser) + parser.add_argument( + '--clean', + help='Clean the build instead of building.', + action='store_true', + ) + parser.add_argument( + '-j', '--nproc', + help='Number of processors to use for the build. Default: use all cores.', + type=int, + default=multiprocessing.cpu_count(), + ) args = this_module.setup(parser) if not this_module.dry_run: start_time = time.time() @@ -131,22 +141,14 @@ class Component: ''' return {} -def add_build_arguments(parser): - parser.add_argument( - '--clean', - help='Clean the build instead of building.', - action='store_true', - ) - parser.add_argument( - '-j', '--nproc', - help='Number of processors to use for the build. Default: use all cores.', - type=int, - default=multiprocessing.cpu_count(), - ) - def add_dry_run_argument(parser): parser.add_argument('--dry-run', default=False, action='store_true', help='''\ - Print the commands that would be run, but don't run them. +Print the commands that would be run, but don't run them. + +We aim display every command that modifies the filesystem state, and generate +Bash equivalents even for actions taken directly in Python without shelling out. + +mkdir are generally omitted since those are obvious. ''') def base64_encode(string): @@ -502,6 +504,7 @@ def resolve_args(defaults, args, extra_args): return argcopy def cp(src, dest): + global this_module print_cmd(['cp', src, dest]) if not this_module.dry_run: shutil.copy2(src, dest) @@ -851,12 +854,17 @@ def write_configs(config_path, configs, config_fragments=None): TODO Can't get rid of these for now with nice fragments: http://stackoverflow.com/questions/44078245/is-it-possible-to-use-config-fragments-with-buildroots-config """ + global this_module if config_fragments is None: config_fragments = [] with open(config_path, 'a') as config_file: for config_fragment in config_fragments: - with open(config_fragment, 'r') as config_fragment: - for line in config_fragment: - config_file.write(line) + with open(config_fragment, 'r') as config_fragment_file: + print_cmd(['cat', config_fragment, '>>', config_path]) + if not this_module.dry_run: + for line in config_fragment_file: + config_file.write(line) for config in configs: - config_file.write(config + '\n') + print_cmd(['echo', config, '>>', config_path]) + if not this_module.dry_run: + config_file.write(config + '\n') diff --git a/kernel_config/buildroot-aarch64 b/linux_config/buildroot-aarch64 similarity index 100% rename from kernel_config/buildroot-aarch64 rename to linux_config/buildroot-aarch64 diff --git a/kernel_config/buildroot-arm b/linux_config/buildroot-arm similarity index 100% rename from kernel_config/buildroot-arm rename to linux_config/buildroot-arm diff --git a/kernel_config/buildroot-x86_64 b/linux_config/buildroot-x86_64 similarity index 100% rename from kernel_config/buildroot-x86_64 rename to linux_config/buildroot-x86_64 diff --git a/kernel_config/default b/linux_config/default similarity index 100% rename from kernel_config/default rename to linux_config/default diff --git a/kernel_config/display b/linux_config/display similarity index 100% rename from kernel_config/display rename to linux_config/display diff --git a/kernel_config/gpio b/linux_config/gpio similarity index 100% rename from kernel_config/gpio rename to linux_config/gpio diff --git a/kernel_config/leds b/linux_config/leds similarity index 100% rename from kernel_config/leds rename to linux_config/leds diff --git a/kernel_config/min b/linux_config/min similarity index 100% rename from kernel_config/min rename to linux_config/min