Move build-buildroot linux configs to build-linux.

Needs a lot of testing, and need to cleanup readme, but seems to work.

build-buildroot

@@ -12,26 +12,13 @@ import common
 
 class BuildrootComponent(common.Component):
     def add_parser_arguments(self, parser):
-        parser.add_argument(
-            '-B', '--buildroot-config', default=self._defaults['buildroot_config'], action='append',
-            help='''Add a single Buildroot config to the current build.
-Example value: 'BR2_TARGET_ROOTFS_EXT2_SIZE="512M"'.
-Can be used multiple times to add multiple configs.
-Takes precedence over any Buildroot config files.
-'''
-        )
-        parser.add_argument(
-            '-b', '--buildroot-config-fragment', default=self._defaults['buildroot_config_fragment'], action='append',
-            help='''Also use the given Buildroot configuration fragment file.
-Pass multiple times to use multiple fragment files.
-'''
-        )
         parser.add_argument(
             '--build-linux', default=self._defaults['build_linux'], action='store_true',
             help='''\
 Enable building the Linux kernel with Buildroot. This is done mostly
 to extract Buildroot's default kernel configurations when updating Buildroot.
-That kernel will not be use by our scripts.
+This kernel will not be use by our other scripts. Configuring this kernel is
+not currently supported, juse use ./build-linux script if you want to do that.
 '''
         )
         parser.add_argument(
@@ -41,38 +28,17 @@ Mostly to track how much slower we are than a basic build.
 '''
         )
         parser.add_argument(
-            '-C', '--kernel-config', default=self._defaults['kernel_config'], action='append',
-            help='''\
-Add a single kernel config configs to the current build.
-Example value: 'CONFIG_FORTIFY_SOURCE=y'.
+            '--config', default=self._defaults['config'], action='append',
+            help='''Add a single Buildroot config to the current build.
+Example value: 'BR2_TARGET_ROOTFS_EXT2_SIZE="512M"'.
 Can be used multiple times to add multiple configs.
 Takes precedence over any Buildroot config files.
 '''
         )
         parser.add_argument(
-            '-c', '--kernel-config-fragment', default=self._defaults['kernel_config_fragment'], action='append',
-            help='''\
-Also use the given kernel configuration fragment file.
+            '--config-fragment', default=self._defaults['config_fragment'], action='append',
+            help='''Also use the given Buildroot configuration fragment file.
 Pass multiple times to use multiple fragment files.
-'''
-        )
-        parser.add_argument(
-            '-I', '--initramfs', default=self._defaults['initramfs'], action='store_true',
-        )
-        parser.add_argument(
-            '-i', '--initrd', default=self._defaults['initrd'], action='store_true',
-        )
-        parser.add_argument(
-            '-K', '--kernel-custom-config-file', default=self._defaults['kernel_custom_config_file'],
-            help='''\
-Ignore all default kernel configurations and use this file instead.
-Still uses options explicitly passed with `-C` and `-c` on top of it.
-'''
-        )
-        parser.add_argument(
-            '-k', '--kernel-modules', default=self._defaults['kernel_modules'], action='store_true',
-            help='''Reconfigure and rebuild the kernel modules package.
-Force --build-linux to true, since building the modules requires building Linux.
 '''
         )
         parser.add_argument(
@@ -80,13 +46,6 @@ Force --build-linux to true, since building the modules requires building Linux.
             help='''\
 Don't build the all target which normally gets build by default.
 That target builds the root filesystem and all its dependencies.
-'''
-        )
-        parser.add_argument(
-            '--skip-configure', default=self._defaults['skip_configure'], action='store_true',
-            help='''\
-Skip the Buildroot configuration. Saves a few seconds,
-but requires you to know what you are doing :-)
 '''
         )
         parser.add_argument(
@@ -101,9 +60,6 @@ usually extra Buildroot targets.
         build_dir = self.get_build_dir(args)
         os.makedirs(common.out_dir, exist_ok=True)
         extra_make_args = args.extra_make_args.copy()
-        if args.kernel_modules:
-            args.build_linux = True
-            extra_make_args.append('lkmc-reconfigure')
         if args.build_linux:
             extra_make_args.append('linux-reconfigure')
         if args.gem5:
@@ -114,9 +70,6 @@ usually extra Buildroot targets.
             defconfig = 'qemu_arm_vexpress_defconfig'
         elif args.arch == 'aarch64':
             defconfig = 'qemu_aarch64_virt_defconfig'
-
-        # Configure.
-        if not args.skip_configure:
         br2_external_dirs = []
         packages_dir = os.path.join(common.root_dir, 'packages')
         for package_dir in os.listdir(packages_dir):
@@ -133,18 +86,18 @@ usually extra Buildroot targets.
             ],
             cwd=common.buildroot_src_dir,
         )
-            buildroot_configs = args.buildroot_config
-            buildroot_configs.extend([
+        configs = args.config
+        configs.extend([
             'BR2_JLEVEL={}'.format(args.nproc),
             'BR2_DL_DIR="{}"'.format(common.buildroot_download_dir),
         ])
-            common.write_configs(common.buildroot_config_file, buildroot_configs)
+        common.write_configs(common.buildroot_config_file, configs)
         if not args.build_linux:
-                buildroot_configs.extend([
+            configs.extend([
                 '# BR2_LINUX_KERNEL is not set',
             ])
         if not args.baseline:
-                buildroot_configs.extend([
+            configs.extend([
                 'BR2_GLOBAL_PATCH_DIR="{}"'.format(
                     self._path_relative_to_buildroot(os.path.join(common.root_dir, 'patches', 'global'))
                 ),
@@ -165,73 +118,10 @@ usually extra Buildroot targets.
                     self._path_relative_to_buildroot(os.path.join(common.root_dir, 'user_table'))
                 ),
             ])
-                if args.kernel_modules:
-                    buildroot_configs.append('BR2_PACKAGE_LKMC=y')
-                if args.gem5:
-                    buildroot_configs.append('BR2_PACKAGE_GEM5=y')
-                if args.initramfs:
-                    buildroot_configs.extend([
-                        'BR2_TARGET_ROOTFS_CPIO=n',
-                        'BR2_TARGET_ROOTFS_EXT2=n',
-                        'BR2_TARGET_ROOTFS_INITRAMFS=y',
-                    ])
-                if args.initrd:
-                    buildroot_configs.extend([
-                        'BR2_TARGET_ROOTFS_CPIO=y',
-                        'BR2_TARGET_ROOTFS_EXT2=n'
-                        'BR2_TARGET_ROOTFS_INITRAMFS=n',
-                    ])
-                buildroot_config_fragments = [
+            config_fragments = [
                 os.path.join(common.root_dir, 'buildroot_config', 'default')
-                ] + args.buildroot_config_fragment
-
-                # Decide kernel configuration.
-                kernel_config_fragments = []
-                if True:
-                    # CLI kernel configurations.
-                    kernel_config_fragment_cli_path = os.path.join(common.buildroot_build_dir, 'lkmc_kernel_config_fragment_cli')
-                    kernel_config_cli_str = '\n'.join(args.kernel_config)
-                    do_write = False
-                    if os.path.exists(kernel_config_fragment_cli_path):
-                        with open(kernel_config_fragment_cli_path, 'r') as kernel_config_fragment_cli_file:
-                            kernel_config_cli_str_old = kernel_config_fragment_cli_file.read()
-                            if kernel_config_cli_str != kernel_config_cli_str_old:
-                                do_write = True
-                    else:
-                        do_write = True
-                    if do_write:
-                        #  Only update if modified, otherwise Buildroot tries to
-                        # rebuilds the kernel every time, which takes a few seconds.
-                        # even when the kernel has already been built.
-                        with open(kernel_config_fragment_cli_path, 'w') as kernel_config_fragment_cli_file:
-                            kernel_config_fragment_cli_file.write(kernel_config_cli_str)
-                    kernel_config_fragments.append(os.path.join(kernel_config_fragment_cli_path))
-                if True:
-                    # Kernel configuration fragments.
-                    if args.kernel_custom_config_file is not None:
-                        if os.path.exists(args.kernel_custom_config_file):
-                            buildroot_configs.extend([
-                                'BR2_LINUX_KERNEL_USE_CUSTOM_CONFIG=y',
-                                'BR2_LINUX_KERNEL_CUSTOM_CONFIG_FILE=\"{}\"'.format(args.kernel_custom_config_file),
-                            ])
-                            if args.build_linux:
-                                pathlib.Path(args.kernel_custom_config_file).touch()
-                        else:
-                            raise Exception('Kernel config fragment file does not exist: {}'.format(args.kernel_custom_config_file))
-                        default_kernel_config_fragments = []
-                    else:
-                        default_kernel_config_fragments = ['min', 'default']
-                        if args.build_linux:
-                            # https://stackoverflow.com/questions/49260466/why-when-i-change-br2-linux-kernel-custom-config-file-and-run-make-linux-reconfi
-                            pathlib.Path(os.path.join(common.linux_config_dir, 'min')).touch()
-                    for i, default_kernel_config_fragment in enumerate(default_kernel_config_fragments):
-                        default_kernel_config_fragments[i] = os.path.join(common.linux_config_dir, default_kernel_config_fragment)
-                kernel_config_fragments.extend(default_kernel_config_fragments)
-                for i, frag in enumerate(kernel_config_fragments):
-                    kernel_config_fragments[i] = self._path_relative_to_buildroot(frag)
-                buildroot_kernel_config_fragment_str = 'BR2_LINUX_KERNEL_CONFIG_FRAGMENT_FILES="{}"'.format(' '.join(kernel_config_fragments))
-                buildroot_configs.append(buildroot_kernel_config_fragment_str)
-                common.write_configs(common.buildroot_config_file, buildroot_configs, buildroot_config_fragments)
+            ] + args.config_fragment
+            common.write_configs(common.buildroot_config_file, configs, config_fragments)
             common.run_cmd(
                 [
                     'make',
@@ -240,8 +130,6 @@ usually extra Buildroot targets.
                 ],
                 cwd=common.buildroot_src_dir,
             )
-
-        # Do the actual build.
         common.make_build_dirs()
         if not args.no_all:
             extra_make_args.append('all')
@@ -259,7 +147,6 @@ usually extra Buildroot targets.
             delete_env=['LD_LIBRARY_PATH'],
             cwd=common.buildroot_src_dir,
         )
-
         # Create the qcow2 from ext2.
         # Skip if qemu is not present, because gem5 does not need the qcow2.
         # so we don't force a QEMU build for gem5.
@@ -279,17 +166,11 @@ Run Linux on an emulator
     _defaults = {
         'baseline': False,
         'build_linux': False,
-        'buildroot_config': [],
-        'buildroot_config_fragment': [],
-        'initramfs': False,
-        'initrd': False,
-        'kernel_config': [],
-        'kernel_config_fragment': [],
-        'kernel_custom_config_file': None,
-        'kernel_modules': False,
+        'config': [],
+        'config_fragment': [],
+        'extra_make_args': [],
         'no_all': False,
         'skip_configure': False,
-        'extra_make_args': [],
     }
 
     def _path_relative_to_buildroot(self, abspath):

build-linux

@@ -7,6 +7,42 @@ import common
 
 class LinuxComponent(common.Component):
     def add_parser_arguments(self, parser):
+        parser.add_argument(
+            '--config', default=[], action='append',
+            help='''\
+Add a single kernel config configs to the current build.
+The `CONFIG_` prefix is added automatically. Sample values:
+'FORTIFY_SOURCE=y', `KGDB=n`. Can be used multiple times to add multiple
+configs. Takes precedence over any config files.
+'''
+        )
+        parser.add_argument(
+            '--config-fragment', default=[], action='append',
+            help='''\
+Also use the given kernel configuration fragment file.
+Pass multiple times to use multiple fragment files.
+'''
+        )
+        parser.add_argument(
+            '--custom-config-file',
+            help='''\
+Ignore all default kernel configurations and use this file instead.
+Still uses options explicitly passed with `--config` and
+`--config-fragment` on top of it.
+'''
+        )
+        parser.add_argument(
+            '--config-only', default=False, action='store_true',
+            help='''\
+Configure the kernel, but don't build it.
+'''
+        )
+        parser.add_argument(
+            '--initramfs', default=False, action='store_true',
+        )
+        parser.add_argument(
+            '--initrd', default=False, action='store_true',
+        )
         parser.add_argument(
             'extra_make_args',
             default=[],
@@ -16,11 +52,9 @@ class LinuxComponent(common.Component):
 
     def do_build(self, args):
         build_dir = self.get_build_dir(args)
+        if args.initrd or args.initramfs:
+            raise Exception('just trolling, --initrd and --initramfs are broken for now')
         os.makedirs(build_dir, exist_ok=True)
-        common.cp(
-            os.path.join(common.linux_config_dir, 'buildroot-{}'.format(args.arch)),
-            os.path.join(build_dir, '.config'),
-        )
         tool = 'gcc'
         gcc = common.get_toolchain_tool(tool)
         prefix = gcc[:-len(tool)]
@@ -33,6 +67,8 @@ class LinuxComponent(common.Component):
         else:
             cc = gcc
         common_make_args = [
+            'make',
+            '-j', str(args.nproc),
             'ARCH={}'.format(common.linux_arch),
             'CROSS_COMPILE={}'.format(prefix),
             'CC={}'.format(cc),
@@ -42,40 +78,50 @@ class LinuxComponent(common.Component):
             verbose = ['V=1']
         else:
             verbose = []
+        if args.custom_config_file is not None:
+            if not os.path.exists(args.custom_config_file):
+                raise Exception('config fragment file does not exist: {}'.format(args.custom_config_file))
+            base_config_file = args.custom_config_file
+            config_fragments = []
+        else:
+            base_config_file = os.path.join(common.linux_config_dir, 'buildroot-{}'.format(args.arch))
+            config_fragments = ['min', 'default']
+            for i, config_fragment in enumerate(config_fragments):
+                config_fragments[i] = os.path.join(common.linux_config_dir, config_fragment)
+        config_fragments.extend(args.config_fragment)
+        if args.config != []:
+            cli_config_fragment_path = os.path.join(build_dir, 'lkmc_cli_config_fragment')
+            cli_config_str = '\n'.join(map(lambda x: 'CONFIG_' + x, args.config))
+            with open(cli_config_fragment_path, 'w') as cli_config_fragment_cli:
+                cli_config_fragment_cli.write(cli_config_str)
+            config_fragments.append(cli_config_fragment_path)
+        common.cp(
+            base_config_file,
+            os.path.join(build_dir, '.config'),
+        )
         common.run_cmd(
             [
                 os.path.join(common.linux_src_dir, 'scripts', 'kconfig', 'merge_config.sh'),
                 '-m',
                 '-O', build_dir,
                 os.path.join(build_dir, '.config'),
-                os.path.join(common.linux_config_dir, 'min'),
-                os.path.join(common.linux_config_dir, 'default'),
-            ],
+            ] +
+            config_fragments
         )
         common.run_cmd(
             (
-                [
-                    'make',
-                    '-j', str(args.nproc),
-                ] +
                 common_make_args +
-                [
-                    'olddefconfig',
-                ]
+                ['olddefconfig']
             ),
-            **common_args,
+            **common_args
         )
+        if not args.config_only:
             common.run_cmd(
                 (
-                [
-                    'make',
-                    '-j', str(args.nproc),
-                ] +
                     common_make_args +
-                verbose +
                     args.extra_make_args
                 ),
-            **common_args,
+                **common_args
             )
 
     def get_argparse_args(self):

common.py

@@ -39,7 +39,7 @@ submodules_dir = os.path.join(root_dir, 'submodules')
 buildroot_src_dir = os.path.join(submodules_dir, 'buildroot')
 crosstool_ng_src_dir = os.path.join(submodules_dir, 'crosstool-ng')
 linux_src_dir = os.path.join(submodules_dir, 'linux')
-linux_config_dir = os.path.join(this_module.root_dir, 'kernel_config')
+linux_config_dir = os.path.join(this_module.root_dir, 'linux_config')
 rootfs_overlay_dir = os.path.join(this_module.root_dir, 'rootfs_overlay')
 extract_vmlinux = os.path.join(linux_src_dir, 'scripts', 'extract-vmlinux')
 qemu_src_dir = os.path.join(submodules_dir, 'qemu')
@@ -89,7 +89,17 @@ class Component:
             default_args=self.get_default_args(),
         )
         self.add_parser_arguments(parser)
-        this_module.add_build_arguments(parser)
+        parser.add_argument(
+            '--clean',
+            help='Clean the build instead of building.',
+            action='store_true',
+        )
+        parser.add_argument(
+            '-j', '--nproc',
+            help='Number of processors to use for the build. Default: use all cores.',
+            type=int,
+            default=multiprocessing.cpu_count(),
+        )
         args = this_module.setup(parser)
         if not this_module.dry_run:
             start_time = time.time()
@@ -131,22 +141,14 @@ class Component:
         '''
         return {}
 
-def add_build_arguments(parser):
-    parser.add_argument(
-        '--clean',
-        help='Clean the build instead of building.',
-        action='store_true',
-    )
-    parser.add_argument(
-        '-j', '--nproc',
-        help='Number of processors to use for the build. Default: use all cores.',
-        type=int,
-        default=multiprocessing.cpu_count(),
-    )
-
 def add_dry_run_argument(parser):
     parser.add_argument('--dry-run', default=False, action='store_true', help='''\
 Print the commands that would be run, but don't run them.
+
+We aim display every command that modifies the filesystem state, and generate
+Bash equivalents even for actions taken directly in Python without shelling out.
+
+mkdir are generally omitted since those are obvious.
 ''')
 
 def base64_encode(string):
@@ -502,6 +504,7 @@ def resolve_args(defaults, args, extra_args):
     return argcopy
 
 def cp(src, dest):
+    global this_module
     print_cmd(['cp', src, dest])
     if not this_module.dry_run:
         shutil.copy2(src, dest)
@@ -851,12 +854,17 @@ def write_configs(config_path, configs, config_fragments=None):
     TODO Can't get rid of these for now with nice fragments:
     http://stackoverflow.com/questions/44078245/is-it-possible-to-use-config-fragments-with-buildroots-config
     """
+    global this_module
     if config_fragments is None:
         config_fragments = []
     with open(config_path, 'a') as config_file:
         for config_fragment in config_fragments:
-            with open(config_fragment, 'r') as config_fragment:
-                for line in config_fragment:
+            with open(config_fragment, 'r') as config_fragment_file:
+                print_cmd(['cat', config_fragment, '>>', config_path])
+                if not this_module.dry_run:
+                    for line in config_fragment_file:
                         config_file.write(line)
         for config in configs:
+            print_cmd(['echo', config, '>>', config_path])
+            if not this_module.dry_run:
                 config_file.write(config + '\n')