From 09659162fbb1133029472b35a600bcb55c7897d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ciro=20Santilli=20=E5=85=AD=E5=9B=9B=E4=BA=8B=E4=BB=B6=20?= =?UTF-8?q?=E6=B3=95=E8=BD=AE=E5=8A=9F?= Date: Tue, 22 Jan 2019 00:00:00 +0000 Subject: [PATCH] common: multi arch everywhere --- build | 52 +++++------------------------------- cli_function.py | 19 +++++++++++++- common.py | 70 +++++++++++++++++++++++++++++++++---------------- release-zip | 14 +++++----- run | 4 +-- test-userland | 41 ++++++++++++++--------------- 6 files changed, 100 insertions(+), 100 deletions(-) diff --git a/build b/build index 61d22de..e2786c8 100755 --- a/build +++ b/build @@ -13,8 +13,8 @@ from shell_helpers import LF class _Component: ''' - Yes, we are re-inventing a crappy dependency resolution system. - I can't believe it. + Yes, we are re-inventing a crappy dependency resolution system, + reminescent of scons or apt or Buildroot. I can't believe it. The hard part is that we have optional dependencies as well... e.g. buildroot optionally depends on m5 to put m5 in the root filesystem, @@ -55,7 +55,7 @@ class Main(cli_function.CliFunction): super().__init__( config_file=common.consts['config_file'], description='''\ -Shallow helper to build everything, or a subset of everything conveniently. +Build a component and all its dependencies. Our build-* scripts don't build any dependencies to make iterative development fast and more predictable. @@ -66,9 +66,8 @@ individual build-* commands which: * build no dependencies, and so are fast and predictable * can take multiple options to custumize the build -Without any args, build only what is necessary for +Without any args, build only what is necessary for: https://github.com/cirosantilli/linux-kernel-module-cheat#qemu-buildroot-setup -for x86_64: .... ./%(prog)s @@ -79,14 +78,6 @@ This is equivalent to: .... ./%(prog)s --arch x86_64 qemu-buildroot .... - -If `--arch` is given, build just for the given archs: - -.... -./%(prog)s --arch arm --arch aarch64 -.... - -This will build `qemu-buildroot` for arm and aarch64 only, but not `x86_64`. ''' ) buildroot_component = _Component( @@ -288,23 +279,6 @@ This will build `qemu-buildroot` for arm and aarch64 only, but not `x86_64`. Build absolutely everything for all archs. ''' ) - self.add_argument( - '-A', - '--all-archs', - default=False, - help='''\ -Build the selected components for all archs. -''') - self.add_argument( - '-a', - '--arch', - choices=common.consts['arch_choices'], - default=[], - action='append', - help='''\ -Build the selected components for this arch. Select multiple archs by -passing this option multiple times. Default: [{}] -'''.format(common.consts['default_arch'])) self.add_argument( '-D', '--download-dependencies', @@ -344,19 +318,6 @@ Which components to build. Default: qemu-buildroot def main(self, **kwargs): self.sh = shell_helpers.ShellHelpers(dry_run=kwargs['dry_run']) - # Decide archs. - if kwargs['arch'] == []: - if kwargs['all'] or kwargs['all_archs']: - archs = kwargs['all_archs'].copy() - else: - archs = set([common.consts['default_arch']]) - else: - archs = set() - for arch in kwargs['arch']: - if arch in common.consts['arch_short_to_long_dict']: - arch = common.consts['arch_short_to_long_dict'][arch] - archs.add(arch) - # Decide components. components = kwargs['components'] if kwargs['all']: @@ -503,9 +464,8 @@ Which components to build. Default: qemu-buildroot ) # Do the build. - for arch in archs: - for component in selected_components: - component.build(arch) + for component in selected_components: + component.build() if __name__ == '__main__': Main().cli() diff --git a/cli_function.py b/cli_function.py index fe126c8..12871df 100755 --- a/cli_function.py +++ b/cli_function.py @@ -13,6 +13,7 @@ class _Argument: long_or_short_1, long_or_short_2=None, default=None, + dest=None, help=None, nargs=None, **kwargs @@ -24,7 +25,8 @@ class _Argument: self.kwargs = {'default': None} shortname, longname, key, is_option = self.get_key( long_or_short_1, - long_or_short_2 + long_or_short_2, + dest ) if shortname is not None: self.args.append(shortname) @@ -35,6 +37,8 @@ class _Argument: self.kwargs['metavar'] = longname if default is not None and nargs is None: self.kwargs['nargs'] = '?' + if dest is not None: + self.kwargs['dest'] = dest if nargs is not None: self.kwargs['nargs'] = nargs if default is True: @@ -80,6 +84,7 @@ class _Argument: def get_key( long_or_short_1, long_or_short_2=None, + dest=None, **kwargs ): if long_or_short_2 is None: @@ -94,6 +99,8 @@ class _Argument: else: key = longname.replace('-', '_') is_option = False + if dest is not None: + key = dest return shortname, longname, key, is_option class CliFunction: @@ -293,6 +300,7 @@ amazing function! self.add_argument('-a', '--asdf', default='A', help='Help for asdf'), self.add_argument('-q', '--qwer', default='Q', help='Help for qwer'), self.add_argument('-b', '--bool', default=True, help='Help for bool'), + self.add_argument('--dest', dest='custom_dest', help='Help for dest'), self.add_argument('--bool-cli', default=False, help='Help for bool'), self.add_argument('--bool-nargs', default=False, nargs='?', action='store', const='') self.add_argument('--no-default', help='Help for no-bool'), @@ -312,6 +320,7 @@ amazing function! 'bool': True, 'bool_nargs': False, 'bool_cli': True, + 'custom_dest': None, 'no_default': None, 'pos_mandatory': 1, 'pos_optional': 0, @@ -355,6 +364,14 @@ amazing function! out['bool_nargs'] = default['bool_nargs'] assert out == default + # --dest + out = one_cli_function(pos_mandatory=1, custom_dest='a') + cli_out = one_cli_function.cli(['--dest', 'a', '1']) + assert out == cli_out + assert out['custom_dest'] == 'a' + out['custom_dest'] = default['custom_dest'] + assert out == default + # Positional out = one_cli_function(pos_mandatory=1, pos_optional=2, args_star=['3', '4']) assert out['pos_mandatory'] == 1 diff --git a/common.py b/common.py index e1aba97..6994531 100644 --- a/common.py +++ b/common.py @@ -72,11 +72,13 @@ consts['arch_short_to_long_dict'] = collections.OrderedDict([ ('a', 'arm'), ('A', 'aarch64'), ]) -consts['all_archs'] = [consts['arch_short_to_long_dict'][k] for k in consts['arch_short_to_long_dict']] -consts['arch_choices'] = [] +# All long arch names. +consts['all_long_archs'] = [consts['arch_short_to_long_dict'][k] for k in consts['arch_short_to_long_dict']] +# All long and short arch names. +consts['arch_choices'] = set() for key in consts['arch_short_to_long_dict']: - consts['arch_choices'].append(key) - consts['arch_choices'].append(consts['arch_short_to_long_dict'][key]) + consts['arch_choices'].add(key) + consts['arch_choices'].add(consts['arch_short_to_long_dict'][key]) consts['default_arch'] = 'x86_64' consts['gem5_cpt_prefix'] = '^cpt\.' def git_sha(repo_path): @@ -115,9 +117,25 @@ class LkmcCliFunction(cli_function.CliFunction): super().__init__(*args, **kwargs) # Args for all scripts. + arches = consts['arch_short_to_long_dict'] + arches_string = [] + for arch_short in arches: + arch_long = arches[arch_short] + arches_string.append('{} ({})'.format(arch_long, arch_short)) + arches_string = ', '.join(arches_string) self.add_argument( - '-a', '--arch', choices=consts['arch_choices'], default=consts['default_arch'], - help='CPU architecture.' + '-A', '--all-archs', default=False, + help='''\ +Run action for all supported --archs archs. Ignore --archs. +'''.format(arches_string) + ) + self.add_argument( + '-a', '--arch', action='append', default=[consts['default_arch']], dest='archs', + help='''\ +CPU architecture to use. If given multiple times, run the action +for each arch sequentially in that order. If one of them fails, stop running. +Valid archs: {} +'''.format(arches_string) ) self.add_argument( '--dry-run', @@ -308,6 +326,8 @@ Use gem5 instead of QEMU. Shortcut for `--emulator gem5`. ''' def join(*paths): return os.path.join(*paths) + if env['arch'] not in consts['arch_choices']: + raise Exception('Unknown arch: ' + env['arch']) if env['emulator'] is None: if env['gem5']: env['emulator'] = 'gem5' @@ -369,7 +389,7 @@ Use gem5 instead of QEMU. Shortcut for `--emulator gem5`. env['buildroot_config_file'] = join(env['buildroot_build_dir'], '.config') env['buildroot_build_build_dir'] = join(env['buildroot_build_dir'], 'build') env['buildroot_linux_build_dir'] = join(env['buildroot_build_build_dir'], 'linux-custom') - env['buildroot_vmlinux'] = join(env['buildroot_linux_build_dir'], "vmlinux") + env['buildroot_vmlinux'] = join(env['buildroot_linux_build_dir'], 'vmlinux') env['host_dir'] = join(env['buildroot_build_dir'], 'host') env['host_bin_dir'] = join(env['host_dir'], 'usr', 'bin') env['buildroot_pkg_config'] = join(env['host_bin_dir'], 'pkg-config') @@ -477,7 +497,7 @@ Use gem5 instead of QEMU. Shortcut for `--emulator gem5`. # Linux kernl. if env['linux_build_dir'] is None: env['linux_build_dir'] = join(env['out_dir'], 'linux', env['linux_build_id'], env['arch']) - env['lkmc_vmlinux'] = join(env['linux_build_dir'], "vmlinux") + env['lkmc_vmlinux'] = join(env['linux_build_dir'], 'vmlinux') if env['arch'] == 'arm': env['linux_arch'] = 'arm' env['linux_image_prefix'] = join('arch', env['linux_arch'], 'boot', 'zImage') @@ -561,9 +581,6 @@ Use gem5 instead of QEMU. Shortcut for `--emulator gem5`. break env['image'] = path - self.env = env - - def add_argument(self, *args, **kwargs): shortname, longname, key, is_option = self.get_key(*args, **kwargs) if key in self._defaults: @@ -696,17 +713,24 @@ Use gem5 instead of QEMU. Shortcut for `--emulator gem5`. ''' Time the main of the derived class. ''' - myargs = kwargs.copy() - if not myargs['dry_run']: - start_time = time.time() - myargs.update(consts) - self._init_env(myargs) - self.sh = shell_helpers.ShellHelpers(dry_run=self.env['dry_run']) - ret = self.timed_main() - if not myargs['dry_run']: - end_time = time.time() - self._print_time(end_time - start_time) - return ret + env = kwargs.copy() + env.update(consts) + if env['all_archs']: + env['archs'] = consts['all_long_archs'] + for arch in env['archs']: + if not env['dry_run']: + start_time = time.time() + env['arch'] = arch + self.env = env.copy() + self._init_env(self.env) + self.sh = shell_helpers.ShellHelpers(dry_run=self.env['dry_run']) + ret = self.timed_main() + if not env['dry_run']: + end_time = time.time() + self._print_time(end_time - start_time) + if ret != 0: + return ret + return 0 def make_build_dirs(self): os.makedirs(self.env['buildroot_build_build_dir'], exist_ok=True) @@ -735,7 +759,7 @@ Use gem5 instead of QEMU. Shortcut for `--emulator gem5`. if self.env['print_time']: hours, rem = divmod(ellapsed_seconds, 3600) minutes, seconds = divmod(rem, 60) - print("time {:02}:{:02}:{:02}".format(int(hours), int(minutes), int(seconds))) + print('time {:02}:{:02}:{:02}'.format(int(hours), int(minutes), int(seconds))) def raw_to_qcow2(self, prebuilt=False, reverse=False): if prebuilt or not os.path.exists(self.env['qemu_img_executable']): diff --git a/release-zip b/release-zip index 55de3c7..a89f8ab 100755 --- a/release-zip +++ b/release-zip @@ -12,14 +12,14 @@ import common from shell_helpers import LF def main(): - os.makedirs(kwargs['release_dir'], exist_ok=True) - if os.path.exists(kwargs['release_zip_file']): - os.unlink(kwargs['release_zip_file']) - zipf = zipfile.ZipFile(kwargs['release_zip_file'], 'w', zipfile.ZIP_DEFLATED) - for arch in kwargs['all_archs']: + os.makedirs(self.env['release_dir'], exist_ok=True) + if os.path.exists(self.env['release_zip_file']): + os.unlink(self.env['release_zip_file']) + zipf = zipfile.ZipFile(self.env['release_zip_file'], 'w', zipfile.ZIP_DEFLATED) + for arch in self.env['all_long_archs']: self.setup(common.get_argparse(default_args={'arch': arch})) - zipf.write(kwargs['qcow2_file'], arcname=os.path.relpath(kwargs['qcow2_file'], kwargs['root_dir'])) - zipf.write(kwargs['linux_image'], arcname=os.path.relpath(kwargs['linux_image'], kwargs['root_dir'])) + zipf.write(self.env['qcow2_file'], arcname=os.path.relpath(self.env['qcow2_file'], self.env['root_dir'])) + zipf.write(self.env['linux_image'], arcname=os.path.relpath(self.env['linux_image'], self.env['root_dir'])) zipf.close() if __name__ == '__main__': diff --git a/run b/run index 1e1767c..7ad94f6 100755 --- a/run +++ b/run @@ -294,11 +294,11 @@ Run QEMU with VNC instead of the default SDL. Connect to it with: def raise_rootfs_not_found(): if not self.env['dry_run']: - raise Exception('Root filesystem not found. Did you build it?\n' \ + raise Exception('Root filesystem not found. Did you build it? ' \ 'Tried to use: ' + self.env['disk_image']) def raise_image_not_found(): if not self.env['dry_run']: - raise Exception('Executable image not found. Did you build it?\n' \ + raise Exception('Executable image not found. Did you build it? ' \ 'Tried to use: ' + self.env['image']) if self.env['image'] is None: raise Exception('Baremetal ELF file not found. Tried:\n' + '\n'.join(paths)) diff --git a/test-userland b/test-userland index 35057d5..5e81337 100755 --- a/test-userland +++ b/test-userland @@ -20,27 +20,26 @@ class Main(common.LkmcCliFunction): } else: extra_args = {} - for arch in self.env['all_archs']: - if arch == 'x86_64': - arch_sources = [ - 'asm_hello' - ] - elif arch == 'aarch64': - arch_sources = [ - 'asm_hello' - ] - else: - arch_sources = [] - arch_sources[:] = [os.path.join('arch', arch, arch_source) for arch_source in arch_sources] - for source in sources + arch_sources: - exit_status = run( - arch=arch, - userland=source, - emulator=emulator, - **extra_args, - ) - if exit_status != 0: - raise Exception('Test failed: {} {} {} {}'.format(emulator, arch, source, exit_status)) + if self.env['arch'] == 'x86_64': + arch_sources = [ + 'asm_hello' + ] + elif self.env['arch'] == 'aarch64': + arch_sources = [ + 'asm_hello' + ] + else: + arch_sources = [] + arch_sources[:] = [os.path.join('arch', self.env['arch'], arch_source) for arch_source in arch_sources] + for source in sources + arch_sources: + exit_status = run( + archs=self.env['archs'], + userland=source, + emulator=emulator, + **extra_args, + ) + if exit_status != 0: + raise Exception('Test failed: {} {} {} {}'.format(emulator, arch, source, exit_status)) if __name__ == '__main__': Main().cli()