common: multi arch everywhere

build

@@ -13,8 +13,8 @@ from shell_helpers import LF
 
 class _Component:
     '''
-    Yes, we are re-inventing a crappy dependency resolution system.
-    I can't believe it.
+    Yes, we are re-inventing a crappy dependency resolution system,
+    reminescent of scons or apt or Buildroot. I can't believe it.
 
     The hard part is that we have optional dependencies as well...
     e.g. buildroot optionally depends on m5 to put m5 in the root filesystem,
@@ -55,7 +55,7 @@ class Main(cli_function.CliFunction):
         super().__init__(
             config_file=common.consts['config_file'],
             description='''\
-Shallow helper to build everything, or a subset of everything conveniently.
+Build a component and all its dependencies.
 
 Our build-* scripts don't build any dependencies to make iterative
 development fast and more predictable.
@@ -66,9 +66,8 @@ individual build-* commands which:
 * build no dependencies, and so are fast and predictable
 * can take multiple options to custumize the build
 
-Without any args, build only what is necessary for
+Without any args, build only what is necessary for:
 https://github.com/cirosantilli/linux-kernel-module-cheat#qemu-buildroot-setup
-for x86_64:
 
 ....
 ./%(prog)s
@@ -79,14 +78,6 @@ This is equivalent to:
 ....
 ./%(prog)s --arch x86_64 qemu-buildroot
 ....
-
-If `--arch` is given, build just for the given archs:
-
-....
-./%(prog)s --arch arm --arch aarch64
-....
-
-This will build `qemu-buildroot` for arm and aarch64 only, but not `x86_64`.
 '''
         )
         buildroot_component = _Component(
@@ -288,23 +279,6 @@ This will build `qemu-buildroot` for arm and aarch64 only, but not `x86_64`.
 Build absolutely everything for all archs.
 '''
         )
-        self.add_argument(
-            '-A',
-            '--all-archs',
-            default=False,
-            help='''\
-Build the selected components for all archs.
-''')
-        self.add_argument(
-            '-a',
-            '--arch',
-            choices=common.consts['arch_choices'],
-            default=[],
-            action='append',
-            help='''\
-Build the selected components for this arch. Select multiple archs by
-passing this option multiple times. Default: [{}]
-'''.format(common.consts['default_arch']))
         self.add_argument(
             '-D',
             '--download-dependencies',
@@ -344,19 +318,6 @@ Which components to build. Default: qemu-buildroot
     def main(self, **kwargs):
         self.sh = shell_helpers.ShellHelpers(dry_run=kwargs['dry_run'])
 
-        # Decide archs.
-        if kwargs['arch'] == []:
-            if kwargs['all'] or kwargs['all_archs']:
-                archs = kwargs['all_archs'].copy()
-            else:
-                archs = set([common.consts['default_arch']])
-        else:
-            archs = set()
-            for arch in kwargs['arch']:
-                if arch in common.consts['arch_short_to_long_dict']:
-                    arch = common.consts['arch_short_to_long_dict'][arch]
-                archs.add(arch)
-
         # Decide components.
         components = kwargs['components']
         if kwargs['all']:
@@ -503,9 +464,8 @@ Which components to build. Default: qemu-buildroot
                 )
 
         # Do the build.
-        for arch in archs:
-            for component in selected_components:
-                component.build(arch)
+        for component in selected_components:
+            component.build()
 
 if __name__ == '__main__':
     Main().cli()

cli_function.py

@@ -13,6 +13,7 @@ class _Argument:
             long_or_short_1,
             long_or_short_2=None,
             default=None,
+            dest=None,
             help=None,
             nargs=None,
             **kwargs
@@ -24,7 +25,8 @@ class _Argument:
             self.kwargs = {'default': None}
             shortname, longname, key, is_option = self.get_key(
                 long_or_short_1,
-                long_or_short_2
+                long_or_short_2,
+                dest
             )
             if shortname is not None:
                 self.args.append(shortname)
@@ -35,6 +37,8 @@ class _Argument:
                 self.kwargs['metavar'] = longname
                 if default is not None and nargs is None:
                     self.kwargs['nargs'] = '?'
+            if dest is not None:
+                self.kwargs['dest'] = dest
             if nargs is not None:
                 self.kwargs['nargs'] = nargs
             if default is True:
@@ -80,6 +84,7 @@ class _Argument:
     def get_key(
         long_or_short_1,
         long_or_short_2=None,
+        dest=None,
         **kwargs
     ):
         if long_or_short_2 is None:
@@ -94,6 +99,8 @@ class _Argument:
         else:
             key = longname.replace('-', '_')
             is_option = False
+        if dest is not None:
+            key = dest
         return shortname, longname, key, is_option
 
 class CliFunction:
@@ -293,6 +300,7 @@ amazing function!
             self.add_argument('-a', '--asdf', default='A', help='Help for asdf'),
             self.add_argument('-q', '--qwer', default='Q', help='Help for qwer'),
             self.add_argument('-b', '--bool', default=True, help='Help for bool'),
+            self.add_argument('--dest', dest='custom_dest', help='Help for dest'),
             self.add_argument('--bool-cli', default=False, help='Help for bool'),
             self.add_argument('--bool-nargs', default=False, nargs='?', action='store', const='')
             self.add_argument('--no-default', help='Help for no-bool'),
@@ -312,6 +320,7 @@ amazing function!
         'bool': True,
         'bool_nargs': False,
         'bool_cli': True,
+        'custom_dest': None,
         'no_default': None,
         'pos_mandatory': 1,
         'pos_optional': 0,
@@ -355,6 +364,14 @@ amazing function!
         out['bool_nargs'] = default['bool_nargs']
         assert out == default
 
+    # --dest
+    out = one_cli_function(pos_mandatory=1, custom_dest='a')
+    cli_out = one_cli_function.cli(['--dest', 'a', '1'])
+    assert out == cli_out
+    assert out['custom_dest'] == 'a'
+    out['custom_dest'] = default['custom_dest']
+    assert out == default
+
     # Positional
     out = one_cli_function(pos_mandatory=1, pos_optional=2, args_star=['3', '4'])
     assert out['pos_mandatory'] == 1

common.py

@@ -72,11 +72,13 @@ consts['arch_short_to_long_dict'] = collections.OrderedDict([
     ('a', 'arm'),
     ('A', 'aarch64'),
 ])
-consts['all_archs'] = [consts['arch_short_to_long_dict'][k] for k in consts['arch_short_to_long_dict']]
-consts['arch_choices'] = []
+# All long arch names.
+consts['all_long_archs'] = [consts['arch_short_to_long_dict'][k] for k in consts['arch_short_to_long_dict']]
+# All long and short arch names.
+consts['arch_choices'] = set()
 for key in consts['arch_short_to_long_dict']:
-    consts['arch_choices'].append(key)
-    consts['arch_choices'].append(consts['arch_short_to_long_dict'][key])
+    consts['arch_choices'].add(key)
+    consts['arch_choices'].add(consts['arch_short_to_long_dict'][key])
 consts['default_arch'] = 'x86_64'
 consts['gem5_cpt_prefix'] = '^cpt\.'
 def git_sha(repo_path):
@@ -115,9 +117,25 @@ class LkmcCliFunction(cli_function.CliFunction):
         super().__init__(*args, **kwargs)
 
         # Args for all scripts.
+        arches = consts['arch_short_to_long_dict']
+        arches_string = []
+        for arch_short in arches:
+            arch_long = arches[arch_short]
+            arches_string.append('{} ({})'.format(arch_long, arch_short))
+        arches_string = ', '.join(arches_string)
         self.add_argument(
-            '-a', '--arch', choices=consts['arch_choices'], default=consts['default_arch'],
-            help='CPU architecture.'
+            '-A', '--all-archs', default=False,
+            help='''\
+Run action for all supported --archs archs. Ignore --archs.
+'''.format(arches_string)
+        )
+        self.add_argument(
+            '-a', '--arch', action='append', default=[consts['default_arch']], dest='archs',
+            help='''\
+CPU architecture to use. If given multiple times, run the action
+for each arch sequentially in that order. If one of them fails, stop running.
+Valid archs: {}
+'''.format(arches_string)
         )
         self.add_argument(
             '--dry-run',
@@ -308,6 +326,8 @@ Use gem5 instead of QEMU. Shortcut for `--emulator gem5`.
         '''
         def join(*paths):
             return os.path.join(*paths)
+        if env['arch'] not in consts['arch_choices']:
+            raise Exception('Unknown arch: ' + env['arch'])
         if env['emulator'] is None:
             if env['gem5']:
                 env['emulator'] = 'gem5'
@@ -369,7 +389,7 @@ Use gem5 instead of QEMU. Shortcut for `--emulator gem5`.
         env['buildroot_config_file'] = join(env['buildroot_build_dir'], '.config')
         env['buildroot_build_build_dir'] = join(env['buildroot_build_dir'], 'build')
         env['buildroot_linux_build_dir'] = join(env['buildroot_build_build_dir'], 'linux-custom')
-        env['buildroot_vmlinux'] = join(env['buildroot_linux_build_dir'], "vmlinux")
+        env['buildroot_vmlinux'] = join(env['buildroot_linux_build_dir'], 'vmlinux')
         env['host_dir'] = join(env['buildroot_build_dir'], 'host')
         env['host_bin_dir'] = join(env['host_dir'], 'usr', 'bin')
         env['buildroot_pkg_config'] = join(env['host_bin_dir'], 'pkg-config')
@@ -477,7 +497,7 @@ Use gem5 instead of QEMU. Shortcut for `--emulator gem5`.
         # Linux kernl.
         if env['linux_build_dir'] is None:
             env['linux_build_dir'] = join(env['out_dir'], 'linux', env['linux_build_id'], env['arch'])
-        env['lkmc_vmlinux'] = join(env['linux_build_dir'], "vmlinux")
+        env['lkmc_vmlinux'] = join(env['linux_build_dir'], 'vmlinux')
         if env['arch'] == 'arm':
             env['linux_arch'] = 'arm'
             env['linux_image_prefix'] = join('arch', env['linux_arch'], 'boot', 'zImage')
@@ -561,9 +581,6 @@ Use gem5 instead of QEMU. Shortcut for `--emulator gem5`.
                         break
             env['image'] = path
 
-        self.env = env
-
-
     def add_argument(self, *args, **kwargs):
         shortname, longname, key, is_option = self.get_key(*args, **kwargs)
         if key in self._defaults:
@@ -696,17 +713,24 @@ Use gem5 instead of QEMU. Shortcut for `--emulator gem5`.
         '''
         Time the main of the derived class.
         '''
-        myargs = kwargs.copy()
-        if not myargs['dry_run']:
-            start_time = time.time()
-        myargs.update(consts)
-        self._init_env(myargs)
-        self.sh = shell_helpers.ShellHelpers(dry_run=self.env['dry_run'])
-        ret = self.timed_main()
-        if not myargs['dry_run']:
-            end_time = time.time()
-            self._print_time(end_time - start_time)
-        return ret
+        env = kwargs.copy()
+        env.update(consts)
+        if env['all_archs']:
+            env['archs'] = consts['all_long_archs']
+        for arch in env['archs']:
+            if not env['dry_run']:
+                start_time = time.time()
+            env['arch'] = arch
+            self.env = env.copy()
+            self._init_env(self.env)
+            self.sh = shell_helpers.ShellHelpers(dry_run=self.env['dry_run'])
+            ret = self.timed_main()
+            if not env['dry_run']:
+                end_time = time.time()
+                self._print_time(end_time - start_time)
+            if ret != 0:
+                return ret
+        return 0
 
     def make_build_dirs(self):
         os.makedirs(self.env['buildroot_build_build_dir'], exist_ok=True)
@@ -735,7 +759,7 @@ Use gem5 instead of QEMU. Shortcut for `--emulator gem5`.
         if self.env['print_time']:
             hours, rem = divmod(ellapsed_seconds, 3600)
             minutes, seconds = divmod(rem, 60)
-            print("time {:02}:{:02}:{:02}".format(int(hours), int(minutes), int(seconds)))
+            print('time {:02}:{:02}:{:02}'.format(int(hours), int(minutes), int(seconds)))
 
     def raw_to_qcow2(self, prebuilt=False, reverse=False):
         if prebuilt or not os.path.exists(self.env['qemu_img_executable']):

release-zip

@@ -12,14 +12,14 @@ import common
 from shell_helpers import LF
 
 def main():
-    os.makedirs(kwargs['release_dir'], exist_ok=True)
-    if os.path.exists(kwargs['release_zip_file']):
-        os.unlink(kwargs['release_zip_file'])
-    zipf = zipfile.ZipFile(kwargs['release_zip_file'], 'w', zipfile.ZIP_DEFLATED)
-    for arch in kwargs['all_archs']:
+    os.makedirs(self.env['release_dir'], exist_ok=True)
+    if os.path.exists(self.env['release_zip_file']):
+        os.unlink(self.env['release_zip_file'])
+    zipf = zipfile.ZipFile(self.env['release_zip_file'], 'w', zipfile.ZIP_DEFLATED)
+    for arch in self.env['all_long_archs']:
         self.setup(common.get_argparse(default_args={'arch': arch}))
-        zipf.write(kwargs['qcow2_file'], arcname=os.path.relpath(kwargs['qcow2_file'], kwargs['root_dir']))
-        zipf.write(kwargs['linux_image'], arcname=os.path.relpath(kwargs['linux_image'], kwargs['root_dir']))
+        zipf.write(self.env['qcow2_file'], arcname=os.path.relpath(self.env['qcow2_file'], self.env['root_dir']))
+        zipf.write(self.env['linux_image'], arcname=os.path.relpath(self.env['linux_image'], self.env['root_dir']))
     zipf.close()
 
 if __name__ == '__main__':

run

@@ -294,11 +294,11 @@ Run QEMU with VNC instead of the default SDL. Connect to it with:
 
         def raise_rootfs_not_found():
             if not self.env['dry_run']:
-                raise Exception('Root filesystem not found. Did you build it?\n' \
+                raise Exception('Root filesystem not found. Did you build it? ' \
                                 'Tried to use: ' + self.env['disk_image'])
         def raise_image_not_found():
             if not self.env['dry_run']:
-                raise Exception('Executable image not found. Did you build it?\n' \
+                raise Exception('Executable image not found. Did you build it? ' \
                                 'Tried to use: ' + self.env['image'])
         if self.env['image'] is None:
             raise Exception('Baremetal ELF file not found. Tried:\n' + '\n'.join(paths))

test-userland

@@ -20,27 +20,26 @@ class Main(common.LkmcCliFunction):
                 }
             else:
                 extra_args = {}
-            for arch in self.env['all_archs']:
-                if arch == 'x86_64':
-                    arch_sources = [
-                        'asm_hello'
-                    ]
-                elif arch == 'aarch64':
-                    arch_sources = [
-                        'asm_hello'
-                    ]
-                else:
-                    arch_sources = []
-                arch_sources[:] = [os.path.join('arch', arch, arch_source) for arch_source in arch_sources]
-                for source in sources + arch_sources:
-                    exit_status = run(
-                        arch=arch,
-                        userland=source,
-                        emulator=emulator,
-                        **extra_args,
-                    )
-                    if exit_status != 0:
-                        raise Exception('Test failed: {} {} {} {}'.format(emulator, arch, source, exit_status))
+            if self.env['arch'] == 'x86_64':
+                arch_sources = [
+                    'asm_hello'
+                ]
+            elif self.env['arch'] == 'aarch64':
+                arch_sources = [
+                    'asm_hello'
+                ]
+            else:
+                arch_sources = []
+            arch_sources[:] = [os.path.join('arch', self.env['arch'], arch_source) for arch_source in arch_sources]
+            for source in sources + arch_sources:
+                exit_status = run(
+                    archs=self.env['archs'],
+                    userland=source,
+                    emulator=emulator,
+                    **extra_args,
+                )
+                if exit_status != 0:
+                    raise Exception('Test failed: {} {} {} {}'.format(emulator, arch, source, exit_status))
 
 if __name__ == '__main__':
     Main().cli()