From 0a33292326739b854455853135ec1fc4cfb0a93b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ciro=20Santilli=20=E5=85=AD=E5=9B=9B=E4=BA=8B=E4=BB=B6=20?= =?UTF-8?q?=E6=B3=95=E8=BD=AE=E5=8A=9F?= Date: Tue, 22 Jan 2019 00:00:00 +0000 Subject: [PATCH] start porting build --- build | 801 ++++++++++++++++++---------------- cli_function.py | 8 +- cli_function_test_config_2.py | 5 + common.py | 2 +- run | 4 +- 5 files changed, 428 insertions(+), 392 deletions(-) create mode 100644 cli_function_test_config_2.py diff --git a/build b/build index 2399d5c..61d22de 100755 --- a/build +++ b/build @@ -6,10 +6,12 @@ import platform import re import os +import cli_function import common +import shell_helpers from shell_helpers import LF -class Component: +class _Component: ''' Yes, we are re-inventing a crappy dependency resolution system. I can't believe it. @@ -40,6 +42,7 @@ class Component: self.submodules_shallow = submodules_shallow or set() self.python2_pkgs = python2_pkgs or set() self.python3_pkgs = python3_pkgs or set() + def build(self, arch): if ( (self.build_callback is not None) and @@ -47,208 +50,11 @@ class Component: ): self.build_callback(arch) -def run_cmd(cmd, arch): - global args - cmd_abs = cmd.copy() - cmd_abs[0] = os.path.join(kwargs['root_dir'], cmd[0]) - cmd_abs.extend(['--arch', arch]) - if kwargs['extra_args']: - cmd_abs.append(kwargs['extra_args']) - self.sh.run_cmd(cmd_abs, dry_run=kwargs['dry_run']) - -buildroot_component = Component( - lambda arch: run_cmd(['build-buildroot'], arch), - submodules = {'buildroot'}, - # https://buildroot.org/downloads/manual/manual.html#requirement - apt_get_pkgs={ - 'bash', - 'bc', - 'binutils', - 'build-essential', - 'bzip2', - 'cpio', - 'g++', - 'gcc', - 'graphviz', - 'gzip', - 'make', - 'patch', - 'perl', - 'python-matplotlib', - 'python3', - 'rsync', - 'sed', - 'tar', - 'unzip', - }, -) - -name_to_component_map = { - # Leaves without dependencies. - 'baremetal-qemu': Component( - lambda arch: run_cmd(['build-baremetal', '--emulator', 'qemu'], arch), - supported_archs=kwargs['crosstool_ng_supported_archs'], - ), - 'baremetal-gem5': Component( - lambda arch: run_cmd(['build-baremetal', '--gem5'], arch), - supported_archs=kwargs['crosstool_ng_supported_archs'], - ), - 'baremetal-gem5-pbx': Component( - lambda arch: run_cmd(['build-baremetal', '--gem5', '--machine', 'RealViewPBX'], arch), - supported_archs=kwargs['crosstool_ng_supported_archs'], - ), - 'buildroot': buildroot_component, - 'buildroot-gcc': buildroot_component, - 'copy-overlay': Component( - lambda arch: run_cmd(['copy-overlay'], arch), - ), - 'crosstool-ng': Component( - lambda arch: run_cmd(['build-crosstool-ng'], arch), - supported_archs=kwargs['crosstool_ng_supported_archs'], - # http://crosstool-ng.github.io/docs/os-setup/ - apt_get_pkgs={ - 'bison', - 'docbook2x', - 'flex', - 'gawk', - 'gcc', - 'gperf', - 'help2man', - 'libncurses5-dev', - 'libtool-bin', - 'make', - 'python-dev', - 'texinfo', - }, - submodules={'crosstool-ng'}, - ), - 'gem5': Component( - lambda arch: run_cmd(['build-gem5'], arch), - # TODO test it out on Docker and answer that question properly: - # https://askubuntu.com/questions/350475/how-can-i-install-gem5 - apt_get_pkgs={ - 'device-tree-compiler', - 'diod', - 'libgoogle-perftools-dev', - 'm4', - 'protobuf-compiler', - 'python-dev', - 'python-pip', - # For prebuilt qcow2 unpack. - 'qemu-utils', - 'scons', - 'zlib1g-dev', - }, - python2_pkgs={ - # Generate graphs of config.ini under m5out. - 'pydot', - }, - submodules={'gem5'}, - ), - 'gem5-debug': Component( - lambda arch: run_cmd(['build-gem5', '--gem5-build-type', 'debug'], arch), - ), - 'gem5-fast': Component( - lambda arch: run_cmd(['build-gem5', '--gem5-build-type', 'fast'], arch), - ), - 'linux': Component( - lambda arch: run_cmd(['build-linux'], arch), - submodules_shallow={'linux'}, - apt_get_pkgs={ - 'bison', - 'flex', - # Without this started failing in kernel 4.15 with: - # Makefile:932: *** "Cannot generate ORC metadata for CONFIG_UNWINDER_ORC=y, please install libelf-dev, libelf-devel or elfutils-libelf-devel". Stop. - 'libelf-dev', - }, - ), - 'modules': Component( - lambda arch: run_cmd(['build-modules'], arch), - ), - 'm5': Component( - lambda arch: run_cmd(['build-m5'], arch), - submodules={'gem5'}, - ), - 'qemu': Component( - lambda arch: run_cmd(['build-qemu'], arch), - apt_build_deps={'qemu'}, - apt_get_pkgs={'libsdl2-dev'}, - submodules={'qemu'}, - ), - 'qemu-user': Component( - lambda arch: run_cmd(['build-qemu', '--userland'], arch), - apt_build_deps = {'qemu'}, - apt_get_pkgs={'libsdl2-dev'}, - submodules = {'qemu'}, - ), - 'parsec-benchmark': Component( - submodules = {'parsec-benchmark'}, - ), - 'userland': Component( - lambda arch: run_cmd(['build-userland'], arch), - ), - - # Dependency only nodes. - 'all': Component(dependencies=[ - 'all-linux', - 'all-baremetal', - ]), - 'all-baremetal': Component(dependencies=[ - 'qemu-baremetal', - 'gem5-baremetal', - 'baremetal-gem5-pbx', - ], - supported_archs=kwargs['crosstool_ng_supported_archs'], - ), - 'all-linux': Component(dependencies=[ - 'qemu-gem5-buildroot', - 'gem5-debug', - 'gem5-fast', - 'qemu-user', - ]), - 'baremetal': Component(dependencies=[ - 'baremetal-gem5', - 'baremetal-qemu', - ]), - 'gem5-buildroot': Component(dependencies=[ - 'buildroot-gcc', - 'linux', - 'm5', - 'overlay', - 'gem5', - ]), - 'gem5-baremetal': Component(dependencies=[ - 'gem5', - 'crosstool-ng', - 'baremetal-gem5', - ]), - 'overlay': Component(dependencies=[ - 'copy-overlay', - 'modules', - 'userland', - 'buildroot', - ]), - 'qemu-baremetal': Component(dependencies=[ - 'qemu', - 'crosstool-ng', - 'baremetal-qemu', - ]), - 'qemu-buildroot': Component(dependencies=[ - 'qemu', - 'buildroot-gcc', - 'overlay', - 'linux', - ]), - 'qemu-gem5-buildroot': Component(dependencies=[ - 'qemu', - 'gem5-buildroot', - ]), - 'release': Component(dependencies=[ - 'qemu-buildroot', - ]), -} -parser = argparse.ArgumentParser( - description= '''\ +class Main(cli_function.CliFunction): + def __init__(self): + super().__init__( + config_file=common.consts['config_file'], + description='''\ Shallow helper to build everything, or a subset of everything conveniently. Our build-* scripts don't build any dependencies to make iterative @@ -281,204 +87,425 @@ If `--arch` is given, build just for the given archs: .... This will build `qemu-buildroot` for arm and aarch64 only, but not `x86_64`. +''' + ) + buildroot_component = _Component( + lambda arch: self._run_cmd(['build-buildroot'], arch), + submodules = {'buildroot'}, + # https://buildroot.org/downloads/manual/manual.html#requirement + apt_get_pkgs={ + 'bash', + 'bc', + 'binutils', + 'build-essential', + 'bzip2', + 'cpio', + 'g++', + 'gcc', + 'graphviz', + 'gzip', + 'make', + 'patch', + 'perl', + 'python-matplotlib', + 'python3', + 'rsync', + 'sed', + 'tar', + 'unzip', + }, + ) -Clean all Linux kernel builds: + self.name_to_component_map = { + # Leaves without dependencies. + 'baremetal-qemu': _Component( + lambda arch: self._run_cmd(['build-baremetal', '--emulator', 'qemu'], arch), + supported_archs=common.consts['crosstool_ng_supported_archs'], + ), + 'baremetal-gem5': _Component( + lambda arch: self._run_cmd(['build-baremetal', '--gem5'], arch), + supported_archs=common.consts['crosstool_ng_supported_archs'], + ), + 'baremetal-gem5-pbx': _Component( + lambda arch: self._run_cmd(['build-baremetal', '--gem5', '--machine', 'RealViewPBX'], arch), + supported_archs=common.consts['crosstool_ng_supported_archs'], + ), + 'buildroot': buildroot_component, + 'buildroot-gcc': buildroot_component, + 'copy-overlay': _Component( + lambda arch: self._run_cmd(['copy-overlay'], arch), + ), + 'crosstool-ng': _Component( + lambda arch: self._run_cmd(['build-crosstool-ng'], arch), + supported_archs=common.consts['crosstool_ng_supported_archs'], + # http://crosstool-ng.github.io/docs/os-setup/ + apt_get_pkgs={ + 'bison', + 'docbook2x', + 'flex', + 'gawk', + 'gcc', + 'gperf', + 'help2man', + 'libncurses5-dev', + 'libtool-bin', + 'make', + 'python-dev', + 'texinfo', + }, + submodules={'crosstool-ng'}, + ), + 'gem5': _Component( + lambda arch: self._run_cmd(['build-gem5'], arch), + # TODO test it out on Docker and answer that question properly: + # https://askubuntu.com/questions/350475/how-can-i-install-gem5 + apt_get_pkgs={ + 'device-tree-compiler', + 'diod', + 'libgoogle-perftools-dev', + 'm4', + 'protobuf-compiler', + 'python-dev', + 'python-pip', + # For prebuilt qcow2 unpack. + 'qemu-utils', + 'scons', + 'zlib1g-dev', + }, + python2_pkgs={ + # Generate graphs of config.ini under m5out. + 'pydot', + }, + submodules={'gem5'}, + ), + 'gem5-debug': _Component( + lambda arch: self._run_cmd(['build-gem5', '--gem5-build-type', 'debug'], arch), + ), + 'gem5-fast': _Component( + lambda arch: self._run_cmd(['build-gem5', '--gem5-build-type', 'fast'], arch), + ), + 'linux': _Component( + lambda arch: self._run_cmd(['build-linux'], arch), + submodules_shallow={'linux'}, + apt_get_pkgs={ + 'bison', + 'flex', + # Without this started failing in kernel 4.15 with: + # Makefile:932: *** "Cannot generate ORC metadata for CONFIG_UNWINDER_ORC=y, please install libelf-dev, libelf-devel or elfutils-libelf-devel". Stop. + 'libelf-dev', + }, + ), + 'modules': _Component( + lambda arch: self._run_cmd(['build-modules'], arch), + ), + 'm5': _Component( + lambda arch: self._run_cmd(['build-m5'], arch), + submodules={'gem5'}, + ), + 'qemu': _Component( + lambda arch: self._run_cmd(['build-qemu'], arch), + apt_build_deps={'qemu'}, + apt_get_pkgs={'libsdl2-dev'}, + submodules={'qemu'}, + ), + 'qemu-user': _Component( + lambda arch: self._run_cmd(['build-qemu', '--userland'], arch), + apt_build_deps = {'qemu'}, + apt_get_pkgs={'libsdl2-dev'}, + submodules = {'qemu'}, + ), + 'parsec-benchmark': _Component( + submodules = {'parsec-benchmark'}, + ), + 'userland': _Component( + lambda arch: self._run_cmd(['build-userland'], arch), + ), -.... -./build --all-archs --extra-args=--clean buildroot -.... -''', - formatter_class=argparse.RawTextHelpFormatter, -) -parser.add_argument('--all', default=False, action='store_true', help='''\ + # Dependency only nodes. + 'all': _Component(dependencies=[ + 'all-linux', + 'all-baremetal', + ]), + 'all-baremetal': _Component(dependencies=[ + 'qemu-baremetal', + 'gem5-baremetal', + 'baremetal-gem5-pbx', + ], + supported_archs=common.consts['crosstool_ng_supported_archs'], + ), + 'all-linux': _Component(dependencies=[ + 'qemu-gem5-buildroot', + 'gem5-debug', + 'gem5-fast', + 'qemu-user', + ]), + 'baremetal': _Component(dependencies=[ + 'baremetal-gem5', + 'baremetal-qemu', + ]), + 'gem5-buildroot': _Component(dependencies=[ + 'buildroot-gcc', + 'linux', + 'm5', + 'overlay', + 'gem5', + ]), + 'gem5-baremetal': _Component(dependencies=[ + 'gem5', + 'crosstool-ng', + 'baremetal-gem5', + ]), + 'overlay': _Component(dependencies=[ + 'copy-overlay', + 'modules', + 'userland', + 'buildroot', + ]), + 'qemu-baremetal': _Component(dependencies=[ + 'qemu', + 'crosstool-ng', + 'baremetal-qemu', + ]), + 'qemu-buildroot': _Component(dependencies=[ + 'qemu', + 'buildroot-gcc', + 'overlay', + 'linux', + ]), + 'qemu-gem5-buildroot': _Component(dependencies=[ + 'qemu', + 'gem5-buildroot', + ]), + 'release': _Component(dependencies=[ + 'qemu-buildroot', + ]), + } + + self.add_argument( + '--all', + default=False, + help='''\ Build absolutely everything for all archs. -''') -group = parser.add_mutually_exclusive_group(required=False) -group.add_argument('-A', '--all-archs', default=False, action='store_true', help='''\ +''' + ) + self.add_argument( + '-A', + '--all-archs', + default=False, + help='''\ Build the selected components for all archs. ''') -group.add_argument('-a', '--arch', choices=kwargs['arch_choices'], default=[], action='append', help='''\ + self.add_argument( + '-a', + '--arch', + choices=common.consts['arch_choices'], + default=[], + action='append', + help='''\ Build the selected components for this arch. Select multiple archs by passing this option multiple times. Default: [{}] -'''.format(kwargs['default_arch'])) -parser.add_argument('-D', '--download-dependencies', default=False, action='store_true', help='''\ +'''.format(common.consts['default_arch'])) + self.add_argument( + '-D', + '--download-dependencies', + default=False, + help='''\ Also download all dependencies required for a given build: Ubuntu packages, Python packages and git submodules. -''') -parser.add_argument('--extra-args', default='', help='''\ +''' + ) + self.add_argument( + '--dry-run', + default=False, + ) + self.add_argument( + '--travis', + default=False, + help='''\ Extra args to pass to all scripts. ''' -) -parser.add_argument('--travis', default=False, action='store_true', help='''\ -Extra args to pass to all scripts. -''' -) -parser.add_argument('components', choices=list(name_to_component_map.keys()) + [[]], default=[], nargs='*', help='''\ + ) + self.add_argument( + 'components', + choices=list(self.name_to_component_map.keys()) + [[]], + default=[], + nargs='*', + help='''\ Which components to build. Default: qemu-buildroot -'''.format(kwargs['default_arch'])) -self.add_dry_run_argument(parser) -args = parser.parse_args() -self.setup_dry_run_arguments(args) +''' + ) -# Decide archs. -if kwargs['arch'] == []: - if kwargs['all'] or kwargs['all_archs']: - archs = kwargs['all_archs'].copy() - else: - archs = set([kwargs['default_arch']]) -else: - archs = set() - for arch in kwargs['arch']: - if arch in kwargs['arch_short_to_long_dict']: - arch = kwargs['arch_short_to_long_dict'][arch] - archs.add(arch) + def _run_cmd(self, python_file, **kwargs): + python_file = os.path.join(common.consts['root_dir'], python_file) + run = common.import_path(python_file).Main() + run(**kwargs) + self.sh.run_cmd(cmd_abs) -# Decide components. -components = kwargs['components'] -if kwargs['all']: - components = ['all'] -elif components == []: - components = ['qemu-buildroot'] -selected_components = [] -selected_component_name_set = set() -for component_name in components: - todo = [component_name] - while todo: - current_name = todo.pop(0) - if current_name not in selected_component_name_set: - selected_component_name_set.add(current_name) - component = name_to_component_map[current_name] - selected_components.append(component) - todo.extend(component.dependencies) + def main(self, **kwargs): + self.sh = shell_helpers.ShellHelpers(dry_run=kwargs['dry_run']) -if kwargs['download_dependencies']: - apt_get_pkgs = { - # Core requirements for this repo. - 'git', - 'moreutils', # ts - 'python3-pip', - 'tmux', - 'vinagre', - 'wget', - } - # E.e. on an ARM host, the package gcc-arm-linux-gnueabihf - # is called just gcc. - processor = platform.processor() - if processor != 'arm': - apt_get_pkgs.update({ - 'gcc-arm-linux-gnueabihf', - 'g++-arm-linux-gnueabihf', - }) - if processor != 'aarch64': - apt_get_pkgs.update({ - 'gcc-aarch64-linux-gnu', - 'g++-aarch64-linux-gnu', - }) - apt_build_deps = set() - submodules = set() - submodules_shallow = set() - python2_pkgs = set() - python3_pkgs = { - 'pexpect==4.6.0', - } - for component in selected_components: - apt_get_pkgs.update(component.apt_get_pkgs) - apt_build_deps.update(component.apt_build_deps) - submodules.update(component.submodules) - submodules_shallow.update(component.submodules_shallow) - python2_pkgs.update(component.python2_pkgs) - python3_pkgs.update(component.python3_pkgs) - if apt_get_pkgs or apt_build_deps: - if kwargs['travis']: - interacive_pkgs = { - 'libsdl2-dev', + # Decide archs. + if kwargs['arch'] == []: + if kwargs['all'] or kwargs['all_archs']: + archs = kwargs['all_archs'].copy() + else: + archs = set([common.consts['default_arch']]) + else: + archs = set() + for arch in kwargs['arch']: + if arch in common.consts['arch_short_to_long_dict']: + arch = common.consts['arch_short_to_long_dict'][arch] + archs.add(arch) + + # Decide components. + components = kwargs['components'] + if kwargs['all']: + components = ['all'] + elif components == []: + components = ['qemu-buildroot'] + selected_components = [] + selected_component_name_set = set() + for component_name in components: + todo = [component_name] + while todo: + current_name = todo.pop(0) + if current_name not in selected_component_name_set: + selected_component_name_set.add(current_name) + component = self.name_to_component_map[current_name] + selected_components.append(component) + todo.extend(component.dependencies) + + if kwargs['download_dependencies']: + apt_get_pkgs = { + # Core requirements for this repo. + 'git', + 'moreutils', # ts + 'python3-pip', + 'tmux', + 'vinagre', + 'wget', } - apt_get_pkgs.difference_update(interacive_pkgs) - if kwargs['in_docker']: - sudo = [] - # https://askubuntu.com/questions/909277/avoiding-user-interaction-with-tzdata-when-installing-certbot-in-a-docker-contai - os.environ['DEBIAN_FRONTEND'] = 'noninteractive' - # https://askubuntu.com/questions/496549/error-you-must-put-some-source-uris-in-your-sources-list - sources_path = os.path.join('/etc', 'apt', 'sources.list') - with open(sources_path, 'r') as f: - sources_txt = f.read() - sources_txt = re.sub('^# deb-src ', 'deb-src ', sources_txt, flags=re.MULTILINE) - with open(sources_path, 'w') as f: - f.write(sources_txt) - else: - sudo = ['sudo'] - if kwargs['in_docker'] or kwargs['travis']: - y = ['-y'] - else: - y = [] - self.sh.run_cmd( - sudo + ['apt-get', 'update', LF] - ) - if apt_get_pkgs: - self.sh.run_cmd( - sudo + ['apt-get', 'install'] + y + [LF] + - self.sh.add_newlines(sorted(apt_get_pkgs)) - ) - if apt_build_deps: - self.sh.run_cmd( - sudo + - ['apt-get', 'build-dep'] + y + [LF] + - self.sh.add_newlines(sorted(apt_build_deps)) - ) - if python2_pkgs: - self.sh.run_cmd( - ['python', '-m', 'pip', 'install', '--user', LF] + - self.sh.add_newlines(sorted(python2_pkgs)) - ) - if python3_pkgs: - # Not with pip executable directly: - # https://stackoverflow.com/questions/49836676/error-after-upgrading-pip-cannot-import-name-main/51846054#51846054 - self.sh.run_cmd( - ['python3', '-m', 'pip', 'install', '--user', LF] + - self.sh.add_newlines(sorted(python3_pkgs)) - ) - git_cmd_common = ['git', 'submodule', 'update', '--init', '--recursive'] - if submodules: - # == Other nice git options for when distros move to newer Git - # - # Currently not on Ubuntu 16.04: - # - # `--progress`: added on Git 2.10: - # - # * https://stackoverflow.com/questions/32944468/how-to-show-progress-for-submodule-fetching - # * https://stackoverflow.com/questions/4640020/progress-indicator-for-git-clone - # - # `--jobs"`: https://stackoverflow.com/questions/26957237/how-to-make-git-clone-faster-with-multiple-threads/52327638#52327638 - self.sh.run_cmd( - git_cmd_common + ['--', LF] + - self.sh.add_newlines([os.path.join(kwargs['submodules_dir'], x) for x in sorted(submodules)]) - ) - if submodules_shallow: - # == Shallow cloning. - # - # TODO Ideally we should shallow clone --depth 1 all of them. - # - # However, most git servers out there are crap or craply configured - # and don't allow shallow cloning except for branches. - # - # So for now, let's shallow clone only the Linux kernel, which has by far - # the largest .git repo history, and full clone the others. - # - # Then we will maintain a GitHub Linux kernel mirror / fork that always has a - # lkmc branch, and point to it, so that it will always succeed. - # - # See also: - # - # * https://stackoverflow.com/questions/3489173/how-to-clone-git-repository-with-specific-revision-changeset - # * https://stackoverflow.com/questions/2144406/git-shallow-submodules/47374702#47374702 - # * https://unix.stackexchange.com/questions/338578/why-is-the-git-clone-of-the-linux-kernel-source-code-much-larger-than-the-extrac - # - self.sh.run_cmd( - git_cmd_common + ['--depth', '1', '--', LF] + - self.sh.add_newlines([os.path.join(kwargs['submodules_dir'], x) for x in sorted(submodules_shallow)]) - ) + # E.g. on an ARM host, the package gcc-arm-linux-gnueabihf + # is called just gcc. + processor = platform.processor() + if processor != 'arm': + apt_get_pkgs.update({ + 'gcc-arm-linux-gnueabihf', + 'g++-arm-linux-gnueabihf', + }) + if processor != 'aarch64': + apt_get_pkgs.update({ + 'gcc-aarch64-linux-gnu', + 'g++-aarch64-linux-gnu', + }) + apt_build_deps = set() + submodules = set() + submodules_shallow = set() + python2_pkgs = set() + python3_pkgs = { + 'pexpect==4.6.0', + } + for component in selected_components: + apt_get_pkgs.update(component.apt_get_pkgs) + apt_build_deps.update(component.apt_build_deps) + submodules.update(component.submodules) + submodules_shallow.update(component.submodules_shallow) + python2_pkgs.update(component.python2_pkgs) + python3_pkgs.update(component.python3_pkgs) + if apt_get_pkgs or apt_build_deps: + if kwargs['travis']: + interacive_pkgs = { + 'libsdl2-dev', + } + apt_get_pkgs.difference_update(interacive_pkgs) + if common.consts['in_docker']: + sudo = [] + # https://askubuntu.com/questions/909277/avoiding-user-interaction-with-tzdata-when-installing-certbot-in-a-docker-contai + os.environ['DEBIAN_FRONTEND'] = 'noninteractive' + # https://askubuntu.com/questions/496549/error-you-must-put-some-source-uris-in-your-sources-list + sources_path = os.path.join('/etc', 'apt', 'sources.list') + with open(sources_path, 'r') as f: + sources_txt = f.read() + sources_txt = re.sub('^# deb-src ', 'deb-src ', sources_txt, flags=re.MULTILINE) + with open(sources_path, 'w') as f: + f.write(sources_txt) + else: + sudo = ['sudo'] + if common.consts['in_docker'] or kwargs['travis']: + y = ['-y'] + else: + y = [] + self.sh.run_cmd( + sudo + ['apt-get', 'update', LF] + ) + if apt_get_pkgs: + self.sh.run_cmd( + sudo + ['apt-get', 'install'] + y + [LF] + + self.sh.add_newlines(sorted(apt_get_pkgs)) + ) + if apt_build_deps: + self.sh.run_cmd( + sudo + + ['apt-get', 'build-dep'] + y + [LF] + + self.sh.add_newlines(sorted(apt_build_deps)) + ) + if python2_pkgs: + self.sh.run_cmd( + ['python', '-m', 'pip', 'install', '--user', LF] + + self.sh.add_newlines(sorted(python2_pkgs)) + ) + if python3_pkgs: + # Not with pip executable directly: + # https://stackoverflow.com/questions/49836676/error-after-upgrading-pip-cannot-import-name-main/51846054#51846054 + self.sh.run_cmd( + ['python3', '-m', 'pip', 'install', '--user', LF] + + self.sh.add_newlines(sorted(python3_pkgs)) + ) + git_cmd_common = ['git', 'submodule', 'update', '--init', '--recursive'] + if submodules: + # == Other nice git options for when distros move to newer Git + # + # Currently not on Ubuntu 16.04: + # + # `--progress`: added on Git 2.10: + # + # * https://stackoverflow.com/questions/32944468/how-to-show-progress-for-submodule-fetching + # * https://stackoverflow.com/questions/4640020/progress-indicator-for-git-clone + # + # `--jobs"`: https://stackoverflow.com/questions/26957237/how-to-make-git-clone-faster-with-multiple-threads/52327638#52327638 + self.sh.run_cmd( + git_cmd_common + ['--', LF] + + self.sh.add_newlines([os.path.join(common.consts['submodules_dir'], x) for x in sorted(submodules)]) + ) + if submodules_shallow: + # == Shallow cloning. + # + # TODO Ideally we should shallow clone --depth 1 all of them. + # + # However, most git servers out there are crap or craply configured + # and don't allow shallow cloning except for branches. + # + # So for now, let's shallow clone only the Linux kernel, which has by far + # the largest .git repo history, and full clone the others. + # + # Then we will maintain a GitHub Linux kernel mirror / fork that always has a + # lkmc branch, and point to it, so that it will always succeed. + # + # See also: + # + # * https://stackoverflow.com/questions/3489173/how-to-clone-git-repository-with-specific-revision-changeset + # * https://stackoverflow.com/questions/2144406/git-shallow-submodules/47374702#47374702 + # * https://unix.stackexchange.com/questions/338578/why-is-the-git-clone-of-the-linux-kernel-source-code-much-larger-than-the-extrac + # + self.sh.run_cmd( + git_cmd_common + ['--depth', '1', '--', LF] + + self.sh.add_newlines([os.path.join(common.consts['submodules_dir'], x) for x in sorted(submodules_shallow)]) + ) -# Do the build. -for arch in archs: - for component in selected_components: - component.build(arch) + # Do the build. + for arch in archs: + for component in selected_components: + component.build(arch) + +if __name__ == '__main__': + Main().cli() diff --git a/cli_function.py b/cli_function.py index b635d77..c6b7d51 100755 --- a/cli_function.py +++ b/cli_function.py @@ -151,7 +151,7 @@ class CliFunction: config_file = args_with_defaults['config_file'] else: config_file = self._config_file - if os.path.exists(config_file): + if config_file is not None and os.path.exists(config_file): config_configs = {} config = imp.load_source('config', config_file) config.set_args(config_configs) @@ -168,7 +168,8 @@ class CliFunction: args_with_defaults[key] = argument.default else: raise Exception('Value not given for mandatory argument: ' + key) - del args_with_defaults['config_file'] + if 'config_file' in args_with_defaults: + del args_with_defaults['config_file'] return args_with_defaults def add_argument( @@ -360,6 +361,9 @@ amazing function! # Force a boolean value set on the config to be False on CLI. assert one_cli_function.cli(['--no-bool-cli', '1'])['bool_cli'] is False + # Pick another config file. + assert one_cli_function.cli(['--config-file', 'cli_function_test_config_2.py', '1'])['bool_cli'] is False + # get_cli assert one_cli_function.get_cli(pos_mandatory=1, asdf='B') == [('--asdf', 'B'), ('--bool-cli',), ('1',)] assert one_cli_function.get_cli(pos_mandatory=1, asdf='B', qwer='R') == [('--asdf', 'B'), ('--bool-cli',), ('--qwer', 'R'), ('1',)] diff --git a/cli_function_test_config_2.py b/cli_function_test_config_2.py new file mode 100644 index 0000000..71a4673 --- /dev/null +++ b/cli_function_test_config_2.py @@ -0,0 +1,5 @@ +def set_args(args): + ''' + :type args: Dict[str, Any] + ''' + args['bool_cli'] = False diff --git a/common.py b/common.py index 2a1f7e9..a604e7a 100644 --- a/common.py +++ b/common.py @@ -577,7 +577,7 @@ Use gem5 instead of QEMU. Shortcut for `--emulator gem5`. def base64_encode(string): return base64.b64encode(string.encode()).decode() - def gem_list_checkpoint_dirs(self): + def gem5_list_checkpoint_dirs(self): ''' List checkpoint directory, oldest first. ''' diff --git a/run b/run index 544248d..1e1767c 100755 --- a/run +++ b/run @@ -351,7 +351,7 @@ Run QEMU with VNC instead of the default SDL. Connect to it with: if self.env['gem5_script'] == 'fs': # TODO port if self.env['gem5_restore'] is not None: - cpt_dirs = self.gem_list_checkpoint_dirs() + cpt_dirs = self.gem5_list_checkpoint_dirs() cpt_dir = cpt_dirs[-self.env['gem5_restore']] extra_emulator_args.extend(['-r', str(sorted(cpt_dirs).index(cpt_dir) + 1)]) cmd.extend([ @@ -401,7 +401,7 @@ Run QEMU with VNC instead of the default SDL. Connect to it with: else: cpu_type = 'atomic' if self.env['gem5_restore'] is not None: - cpt_dir = self.gem_list_checkpoint_dirs()[-self.env['gem5_restore']] + cpt_dir = self.gem5_list_checkpoint_dirs()[-self.env['gem5_restore']] extra_emulator_args.extend(['--restore-from', os.path.join(self.env['m5out_dir'], cpt_dir)]) cmd.extend([ os.path.join(self.env['gem5_source_dir'], 'configs', 'example', 'arm', 'fs_bigLITTLE.py'), LF,