From 584a90eae2e643285ab647e9d133c403fc58674d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ciro=20Santilli=20=E5=85=AD=E5=9B=9B=E4=BA=8B=E4=BB=B6=20?= =?UTF-8?q?=E6=B3=95=E8=BD=AE=E5=8A=9F?= Date: Tue, 15 Sep 2020 01:00:00 +0000 Subject: [PATCH] Docker: start migrating to 20.04 Still failing with: Traceback (most recent call last): File "util/cpt_upgrader.py", line 73, in from six.moves import configparser ImportError: No module named six.moves Also fix some issues noticed: - userland/c/atomic was not ignoring arch specific examples - ./build would not stop on the first error, now it does - add libhdf5-dev as a dependency of gem5 --- Dockerfile | 2 +- build | 52 +++++++++++++++++++++++++++++++++------------- build-buildroot | 10 +++++++-- build-docker | 2 +- build-gem5 | 2 ++ common.py | 2 +- path_properties.py | 13 +++++++++++- requirements.txt | 2 +- 8 files changed, 63 insertions(+), 22 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7f96735..7f31563 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # https://cirosantilli.com/linux-kernel-module-cheat#docker -FROM ubuntu:18.04 +FROM ubuntu:20.04 COPY setup / RUN /setup -y CMD bash diff --git a/build b/build index 3bf160b..042816c 100755 --- a/build +++ b/build @@ -54,7 +54,10 @@ class _Component: (self.build_callback is not None) and (self.supported_archs is None or arch in self.supported_archs) ): - self.build_callback() + return self.build_callback() + else: + # Component that does not build anything itself, only has dependencies. + return 0 submodule_extra_remotes = { 'binutils-gdb': { @@ -168,13 +171,16 @@ so looping over all of them would waste time. 'make', 'patch', 'perl', - 'python-matplotlib', 'python3', 'rsync', 'sed', 'tar', 'unzip', }, + python3_pkgs={ + # Generate graphs of config.ini under m5out. + 'matplotlib', + }, ) buildroot_overlay_qemu_component = copy.copy(buildroot_component) # We need to build QEMU before the final Buildroot to get qemu-img. @@ -182,24 +188,36 @@ so looping over all of them would waste time. buildroot_overlay_gem5_component = copy.copy(buildroot_component) buildroot_overlay_gem5_component.dependencies = ['overlay-gem5'] gem5_deps = { - # TODO test it out on Docker and answer that question properly: - # https://askubuntu.com/questions/350475/how-can-i-install-gem5 'apt_get_pkgs': { - 'device-tree-compiler', - 'diod', - 'libgoogle-perftools-dev', + # https://askubuntu.com/questions/350475/how-can-i-install-gem5/1275773#1275773 + 'build-essential', + 'doxygen', + 'git', 'libboost-all-dev', + 'libelf-dev', + 'libgoogle-perftools-dev', + 'libhdf5-serial-dev', + 'libpng-dev', + 'libprotobuf-dev', + 'libprotoc-dev', 'm4', 'protobuf-compiler', - 'python-dev', - 'python-pip', + 'python-is-python3', + 'python3-dev', + 'python3-pydot', + 'python3-six', + 'scons', + 'zlib1g', + 'zlib1g-dev', + + # Some extra ones. + 'device-tree-compiler', + 'diod', # For prebuilt qcow2 unpack. 'qemu-utils', - 'scons', - 'zlib1g-dev', }, - 'python2_pkgs': { - # Generate graphs of config.ini under m5out. + 'python3_pkgs': { + # https://cirosantilli.com/linux-kernel-module-cheat#gem5-config-dot 'pydot', }, 'submodules_shallow': {'gem5'}, @@ -446,7 +464,7 @@ Which components to build. Default: qemu-buildroot args = self.get_common_args() args.update(extra_args) args['show_time'] = False - lkmc.import_path.import_path_main(component_file)(**args) + return lkmc.import_path.import_path_main(component_file)(**args) return f def timed_main(self): @@ -511,6 +529,8 @@ Which components to build. Default: qemu-buildroot ruby_pkgs.update(component.ruby_pkgs) if ruby_pkgs: apt_get_pkgs.add('ruby') + if python3_pkgs: + apt_get_pkgs.add('python3-pip') if apt_get_pkgs or apt_build_deps: if self.env['travis']: interacive_pkgs = { @@ -653,7 +673,9 @@ Which components to build. Default: qemu-buildroot if self.env['print_components']: print(self.component_to_name_map[component]) else: - component.build(self.env['arch']) + ret = component.build(self.env['arch']) + if (ret != 0): + return ret if __name__ == '__main__': Main().cli() diff --git a/build-buildroot b/build-buildroot index 00b97f0..b64be45 100755 --- a/build-buildroot +++ b/build-buildroot @@ -152,9 +152,15 @@ files on the root filesystem. ] + extra_make_args , - out_file=os.path.join(self.env['buildroot_build_dir'], self.env['repo_short_id'] + '.log'), - delete_env=['LD_LIBRARY_PATH', 'PERL_MM_OPT'], cwd=self.env['buildroot_source_dir'], + delete_env=['LD_LIBRARY_PATH', 'PERL_MM_OPT'], + extra_env={ + # In Docker, >>> host-tar 1.29 Configuring + # checking whether mknod can create fifo without root privileges... configure: error: in `/root/lkmc/out.docker/buildroot/build/default/aarch64/build/host-tar-1.29': + # configure: error: you should not run configure as root (set FORCE_UNSAFE_CONFIGURE=1 in environment to bypass this check) + 'FORCE_UNSAFE_CONFIGURE': '1', + }, + out_file=os.path.join(self.env['buildroot_build_dir'], self.env['repo_short_id'] + '.log'), ) # Create the qcow2 from ext2. # Skip if qemu is not present, because gem5 does not need the qcow2. diff --git a/build-docker b/build-docker index b68979d..d3636e1 100755 --- a/build-docker +++ b/build-docker @@ -45,7 +45,7 @@ See also: https://github.com/cirosantilli/linux-kernel-module-cheatTODO#ubuntu-g '-t', '-w', target_dir, '-v', '{}:{}'.format(kwargs['root_dir'], target_dir), - 'ubuntu:18.04', + 'ubuntu:20.04', 'bash', ]) self.sh.run_cmd([ diff --git a/build-gem5 b/build-gem5 index b0af5d6..3f17f76 100755 --- a/build-gem5 +++ b/build-gem5 @@ -133,6 +133,8 @@ https://github.com/cirosantilli/linux-kernel-module-cheat-regression#gem5-unit-t [ # TODO reenable, broken, had enough of this. # https://gem5.atlassian.net/browse/GEM5-357 + # https://gem5.atlassian.net/browse/GEM5-656 + # https://gem5.atlassian.net/browse/GEM5-778 #'SLICC_HTML=True', LF, ] + self.sh.add_newlines(targets) + diff --git a/common.py b/common.py index 30f2746..12df835 100644 --- a/common.py +++ b/common.py @@ -1453,7 +1453,7 @@ lunch aosp_{}-eng ''' Run timed_main across all selected archs and emulators. - :return: if any of the timed_mains exits non-zero and non-null, + :return: if any of the timed_mains exits non-zero and non-None, return that. Otherwise, return 0. ''' env = kwargs.copy() diff --git a/path_properties.py b/path_properties.py index 9abe4f0..1a23114 100644 --- a/path_properties.py +++ b/path_properties.py @@ -690,7 +690,18 @@ path_properties_tuples = ( # This has complex failure modes, too hard to assert. 'smash_stack.c': {'skip_run_unclassified': True}, 'std_atomic.c': {'baremetal': False}, - 'atomic': {'baremetal': False}, + 'atomic': ( + { + 'test_run_args': {'cpus': 3}, + }, + { + 'aarch64_add.c': {'allowed_archs': {'aarch64'}}, + 'aarch64_ldadd.c': {'allowed_archs': {'aarch64'}}, + 'aarch64_ldaxr_stlxr.c': {'allowed_archs': {'aarch64'}}, + 'x86_64_inc.c': {'allowed_archs': {'x86_64'}}, + 'x86_64_lock_inc.c': {'allowed_archs': {'x86_64'}}, + }, + ), # Wrapper not defined by newlib. 'timespec_get.c': {'baremetal': False}, } diff --git a/requirements.txt b/requirements.txt index f802773..71f95ee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ Cython==0.29.15 -china-dictatorship==0.0.52 +china-dictatorship==0.0.67 pexpect==4.6.0