Docker: start migrating to 20.04

Still failing with:

Traceback (most recent call last):
  File "util/cpt_upgrader.py", line 73, in <module>
    from six.moves import configparser
ImportError: No module named six.moves

Also fix some issues noticed:

- userland/c/atomic was not ignoring arch specific examples
- ./build would not stop on the first error, now it does
- add libhdf5-dev as a dependency of gem5

Dockerfile

@@ -1,5 +1,5 @@
 # https://cirosantilli.com/linux-kernel-module-cheat#docker
-FROM ubuntu:18.04
+FROM ubuntu:20.04
 COPY setup /
 RUN /setup -y
 CMD bash

build

@@ -54,7 +54,10 @@ class _Component:
             (self.build_callback is not None) and
             (self.supported_archs is None or arch in self.supported_archs)
         ):
-            self.build_callback()
+            return self.build_callback()
+        else:
+            # Component that does not build anything itself, only has dependencies.
+            return 0
 
 submodule_extra_remotes = {
     'binutils-gdb': {
@@ -168,13 +171,16 @@ so looping over all of them would waste time.
                 'make',
                 'patch',
                 'perl',
-                'python-matplotlib',
                 'python3',
                 'rsync',
                 'sed',
                 'tar',
                 'unzip',
             },
+            python3_pkgs={
+                # Generate graphs of config.ini under m5out.
+                'matplotlib',
+            },
         )
         buildroot_overlay_qemu_component = copy.copy(buildroot_component)
         # We need to build QEMU before the final Buildroot to get qemu-img.
@@ -182,24 +188,36 @@ so looping over all of them would waste time.
         buildroot_overlay_gem5_component = copy.copy(buildroot_component)
         buildroot_overlay_gem5_component.dependencies = ['overlay-gem5']
         gem5_deps = {
-            # TODO test it out on Docker and answer that question properly:
-            # https://askubuntu.com/questions/350475/how-can-i-install-gem5
             'apt_get_pkgs': {
-                'device-tree-compiler',
+                # https://askubuntu.com/questions/350475/how-can-i-install-gem5/1275773#1275773
-                'diod',
+                'build-essential',
-                'libgoogle-perftools-dev',
+                'doxygen',
+                'git',
                 'libboost-all-dev',
+                'libelf-dev',
+                'libgoogle-perftools-dev',
+                'libhdf5-serial-dev',
+                'libpng-dev',
+                'libprotobuf-dev',
+                'libprotoc-dev',
                 'm4',
                 'protobuf-compiler',
-                'python-dev',
+                'python-is-python3',
-                'python-pip',
+                'python3-dev',
+                'python3-pydot',
+                'python3-six',
+                'scons',
+                'zlib1g',
+                'zlib1g-dev',
+
+                # Some extra ones.
+                'device-tree-compiler',
+                'diod',
                 # For prebuilt qcow2 unpack.
                 'qemu-utils',
-                'scons',
-                'zlib1g-dev',
             },
-            'python2_pkgs': {
+            'python3_pkgs': {
-                # Generate graphs of config.ini under m5out.
+                # https://cirosantilli.com/linux-kernel-module-cheat#gem5-config-dot
                 'pydot',
             },
             'submodules_shallow': {'gem5'},
@@ -446,7 +464,7 @@ Which components to build. Default: qemu-buildroot
             args = self.get_common_args()
             args.update(extra_args)
             args['show_time'] = False
-            lkmc.import_path.import_path_main(component_file)(**args)
+            return lkmc.import_path.import_path_main(component_file)(**args)
         return f
 
     def timed_main(self):
@@ -511,6 +529,8 @@ Which components to build. Default: qemu-buildroot
                 ruby_pkgs.update(component.ruby_pkgs)
             if ruby_pkgs:
                 apt_get_pkgs.add('ruby')
+            if python3_pkgs:
+                apt_get_pkgs.add('python3-pip')
             if apt_get_pkgs or apt_build_deps:
                 if self.env['travis']:
                     interacive_pkgs = {
@@ -653,7 +673,9 @@ Which components to build. Default: qemu-buildroot
             if self.env['print_components']:
                 print(self.component_to_name_map[component])
             else:
-                component.build(self.env['arch'])
+                ret = component.build(self.env['arch'])
+                if (ret != 0):
+                    return ret
 
 if __name__ == '__main__':
     Main().cli()

build-buildroot

@@ -152,9 +152,15 @@ files on the root filesystem.
             ] +
             extra_make_args
             ,
-            out_file=os.path.join(self.env['buildroot_build_dir'], self.env['repo_short_id'] + '.log'),
-            delete_env=['LD_LIBRARY_PATH', 'PERL_MM_OPT'],
             cwd=self.env['buildroot_source_dir'],
+            delete_env=['LD_LIBRARY_PATH', 'PERL_MM_OPT'],
+            extra_env={
+                # In Docker, >>> host-tar 1.29 Configuring
+                # checking whether mknod can create fifo without root privileges... configure: error: in `/root/lkmc/out.docker/buildroot/build/default/aarch64/build/host-tar-1.29':
+                # configure: error: you should not run configure as root (set FORCE_UNSAFE_CONFIGURE=1 in environment to bypass this check)
+                'FORCE_UNSAFE_CONFIGURE': '1',
+            },
+            out_file=os.path.join(self.env['buildroot_build_dir'], self.env['repo_short_id'] + '.log'),
         )
         # Create the qcow2 from ext2.
         # Skip if qemu is not present, because gem5 does not need the qcow2.

build-docker

@@ -45,7 +45,7 @@ See also: https://github.com/cirosantilli/linux-kernel-module-cheatTODO#ubuntu-g
             '-t',
             '-w', target_dir,
             '-v', '{}:{}'.format(kwargs['root_dir'], target_dir),
-            'ubuntu:18.04',
+            'ubuntu:20.04',
             'bash',
         ])
         self.sh.run_cmd([

build-gem5

@@ -133,6 +133,8 @@ https://github.com/cirosantilli/linux-kernel-module-cheat-regression#gem5-unit-t
                 [
                     # TODO reenable, broken, had enough of this.
                     # https://gem5.atlassian.net/browse/GEM5-357
+                    # https://gem5.atlassian.net/browse/GEM5-656
+                    # https://gem5.atlassian.net/browse/GEM5-778
                     #'SLICC_HTML=True', LF,
                 ] +
                 self.sh.add_newlines(targets) +

common.py

@@ -1453,7 +1453,7 @@ lunch aosp_{}-eng
         '''
         Run timed_main across all selected archs and emulators.
 
-        :return: if any of the timed_mains exits non-zero and non-null,
+        :return: if any of the timed_mains exits non-zero and non-None,
                  return that. Otherwise, return 0.
         '''
         env = kwargs.copy()

path_properties.py

@@ -690,7 +690,18 @@ path_properties_tuples = (
                         # This has complex failure modes, too hard to assert.
                         'smash_stack.c': {'skip_run_unclassified': True},
                         'std_atomic.c': {'baremetal': False},
-                        'atomic': {'baremetal': False},
+                        'atomic': (
+                            {
+                                'test_run_args': {'cpus': 3},
+                            },
+                            {
+                                'aarch64_add.c': {'allowed_archs': {'aarch64'}},
+                                'aarch64_ldadd.c': {'allowed_archs': {'aarch64'}},
+                                'aarch64_ldaxr_stlxr.c': {'allowed_archs': {'aarch64'}},
+                                'x86_64_inc.c': {'allowed_archs': {'x86_64'}},
+                                'x86_64_lock_inc.c': {'allowed_archs': {'x86_64'}},
+                            },
+                        ),
                         # Wrapper not defined by newlib.
                         'timespec_get.c': {'baremetal': False},
                     }

requirements.txt

@@ -1,3 +1,3 @@
 Cython==0.29.15
-china-dictatorship==0.0.52
+china-dictatorship==0.0.67
 pexpect==4.6.0