Get rid of imp, started giving deprecation warning every time in Python 3.7 in Ubuntu 19.04.

Please python stop torturing me with refactors.

Make ./run -u blow up if executable not found, otherwise I go crazy.

Get ./test-gdb back to life after the ./run relative path refactor, forgot to test this.

README.adoc

@@ -3910,20 +3910,10 @@ Similarly to <<qemu-user-mode-does-not-show-stdout-immediately>>, QEMU error mes
 In particular, it does not say anything if you pass it a non-existing executable:
 
 ....
-./build-userland --clean userland/c/hello.c
-./run --userland userland/c/hello.c
-echo $?
+qemu-x86_64 asdf | cat
 ....
 
-does not output anything, except for the `1` exit status.
-
-If you run however the raw command without a pipe manually, it shows a helpful error message:
-
-....
-Error while loading /path/to/linux-kernel-module-cheat/out/userland/default/x86_64/c/hello.out: No such file or directory
-....
-
-Tested in de77c62c091f6418e73b64e8a0a19639c587a103 + 1.
+So we just check ourselves manually
 
 == Kernel module utilities
 

bisect-linux-boot-gem5

@@ -1,13 +1,13 @@
 #!/usr/bin/env python3
 
-import imp
 import os
 import shutil
 import sys
 
 import common
-build_linux = imp.load_source('build-linux', os.path.join(kwargs['root_dir'], 'build_linux'))
-run = imp.load_source('run', os.path.join(kwargs['root_dir'], 'run'))
+
+build_linux = common.import_path_relative_root('build-linux')
+run = common.import_path_relative_root('run')
 
 parser = self.get_argparse(
     argparse_args={

build

@@ -403,7 +403,7 @@ Which components to build. Default: qemu-buildroot
             args = self.get_common_args()
             args.update(extra_args)
             args['show_time'] = False
-            self.import_path_main(component_file)(**args)
+            common.import_path_main(component_file)(**args)
         return f
 
     def timed_main(self):

build-baremetal

@@ -133,7 +133,7 @@ Build the baremetal examples with crosstool-NG.
                     in_name, in_ext = os.path.splitext(in_basename)
                     if  (
                         os.path.isfile(in_path) and
-                        in_ext in (self.env['c_ext'], self.env['asm_ext'])
+                        in_ext in self.env['build_in_exts']
                     ):
                         out = os.path.join(out_dir, in_name + self.env['baremetal_build_ext'])
                         src = os.path.join(self.env['baremetal_source_dir'], in_path)

build-userland

@@ -88,7 +88,7 @@ Default: build all examples that have their package dependencies met, e.g.:
                     for path, in_dirnames, in_filenames in self.sh.walk(target):
                         for in_filename in in_filenames:
                             in_ext = os.path.splitext(in_filename)[1]
-                            if not in_ext in self.env['userland_in_exts']:
+                            if not in_ext in self.env['build_in_exts']:
                                 continue
                             in_path = os.path.join(path, in_filename)
                             error = thread_pool.submit({

build-userland-in-tree

@@ -1,18 +1,11 @@
 #!/usr/bin/env python3
 
-import imp
 import os
 import subprocess
 
-git_root = subprocess.check_output([
-    'git',
-    'rev-parse',
-    '--show-toplevel',
-]).decode().rstrip()
-build_userland = imp.load_source(
-    'build_userland',
-    os.path.join(git_root, 'build-userland')
-)
+import common
+
+build_userland = common.import_path_relative_root('build-userland')
 
 class Main(build_userland.Main):
     def __init__(self):

cli_function.py

@@ -11,10 +11,11 @@ made to this file.
 import argparse
 import bisect
 import collections
-import imp
 import os
 import sys
 
+import common
+
 class _Argument:
     def __init__(
             self,
@@ -188,7 +189,7 @@ class CliFunction:
         if config_file is not None:
             if os.path.exists(config_file):
                 config_configs = {}
-                config = imp.load_source('config', config_file)
+                config = common.import_path(config_file)
                 if self.extra_config_params is None:
                     config.set_args(config_configs)
                 else:

common.py

@@ -8,7 +8,7 @@ import datetime
 import enum
 import functools
 import glob
-import imp
+import importlib
 import inspect
 import itertools
 import json
@@ -105,7 +105,7 @@ consts['cxx_ext'] = '.cpp'
 consts['header_ext'] = '.h'
 consts['kernel_module_ext'] = '.ko'
 consts['obj_ext'] = '.o'
-consts['userland_in_exts'] = [
+consts['build_in_exts'] = [
     consts['asm_ext'],
     consts['c_ext'],
     consts['cxx_ext'],
@@ -136,6 +136,32 @@ for key in consts['emulator_short_to_long_dict']:
     consts['emulator_choices'].add(consts['emulator_short_to_long_dict'][key])
 consts['host_arch'] = platform.processor()
 
+def import_path(path):
+    '''
+    https://stackoverflow.com/questions/2601047/import-a-python-module-without-the-py-extension
+    https://stackoverflow.com/questions/31773310/what-does-the-first-argument-of-the-imp-load-source-method-do
+    '''
+    module_name = os.path.basename(path).replace('-', '_')
+    spec = importlib.util.spec_from_loader(
+        module_name,
+        importlib.machinery.SourceFileLoader(module_name, path)
+    )
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    sys.modules[module_name] = module
+    return module
+
+def import_path_relative_root(basename):
+    return import_path(os.path.join(consts['root_dir'], basename))
+
+def import_path_main(basename):
+    '''
+    Import an object of the Main class of a given file.
+
+    By convention, we call the main object of all our CLI scripts as Main.
+    '''
+    return import_path_relative_root(basename).Main()
+
 class ExitLoop(Exception):
     pass
 
@@ -905,6 +931,8 @@ Incompatible archs are skipped.
                         env['source_path'] = source_path
                         break
             env['image'] = path
+        elif env['userland'] is not None:
+            env['image'] = self.resolve_userland_executable(env['userland'])
         else:
             if env['emulator'] == 'gem5':
                 env['image'] = env['vmlinux']
@@ -1084,24 +1112,6 @@ lunch aosp_{}-eng
             _json = {}
         return _json
 
-    def import_path(self, basename):
-        '''
-        https://stackoverflow.com/questions/2601047/import-a-python-module-without-the-py-extension
-        https://stackoverflow.com/questions/31773310/what-does-the-first-argument-of-the-imp-load-source-method-do
-        '''
-        return imp.load_source(
-            basename.replace('-', '_'),
-            os.path.join(self.env['root_dir'], basename)
-        )
-
-    def import_path_main(self, path):
-        '''
-        Import an object of the Main class of a given file.
-
-        By convention, we call the main object of all our CLI scripts as Main.
-        '''
-        return self.import_path(path).Main()
-
     def is_arch_supported(self, arch):
         return self.supported_archs is None or arch in self.supported_archs
 
@@ -1270,6 +1280,8 @@ lunch aosp_{}-eng
 
         If the input path is a file, add the executable extension automatically.
         '''
+        if not self.env['dry_run'] and not os.path.exists(in_path):
+            raise Exception('Input path does not exist: ' + in_path)
         if self.is_subpath(in_path, magic_in_dir):
             # Abspath needed to remove the trailing `/.` which makes e.g. rmrf fail.
             out = os.path.abspath(os.path.join(

run

@@ -404,12 +404,12 @@ Extra options to append at the end of the emulator command line.
             if not self.env['dry_run']:
                 raise Exception('Root filesystem not found. Did you build it? ' \
                                 'Tried to use: ' + self.env['disk_image'])
-        def raise_image_not_found():
+        def raise_image_not_found(image):
             if not self.env['dry_run']:
                 raise Exception('Executable image not found. Did you build it? ' \
-                                'Tried to use: ' + self.env['image'])
-        if self.env['image'] is None:
-            raise Exception('Baremetal ELF file not found. Tried:\n' + '\n'.join(paths))
+                                'Tried to use: ' + image)
+        if not os.path.exists(self.env['image']):
+            raise_image_not_found(self.env['image'])
         cmd = debug_vm.copy()
         if self.env['emulator'] == 'gem5':
             if self.env['quiet']:
@@ -430,7 +430,7 @@ Extra options to append at the end of the emulator command line.
                     self.env['userland'] is None
                 ):
                     if not os.path.exists(self.env['linux_image']):
-                        raise_image_not_found()
+                        raise_image_not_found(self.env['image'])
                     self.sh.run_cmd([os.path.join(self.env['extract_vmlinux'], self.env['linux_image'])])
             os.makedirs(os.path.dirname(self.env['gem5_readfile']), exist_ok=True)
             self.sh.write_string_to_file(self.env['gem5_readfile'], self.env['gem5_readfile'])
@@ -457,7 +457,7 @@ Extra options to append at the end of the emulator command line.
             if self.env['userland'] is not None:
                 cmd.extend([
                     self.env['gem5_se_file'], LF,
-                    '--cmd', self.resolve_userland_executable(self.env['userland']), LF,
+                    '--cmd', self.env['image'], LF,
                 ])
                 if self.env['userland_args'] is not None:
                     cmd.extend(['--options', self.env['userland_args'], LF])
@@ -551,8 +551,6 @@ Extra options to append at the end of the emulator command line.
                     debug_args
                 )
             else:
-                if not os.path.exists(self.env['image']):
-                    raise_image_not_found()
                 extra_emulator_args.extend(extra_qemu_args)
                 self.make_run_dirs()
                 if self.env['debug_vm']:
@@ -725,7 +723,7 @@ Extra options to append at the end of the emulator command line.
         cmd.extend(self.env['extra_emulator_args'])
         if self.env['userland'] and self.env['emulator'] in ('qemu', 'native'):
             # The program and arguments must come at the every end of the CLI.
-            cmd.extend([self.resolve_userland_executable(self.env['userland']), LF])
+            cmd.extend([self.env['image'], LF])
             if self.env['userland_args'] is not None:
                 cmd.extend(self.sh.shlex_split(self.env['userland_args']))
         if debug_vm or self.env['terminal']:

run-gdb

@@ -1,6 +1,5 @@
 #!/usr/bin/env python3
 
-import imp
 import os
 import signal
 import subprocess
@@ -34,7 +33,7 @@ class GdbTestcase:
         self.child.setecho(False)
         self.child.waitnoecho()
         self.child.expect(self.prompt)
-        test = imp.load_source('test', test_script_path)
+        test = common.import_path(test_script_path)
         exception = None
         try:
             test.test(self)

run-gdb-user

@@ -1,11 +1,10 @@
 #!/usr/bin/env python3
 
-import imp
 import os
 import sys
 
 import common
-rungdb = imp.load_source('run_gdb', os.path.join(kwargs['root_dir'], 'run-gdb'))
+rungdb = common.import_path_relative_root('run-gdb')
 
 parser = self.get_argparse(argparse_args={
     'description': '''GDB step debug guest userland processes without gdbserver.

test

@@ -28,15 +28,15 @@ Size of the tests to run. Scale:
         run_args = self.get_common_args()
         test_boot_args = run_args.copy()
         test_boot_args['size'] = self.env['size']
-        self.run_test(self.import_path_main('test-boot'), test_boot_args, 'test-boot')
-        self.run_test(self.import_path_main('test-userland-full-system'), run_args, 'test-userland')
-        self.run_test(self.import_path_main('test-baremetal'), run_args, 'test-baremetal')
-        self.run_test(self.import_path_main('test-user-mode'), run_args, 'test-user-mode')
-        self.run_test(self.import_path_main('test-gdb'), run_args, 'test-gdb')
+        self.run_test(common.import_path_main('test-boot'), test_boot_args, 'test-boot')
+        self.run_test(common.import_path_main('test-userland-full-system'), run_args, 'test-userland')
+        self.run_test(common.import_path_main('test-baremetal'), run_args, 'test-baremetal')
+        self.run_test(common.import_path_main('test-user-mode'), run_args, 'test-user-mode')
+        self.run_test(common.import_path_main('test-gdb'), run_args, 'test-gdb')
         if self.env['emulator'] == 'gem5':
             gem5_unit_test_args = run_args.copy()
             gem5_unit_test_args['unit_tests'] = True
-            self.run_test(self.import_path_main('build-gem5'), gem5_unit_test_args, 'gem5-unit-tests')
+            self.run_test(common.import_path_main('build-gem5'), gem5_unit_test_args, 'gem5-unit-tests')
 
 if __name__ == '__main__':
     Main().cli()

test-baremetal

@@ -52,7 +52,7 @@ If given, run only the given tests. Otherwise, run all tests.
                                     test_args = {
                                         'expected_exit_status': my_path_properties['exit_status'],
                                         'run_args': cur_run_args,
-                                        'run_obj': self.import_path_main('run'),
+                                        'run_obj': common.import_path_main('run'),
                                         'test_id': path_relative_root,
                                     }
                                     error = thread_pool.submit(test_args)

test-boot

@@ -45,7 +45,7 @@ See ./test --help for --size.
         #)
         #
         #rm -f "${self.env['test_boot_benchmark_file']}"
-        self.run = self.import_path_main('run')
+        self.run = common.import_path_main('run')
         self.common_args = self.get_common_args()
         self.common_args['ctrl_c_host'] = True
         self.common_args['quit_after_boot'] = True

test-gdb

@@ -22,38 +22,51 @@ found by searching for the Python test files.
         )
 
     def timed_main(self):
-        run = self.import_path_main('run')
-        run_gdb = self.import_path_main('run-gdb')
+        run = common.import_path_main('run')
+        run_gdb = common.import_path_main('run-gdb')
         if self.env['arch'] in self.env['crosstool_ng_supported_archs']:
+            test_sources = []
             if self.env['tests'] == []:
-                test_scripts_noext = []
+                source_paths = []
                 for filename in sorted(os.listdir(self.env['baremetal_source_dir'])):
                     base, ext = os.path.splitext(filename)
-                    if ext == '.py':
-                        test_scripts_noext.append(base)
-                for root, dirnames, filenames in os.walk(os.path.join(self.env['baremetal_source_dir'], 'arch', self.env['arch'])):
+                    if ext in self.env['build_in_exts']:
+                        test_sources.append(
+                            os.path.join(
+                                self.env['baremetal_source_dir'],
+                                filename
+                            )
+                        )
+                for root, dirnames, filenames in os.walk(
+                    os.path.join(
+                        self.env['baremetal_source_dir'],
+                        'arch',
+                        self.env['arch']
+                    )
+                ):
                     for filename in filenames:
                         base, ext = os.path.splitext(filename)
-                        if ext == '.py':
-                            full_path = os.path.join(root, base)
-                            relpath = os.path.relpath(full_path, self.env['baremetal_source_dir'])
-                            test_scripts_noext.append(relpath)
+                        if ext in self.env['build_in_exts']:
+                            test_sources.append(os.path.join(root, filename))
             else:
-                test_scripts_noext = self.env['tests']
-            for test_script_noext in test_scripts_noext:
-                common_args = self.get_common_args()
-                common_args['baremetal'] = test_script_noext
-                test_id_string = self.test_setup(test_script_noext)
-                run_args = common_args.copy()
-                run_args['gdb_wait'] = True
-                run_args['background'] = True
-                run_thread = threading.Thread(target=lambda: run(**run_args))
-                run_thread.start()
-                gdb_args = common_args.copy()
-                gdb_args['test'] = True
-                run_gdb(**gdb_args)
-                run_thread.join()
-                self.test_teardown(run, 0, test_id_string)
+                test_sources = self.env['tests']
+            for test_source_full in test_sources:
+                base, ext = os.path.splitext(test_source_full)
+                if os.path.exists(base + '.py'):
+                    test_source_base = os.path.relpath(base, self.env['root_dir'])
+                    common_args = self.get_common_args()
+                    common_args['baremetal'] = test_source_base + ext
+                    test_id_string = self.test_setup(test_source_base)
+                    run_args = common_args.copy()
+                    run_args['gdb_wait'] = True
+                    run_args['background'] = True
+                    run_thread = threading.Thread(target=lambda: run(**run_args))
+                    run_thread.start()
+                    gdb_args = common_args.copy()
+                    gdb_args['test'] = True
+                    run_gdb(**gdb_args)
+                    run_thread.join()
+                    self.test_teardown(run, 0, test_id_string)
 
 if __name__ == '__main__':
     Main().cli()

test-user-mode

@@ -50,7 +50,7 @@ If given, run only the given tests. Otherwise, run all tests.
                         path_abs = os.path.abspath(path)
                         dirpath_relative_root = path_abs[rootdir_abs_len + 1:]
                         for in_filename in in_filenames:
-                            if os.path.splitext(in_filename)[1] in self.env['userland_in_exts']:
+                            if os.path.splitext(in_filename)[1] in self.env['build_in_exts']:
                                 path_relative_root = os.path.join(dirpath_relative_root, in_filename)
                                 my_path_properties = path_properties.get(path_relative_root)
                                 if my_path_properties.should_be_tested(self.env):
@@ -62,7 +62,7 @@ If given, run only the given tests. Otherwise, run all tests.
                                     run_test_args = {
                                         'expected_exit_status': my_path_properties['exit_status'],
                                         'run_args': cur_run_args,
-                                        'run_obj':  self.import_path_main('run'),
+                                        'run_obj':  common.import_path_main('run'),
                                         'test_id': path_relative_root,
                                     }
                                     if my_path_properties['receives_signal']:

test-user-mode-in-tree

@@ -1,18 +1,8 @@
 #!/usr/bin/env python3
 
-import imp
-import os
-import subprocess
+import common
 
-git_root = subprocess.check_output([
-    'git',
-    'rev-parse',
-    '--show-toplevel',
-]).decode().rstrip()
-test_user_mode = imp.load_source(
-    'test_user_mode',
-    os.path.join(git_root, 'test-user-mode')
-)
+test_user_mode = common.import_path_relative_root('test-user-mode')
 
 class Main(test_user_mode.Main):
     def __init__(self):

test-userland-full-system

@@ -13,7 +13,7 @@ https://github.com/cirosantilli/linux-kernel-module-cheat#test-userland-in-full-
 '''
         )
     def timed_main(self):
-        run = self.import_path_main('run')
+        run = common.import_path_main('run')
         run_args = self.get_common_args()
         run_args['eval_after'] = './test_all.sh;{};'.format(self.env['userland_quit_cmd'])
         self.run_test(run, run_args)

trace-boot

@@ -14,7 +14,7 @@ More information at: https://github.com/cirosantilli/linux-kernel-module-cheat#t
 
     def timed_main(self):
         args = self.get_common_args()
-        run = self.import_path_main('run')
+        run = common.import_path_main('run')
         if self.env['emulator'] == 'gem5':
             args['trace'] = 'Exec,-ExecSymbol,-ExecMicro'
             run.main(**args)
@@ -23,7 +23,7 @@ More information at: https://github.com/cirosantilli/linux-kernel-module-cheat#t
             run_args['trace'] = 'exec_tb'
             run_args['quit_after_boot'] = True
             run.main(**run_args)
-            qemu_trace2txt = self.import_path_main('qemu-trace2txt')
+            qemu_trace2txt = common.import_path_main('qemu-trace2txt')
             qemu_trace2txt.main(**args)
             # Instruction count.
             # We could put this on a separate script, but it just adds more arch boilerplate to a new script.