From 3cc1b793cbde8dfdb87fa495c48e3bf00706caaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ciro=20Santilli=20=E5=85=AD=E5=9B=9B=E4=BA=8B=E4=BB=B6=20?= =?UTF-8?q?=E6=B3=95=E8=BD=AE=E5=8A=9F?= Date: Sat, 11 May 2019 00:00:01 +0000 Subject: [PATCH] Get rid of imp, started giving deprecation warning every time in Python 3.7 in Ubuntu 19.04. Please python stop torturing me with refactors. Make ./run -u blow up if executable not found, otherwise I go crazy. Get ./test-gdb back to life after the ./run relative path refactor, forgot to test this. --- README.adoc | 14 ++------- bisect-linux-boot-gem5 | 6 ++-- build | 2 +- build-baremetal | 2 +- build-userland | 2 +- build-userland-in-tree | 13 ++------ cli_function.py | 5 ++-- common.py | 52 +++++++++++++++++++------------- run | 16 +++++----- run-gdb | 3 +- run-gdb-user | 3 +- test | 12 ++++---- test-baremetal | 2 +- test-boot | 2 +- test-gdb | 63 +++++++++++++++++++++++---------------- test-user-mode | 4 +-- test-user-mode-in-tree | 14 ++------- test-userland-full-system | 2 +- trace-boot | 4 +-- 19 files changed, 108 insertions(+), 113 deletions(-) diff --git a/README.adoc b/README.adoc index 273a380..4587ae8 100644 --- a/README.adoc +++ b/README.adoc @@ -3910,20 +3910,10 @@ Similarly to <>, QEMU error mes In particular, it does not say anything if you pass it a non-existing executable: .... -./build-userland --clean userland/c/hello.c -./run --userland userland/c/hello.c -echo $? +qemu-x86_64 asdf | cat .... -does not output anything, except for the `1` exit status. - -If you run however the raw command without a pipe manually, it shows a helpful error message: - -.... -Error while loading /path/to/linux-kernel-module-cheat/out/userland/default/x86_64/c/hello.out: No such file or directory -.... - -Tested in de77c62c091f6418e73b64e8a0a19639c587a103 + 1. +So we just check ourselves manually == Kernel module utilities diff --git a/bisect-linux-boot-gem5 b/bisect-linux-boot-gem5 index 6d9a43c..f438c62 100755 --- a/bisect-linux-boot-gem5 +++ b/bisect-linux-boot-gem5 @@ -1,13 +1,13 @@ #!/usr/bin/env python3 -import imp import os import shutil import sys import common -build_linux = imp.load_source('build-linux', os.path.join(kwargs['root_dir'], 'build_linux')) -run = imp.load_source('run', os.path.join(kwargs['root_dir'], 'run')) + +build_linux = common.import_path_relative_root('build-linux') +run = common.import_path_relative_root('run') parser = self.get_argparse( argparse_args={ diff --git a/build b/build index 9769b24..70b733f 100755 --- a/build +++ b/build @@ -403,7 +403,7 @@ Which components to build. Default: qemu-buildroot args = self.get_common_args() args.update(extra_args) args['show_time'] = False - self.import_path_main(component_file)(**args) + common.import_path_main(component_file)(**args) return f def timed_main(self): diff --git a/build-baremetal b/build-baremetal index 879e43c..9f81970 100755 --- a/build-baremetal +++ b/build-baremetal @@ -133,7 +133,7 @@ Build the baremetal examples with crosstool-NG. in_name, in_ext = os.path.splitext(in_basename) if ( os.path.isfile(in_path) and - in_ext in (self.env['c_ext'], self.env['asm_ext']) + in_ext in self.env['build_in_exts'] ): out = os.path.join(out_dir, in_name + self.env['baremetal_build_ext']) src = os.path.join(self.env['baremetal_source_dir'], in_path) diff --git a/build-userland b/build-userland index 6251c61..f6c5384 100755 --- a/build-userland +++ b/build-userland @@ -88,7 +88,7 @@ Default: build all examples that have their package dependencies met, e.g.: for path, in_dirnames, in_filenames in self.sh.walk(target): for in_filename in in_filenames: in_ext = os.path.splitext(in_filename)[1] - if not in_ext in self.env['userland_in_exts']: + if not in_ext in self.env['build_in_exts']: continue in_path = os.path.join(path, in_filename) error = thread_pool.submit({ diff --git a/build-userland-in-tree b/build-userland-in-tree index c1f354f..7d6b56f 100755 --- a/build-userland-in-tree +++ b/build-userland-in-tree @@ -1,18 +1,11 @@ #!/usr/bin/env python3 -import imp import os import subprocess -git_root = subprocess.check_output([ - 'git', - 'rev-parse', - '--show-toplevel', -]).decode().rstrip() -build_userland = imp.load_source( - 'build_userland', - os.path.join(git_root, 'build-userland') -) +import common + +build_userland = common.import_path_relative_root('build-userland') class Main(build_userland.Main): def __init__(self): diff --git a/cli_function.py b/cli_function.py index 6dee84d..e7747f8 100755 --- a/cli_function.py +++ b/cli_function.py @@ -11,10 +11,11 @@ made to this file. import argparse import bisect import collections -import imp import os import sys +import common + class _Argument: def __init__( self, @@ -188,7 +189,7 @@ class CliFunction: if config_file is not None: if os.path.exists(config_file): config_configs = {} - config = imp.load_source('config', config_file) + config = common.import_path(config_file) if self.extra_config_params is None: config.set_args(config_configs) else: diff --git a/common.py b/common.py index 0fb460a..c1a355c 100644 --- a/common.py +++ b/common.py @@ -8,7 +8,7 @@ import datetime import enum import functools import glob -import imp +import importlib import inspect import itertools import json @@ -105,7 +105,7 @@ consts['cxx_ext'] = '.cpp' consts['header_ext'] = '.h' consts['kernel_module_ext'] = '.ko' consts['obj_ext'] = '.o' -consts['userland_in_exts'] = [ +consts['build_in_exts'] = [ consts['asm_ext'], consts['c_ext'], consts['cxx_ext'], @@ -136,6 +136,32 @@ for key in consts['emulator_short_to_long_dict']: consts['emulator_choices'].add(consts['emulator_short_to_long_dict'][key]) consts['host_arch'] = platform.processor() +def import_path(path): + ''' + https://stackoverflow.com/questions/2601047/import-a-python-module-without-the-py-extension + https://stackoverflow.com/questions/31773310/what-does-the-first-argument-of-the-imp-load-source-method-do + ''' + module_name = os.path.basename(path).replace('-', '_') + spec = importlib.util.spec_from_loader( + module_name, + importlib.machinery.SourceFileLoader(module_name, path) + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + sys.modules[module_name] = module + return module + +def import_path_relative_root(basename): + return import_path(os.path.join(consts['root_dir'], basename)) + +def import_path_main(basename): + ''' + Import an object of the Main class of a given file. + + By convention, we call the main object of all our CLI scripts as Main. + ''' + return import_path_relative_root(basename).Main() + class ExitLoop(Exception): pass @@ -905,6 +931,8 @@ Incompatible archs are skipped. env['source_path'] = source_path break env['image'] = path + elif env['userland'] is not None: + env['image'] = self.resolve_userland_executable(env['userland']) else: if env['emulator'] == 'gem5': env['image'] = env['vmlinux'] @@ -1084,24 +1112,6 @@ lunch aosp_{}-eng _json = {} return _json - def import_path(self, basename): - ''' - https://stackoverflow.com/questions/2601047/import-a-python-module-without-the-py-extension - https://stackoverflow.com/questions/31773310/what-does-the-first-argument-of-the-imp-load-source-method-do - ''' - return imp.load_source( - basename.replace('-', '_'), - os.path.join(self.env['root_dir'], basename) - ) - - def import_path_main(self, path): - ''' - Import an object of the Main class of a given file. - - By convention, we call the main object of all our CLI scripts as Main. - ''' - return self.import_path(path).Main() - def is_arch_supported(self, arch): return self.supported_archs is None or arch in self.supported_archs @@ -1270,6 +1280,8 @@ lunch aosp_{}-eng If the input path is a file, add the executable extension automatically. ''' + if not self.env['dry_run'] and not os.path.exists(in_path): + raise Exception('Input path does not exist: ' + in_path) if self.is_subpath(in_path, magic_in_dir): # Abspath needed to remove the trailing `/.` which makes e.g. rmrf fail. out = os.path.abspath(os.path.join( diff --git a/run b/run index 3e1466e..2447ecd 100755 --- a/run +++ b/run @@ -404,12 +404,12 @@ Extra options to append at the end of the emulator command line. if not self.env['dry_run']: raise Exception('Root filesystem not found. Did you build it? ' \ 'Tried to use: ' + self.env['disk_image']) - def raise_image_not_found(): + def raise_image_not_found(image): if not self.env['dry_run']: raise Exception('Executable image not found. Did you build it? ' \ - 'Tried to use: ' + self.env['image']) - if self.env['image'] is None: - raise Exception('Baremetal ELF file not found. Tried:\n' + '\n'.join(paths)) + 'Tried to use: ' + image) + if not os.path.exists(self.env['image']): + raise_image_not_found(self.env['image']) cmd = debug_vm.copy() if self.env['emulator'] == 'gem5': if self.env['quiet']: @@ -430,7 +430,7 @@ Extra options to append at the end of the emulator command line. self.env['userland'] is None ): if not os.path.exists(self.env['linux_image']): - raise_image_not_found() + raise_image_not_found(self.env['image']) self.sh.run_cmd([os.path.join(self.env['extract_vmlinux'], self.env['linux_image'])]) os.makedirs(os.path.dirname(self.env['gem5_readfile']), exist_ok=True) self.sh.write_string_to_file(self.env['gem5_readfile'], self.env['gem5_readfile']) @@ -457,7 +457,7 @@ Extra options to append at the end of the emulator command line. if self.env['userland'] is not None: cmd.extend([ self.env['gem5_se_file'], LF, - '--cmd', self.resolve_userland_executable(self.env['userland']), LF, + '--cmd', self.env['image'], LF, ]) if self.env['userland_args'] is not None: cmd.extend(['--options', self.env['userland_args'], LF]) @@ -551,8 +551,6 @@ Extra options to append at the end of the emulator command line. debug_args ) else: - if not os.path.exists(self.env['image']): - raise_image_not_found() extra_emulator_args.extend(extra_qemu_args) self.make_run_dirs() if self.env['debug_vm']: @@ -725,7 +723,7 @@ Extra options to append at the end of the emulator command line. cmd.extend(self.env['extra_emulator_args']) if self.env['userland'] and self.env['emulator'] in ('qemu', 'native'): # The program and arguments must come at the every end of the CLI. - cmd.extend([self.resolve_userland_executable(self.env['userland']), LF]) + cmd.extend([self.env['image'], LF]) if self.env['userland_args'] is not None: cmd.extend(self.sh.shlex_split(self.env['userland_args'])) if debug_vm or self.env['terminal']: diff --git a/run-gdb b/run-gdb index afbaa71..4ad509b 100755 --- a/run-gdb +++ b/run-gdb @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -import imp import os import signal import subprocess @@ -34,7 +33,7 @@ class GdbTestcase: self.child.setecho(False) self.child.waitnoecho() self.child.expect(self.prompt) - test = imp.load_source('test', test_script_path) + test = common.import_path(test_script_path) exception = None try: test.test(self) diff --git a/run-gdb-user b/run-gdb-user index 0e2d406..a7f71c6 100755 --- a/run-gdb-user +++ b/run-gdb-user @@ -1,11 +1,10 @@ #!/usr/bin/env python3 -import imp import os import sys import common -rungdb = imp.load_source('run_gdb', os.path.join(kwargs['root_dir'], 'run-gdb')) +rungdb = common.import_path_relative_root('run-gdb') parser = self.get_argparse(argparse_args={ 'description': '''GDB step debug guest userland processes without gdbserver. diff --git a/test b/test index 3389277..1d8084e 100755 --- a/test +++ b/test @@ -28,15 +28,15 @@ Size of the tests to run. Scale: run_args = self.get_common_args() test_boot_args = run_args.copy() test_boot_args['size'] = self.env['size'] - self.run_test(self.import_path_main('test-boot'), test_boot_args, 'test-boot') - self.run_test(self.import_path_main('test-userland-full-system'), run_args, 'test-userland') - self.run_test(self.import_path_main('test-baremetal'), run_args, 'test-baremetal') - self.run_test(self.import_path_main('test-user-mode'), run_args, 'test-user-mode') - self.run_test(self.import_path_main('test-gdb'), run_args, 'test-gdb') + self.run_test(common.import_path_main('test-boot'), test_boot_args, 'test-boot') + self.run_test(common.import_path_main('test-userland-full-system'), run_args, 'test-userland') + self.run_test(common.import_path_main('test-baremetal'), run_args, 'test-baremetal') + self.run_test(common.import_path_main('test-user-mode'), run_args, 'test-user-mode') + self.run_test(common.import_path_main('test-gdb'), run_args, 'test-gdb') if self.env['emulator'] == 'gem5': gem5_unit_test_args = run_args.copy() gem5_unit_test_args['unit_tests'] = True - self.run_test(self.import_path_main('build-gem5'), gem5_unit_test_args, 'gem5-unit-tests') + self.run_test(common.import_path_main('build-gem5'), gem5_unit_test_args, 'gem5-unit-tests') if __name__ == '__main__': Main().cli() diff --git a/test-baremetal b/test-baremetal index dd3afb5..670a05b 100755 --- a/test-baremetal +++ b/test-baremetal @@ -52,7 +52,7 @@ If given, run only the given tests. Otherwise, run all tests. test_args = { 'expected_exit_status': my_path_properties['exit_status'], 'run_args': cur_run_args, - 'run_obj': self.import_path_main('run'), + 'run_obj': common.import_path_main('run'), 'test_id': path_relative_root, } error = thread_pool.submit(test_args) diff --git a/test-boot b/test-boot index d373277..8733e7c 100755 --- a/test-boot +++ b/test-boot @@ -45,7 +45,7 @@ See ./test --help for --size. #) # #rm -f "${self.env['test_boot_benchmark_file']}" - self.run = self.import_path_main('run') + self.run = common.import_path_main('run') self.common_args = self.get_common_args() self.common_args['ctrl_c_host'] = True self.common_args['quit_after_boot'] = True diff --git a/test-gdb b/test-gdb index a4532ca..6ecf577 100755 --- a/test-gdb +++ b/test-gdb @@ -22,38 +22,51 @@ found by searching for the Python test files. ) def timed_main(self): - run = self.import_path_main('run') - run_gdb = self.import_path_main('run-gdb') + run = common.import_path_main('run') + run_gdb = common.import_path_main('run-gdb') if self.env['arch'] in self.env['crosstool_ng_supported_archs']: + test_sources = [] if self.env['tests'] == []: - test_scripts_noext = [] + source_paths = [] for filename in sorted(os.listdir(self.env['baremetal_source_dir'])): base, ext = os.path.splitext(filename) - if ext == '.py': - test_scripts_noext.append(base) - for root, dirnames, filenames in os.walk(os.path.join(self.env['baremetal_source_dir'], 'arch', self.env['arch'])): + if ext in self.env['build_in_exts']: + test_sources.append( + os.path.join( + self.env['baremetal_source_dir'], + filename + ) + ) + for root, dirnames, filenames in os.walk( + os.path.join( + self.env['baremetal_source_dir'], + 'arch', + self.env['arch'] + ) + ): for filename in filenames: base, ext = os.path.splitext(filename) - if ext == '.py': - full_path = os.path.join(root, base) - relpath = os.path.relpath(full_path, self.env['baremetal_source_dir']) - test_scripts_noext.append(relpath) + if ext in self.env['build_in_exts']: + test_sources.append(os.path.join(root, filename)) else: - test_scripts_noext = self.env['tests'] - for test_script_noext in test_scripts_noext: - common_args = self.get_common_args() - common_args['baremetal'] = test_script_noext - test_id_string = self.test_setup(test_script_noext) - run_args = common_args.copy() - run_args['gdb_wait'] = True - run_args['background'] = True - run_thread = threading.Thread(target=lambda: run(**run_args)) - run_thread.start() - gdb_args = common_args.copy() - gdb_args['test'] = True - run_gdb(**gdb_args) - run_thread.join() - self.test_teardown(run, 0, test_id_string) + test_sources = self.env['tests'] + for test_source_full in test_sources: + base, ext = os.path.splitext(test_source_full) + if os.path.exists(base + '.py'): + test_source_base = os.path.relpath(base, self.env['root_dir']) + common_args = self.get_common_args() + common_args['baremetal'] = test_source_base + ext + test_id_string = self.test_setup(test_source_base) + run_args = common_args.copy() + run_args['gdb_wait'] = True + run_args['background'] = True + run_thread = threading.Thread(target=lambda: run(**run_args)) + run_thread.start() + gdb_args = common_args.copy() + gdb_args['test'] = True + run_gdb(**gdb_args) + run_thread.join() + self.test_teardown(run, 0, test_id_string) if __name__ == '__main__': Main().cli() diff --git a/test-user-mode b/test-user-mode index 645cfb7..6d5848f 100755 --- a/test-user-mode +++ b/test-user-mode @@ -50,7 +50,7 @@ If given, run only the given tests. Otherwise, run all tests. path_abs = os.path.abspath(path) dirpath_relative_root = path_abs[rootdir_abs_len + 1:] for in_filename in in_filenames: - if os.path.splitext(in_filename)[1] in self.env['userland_in_exts']: + if os.path.splitext(in_filename)[1] in self.env['build_in_exts']: path_relative_root = os.path.join(dirpath_relative_root, in_filename) my_path_properties = path_properties.get(path_relative_root) if my_path_properties.should_be_tested(self.env): @@ -62,7 +62,7 @@ If given, run only the given tests. Otherwise, run all tests. run_test_args = { 'expected_exit_status': my_path_properties['exit_status'], 'run_args': cur_run_args, - 'run_obj': self.import_path_main('run'), + 'run_obj': common.import_path_main('run'), 'test_id': path_relative_root, } if my_path_properties['receives_signal']: diff --git a/test-user-mode-in-tree b/test-user-mode-in-tree index b5c1fa0..aa3cd08 100755 --- a/test-user-mode-in-tree +++ b/test-user-mode-in-tree @@ -1,18 +1,8 @@ #!/usr/bin/env python3 -import imp -import os -import subprocess +import common -git_root = subprocess.check_output([ - 'git', - 'rev-parse', - '--show-toplevel', -]).decode().rstrip() -test_user_mode = imp.load_source( - 'test_user_mode', - os.path.join(git_root, 'test-user-mode') -) +test_user_mode = common.import_path_relative_root('test-user-mode') class Main(test_user_mode.Main): def __init__(self): diff --git a/test-userland-full-system b/test-userland-full-system index cc69376..ea2b6b2 100755 --- a/test-userland-full-system +++ b/test-userland-full-system @@ -13,7 +13,7 @@ https://github.com/cirosantilli/linux-kernel-module-cheat#test-userland-in-full- ''' ) def timed_main(self): - run = self.import_path_main('run') + run = common.import_path_main('run') run_args = self.get_common_args() run_args['eval_after'] = './test_all.sh;{};'.format(self.env['userland_quit_cmd']) self.run_test(run, run_args) diff --git a/trace-boot b/trace-boot index 4336979..9d442d8 100755 --- a/trace-boot +++ b/trace-boot @@ -14,7 +14,7 @@ More information at: https://github.com/cirosantilli/linux-kernel-module-cheat#t def timed_main(self): args = self.get_common_args() - run = self.import_path_main('run') + run = common.import_path_main('run') if self.env['emulator'] == 'gem5': args['trace'] = 'Exec,-ExecSymbol,-ExecMicro' run.main(**args) @@ -23,7 +23,7 @@ More information at: https://github.com/cirosantilli/linux-kernel-module-cheat#t run_args['trace'] = 'exec_tb' run_args['quit_after_boot'] = True run.main(**run_args) - qemu_trace2txt = self.import_path_main('qemu-trace2txt') + qemu_trace2txt = common.import_path_main('qemu-trace2txt') qemu_trace2txt.main(**args) # Instruction count. # We could put this on a separate script, but it just adds more arch boilerplate to a new script.