Fix import_path circular dependency by splitting it out.

Use import thread_pool instead from, from is evil.

Fix poweroff.out path for ./trace-boot.
This commit is contained in:
Ciro Santilli 六四事件 法轮功
2019-05-12 00:00:02 +00:00
parent 7cf3c20a40
commit 1ab7fbf607
26 changed files with 149 additions and 93 deletions

View File

@@ -14198,11 +14198,12 @@ You should also test that the Internet works:
./run --arch x86_64 --kernel-cli '- lkmc_eval="ifup -a;wget -S google.com;poweroff;"'
....
===== test-build-userland
===== CLI script tests
`build-userland` has a wide variety of target selection modes, and it was hard to keep them all working without a test.
`build-userland` and `test-user-mode` have a wide variety of target selection modes, and it was hard to keep them all working without some tests:
So we've created the simple: link:test-build-userland[] to ensure that at least none of the build modes blows up.
* link:test-build-userland[]
* link:test-test-user-mode[]
=== Bisection

4
build
View File

@@ -10,6 +10,8 @@ import copy
import shell_helpers
from shell_helpers import LF
import lkmc
class _Component:
'''
Yes, we are re-inventing a crappy dependency resolution system,
@@ -403,7 +405,7 @@ Which components to build. Default: qemu-buildroot
args = self.get_common_args()
args.update(extra_args)
args['show_time'] = False
common.import_path_main(component_file)(**args)
lkmc.import_path.import_path_main(component_file)(**args)
return f
def timed_main(self):

View File

@@ -161,7 +161,7 @@ usually extra Buildroot targets.
] +
extra_make_args
,
out_file=os.path.join(self.env['buildroot_build_dir'], 'lkmc.log'),
out_file=os.path.join(self.env['buildroot_build_dir'], self.env['repo_short_id'] + '.log'),
delete_env=['LD_LIBRARY_PATH'],
cwd=self.env['buildroot_source_dir'],
)

View File

@@ -61,7 +61,7 @@ Build crosstool-NG with Newlib for bare metal compilation
'build', LF,
'CT_JOBS={}'.format(str(self.env['nproc'])), LF,
],
out_file=os.path.join(build_dir, 'lkmc.log'),
out_file=os.path.join(build_dir, self.env['repo_short_id'] + '.log'),
delete_env=['LD_LIBRARY_PATH'],
extra_paths=[self.env['ccache_dir']],
)

View File

@@ -202,7 +202,7 @@ Run `make modules_install` after `make`.
extra_env={
'KBUILD_BUILD_VERSION': '1',
'KBUILD_BUILD_TIMESTAMP': 'Thu Jan 1 00:00:00 UTC 1970',
'KBUILD_BUILD_USER': 'lkmc',
'KBUILD_BUILD_USER': self.env['repo_short_id'],
'KBUILD_BUILD_HOST': common.git_sha(self.env['linux_source_dir']),
},
**common_args

View File

@@ -7,7 +7,7 @@ import threading
from shell_helpers import LF
import common
from thread_pool import ThreadPool
import thread_pool
class Main(common.BuildCliFunction):
def __init__(self, *args, **kwargs):
@@ -79,10 +79,10 @@ Default: build all examples that have their package dependencies met, e.g.:
extra_deps=[self.env['common_h']],
link=False,
)
with ThreadPool(
with thread_pool.ThreadPool(
self._build_one,
nthreads=self.env['nproc'],
) as thread_pool:
) as my_thread_pool:
try:
for target in self.env['targets']:
for path, in_dirnames, in_filenames in self.sh.walk(target):
@@ -91,7 +91,7 @@ Default: build all examples that have their package dependencies met, e.g.:
if not in_ext in self.env['build_in_exts']:
continue
in_path = os.path.join(path, in_filename)
error = thread_pool.submit({
error = my_thread_pool.submit({
'in_path': in_path,
'out_path': self.resolve_userland_executable(in_path),
'cc_flags': cc_flags,
@@ -102,7 +102,7 @@ Default: build all examples that have their package dependencies met, e.g.:
raise common.ExitLoop()
except common.ExitLoop:
pass
error = thread_pool.get_error()
error = my_thread_pool.get_error()
if error is not None:
print(error)
return 1

View File

@@ -3,9 +3,9 @@
import os
import subprocess
import common
import lkmc.import_path
build_userland = common.import_path_relative_root('build-userland')
build_userland = lkmc.import_path.import_path_relative_root('build-userland')
class Main(build_userland.Main):
def __init__(self):

View File

@@ -14,7 +14,7 @@ import collections
import os
import sys
import common
import lkmc.import_path
class _Argument:
def __init__(
@@ -192,7 +192,7 @@ class CliFunction:
if config_file is not None:
if os.path.exists(config_file):
config_configs = {}
config = common.import_path(config_file)
config = lkmc.import_path.import_path(config_file)
if self.extra_config_params is None:
config.set_args(config_configs)
else:

View File

@@ -8,7 +8,6 @@ import datetime
import enum
import functools
import glob
import importlib
import inspect
import itertools
import json
@@ -64,7 +63,7 @@ consts['userland_subdir'] = 'userland'
consts['userland_source_dir'] = os.path.join(consts['root_dir'], consts['userland_subdir'])
consts['userland_source_arch_dir'] = os.path.join(consts['userland_source_dir'], 'arch')
consts['userland_executable_ext'] = '.out'
consts['include_subdir'] = 'lkmc'
consts['include_subdir'] = consts['repo_short_id']
consts['include_source_dir'] = os.path.join(consts['root_dir'], consts['include_subdir'])
consts['submodules_dir'] = os.path.join(consts['root_dir'], 'submodules')
consts['buildroot_source_dir'] = os.path.join(consts['submodules_dir'], 'buildroot')
@@ -135,32 +134,7 @@ for key in consts['emulator_short_to_long_dict']:
consts['emulator_choices'].add(key)
consts['emulator_choices'].add(consts['emulator_short_to_long_dict'][key])
consts['host_arch'] = platform.processor()
def import_path(path):
'''
https://stackoverflow.com/questions/2601047/import-a-python-module-without-the-py-extension
https://stackoverflow.com/questions/31773310/what-does-the-first-argument-of-the-imp-load-source-method-do
'''
module_name = os.path.basename(path).replace('-', '_')
spec = importlib.util.spec_from_loader(
module_name,
importlib.machinery.SourceFileLoader(module_name, path)
)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
sys.modules[module_name] = module
return module
def import_path_relative_root(basename):
return import_path(os.path.join(consts['root_dir'], basename))
def import_path_main(basename):
'''
Import an object of the Main class of a given file.
By convention, we call the main object of all our CLI scripts as Main.
'''
return import_path_relative_root(basename).Main()
consts['guest_lkmc_home'] = os.sep + consts['repo_short_id']
class ExitLoop(Exception):
pass
@@ -452,8 +426,8 @@ Use the docker download Ubuntu root filesystem instead of the default Buildroot
)
self.add_argument(
'--qemu-which',
choices=['lkmc', 'host'],
default='lkmc',
choices=[consts['repo_short_id'], 'host'],
default=consts['repo_short_id'],
help='''\
Which qemu binaries to use: qemu-system-, qemu-, qemu-img, etc.:
- lkmc: the ones we built with ./build-qemu
@@ -850,7 +824,11 @@ Incompatible archs are skipped.
if env['emulator']== 'gem5':
env['userland_quit_cmd'] = './gem5_exit.sh'
else:
env['userland_quit_cmd'] = './poweroff.out'
env['userland_quit_cmd'] = join(
env['guest_lkmc_home'],
'linux',
'poweroff' + env['userland_executable_ext']
)
env['ramfs'] = env['initrd'] or env['initramfs']
if env['ramfs']:
env['initarg'] = 'rdinit'
@@ -874,9 +852,8 @@ Incompatible archs are skipped.
# Overlay.
env['out_rootfs_overlay_dir'] = join(env['out_dir'], 'rootfs_overlay', env['arch'])
env['out_rootfs_overlay_lkmc_dir'] = join(env['out_rootfs_overlay_dir'], 'lkmc')
env['out_rootfs_overlay_lkmc_dir'] = join(env['out_rootfs_overlay_dir'], env['repo_short_id'])
env['out_rootfs_overlay_bin_dir'] = join(env['out_rootfs_overlay_lkmc_dir'], 'bin')
env['guest_lkmc_home'] = os.sep + 'lkmc'
# Baremetal.
env['baremetal_source_dir'] = join(env['root_dir'], 'baremetal')
@@ -893,7 +870,7 @@ Incompatible archs are skipped.
env['baremetal_build_ext'] = '.elf'
# Userland / baremetal common source.
env['common_basename_noext'] = 'lkmc'
env['common_basename_noext'] = env['repo_short_id']
env['common_c'] = common_c = os.path.join(
env['root_dir'],
env['common_basename_noext'] + env['c_ext']
@@ -1165,8 +1142,6 @@ lunch aosp_{}-eng
if arch in env['arch_short_to_long_dict']:
arch = env['arch_short_to_long_dict'][arch]
if emulator == 'native':
if env['userland'] is None:
raise Exception('Emulator only supported in user mode: {}'.format(emulator))
if arch != env['host_arch']:
continue
if self.is_arch_supported(arch):
@@ -1286,6 +1261,8 @@ lunch aosp_{}-eng
If it is out of tree, return the same exact path as input.
If the input path is a file, add the executable extension automatically.
Directories map to the directories that would contain executable in that directory.
'''
if not self.env['dry_run'] and not os.path.exists(in_path):
raise Exception('Input path does not exist: ' + in_path)

0
lkmc/__init__.py Normal file
View File

33
lkmc/import_path.py Normal file
View File

@@ -0,0 +1,33 @@
#!/usr/bin/env python3
import importlib
import os
import sys
root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
def import_path(path):
'''
https://stackoverflow.com/questions/2601047/import-a-python-module-without-the-py-extension
https://stackoverflow.com/questions/31773310/what-does-the-first-argument-of-the-imp-load-source-method-do
'''
module_name = os.path.basename(path).replace('-', '_')
spec = importlib.util.spec_from_loader(
module_name,
importlib.machinery.SourceFileLoader(module_name, path)
)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
sys.modules[module_name] = module
return module
def import_path_relative_root(basename):
return import_path(os.path.join(root_dir, basename))
def import_path_main(basename):
'''
Import an object of the Main class of a given file.
By convention, we call the main object of all our CLI scripts as Main.
'''
return import_path_relative_root(basename).Main()

2
run
View File

@@ -309,6 +309,8 @@ Extra options to append at the end of the emulator command line.
)
def timed_main(self):
if self.env['emulator'] == 'native' and self.env['userland'] is None:
raise Exception('native emulator only supported in user mode')
show_stdout = self.env['show_stdout']
# Common qemu / gem5 logic.
# nokaslr:

View File

@@ -6,6 +6,7 @@ import subprocess
import sys
import common
import lkmc.import_path
from shell_helpers import LF
class GdbTestcase:
@@ -33,7 +34,7 @@ class GdbTestcase:
self.child.setecho(False)
self.child.waitnoecho()
self.child.expect(self.prompt)
test = common.import_path(test_script_path)
test = lkmc.import_path.import_path(test_script_path)
exception = None
try:
test.test(self)

View File

@@ -3,6 +3,7 @@
import os
import common
import lkmc.import_path
class Main(common.LkmcCliFunction):
def __init__(self):
@@ -35,7 +36,7 @@ More information at: https://github.com/cirosantilli/linux-kernel-module-cheat#g
# Also, lx-symbols overrides the add-symbol-file commands.
args['no_lxsymbols'] = True
args['break_at'] = self.env['break_at']
rungdb = common.import_path_main('run-gdb')
rungdb = lkmc.import_path.import_path_main('run-gdb')
return rungdb(**args)
if __name__ == '__main__':

14
test
View File

@@ -1,7 +1,7 @@
#!/usr/bin/env python3
import common
import shell_helpers
import lkmc.import_path
from shell_helpers import LF
class Main(common.TestCliFunction):
@@ -28,15 +28,15 @@ Size of the tests to run. Scale:
run_args = self.get_common_args()
test_boot_args = run_args.copy()
test_boot_args['size'] = self.env['size']
self.run_test(common.import_path_main('test-boot'), test_boot_args, 'test-boot')
self.run_test(common.import_path_main('test-userland-full-system'), run_args, 'test-userland')
self.run_test(common.import_path_main('test-baremetal'), run_args, 'test-baremetal')
self.run_test(common.import_path_main('test-user-mode'), run_args, 'test-user-mode')
self.run_test(common.import_path_main('test-gdb'), run_args, 'test-gdb')
self.run_test(lkmc.import_path.import_path_main('test-boot'), test_boot_args, 'test-boot')
self.run_test(lkmc.import_path.import_path_main('test-userland-full-system'), run_args, 'test-userland')
self.run_test(lkmc.import_path.import_path_main('test-baremetal'), run_args, 'test-baremetal')
self.run_test(lkmc.import_path.import_path_main('test-user-mode'), run_args, 'test-user-mode')
self.run_test(lkmc.import_path.import_path_main('test-gdb'), run_args, 'test-gdb')
if self.env['emulator'] == 'gem5':
gem5_unit_test_args = run_args.copy()
gem5_unit_test_args['unit_tests'] = True
self.run_test(common.import_path_main('build-gem5'), gem5_unit_test_args, 'gem5-unit-tests')
self.run_test(lkmc.import_path.import_path_main('build-gem5'), gem5_unit_test_args, 'gem5-unit-tests')
if __name__ == '__main__':
Main().cli()

View File

@@ -4,8 +4,9 @@ import os
import sys
import common
import lkmc.import_path
import path_properties
from thread_pool import ThreadPool
import thread_pool
class Main(common.TestCliFunction):
def __init__(self):
@@ -29,11 +30,11 @@ If given, run only the given tests. Otherwise, run all tests.
def timed_main(self):
run_args = self.get_common_args()
rootdir_abs_len = len(self.env['root_dir'])
with ThreadPool(
with thread_pool.ThreadPool(
self.run_test,
nthreads=self.env['nproc'],
thread_id_arg='thread_id',
) as thread_pool:
) as my_thread_pool:
try:
for test in self.env['tests']:
for path, in_dirnames, in_filenames in self.sh.walk(test):
@@ -52,17 +53,17 @@ If given, run only the given tests. Otherwise, run all tests.
test_args = {
'expected_exit_status': my_path_properties['exit_status'],
'run_args': cur_run_args,
'run_obj': common.import_path_main('run'),
'run_obj': lkmc.import_path.import_path_main('run'),
'test_id': path_relative_root,
}
error = thread_pool.submit(test_args)
error = my_thread_pool.submit(test_args)
if error is not None:
if self.env['quit_on_fail']:
raise common.ExitLoop()
except common.ExitLoop:
pass
error = thread_pool.get_error()
error = my_thread_pool.get_error()
if error is not None:
print(error)
return 1

View File

@@ -1,6 +1,7 @@
#!/usr/bin/env python3
import common
import lkmc.import_path
import shell_helpers
from shell_helpers import LF
@@ -45,7 +46,7 @@ See ./test --help for --size.
#)
#
#rm -f "${self.env['test_boot_benchmark_file']}"
self.run = common.import_path_main('run')
self.run = lkmc.import_path.import_path_main('run')
self.common_args = self.get_common_args()
self.common_args['ctrl_c_host'] = True
self.common_args['quit_after_boot'] = True

View File

@@ -1,6 +1,6 @@
#!/usr/bin/env bash
# Quick sanity check that userland target resolution works.
# https://github.com/cirosantilli/linux-kernel-module-cheat#cli-script-tests
set -eux
@@ -44,13 +44,12 @@ for in_tree in '' --in-tree; do
! ./build-userland $in_tree "${userland_build_dir}/c/hello.out"
tmpfile="$(mktemp)"
! ./build-userland $in_tree "$tmpfile"
! ./build-userland --clean $in_tree "$tmpfile"
rm "$tmpfile"
! ./build-userland $in_tree ..
! ./build-userland $in_tree kernel_modules
! ./build-userland --clean $in_tree userland/does_not_exist
./build-userland --clean $in_tree
# Clean is however more forgiving and accepts paths that don't exist.
./build-userland --clean $in_tree userland/does_not_exist
done
./build-userland-in-tree

View File

@@ -4,6 +4,7 @@ import threading
import os
import common
import lkmc.import_path
class Main(common.TestCliFunction):
def __init__(self):
@@ -22,8 +23,8 @@ found by searching for the Python test files.
)
def timed_main(self):
run = common.import_path_main('run')
run_gdb = common.import_path_main('run-gdb')
run = lkmc.import_path.import_path_main('run')
run_gdb = lkmc.import_path.import_path_main('run-gdb')
if self.env['arch'] in self.env['crosstool_ng_supported_archs']:
test_sources = []
if self.env['tests'] == []:

39
test-test-user-mode Executable file
View File

@@ -0,0 +1,39 @@
#!/usr/bin/env bash
# https://github.com/cirosantilli/linux-kernel-module-cheat#cli-script-tests
set -eux
./build-userland
./build-userland-in-tree
f="$(tempfile)"
./test-user-mode | tee "$f"
grep -E '^PASS .* userland/c/hello' "$f"
grep -E '^PASS .* userland/posix/uname' "$f"
./test-user-mode userland | tee "$f"
grep -E '^PASS .* userland/c/hello' "$f"
grep -E '^PASS .* userland/posix/uname' "$f"
./test-user-mode userland/c | tee "$f"
grep -E '^PASS .* userland/c/hello' "$f"
! grep -E '^PASS .* userland/posix/uname' "$f"
./test-user-mode userland/c/hello.c | tee "$f"
grep -E '^PASS .* userland/c/hello' "$f"
! grep -E '^PASS .* userland/c/false' "$f"
! grep -E '^PASS .* userland/posix/uname' "$f"
./test-user-mode-in-tree | tee "$f"
grep -E '^PASS .* userland/c/hello' "$f"
grep -E '^PASS .* userland/posix/uname' "$f"
cd userland
./test
grep -E '^PASS .* userland/c/hello' "$f"
grep -E '^PASS .* userland/posix/uname' "$f"
cd ..
rm "$f"

View File

@@ -4,8 +4,9 @@ import os
import sys
import common
import lkmc.import_path
import path_properties
from thread_pool import ThreadPool
import thread_pool
class Main(common.TestCliFunction):
def __init__(self, *args, **kwargs):
@@ -16,8 +17,6 @@ TODO: expose all userland relevant ./run args here as well somehow.
'''
if not 'defaults' in kwargs:
kwargs['defaults'] = {}
if not 'userland' in kwargs['defaults']:
kwargs['defaults']['userland'] = ''
super().__init__(*args, **kwargs)
self.add_argument(
'tests',
@@ -39,11 +38,11 @@ If given, run only the given tests. Otherwise, run all tests.
run_args['userland_build_id'] = 'static'
had_failure = False
rootdir_abs_len = len(self.env['root_dir'])
with ThreadPool(
with thread_pool.ThreadPool(
self.run_test,
nthreads=self.env['nproc'],
thread_id_arg='thread_id',
) as thread_pool:
) as my_thread_pool:
try:
for test in self.env['tests']:
for path, in_dirnames, in_filenames in self.sh.walk(test):
@@ -62,18 +61,18 @@ If given, run only the given tests. Otherwise, run all tests.
run_test_args = {
'expected_exit_status': my_path_properties['exit_status'],
'run_args': cur_run_args,
'run_obj': common.import_path_main('run'),
'run_obj': lkmc.import_path.import_path_main('run'),
'test_id': path_relative_root,
}
if my_path_properties['receives_signal']:
run_test_args['expected_exit_status'] = 128 - my_path_properties['exit_status']
error = thread_pool.submit(run_test_args)
error = my_thread_pool.submit(run_test_args)
if error is not None:
if self.env['quit_on_fail']:
raise common.ExitLoop()
except common.ExitLoop:
pass
error = thread_pool.get_error()
error = my_thread_pool.get_error()
if error is not None:
print(error)
return 1

View File

@@ -1,8 +1,8 @@
#!/usr/bin/env python3
import common
import lkmc.import_path
test_user_mode = common.import_path_relative_root('test-user-mode')
test_user_mode = lkmc.import_path.import_path_relative_root('test-user-mode')
class Main(test_user_mode.Main):
def __init__(self):

View File

@@ -1,9 +1,9 @@
#!/usr/bin/env python3
import os
import sys
import common
import lkmc.import_path
class Main(common.TestCliFunction):
def __init__(self):
@@ -13,7 +13,7 @@ https://github.com/cirosantilli/linux-kernel-module-cheat#test-userland-in-full-
'''
)
def timed_main(self):
run = common.import_path_main('run')
run = lkmc.import_path.import_path_main('run')
run_args = self.get_common_args()
run_args['eval_after'] = './test_all.sh;{};'.format(self.env['userland_quit_cmd'])
self.run_test(run, run_args)

View File

@@ -1,7 +1,8 @@
#!/usr/bin/env python3
import common
from shell_helpers import LF
import common
import lkmc.import_path
class Main(common.LkmcCliFunction):
def __init__(self):
@@ -14,7 +15,7 @@ More information at: https://github.com/cirosantilli/linux-kernel-module-cheat#t
def timed_main(self):
args = self.get_common_args()
run = common.import_path_main('run')
run = lkmc.import_path.import_path_main('run')
if self.env['emulator'] == 'gem5':
args['trace'] = 'Exec,-ExecSymbol,-ExecMicro'
run(**args)
@@ -23,7 +24,7 @@ More information at: https://github.com/cirosantilli/linux-kernel-module-cheat#t
run_args['trace'] = 'exec_tb'
run_args['quit_after_boot'] = True
run(**run_args)
qemu_trace2txt = common.import_path_main('qemu-trace2txt')
qemu_trace2txt = lkmc.import_path.import_path_main('qemu-trace2txt')
qemu_trace2txt(**args)
# Instruction count.
# We could put this on a separate script, but it just adds more arch boilerplate to a new script.

View File

@@ -10,6 +10,7 @@ now...
import os
import common
import lkmc.import_path
from shell_helpers import LF
class Main(common.LkmcCliFunction):

View File

@@ -7,10 +7,7 @@
ENTRY
.bss
output: .skip 16
#define t
output: .skip 16
.data
addps_input0: .float 1.5, 2.5, 3.5, 4.5
addps_input1: .float 5.5, 6.5, 7.5, 8.5