From 8509f17a8476396f75e7d71a5f6ea456790b3bf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ciro=20Santilli=20=E5=85=AD=E5=9B=9B=E4=BA=8B=E4=BB=B6=20?= =?UTF-8?q?=E6=B3=95=E8=BD=AE=E5=8A=9F?= Date: Sun, 5 May 2019 00:00:00 +0000 Subject: [PATCH] userland: native testing --- README.adoc | 60 +++++++++++++------ common.py | 39 ++++++------ run | 11 +++- test-user-mode | 14 +++-- test-user-mode-in-tree | 31 ++++++++++ .../arch/aarch64/c/freestanding/linux/test | 1 + userland/arch/aarch64/c/freestanding/test | 1 + userland/arch/aarch64/c/linux/test | 1 + userland/arch/aarch64/c/test | 1 + userland/arch/aarch64/freestanding/linux/test | 1 + userland/arch/aarch64/freestanding/test | 1 + userland/arch/aarch64/test | 1 + userland/arch/arm/c/freestanding/linux/test | 1 + userland/arch/arm/c/freestanding/test | 1 + userland/arch/arm/c/test | 1 + userland/arch/arm/freestanding/linux/test | 1 + userland/arch/arm/freestanding/test | 1 + userland/arch/arm/linux/test | 1 + userland/arch/arm/test | 1 + userland/arch/test | 1 + .../arch/x86_64/c/freestanding/linux/test | 1 + userland/arch/x86_64/c/freestanding/test | 1 + userland/arch/x86_64/c/test | 1 + userland/arch/x86_64/freestanding/linux/test | 1 + userland/arch/x86_64/freestanding/test | 1 + userland/arch/x86_64/test | 1 + userland/c/test | 1 + userland/cpp/test | 1 + userland/gcc/test | 1 + userland/kernel_modules/test | 1 + userland/libs/eigen/test | 1 + userland/libs/libdrm/test | 1 + userland/libs/openblas/test | 1 + userland/libs/test | 1 + userland/linux/test | 1 + userland/posix/test | 1 + userland/test | 1 + 37 files changed, 142 insertions(+), 45 deletions(-) create mode 100755 test-user-mode-in-tree create mode 120000 userland/arch/aarch64/c/freestanding/linux/test create mode 120000 userland/arch/aarch64/c/freestanding/test create mode 120000 userland/arch/aarch64/c/linux/test create mode 120000 userland/arch/aarch64/c/test create mode 120000 userland/arch/aarch64/freestanding/linux/test create mode 120000 userland/arch/aarch64/freestanding/test create mode 120000 userland/arch/aarch64/test create mode 120000 userland/arch/arm/c/freestanding/linux/test create mode 120000 userland/arch/arm/c/freestanding/test create mode 120000 userland/arch/arm/c/test create mode 120000 userland/arch/arm/freestanding/linux/test create mode 120000 userland/arch/arm/freestanding/test create mode 120000 userland/arch/arm/linux/test create mode 120000 userland/arch/arm/test create mode 120000 userland/arch/test create mode 120000 userland/arch/x86_64/c/freestanding/linux/test create mode 120000 userland/arch/x86_64/c/freestanding/test create mode 120000 userland/arch/x86_64/c/test create mode 120000 userland/arch/x86_64/freestanding/linux/test create mode 120000 userland/arch/x86_64/freestanding/test create mode 120000 userland/arch/x86_64/test create mode 120000 userland/c/test create mode 120000 userland/cpp/test create mode 120000 userland/gcc/test create mode 120000 userland/kernel_modules/test create mode 120000 userland/libs/eigen/test create mode 120000 userland/libs/libdrm/test create mode 120000 userland/libs/openblas/test create mode 120000 userland/libs/test create mode 120000 userland/linux/test create mode 120000 userland/posix/test create mode 120000 userland/test diff --git a/README.adoc b/README.adoc index 9f5dfe4..f59359e 100644 --- a/README.adoc +++ b/README.adoc @@ -984,16 +984,20 @@ cd userland Source: link:userland/c/hello.c[]. -Build an entire directory: +Build an entire directory and test it: .... +cd userland ./build c +./test c .... -Build the current directory: +Build the current directory and test it: .... +cd userland/c ./build +./test .... Note however that if you run this from link:userland/[] toplevel, it would try to build the link:userland/libs/[] folder, which depends on certain libraries being installed on the host, e.g. <>. @@ -1005,21 +1009,10 @@ cd linux-kernel-module-cheat ./build --download-dependencies userland-host .... -The `build` scripts inside link:userland/[] are just symlinks to link:build-userland-in-tree[] which you can also use from toplevel as: +Pass custom compiler options: .... -./build-userland-in-tree -./build-userland-in-tree c -./build-userland-in-tree c/hello.c -./build-userland-in-tree userland/c/hello.c -.... - -which is in turn just a thin wrapper around link:build-userland[], so you can use any option supported by that script freely. - -Pass custom compiler options with: - -.... -./build-userland --ccflags='-foptimize-sibling-calls -foptimize-strlen' --force-rebuild +./build --ccflags='-foptimize-sibling-calls -foptimize-strlen' --force-rebuild .... Here we used `--force-rebuild` to force rebuild since the sources weren't modified since the last build. @@ -1027,19 +1020,48 @@ Here we used `--force-rebuild` to force rebuild since the sources weren't modifi Some CLI options have more specialized flags, e.g. `-O` optimization level: .... -./build-userland --optimization-level 3 --force-rebuild +./build --optimization-level 3 --force-rebuild .... See also <> for `--static`. -Do a more clean out-of-tree build and run the program instead: +The `build` scripts inside link:userland/[] are just symlinks to link:build-userland-in-tree[] which you can also use from toplevel as: + +.... +./build-userland-in-tree +./build-userland-in-tree userland/c +./build-userland-in-tree userland/c/hello.c +.... + +`build-userland-in-tre` is in turn just a thin wrapper around link:build-userland[]: + +.... +./build-userland --gcc-which host --in-tree userland/c +.... + +So you can use any option supported by `build-userland` script freely with `build-userland-in-tree` and `build`. + +The situation is analogous for link:userland/test[], link:test-user-mode-in-tree[] and link:test-user-mode[]. + +Do a more clean out-of-tree build instead and run the program: .... ./build-userland --gcc-which host --userland-build-id host -"$(./getvar --userland-build-id host userland_build_dir)/hello.out" +./run --emulator native --userland userland/c/hello.c --userland-build-id host .... -Here we put the host executables in a separate <> to avoid conflict with Buildroot builds. +Here we: + +* put the host executables in a separate <> to avoid conflict with Buildroot builds. +* ran with the `--emulator native` option to run the program natively + +In this case you can debub the program with: + +.... +./run --debug-vm --emulator native --userland userland/c/hello.c --userland-build-id host +.... + +as shown at: <>, although direct GDB host usage works as well of course. ===== Userland setup getting started full system diff --git a/common.py b/common.py index a08e2e5..a013e34 100644 --- a/common.py +++ b/common.py @@ -116,10 +116,18 @@ consts['userland_out_exts'] = [ consts['config_file'] = os.path.join(consts['data_dir'], 'config.py') consts['magic_fail_string'] = b'lkmc_test_fail' consts['baremetal_lib_basename'] = 'lib' +consts['emulator_userland_only_short_to_long_dict'] = collections.OrderedDict([ + ('n', 'native'), +]) +consts['all_userland_only_emulators'] = set() +for key in consts['emulator_userland_only_short_to_long_dict']: + consts['all_userland_only_emulators'].add(key) + consts['all_userland_only_emulators'].add(consts['emulator_userland_only_short_to_long_dict'][key]) consts['emulator_short_to_long_dict'] = collections.OrderedDict([ ('q', 'qemu'), ('g', 'gem5'), ]) +consts['emulator_short_to_long_dict'].update(consts['emulator_userland_only_short_to_long_dict']) consts['all_long_emulators'] = [consts['emulator_short_to_long_dict'][k] for k in consts['emulator_short_to_long_dict']] consts['emulator_choices'] = set() for key in consts['emulator_short_to_long_dict']: @@ -495,7 +503,7 @@ and then inspect separate outputs later in different output directories. self.add_argument( '--all-emulators', default=False, help='''\ -Run action for all supported --emulators emulators. Ignore --emulators. +Run action for all supported emulators. Ignore --emulator. '''.format(emulators_string) ) self.add_argument( @@ -508,6 +516,11 @@ Run action for all supported --emulators emulators. Ignore --emulators. help='''\ Emulator to use. If given multiple times, semantics are similar to --arch. Valid emulators: {} + +"native" means running natively on host. It is only supported for userland, +and you must have built the program for native running, see: +https://github.com/cirosantilli/linux-kernel-module-cheat#userland-setup-getting-started-natively +Incompatible archs are skipped. '''.format(emulators_string) ) self._is_common = False @@ -1044,6 +1057,11 @@ lunch aosp_{}-eng for arch in real_archs: if arch in env['arch_short_to_long_dict']: arch = env['arch_short_to_long_dict'][arch] + if emulator == 'native': + if env['userland'] is None: + raise Exception('Emulator only supported in user mode: {}'.format(emulator)) + if arch != env['host_arch']: + continue if self.is_arch_supported(arch): if not env['dry_run']: start_time = time.time() @@ -1150,22 +1168,7 @@ lunch aosp_{}-eng Convert a convenient shorthand user input string to paths of existing files in the source tree. - Input path file extensions are ignored. - - All the following input paths would be equivalent for - magic_in_dir == '/full/path/to/userland' - - - hello - - hello. - - hello.c - - hello.out - - userland/hello - - userland/hello. - - userland/hello.c - - userland/hello.out - - /full/path/to/userland/hello - - /full/path/to/userland/hello. - - /full/path/to/userland/hello.c + Ensure that the path lies inside source_tree_root. Multiple matches may happen if multiple multiple exts files exist. E.g., after an in-tree build, in_path='hello' and exts=['.c', '.out'] @@ -1183,7 +1186,7 @@ lunch aosp_{}-eng - /full/path/to/userland/arch Note however that this potentially prevents differentiation between - files and directories: e.g. if you had both a file arch.c and a directory arch, + files and directories: e.g. if you had both a file arch.c and a directory arch/, and exts=['', '.c'], then both would get matched. ''' in_path = os.path.abspath(in_path) diff --git a/run b/run index b930171..ca980b8 100755 --- a/run +++ b/run @@ -39,8 +39,13 @@ Ctrl +C kills the QEMU simulator instead of being passed to the guest. ''' ) self.add_argument( - '-D', '--debug-vm', default=False, - help='Run GDB on the emulator itself.' + '-D', + '--debug-vm', + default=False, + help='''\ +Run GDB on the emulator itself. +For --emulator native, this debugs the target program. +''' ) self.add_argument( '--debug-vm-args', default='', @@ -669,7 +674,7 @@ Extra options to append at the end of the emulator command line. ]) cmd.extend(extra_emulator_args) cmd.extend(self.env['extra_emulator_args']) - if self.env['emulator'] == 'qemu' and self.env['userland']: + if self.env['userland'] and self.env['emulator'] in ('qemu', 'native'): # The program and arguments must come at the every end of the CLI. cmd.extend([self.resolve_userland_executable(self.env['userland']), LF]) if self.env['userland_args'] is not None: diff --git a/test-user-mode b/test-user-mode index 9cb058b..77bf7c3 100755 --- a/test-user-mode +++ b/test-user-mode @@ -8,13 +8,17 @@ import example_properties from thread_pool import ThreadPool class Main(common.TestCliFunction): - def __init__(self): - super().__init__( - description='''\ + def __init__(self, *args, **kwargs): + if not 'description' in kwargs: + kwargs['description'] = '''\ https://github.com/cirosantilli/linux-kernel-module-cheat#user-mode-tests TODO: expose all userland relevant ./run args here as well somehow. -''', - ) +''' + if not 'defaults' in kwargs: + kwargs['defaults'] = {} + if not 'userland' in kwargs['defaults']: + kwargs['defaults']['userland'] = '' + super().__init__(*args, **kwargs) self.add_argument( 'tests', nargs='*', diff --git a/test-user-mode-in-tree b/test-user-mode-in-tree new file mode 100755 index 0000000..b5c1fa0 --- /dev/null +++ b/test-user-mode-in-tree @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 + +import imp +import os +import subprocess + +git_root = subprocess.check_output([ + 'git', + 'rev-parse', + '--show-toplevel', +]).decode().rstrip() +test_user_mode = imp.load_source( + 'test_user_mode', + os.path.join(git_root, 'test-user-mode') +) + +class Main(test_user_mode.Main): + def __init__(self): + super().__init__( + description='''\ +https://github.com/cirosantilli/linux-kernel-module-cheat#userland-setup-getting-started-natively +''', + defaults={ + 'emulators': ['native'], + 'in_tree': True, + 'tests': ['.'], + } + ) + +if __name__ == '__main__': + Main().cli() diff --git a/userland/arch/aarch64/c/freestanding/linux/test b/userland/arch/aarch64/c/freestanding/linux/test new file mode 120000 index 0000000..419df4f --- /dev/null +++ b/userland/arch/aarch64/c/freestanding/linux/test @@ -0,0 +1 @@ +../test \ No newline at end of file diff --git a/userland/arch/aarch64/c/freestanding/test b/userland/arch/aarch64/c/freestanding/test new file mode 120000 index 0000000..419df4f --- /dev/null +++ b/userland/arch/aarch64/c/freestanding/test @@ -0,0 +1 @@ +../test \ No newline at end of file diff --git a/userland/arch/aarch64/c/linux/test b/userland/arch/aarch64/c/linux/test new file mode 120000 index 0000000..419df4f --- /dev/null +++ b/userland/arch/aarch64/c/linux/test @@ -0,0 +1 @@ +../test \ No newline at end of file diff --git a/userland/arch/aarch64/c/test b/userland/arch/aarch64/c/test new file mode 120000 index 0000000..419df4f --- /dev/null +++ b/userland/arch/aarch64/c/test @@ -0,0 +1 @@ +../test \ No newline at end of file diff --git a/userland/arch/aarch64/freestanding/linux/test b/userland/arch/aarch64/freestanding/linux/test new file mode 120000 index 0000000..419df4f --- /dev/null +++ b/userland/arch/aarch64/freestanding/linux/test @@ -0,0 +1 @@ +../test \ No newline at end of file diff --git a/userland/arch/aarch64/freestanding/test b/userland/arch/aarch64/freestanding/test new file mode 120000 index 0000000..419df4f --- /dev/null +++ b/userland/arch/aarch64/freestanding/test @@ -0,0 +1 @@ +../test \ No newline at end of file diff --git a/userland/arch/aarch64/test b/userland/arch/aarch64/test new file mode 120000 index 0000000..419df4f --- /dev/null +++ b/userland/arch/aarch64/test @@ -0,0 +1 @@ +../test \ No newline at end of file diff --git a/userland/arch/arm/c/freestanding/linux/test b/userland/arch/arm/c/freestanding/linux/test new file mode 120000 index 0000000..419df4f --- /dev/null +++ b/userland/arch/arm/c/freestanding/linux/test @@ -0,0 +1 @@ +../test \ No newline at end of file diff --git a/userland/arch/arm/c/freestanding/test b/userland/arch/arm/c/freestanding/test new file mode 120000 index 0000000..419df4f --- /dev/null +++ b/userland/arch/arm/c/freestanding/test @@ -0,0 +1 @@ +../test \ No newline at end of file diff --git a/userland/arch/arm/c/test b/userland/arch/arm/c/test new file mode 120000 index 0000000..419df4f --- /dev/null +++ b/userland/arch/arm/c/test @@ -0,0 +1 @@ +../test \ No newline at end of file diff --git a/userland/arch/arm/freestanding/linux/test b/userland/arch/arm/freestanding/linux/test new file mode 120000 index 0000000..419df4f --- /dev/null +++ b/userland/arch/arm/freestanding/linux/test @@ -0,0 +1 @@ +../test \ No newline at end of file diff --git a/userland/arch/arm/freestanding/test b/userland/arch/arm/freestanding/test new file mode 120000 index 0000000..419df4f --- /dev/null +++ b/userland/arch/arm/freestanding/test @@ -0,0 +1 @@ +../test \ No newline at end of file diff --git a/userland/arch/arm/linux/test b/userland/arch/arm/linux/test new file mode 120000 index 0000000..419df4f --- /dev/null +++ b/userland/arch/arm/linux/test @@ -0,0 +1 @@ +../test \ No newline at end of file diff --git a/userland/arch/arm/test b/userland/arch/arm/test new file mode 120000 index 0000000..419df4f --- /dev/null +++ b/userland/arch/arm/test @@ -0,0 +1 @@ +../test \ No newline at end of file diff --git a/userland/arch/test b/userland/arch/test new file mode 120000 index 0000000..419df4f --- /dev/null +++ b/userland/arch/test @@ -0,0 +1 @@ +../test \ No newline at end of file diff --git a/userland/arch/x86_64/c/freestanding/linux/test b/userland/arch/x86_64/c/freestanding/linux/test new file mode 120000 index 0000000..419df4f --- /dev/null +++ b/userland/arch/x86_64/c/freestanding/linux/test @@ -0,0 +1 @@ +../test \ No newline at end of file diff --git a/userland/arch/x86_64/c/freestanding/test b/userland/arch/x86_64/c/freestanding/test new file mode 120000 index 0000000..419df4f --- /dev/null +++ b/userland/arch/x86_64/c/freestanding/test @@ -0,0 +1 @@ +../test \ No newline at end of file diff --git a/userland/arch/x86_64/c/test b/userland/arch/x86_64/c/test new file mode 120000 index 0000000..419df4f --- /dev/null +++ b/userland/arch/x86_64/c/test @@ -0,0 +1 @@ +../test \ No newline at end of file diff --git a/userland/arch/x86_64/freestanding/linux/test b/userland/arch/x86_64/freestanding/linux/test new file mode 120000 index 0000000..419df4f --- /dev/null +++ b/userland/arch/x86_64/freestanding/linux/test @@ -0,0 +1 @@ +../test \ No newline at end of file diff --git a/userland/arch/x86_64/freestanding/test b/userland/arch/x86_64/freestanding/test new file mode 120000 index 0000000..419df4f --- /dev/null +++ b/userland/arch/x86_64/freestanding/test @@ -0,0 +1 @@ +../test \ No newline at end of file diff --git a/userland/arch/x86_64/test b/userland/arch/x86_64/test new file mode 120000 index 0000000..419df4f --- /dev/null +++ b/userland/arch/x86_64/test @@ -0,0 +1 @@ +../test \ No newline at end of file diff --git a/userland/c/test b/userland/c/test new file mode 120000 index 0000000..419df4f --- /dev/null +++ b/userland/c/test @@ -0,0 +1 @@ +../test \ No newline at end of file diff --git a/userland/cpp/test b/userland/cpp/test new file mode 120000 index 0000000..419df4f --- /dev/null +++ b/userland/cpp/test @@ -0,0 +1 @@ +../test \ No newline at end of file diff --git a/userland/gcc/test b/userland/gcc/test new file mode 120000 index 0000000..419df4f --- /dev/null +++ b/userland/gcc/test @@ -0,0 +1 @@ +../test \ No newline at end of file diff --git a/userland/kernel_modules/test b/userland/kernel_modules/test new file mode 120000 index 0000000..419df4f --- /dev/null +++ b/userland/kernel_modules/test @@ -0,0 +1 @@ +../test \ No newline at end of file diff --git a/userland/libs/eigen/test b/userland/libs/eigen/test new file mode 120000 index 0000000..419df4f --- /dev/null +++ b/userland/libs/eigen/test @@ -0,0 +1 @@ +../test \ No newline at end of file diff --git a/userland/libs/libdrm/test b/userland/libs/libdrm/test new file mode 120000 index 0000000..419df4f --- /dev/null +++ b/userland/libs/libdrm/test @@ -0,0 +1 @@ +../test \ No newline at end of file diff --git a/userland/libs/openblas/test b/userland/libs/openblas/test new file mode 120000 index 0000000..419df4f --- /dev/null +++ b/userland/libs/openblas/test @@ -0,0 +1 @@ +../test \ No newline at end of file diff --git a/userland/libs/test b/userland/libs/test new file mode 120000 index 0000000..419df4f --- /dev/null +++ b/userland/libs/test @@ -0,0 +1 @@ +../test \ No newline at end of file diff --git a/userland/linux/test b/userland/linux/test new file mode 120000 index 0000000..419df4f --- /dev/null +++ b/userland/linux/test @@ -0,0 +1 @@ +../test \ No newline at end of file diff --git a/userland/posix/test b/userland/posix/test new file mode 120000 index 0000000..419df4f --- /dev/null +++ b/userland/posix/test @@ -0,0 +1 @@ +../test \ No newline at end of file diff --git a/userland/test b/userland/test new file mode 120000 index 0000000..2eb7181 --- /dev/null +++ b/userland/test @@ -0,0 +1 @@ +../test-user-mode-in-tree \ No newline at end of file