From abb67c14b845cdb06bc7323b239bdb0763c6e35e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ciro=20Santilli=20=E5=85=AD=E5=9B=9B=E4=BA=8B=E4=BB=B6=20?= =?UTF-8?q?=E6=B3=95=E8=BD=AE=E5=8A=9F?= Date: Sun, 5 May 2019 00:00:00 +0000 Subject: [PATCH] preparing test_user_mode, need to generalize stuff as usual --- README.adoc | 11 ++++--- build-userland | 75 ++++++++++++++++++++++++++++--------------- common.py | 51 +++++++++++++++++++---------- example_properties.py | 25 +++++++++++++++ file_properties.py | 14 -------- test-build-userland | 42 ++++++++++++++++++++++++ test-user-mode | 16 +++++---- thread_pool.py | 6 ++++ 8 files changed, 173 insertions(+), 67 deletions(-) create mode 100644 example_properties.py delete mode 100644 file_properties.py create mode 100755 test-build-userland diff --git a/README.adoc b/README.adoc index 99a5212..4797f78 100644 --- a/README.adoc +++ b/README.adoc @@ -1011,6 +1011,7 @@ The `build` scripts inside link:userland/[] are just symlinks to link:build-user ./build-userland-in-tree ./build-userland-in-tree c ./build-userland-in-tree c/hello.c +./build-userland-in-tree userland/c/hello.c .... which is in turn just a thin wrapper around link:build-userland[], so you can use any option supported by that script freely. @@ -3641,7 +3642,7 @@ So programs that rely on those libraries might not compile as GCC can't find the For example, if we try to build <> statically: .... -./build-userland --has-package openblas --static -- libs/openblas/hello +./build-userland --has-package openblas --static -- userland/libs/openblas/hello.c .... it fails with: @@ -8182,7 +8183,7 @@ DRM / DRI is the new interface that supersedes `fbdev`: .... ./build-buildroot --config 'BR2_PACKAGE_LIBDRM=y' -./build-userland --has-package libdrm -- libs/libdrm/modeset +./build-userland --has-package libdrm -- userland/libs/libdrm/modeset.c ./run --eval-after './libs/libdrm/modeset.out' --graphic .... @@ -9931,7 +9932,7 @@ Buildroot supports it, which makes everything just trivial: .... ./build-buildroot --config 'BR2_PACKAGE_OPENBLAS=y' -./build-userland --has-package openblas -- libs/openblas/hello +./build-userland --has-package openblas -- userland/libs/openblas/hello.c ./run --eval-after './libs/openblas/hello.out; echo $?' .... @@ -9971,7 +9972,7 @@ Header only linear algebra library with a mainline Buildroot package: .... ./build-buildroot --config 'BR2_PACKAGE_EIGEN=y' -./build-userland --has-package eigen -- libs/eigen/hello +./build-userland --has-package eigen -- userland/libs/eigen/hello.cpp .... Just create an array and print it: @@ -11622,7 +11623,7 @@ TODO: review this section, make a more controlled userland experiment with <> `system.cpu.numCycles` cycle count with the link:https://en.wikipedia.org/wiki/Time_Stamp_Counter[x86 `rdtsc` instruction] that is supposed to do the same thing: .... -./build-userland --static arch/x86_64/c/rdtsc +./build-userland --static userland/arch/x86_64/c/rdtsc.c ./run --eval './arch/x86_64/c/rdtsc.out;m5 exit;' --emulator gem5 ./gem5-stat .... diff --git a/build-userland b/build-userland index cd485e3..5f06511 100755 --- a/build-userland +++ b/build-userland @@ -47,7 +47,7 @@ building with the host toolchain. default=False, help='''\ Treat targets as relative to the current working directory. If the current working -directory is outside of userland/, use userland/ instead. +directory is outside of userland/, this has no effect. ''', ) self.add_argument( @@ -141,17 +141,46 @@ Default: build all examples that have their package dependencies met, e.g.: return self.env['userland_source_dir'] def _get_targets(self): + ''' + Resolve target_relative_cwd and default targets. + ''' if self.env['_args_given']['targets']: targets = self.env['targets'] if self.env['target_relative_cwd']: cwd = self._get_cwd() - targets = [os.path.join(cwd, target) for target in targets] - return targets + for target in targets: + yield os.path.join(cwd, target) + else: + for target in targets: + yield target else: if self.env['target_relative_cwd']: - return [self._get_cwd()] + yield self._get_cwd() else: - return [self.env['userland_source_dir']] + yield self.env['userland_source_dir'] + + def _walk_targets(self): + ''' + Walk existing directories and files pointed to by the targets + from the command line from under the userland/ source tree. + + This may include outputs of in-tree builds. + + File extensions are ignored, e.g.: + + c/hello + + will find both: + + c/hello.c + c/hello.out + ''' + for target in self._get_targets(): + target = self.resolve_userland_source(target) + noext, ext = os.path.splitext(filename) + + for path, in_dirnames, in_filenames in self.sh.walk(target): + yield path, in_dirnames, in_filenames def build(self): build_dir = self.get_build_dir() @@ -225,15 +254,13 @@ Default: build all examples that have their package dependencies met, e.g.: 'openblas': {}, } rootdir_abs_len = len(self.env['userland_source_dir']) - thread_pool = ThreadPool( + with ThreadPool( self._build_one, nthreads=self.env['nproc'], - ) - class ExitLoop(Exception): pass - try: - for target in self._get_targets(): - target = self.resolve_userland_source(target) - for path, in_dirnames, in_filenames in self.sh.walk(target): + ) as thread_pool: + class ExitLoop(Exception): pass + try: + for path, in_dirnames, in_filenames in self._walk_targets(): in_dirnames.sort() in_filenames.sort() path_abs = os.path.abspath(path) @@ -348,9 +375,8 @@ Default: build all examples that have their package dependencies met, e.g.: }) if error is not None: raise ExitLoop() - except ExitLoop: - pass - error = thread_pool.join() + except ExitLoop: + pass if error is not None: print(error) return 1 @@ -364,16 +390,15 @@ Default: build all examples that have their package dependencies met, e.g.: def clean(self): if self.env['in_tree']: - for target in self._get_targets(): - if os.path.exists(target): - for path, dirnames, filenames in os.walk(target): - filenames.sort() - dirnames.sort() - for filename in filenames: - if os.path.splitext(filename)[1] in self.env['userland_out_exts']: - self.sh.rmrf(os.path.join(path, filename)) - else: - raise Exception('Path does not exist: ' + target) + for path, dirnames, filenames in self._walk_targets(): + filenames.sort() + dirnames.sort() + for filename in filenames: + noext, ext = os.path.splitext(filename) + for out_ext in self.env['userland_out_exts']: + out_path = os.path.join(path, noext + out_ext) + if os.path.exists(out_path): + self.sh.rmrf(out_path) else: self.sh.rmrf(self.get_build_dir()) diff --git a/common.py b/common.py index 4d51771..2b1c9b3 100644 --- a/common.py +++ b/common.py @@ -784,7 +784,7 @@ Valid emulators: {} env['baremetal'], env['baremetal_source_dir'], env['baremetal_build_dir'], - env['baremetal_build_ext'], + [env['baremetal_build_ext']], ) source_path_noext = os.path.splitext(join( env['baremetal_source_dir'], @@ -1121,26 +1121,46 @@ lunch aosp_{}-eng ] ) - def resolve_source(self, in_path, magic_in_dir, in_exts): + def resolve_source(self, in_path, magic_in_dir, exts): ''' - Convert a path-like string to a source file to the full source path, - e.g. all follogin work and to do the same: + Convert a convenient shorthand user input string to paths of existing files + in the source tree. + + Input path file extensions are ignored. + + All the following input paths would be equivalent for + magic_in_dir == '/full/path/to/userland' - hello - hello. - hello.c + - hello.out - userland/hello - userland/hello. - userland/hello.c + - userland/hello.out - /full/path/to/userland/hello - /full/path/to/userland/hello. - /full/path/to/userland/hello.c - Also works on directories: + Multiple matches may happen if multiple multiple exts files exist. + E.g., after an in-tree build, in_path='hello' and exts=['.c', '.out'] + would match both: + + - userland/hello.c + - userland/hello.out + + If you also want directories to be matched, just add an empty string + `''` to exts, which leads all of the following to match the arch directory: - arch + - arch.c - userland/arch - /full/path/to/userland/arch + + Note however that this potentially prevents differentiation between + files and directories: e.g. if you had both a file arch.c and a directory arch, + and exts=['', '.c'], then both would get matched. ''' if os.path.isabs(in_path): return in_path @@ -1154,18 +1174,14 @@ lunch aosp_{}-eng ] for path in paths: name, ext = os.path.splitext(path) - if len(ext) > 1: - try_exts = [ext] - else: - try_exts = in_exts + [''] - for in_ext in try_exts: - path = name + in_ext + for try_ext in exts: + path = name + try_ext if os.path.exists(path): return path if not self.env['dry_run']: - raise Exception('Source file not found for input: ' + in_path) + raise Exception('No file not found for input: ' + in_path) - def resolve_executable(self, in_path, magic_in_dir, magic_out_dir, out_ext): + def resolve_executable(self, in_path, magic_in_dir, magic_out_dir, out_exts): if os.path.isabs(in_path): return in_path else: @@ -1177,9 +1193,10 @@ lunch aosp_{}-eng ) ] for path in paths: - path = os.path.splitext(path)[0] + out_ext - if os.path.exists(path): - return path + for out_ext in out_exts: + path = os.path.splitext(path)[0] + out_ext + if os.path.exists(path): + return path if not self.env['dry_run']: raise Exception('Executable file not found. Tried:\n' + '\n'.join(paths)) @@ -1192,7 +1209,7 @@ lunch aosp_{}-eng path, self.env['userland_source_dir'], self.env['userland_build_dir'], - self.env['userland_build_ext'], + [self.env['userland_build_ext']], ) def resolve_userland_source(self, path): diff --git a/example_properties.py b/example_properties.py new file mode 100644 index 0000000..4a0d85b --- /dev/null +++ b/example_properties.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 + +class ExampleProperties: + ''' + Encodes properties of userland and baremetal examples. + For directories, it applies to all files under the directory. + Used to determine how to build and test the examples. + ''' + def __init__( + exit_status=0, + interactive=False, + more_than_1s=False, + ): + self.exit_status = exit_status + self.interactive = interactive + self.more_than_1s = more_than_1s + + def should_be_tested(self): + return \ + not self.interactive and \ + not self.more_than_1s + +executable_properties = { + 'userland/arch/x86_64/c/ring0.c': ExecutableProperties(exits_nonzero=True), +} diff --git a/file_properties.py b/file_properties.py deleted file mode 100644 index a56ecac..0000000 --- a/file_properties.py +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env python3 - -class FileProperties: - def __init__( - more_than_1s=False, - exits_nonzero=False, - interactive=False, - ): - self.more_than_1s = more_than_1s - self.exits_nonzero = exits_nonzero - -executable_properties = { - 'userland/arch/x86_64/c/ring0.c': ExecutableProperties(exits_nonzero=True), -} diff --git a/test-build-userland b/test-build-userland new file mode 100755 index 0000000..1af5ae3 --- /dev/null +++ b/test-build-userland @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +set -eux + +./build-userland +./build-userland --clean +./build-userland c +./build-userland --clean +./build-userland userland/c +./build-userland --clean +./build-userland userland/c/hello +./build-userland --clean +./build-userland userland/c/hello.c +./build-userland --clean +./build-userland userland/c/hello.out +./build-userland --clean + +./build-userland --in-tree +./build-userland --in-tree --clean +./build-userland --in-tree c +./build-userland --in-tree --clean +./build-userland --in-tree userland/c +./build-userland --in-tree --clean +./build-userland --in-tree userland/c/hello +./build-userland --in-tree --clean +./build-userland --in-tree userland/c/hello.c +./build-userland --in-tree --clean +./build-userland --in-tree userland/c/hello.out +./build-userland --in-tree --clean + +cd userland + +./build-userland --in-tree +./build-userland --in-tree --clean +./build-userland --in-tree c +./build-userland --in-tree --clean +./build-userland --in-tree c/hello +./build-userland --in-tree --clean +./build-userland --in-tree c/hello.c +./build-userland --in-tree --clean +./build-userland --in-tree c/hello.out +./build-userland --in-tree --clean diff --git a/test-user-mode b/test-user-mode index 651a770..a334a5a 100755 --- a/test-user-mode +++ b/test-user-mode @@ -27,7 +27,7 @@ If given, run only the given tests. Otherwise, run all tests. if self.env['emulator'] == 'gem5': run_args['userland_build_id'] = 'static' if self.env['tests'] == []: - sources = [ + tests = [ 'add.c', 'hello.c', 'hello_cpp.cpp', @@ -48,12 +48,16 @@ If given, run only the given tests. Otherwise, run all tests. for arch_source in arch_sources ] - sources.extend(arch_sources) + tests.extend(arch_sources) else: - sources = self.env['tests'] - for source in sources: - run_args['userland'] = source - self.run_test(run, run_args, source) + tests = self.env['tests'] + for test_dir_or_file in tests: + for test in self.sh.walk(self.resolve_userland_source(test_dir_or_file)): + + consts['userland_in_exts'] = [ + + run_args['userland'] = test + self.run_test(run, run_args) if __name__ == '__main__': Main().cli() diff --git a/thread_pool.py b/thread_pool.py index c88240d..9f22e6a 100644 --- a/thread_pool.py +++ b/thread_pool.py @@ -78,6 +78,12 @@ class ThreadPool: self.threads.append(thread) thread.start() + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + self.join() + def submit(self, work): ''' Submit work. Block if there is already enough work scheduled (~nthreads).