From eba97f9cef8cb7a8e50dfd754bdd0f2c7edeaeda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ciro=20Santilli=20=E5=85=AD=E5=9B=9B=E4=BA=8B=E4=BB=B6=20?= =?UTF-8?q?=E6=B3=95=E8=BD=AE=E5=8A=9F?= Date: Sun, 5 May 2019 00:00:00 +0000 Subject: [PATCH] userland: try to make userland executable selection saner Only allow existing files to be built, stop extension expansion madness. cli_function: get_cli print booleans properly, was printing without --no- for negations. --- README.adoc | 190 +++++++++++++++++------------ build-userland | 267 ++++++++++++++++++----------------------- build-userland-in-tree | 1 - cli_function.py | 8 +- common.py | 190 +++++++++++++---------------- path_properties.py | 185 +++++++++++++++++++++------- run | 29 +++-- shell_helpers.py | 5 +- test-baremetal | 2 +- test-build-userland | 97 +++++++++------ test-user-mode | 55 +++++---- 11 files changed, 570 insertions(+), 459 deletions(-) diff --git a/README.adoc b/README.adoc index 71196e1..40d562c 100644 --- a/README.adoc +++ b/README.adoc @@ -350,7 +350,7 @@ Lol! We can also test our hacked glibc on <> with: .... -./run --userland c/hello +./run --userland userland/c/hello.c .... I just noticed that this is actually a good way to develop glibc for other archs. @@ -427,7 +427,7 @@ Finally, rebuild Binutils, userland and test our program with <>. - You can install those libraries and do the build in one go with: .... @@ -1043,7 +1041,7 @@ The `build` scripts inside link:userland/[] are just symlinks to link:build-user So you can use any option supported by `build-userland` script freely with `build-userland-in-tree` and `build`. -The situation is analogous for link:userland/test[], link:test-user-mode-in-tree[] and link:test-user-mode[]. +The situation is analogous for link:userland/test[], link:test-user-mode-in-tree[] and link:test-user-mode[], which are further documented at: <>. Do a more clean out-of-tree build instead and run the program: @@ -1123,7 +1121,7 @@ For example, to run link:baremetal/hello.c[] in QEMU do: .... ./build --arch aarch64 --download-dependencies qemu-baremetal -./run --arch aarch64 --baremetal hello +./run --arch aarch64 --baremetal baremetal/hello.c .... The terminal prints: @@ -1135,7 +1133,7 @@ hello Now let's run link:baremetal/arch/aarch64/add.S[]: .... -./run --arch aarch64 --baremetal arch/aarch64/add +./run --arch aarch64 --baremetal baremetal/arch/aarch64/add.S .... This time, the terminal does not print anything, which indicates success. @@ -1145,7 +1143,7 @@ If you look into the source, you will see that we just have an assertion there. You can see a sample assertion fail in link:baremetal/interactive/assert_fail.c[]: .... -./run --arch aarch64 --baremetal interactive/assert_fail +./run --arch aarch64 --baremetal baremetal/interactive/assert_fail.c .... and the terminal contains: @@ -1161,7 +1159,7 @@ and the exit status of our script is 1: echo $? .... -To modify a baremetal program, simply edit the file, .g. +To modify a baremetal program, simply edit the file, e.g. .... vim baremetal/hello.c @@ -1170,11 +1168,11 @@ vim baremetal/hello.c and rebuild: .... -./build --arch aarch64 --download-dependencies qemu-baremetal -./run --arch aarch64 --baremetal hello +./build-baremetal --arch aarch64 +./run --arch aarch64 --baremetal baremetal/hello.c .... -`./build qemu-baremetal` had called link:build-baremetal[] for us previously, in addition to its requirements. +`./build qemu-baremetal` that we run previously is only needed for the initial build. That script calls link:build-baremetal[] for us, in addition to building prerequisites such as QEMU and crosstool-NG. `./build-baremetal` uses crosstool-NG, and so it must be preceded by link:build-crosstool-ng[], which `./build qemu-baremetal` also calls. @@ -1188,14 +1186,14 @@ Alternatively, for the sake of tab completion, we also accept relative paths ins Absolute paths however are used as is and must point to the actual executable: .... -./run --arch aarch64 --baremetal "$(./getvar --arch aarch64 baremetal_build_dir)/exit.elf" +./run --arch aarch64 --baremetal "$(./getvar --arch aarch64 baremetal_build_dir)/hello.elf" .... To use gem5 instead of QEMU do: .... ./build --download-dependencies gem5-baremetal -./run --arch aarch64 --baremetal interactive/prompt --emulator gem5 +./run --arch aarch64 --baremetal baremetal/hello.c --emulator gem5 .... and then <> open a shell with: @@ -1207,7 +1205,7 @@ and then <> open a shell with: Or as usual, <> users can do both in one go with: .... -./run --arch aarch64 --baremetal interactive/prompt --emulator gem5 --tmux +./run --arch aarch64 --baremetal baremetal/hello.c --emulator gem5 --tmux .... TODO: the carriage returns are a bit different than in QEMU, see: <>. @@ -1215,8 +1213,8 @@ TODO: the carriage returns are a bit different than in QEMU, see: <> just works, right? ./run \ --arch aarch64 \ --gdb-wait \ - --userland c/print_argv \ + --userland userland/c/print_argv.c \ --userland-args 'asdf "qw er"' \ ; .... @@ -3495,7 +3493,7 @@ and on another shell: .... ./run-gdb \ --arch aarch64 \ - --userland c/print_argv \ + --userland userland/c/print_argv.c \ main \ ; .... @@ -3506,7 +3504,7 @@ Or alternatively, if you are using <>, do everything in one go with: ./run \ --arch aarch64 \ --gdb \ - --userland c/print_argv \ + --userland userland/c/print_argv.c \ --userland-args 'asdf "qw er"' \ ; .... @@ -3538,6 +3536,8 @@ This script skips a manually configured list of tests, notably: * tests that take perceptible ammounts of time * known bugs we didn't have time to fix ;-) +Tests under link:userland/libs/[] depend on certain libraries being available on the target, e.g. <> for link:userland/libs/blas[]. They are not run by default, but can be enabled with `--has-package` and `--has-all-packages`. + The gem5 tests require building statically with build id `static`, see also: <>. TODO automate this better. See: <> for more useful testing tips. @@ -3570,7 +3570,7 @@ sudo apt-get install gcc-aarch64-linux-gnu qemu-system-aarch64 --arch aarch64 \ --qemu-which host --userland-build-id host \ - --userland c/print_argv \ + --userland userland/c/print_argv.c \ --userland-args 'asdf "qw er"' \ ; .... @@ -3603,7 +3603,7 @@ We don't have this failure for QEMU on an 18.04 host, only gem5. QEMU by default copies the host `uname` value. However, our scripts set it by default to our the latest Buildroot kernel version with QEMU's `-r` option, which is exposed as `--kernel-version`: .... -./run --arch aarch64 --kernel-version 4.18 --userland ./posix/uname +./run --arch aarch64 --kernel-version 4.18 --userland userland/posix/uname.c .... Source: link:userland/posix/uname.c[]. @@ -3633,8 +3633,8 @@ Reproduction: .... rm -f "$(./getvar buildroot_target_dir)/etc/ld.so.cache" -./run --userland c/hello -./run --userland c/hello --qemu-which host +./run --userland userland/c/hello.c +./run --userland userland/c/hello.c --qemu-which host .... Outcome: @@ -3672,7 +3672,7 @@ Example: ./run \ --arch aarch64 \ --static \ - --userland c/print_argv \ + --userland userland/c/print_argv.c \ --userland-args 'asdf "qw er"' \ ; .... @@ -3734,7 +3734,7 @@ So let's just play with some static ones: ./run \ --arch aarch64 \ --emulator gem5 \ - --userland c/print_argv \ + --userland userland/c/print_argv.c \ --userland-args 'asdf "qw er"' \ ; .... @@ -3749,14 +3749,14 @@ TODO: how to escape spaces on the command line arguments? --emulator gem5 \ --gdb-wait \ --static \ - --userland c/print_argv \ + --userland userland/c/print_argv.c \ --userland-args 'asdf "qw er"' \ ; ./run-gdb \ --arch aarch64 \ --emulator gem5 \ --static \ - --userland c/print_argv \ + --userland userland/c/print_argv.c \ main \ ; .... @@ -3766,7 +3766,7 @@ TODO: how to escape spaces on the command line arguments? As of gem5 7fa4c946386e7207ad5859e8ade0bbfc14000d91, the crappy `se.py` script does not forward the exit status of syscall emulation mode, you can test it with: .... -./run --dry-run --emulator gem5 --static --userland c/false +./run --dry-run --emulator gem5 --static --userland userland/c/false.c .... Source: link:userland/c/false.c[]. @@ -3890,6 +3890,28 @@ The same can be reproduced by copying the raw QEMU command and piping it through TODO: investigate further and then possibly post on QEMU mailing list. +===== QEMU user mode does not show errors + +Similarly to <>, QEMU error messages do not show at all through pipes. + +In particular, it does not say anything if you pass it a non-existing executable: + +.... +./build-userland --clean userland/c/hello.c +./run --userland userland/c/hello.c +echo $? +.... + +does not output anything, except for the `1` exit status. + +If you run however the raw command without a pipe manually, it shows a helpful error message: + +.... +Error while loading /path/to/linux-kernel-module-cheat/out/userland/default/x86_64/c/hello.out: No such file or directory +.... + +Tested in de77c62c091f6418e73b64e8a0a19639c587a103 + 1. + == Kernel module utilities === insmod @@ -10017,7 +10039,7 @@ Usage: --arch aarch64 \ --emulator gem5 \ --static \ - --userland cpp/bst_vs_heap \ + --userland userland/cpp/bst_vs_heap.cpp \ --userland-args='1000' \ ; ./bst-vs-heap --arch aarch64 > bst_vs_heap.dat @@ -11512,13 +11534,13 @@ This section will document ISA generic ideas. ISA specifics are documented on th The first example that you want to run for each arch is: .... -./run --userland arch//add +./run --userland userland/arch//add.S .... e.g.: .... -./run --userland arch/x86_64/add +./run --userland userland/arch/x86_64/add.S .... Sources: @@ -11541,7 +11563,7 @@ And then watch the assertion fail: .... ./build-userland -./run --userland arch/x86_64/add +./run --userland userland/arch/x86_64/add.S .... with error message: @@ -11598,8 +11620,8 @@ Such executables are called freestanding because they don't execute the glibc in In order to GDB step debug those executables, you will want to use `--no-continue`, e.g.: .... -./run --arch aarch64 --userland arch/aarch64/freestanding/hello --gdb-wait -./run-gdb --arch aarch64 --no-continue --userland arch/aarch64/freestanding/hello +./run --arch aarch64 --userland userland/arch/aarch64/freestanding/linux/hello.S --gdb-wait +./run-gdb --arch aarch64 --no-continue --userland userland/arch/aarch64/freestanding/linux/hello.S .... You are now left on the very first instruction of our tiny executable! @@ -11774,24 +11796,26 @@ Getting started at: <> === Baremetal GDB step debug -GDB step debug works on baremetal exactly as it does on the Linux kernel, except that is is even cooler here since we can easily control and understand every single instruction that is being run! +GDB step debug works on baremetal exactly as it does on the Linux kernel: <>. + +Except that is is even cooler here since we can easily control and understand every single instruction that is being run! For example, on the first shell: .... -./run --arch arm --baremetal interactive/prompt --gdb-wait +./run --arch arm --baremetal baremetal/hello.c --gdb-wait .... then on the second shell: .... -./run-gdb --arch arm --baremetal interactive/prompt -- main +./run-gdb --arch arm --baremetal baremetal/hello.c -- main .... Or if you are a <>, do everything in one go with: .... -./run --arch arm --baremetal interactive/prompt --gdb-wait --tmux-args main +./run --arch arm --baremetal baremetal/hello.c --gdb .... Alternatively, to start from the very first executed instruction of our tiny <>: @@ -11799,20 +11823,20 @@ Alternatively, to start from the very first executed instruction of our tiny </no_bootloader/*.S`, e.g.: .... ./run \ --arch arm \ - --baremetal arch/arm/no_bootloader/semihost_exit \ + --baremetal baremetal/arch/arm/no_bootloader/semihost_exit.S \ --gdb-wait \ --tmux-args=--no-continue \ ; @@ -11858,7 +11882,7 @@ It is documented at: https://developer.arm.com/docs/100863/latest/introduction For example, the following code makes QEMU exit: .... -./run --arch arm --baremetal arch/arm/semihost_exit +./run --arch arm --baremetal baremetal/arch/arm/semihost_exit.S .... Source: link:baremetal/arch/arm/no_bootloader/semihost_exit.S[] @@ -11942,7 +11966,7 @@ For `arm`, some baremetal examples compile fine with: .... sudo apt-get install gcc-arm-none-eabi qemu-system-arm ./build-baremetal --arch arm --gcc-which host-baremetal -./run --arch arm --baremetal interactive/prompt --qemu-which host +./run --arch arm --baremetal baremetal/hello.c --qemu-which host .... However, there are as usual limitations to using prebuilts: @@ -11971,7 +11995,7 @@ TODO: any advantage over QEMU? I doubt it, mostly using it as as toy for now: Without running `./run`, do directly: .... -./run-gdb --arch arm --baremetal interactive/prompt --sim +./run-gdb --arch arm --baremetal baremetal/hello.c --sim .... Then inside GDB: @@ -12039,8 +12063,8 @@ ARM exception levels are analogous to x86 <>. Print the EL at the beginning of a baremetal simulation: .... -./run --arch arm --baremetal arch/arm/el -./run --arch aarch64 --baremetal arch/aarch64/el +./run --arch arm --baremetal baremetal/arch/arm/el.c +./run --arch aarch64 --baremetal baremetal/arch/aarch64/el.c .... Sources: @@ -12055,12 +12079,12 @@ The lower ELs are not mandated by the architecture, and can be controlled throug In QEMU, you can configure the lowest EL as explained at https://stackoverflow.com/questions/42824706/qemu-system-aarch64-entering-el1-when-emulating-a53-power-up .... -./run --arch arm --baremetal arch/arm/el -./run --arch arm --baremetal arch/arm/el -- -machine virtualization=on -./run --arch arm --baremetal arch/arm/el -- -machine secure=on -./run --arch aarch64 --baremetal arch/aarch64/el -./run --arch aarch64 --baremetal arch/aarch64/el -- -machine virtualization=on -./run --arch aarch64 --baremetal arch/aarch64/el -- -machine secure=on +./run --arch arm --baremetal baremetal/arch/arm/el.c +./run --arch arm --baremetal baremetal/arch/arm/el.c -- -machine virtualization=on +./run --arch arm --baremetal baremetal/arch/arm/el.c -- -machine secure=on +./run --arch aarch64 --baremetal baremetal/arch/aarch64/el.c +./run --arch aarch64 --baremetal baremetal/arch/aarch64/el.c -- -machine virtualization=on +./run --arch aarch64 --baremetal baremetal/arch/aarch64/el.c -- -machine secure=on .... outputs respectively: @@ -12079,17 +12103,17 @@ TODO: why is `arm` stuck at `19` which equals Supervisor mode? In gem5, you can configure the lowest EL with: .... -./run --arch arm --baremetal arch/arm/el --emulator gem5 +./run --arch arm --baremetal baremeta/arch/arm/el.c --emulator gem5 cat "$(./getvar --arch arm --emulator gem5 gem5_guest_terminal_file)" -./run --arch arm --baremetal arch/arm/el --emulator gem5 -- --param 'system.have_virtualization = True' +./run --arch arm --baremetal baremetal/arch/arm/el.c --emulator gem5 -- --param 'system.have_virtualization = True' cat "$(./getvar --arch arm --emulator gem5 gem5_guest_terminal_file)" -./run --arch arm --baremetal arch/arm/el --emulator gem5 -- --param 'system.have_security = True' +./run --arch arm --baremetal baremetal/arch/arm/el.c --emulator gem5 -- --param 'system.have_security = True' cat "$(./getvar --arch arm --emulator gem5 gem5_guest_terminal_file)" -./run --arch aarch64 --baremetal arch/aarch64/el --emulator gem5 +./run --arch aarch64 --baremetal baremetal/arch/aarch64/el.c --emulator gem5 cat "$(./getvar --arch aarch64 --emulator gem5 gem5_guest_terminal_file)" -./run --arch aarch64 --baremetal arch/aarch64/el --emulator gem5 -- --param 'system.have_virtualization = True' +./run --arch aarch64 --baremetal baremetal/arch/aarch64/el.c --emulator gem5 -- --param 'system.have_virtualization = True' cat "$(./getvar --arch aarch64 --emulator gem5 gem5_guest_terminal_file)" -./run --arch aarch64 --baremetal arch/aarch64/el --emulator gem5 -- --param 'system.have_security = True' +./run --arch aarch64 --baremetal baremetal/arch/aarch64/el.c --emulator gem5 -- --param 'system.have_security = True' cat "$(./getvar --arch aarch64 --emulator gem5 gem5_guest_terminal_file)" .... @@ -12111,14 +12135,14 @@ This is the most basic example of exception handling we have. We a handler for `svc`, do an `svc`, and observe that the handler got called and returned from C and assembly: .... -./run --arch aarch64 --baremetal arch/aarch64/svc -./run --arch aarch64 --baremetal arch/aarch64/svc_asm +./run --arch aarch64 --baremetal baremetal/arch/aarch64/svc.c +./run --arch aarch64 --baremetal baremetal/arch/aarch64/svc_asm.S .... Sources: -* link:baremetal/arch/aarch64/svc_asm.S[] * link:baremetal/arch/aarch64/svc.c[] +* link:baremetal/arch/aarch64/svc_asm.S[] Sample output for the C one: @@ -12171,7 +12195,7 @@ Both QEMU and gem5 are able to trace interrupts in addition to instructions, and .... ./run \ --arch aarch64 \ - --baremetal arch/aarch64/svc_asm + --baremetal baremetal/arch/aarch64/svc_asm.S -- -d in_asm,int \ ; .... @@ -12198,7 +12222,7 @@ and: .... ./run \ --arch aarch64 \ - --baremetal arch/aarch64/svc_asm \ + --baremetal baremetal/arch/aarch64/svc_asm.S \ --trace ExecAll,Faults \ --trace-stdout \ ; @@ -12270,10 +12294,10 @@ Bibliography: ==== ARM multicore .... -./run --arch aarch64 --baremetal arch/aarch64/multicore --cpus 2 -./run --arch aarch64 --baremetal arch/aarch64/multicore --cpus 2 --emulator gem5 -./run --arch arm --baremetal arch/aarch64/multicore --cpus 2 -./run --arch arm --baremetal arch/aarch64/multicore --cpus 2 --emulator gem5 +./run --arch aarch64 --baremetal baremetal/arch/aarch64/multicore.S --cpus 2 +./run --arch aarch64 --baremetal baremetal/arch/aarch64/multicore.S --cpus 2 --emulator gem5 +./run --arch arm --baremetal baremetal/arch/aarch64/multicore.S --cpus 2 +./run --arch arm --baremetal baremetal/arch/aarch64/multicore.S --cpus 2 --emulator gem5 .... Sources: @@ -12288,7 +12312,7 @@ So, we need CPU 1 to come to the rescue and set that memory address to `1`, othe Don't believe me? Then try: .... -./run --arch aarch64 --baremetal arch/aarch64/multicore --cpus 1 +./run --arch aarch64 --baremetal baremetal/arch/aarch64/multicore.S --cpus 1 .... and watch it hang forever. @@ -12296,7 +12320,7 @@ and watch it hang forever. Note that if you try the same thing on gem5: .... -./run --arch aarch64 --baremetal arch/aarch64/multicore --cpus 1 --emulator gem5 +./run --arch aarch64 --baremetal baremetal/arch/aarch64/multicore.S --cpus 1 --emulator gem5 .... then the gem5 actually exits, but with a different message: @@ -13888,14 +13912,14 @@ Sources: If a test fails, re-run the test commands manually and use `--verbose` to understand what happened: .... -./run --arch arm --background --baremetal add --gdb-wait & -./run-gdb --arch arm --baremetal add --verbose -- main +./run --arch arm --background --baremetal baremetal/add.c --gdb-wait & +./run-gdb --arch arm --baremetal baremetal/add.c --verbose -- main .... and possibly repeat the GDB steps manually with the usual: .... -./run-gdb --arch arm --baremetal add --no-continue --verbose +./run-gdb --arch arm --baremetal baremetal/add.c --no-continue --verbose .... To debug GDB problems on gem5, you might want to enable the following <> options: @@ -13903,7 +13927,7 @@ To debug GDB problems on gem5, you might want to enable the following < 0: - if dirpath_relative_root_components[0] in ( - 'gcc', - 'kernel_modules', - 'linux', - ): - cstd = 'gnu11' - cxxstd = 'gnu++17' - else: - cstd = self.default_cstd - cxxstd = self.default_cxxstd - # -pedantic complains even if we use -std=gnu11. - ccflags_dir.extend(['-pedantic', LF]) + for target in self.env['targets']: + for path, in_dirnames, in_filenames in self.sh.walk(target): + path_abs = os.path.abspath(path) + dirpath_relative_root = path_abs[rootdir_abs_len + 1:] + dirpath_relative_root_components = dirpath_relative_root.split(os.sep) + dirpath_relative_root_components_len = len(dirpath_relative_root_components) + out_dir = os.path.join( + build_dir, + dirpath_relative_root + ) + common_objs_dir = [common_obj] + cc_flags_after = [] + cc_flags_dir = cc_flags.copy() + if dirpath_relative_root_components_len > 0: if dirpath_relative_root_components[0] == 'arch': if dirpath_relative_root_components_len > 1: if dirpath_relative_root_components[1] == self.env['arch']: - ccflags_dir.extend([ + cc_flags_dir.extend([ '-I', os.path.join(self.env['userland_source_arch_arch_dir']), LF, '-I', os.path.join(self.env['userland_source_arch_dir']), LF, - '-fno-pie', LF, - '-no-pie', LF, ]) if 'freestanding' in dirpath_relative_root_components: common_objs_dir = [] - ccflags_dir.extend([ + cc_flags_dir.extend([ '-ffreestanding', LF, '-nostdlib', LF, '-static', LF, @@ -251,24 +229,6 @@ Default: build all examples that have their package dependencies met, e.g.: common_objs_dir = [] else: common_objs_dir = [common_obj_asm] - if self.env['arch'] == 'arm': - ccflags_dir.extend([ - '-Xassembler', '-mcpu=cortex-a72', LF, - # To prevent: - # > vfp.S: Error: selected processor does not support in ARM mode - # https://stackoverflow.com/questions/41131432/cross-compiling-error-selected-processor-does-not-support-fmrx-r3-fpexc-in/52875732#52875732 - # We aim to take the most extended mode currently available that works on QEMU. - '-Xassembler', '-mfpu=crypto-neon-fp-armv8.1', LF, - '-Xassembler', '-meabi=5', LF, - # Treat inline assembly as arm instead of thumb - # The opposite of -mthumb. - '-marm', LF, - # Make gcc generate .syntax unified for inline assembly. - # However, it gets ignored if -marm is given, which a GCC bug that was recently fixed: - # https://stackoverflow.com/questions/54078112/how-to-write-syntax-unified-ual-armv7-inline-assembly-in-gcc/54132097#54132097 - # So we just write divided inline assembly for now. - '-masm-syntax-unified', LF, - ]) else: continue else: @@ -279,45 +239,48 @@ Default: build all examples that have their package dependencies met, e.g.: if not (has_all_packages or pkg_key in has_packages): continue pkg = pkgs[pkg_key] - if 'ccflags' in pkg: - ccflags_dir.extend(pkg['ccflags']) + if 'cc_flags' in pkg: + cc_flags_dir.extend(pkg['cc_flags']) else: pkg_config_output = subprocess.check_output([ self.env['pkg_config'], '--cflags', pkg_key ]).decode() - ccflags_dir.extend(self.sh.shlex_split(pkg_config_output)) - if 'ccflags_after' in pkg: - ccflags_dir.extend(pkg['ccflags_after']) + cc_flags_dir.extend(self.sh.shlex_split(pkg_config_output)) + if 'cc_flags_after' in pkg: + cc_flags_dir.extend(pkg['cc_flags_after']) else: pkg_config_output = subprocess.check_output([ self.env['pkg_config'], '--libs', pkg_key ]).decode() - ccflags_after.extend(self.sh.shlex_split(pkg_config_output)) - for in_filename in in_filenames: - path_relative_root = os.path.join(dirpath_relative_root, in_filename) - if path_relative_root == common_obj_asm_relpath: - continue - in_path = os.path.join(path, in_filename) - in_name, in_ext = os.path.splitext(in_filename) - out_path = os.path.join( - out_dir, - in_name + self.env['userland_build_ext'] - ) - error = thread_pool.submit({ - 'in_path': in_path, - 'out_path': out_path, - 'ccflags': ccflags_dir, - 'cstd': cstd, - 'cxxstd': cxxstd, - 'extra_objs': common_objs_dir, - 'ccflags_after': ccflags_after, - }) - if error is not None: - raise common.ExitLoop() + cc_flags_after.extend(self.sh.shlex_split(pkg_config_output)) + for in_filename in in_filenames: + in_path = os.path.join(path, in_filename) + cc_flags_file = cc_flags_dir.copy() + in_ext = os.path.splitext(in_filename)[1] + if not in_ext in self.env['userland_in_exts']: + continue + my_path_properties = path_properties.get(os.path.join( + self.env['userland_subdir'], + dirpath_relative_root, + in_filename + )) + if my_path_properties['pedantic']: + cc_flags_file.extend(['-pedantic', LF]) + error = thread_pool.submit({ + 'c_std': my_path_properties['c_std'], + 'cc_flags': cc_flags_file + my_path_properties['cc_flags'], + 'cc_flags_after': cc_flags_after, + 'cxx_std': my_path_properties['cxx_std'], + 'extra_objs': common_objs_dir, + 'in_path': in_path, + 'out_path': self.resolve_userland_executable(in_path), + }) + if error is not None: + raise common.ExitLoop() except common.ExitLoop: pass error = thread_pool.get_error() @@ -328,27 +291,33 @@ Default: build all examples that have their package dependencies met, e.g.: self.sh.copy_dir_if_update( srcdir=build_dir, destdir=self.env['out_rootfs_overlay_lkmc_dir'], - filter_ext=self.env['userland_build_ext'], + filter_ext=self.env['userland_executable_ext'], ) return 0 def clean(self): if self.env['in_tree']: - for path, dirnames, filenames in self.walk_source_targets( - self.env['targets'], - self.env['userland_out_exts'], - empty_ok=True - ): - for filename in filenames: - self.sh.rmrf(os.path.join(path, filename)) + for target in self.env['targets']: + if os.path.exists(target): + if os.path.isfile(target): + self.sh.rmrf(self.resolve_userland_executable(target)) + else: + for path, dirnames, filenames in self.sh.walk(target): + for filename in filenames: + if os.path.splitext(filename)[1] in self.env['userland_out_exts']: + self.sh.rmrf(os.path.join(path, filename)) else: - self.sh.rmrf(self.get_build_dir()) + for target in self.env['targets']: + self.sh.rmrf(self.resolve_userland_executable(target)) def get_build_dir(self): - if self.env['in_tree']: - return self.env['userland_source_dir'] - else: - return self.env['userland_build_dir'] + return self.env['userland_build_dir'] + + def setup_one(self): + self.env['targets'] = self.resolve_targets( + self.env['userland_source_dir'], + self.env['targets'] + ) if __name__ == '__main__': Main().cli() diff --git a/build-userland-in-tree b/build-userland-in-tree index 754d3e1..c1f354f 100755 --- a/build-userland-in-tree +++ b/build-userland-in-tree @@ -22,7 +22,6 @@ https://github.com/cirosantilli/linux-kernel-module-cheat#userland-setup-getting ''', defaults={ 'gcc_which': 'host', - 'has_all_packages': True, 'in_tree': True, 'targets': ['.'], } diff --git a/cli_function.py b/cli_function.py index 7a96efe..1d163bd 100755 --- a/cli_function.py +++ b/cli_function.py @@ -290,7 +290,10 @@ class CliFunction: if value != default: if argument.is_option: if argument.is_bool: - vals = [(argument.longname,)] + if value: + vals = [(argument.longname,)] + else: + vals = [('--no-' + argument.longname[2:],)] elif 'action' in argument.kwargs and argument.kwargs['action'] == 'append': vals = [(argument.longname, str(val)) for val in value] else: @@ -454,7 +457,8 @@ amazing function! # get_cli assert one_cli_function.get_cli(pos_mandatory=1, asdf='B') == [('--asdf', 'B'), ('--bool-cli',), ('1',)] assert one_cli_function.get_cli(pos_mandatory=1, asdf='B', qwer='R') == [('--asdf', 'B'), ('--bool-cli',), ('--qwer', 'R'), ('1',)] - assert one_cli_function.get_cli(pos_mandatory=1, bool_true=False) == [('--bool-cli',), ('--bool-true',), ('1',)] + assert one_cli_function.get_cli(pos_mandatory=1, bool_true=False) == [('--bool-cli',), ('--no-bool-true',), ('1',)] + assert one_cli_function.get_cli(pos_mandatory=1, bool_false=True) == [('--bool-cli',), ('--bool-false',), ('1',)] assert one_cli_function.get_cli(pos_mandatory=1, pos_optional=2, args_star=['asdf', 'qwer']) == [('--bool-cli',), ('1',), ('2',), ('asdf',), ('qwer',)] assert one_cli_function.get_cli(pos_mandatory=1, append=['2', '3']) == [('--append', '2'), ('--append', '3',), ('--bool-cli',), ('1',)] diff --git a/common.py b/common.py index a013e34..909220a 100644 --- a/common.py +++ b/common.py @@ -62,7 +62,7 @@ consts['kernel_modules_source_dir'] = os.path.join(consts['root_dir'], consts['k consts['userland_subdir'] = 'userland' consts['userland_source_dir'] = os.path.join(consts['root_dir'], consts['userland_subdir']) consts['userland_source_arch_dir'] = os.path.join(consts['userland_source_dir'], 'arch') -consts['userland_build_ext'] = '.out' +consts['userland_executable_ext'] = '.out' consts['include_subdir'] = 'lkmc' consts['include_source_dir'] = os.path.join(consts['root_dir'], consts['include_subdir']) consts['submodules_dir'] = os.path.join(consts['root_dir'], 'submodules') @@ -110,7 +110,7 @@ consts['userland_in_exts'] = [ consts['cxx_ext'], ] consts['userland_out_exts'] = [ - consts['userland_build_ext'], + consts['userland_executable_ext'], consts['obj_ext'], ] consts['config_file'] = os.path.join(consts['data_dir'], 'config.py') @@ -358,10 +358,8 @@ See: https://github.com/cirosantilli/linux-kernel-module-cheat#initrd help='''\ Use the given baremetal executable instead of the Linux kernel. -If the path is absolute, it is used as is. - -If the path is relative, we assume that it points to a source code -inside baremetal/ and then try to use corresponding executable. +If the path points to a source code inside baremetal/, then the +corresponding executable is automatically found. ''' ) @@ -467,6 +465,17 @@ CLI arguments to pass to the userland executable. ) # Run. + self.add_argument( + '--in-tree', + default=False, + help='''\ +Place build output inside source tree to conveniently run it, especially when +building with the host native toolchain. Currently only supported by ./build-userland. + +When running, prefer in-tree executables instead of out-of-tree ones, e.g.: +userland/c/hello resolves userland/c/hello.out instead of the out-of-tree one. +''', + ) self.add_argument( '--port-offset', type=int, @@ -759,7 +768,10 @@ Incompatible archs are skipped. # Userland env['userland_source_arch_arch_dir'] = join(env['userland_source_arch_dir'], env['arch']) - env['userland_build_dir'] = join(env['out_dir'], 'userland', env['userland_build_id'], env['arch']) + if env['in_tree']: + env['userland_build_dir'] = self.env['userland_source_dir'] + else: + env['userland_build_dir'] = join(env['out_dir'], 'userland', env['userland_build_id'], env['arch']) # Kernel modules. env['kernel_modules_build_dir'] = join(env['kernel_modules_build_base_dir'], env['arch']) @@ -821,7 +833,7 @@ Incompatible archs are skipped. env['baremetal'], env['baremetal_source_dir'], env['baremetal_build_dir'], - [env['baremetal_build_ext']], + env['baremetal_build_ext'], ) source_path_noext = os.path.splitext(join( env['baremetal_source_dir'], @@ -905,6 +917,15 @@ lunch aosp_{}-eng self._common_args.add(key) super().add_argument(*args, **kwargs) + def assert_is_subpath(self, subpath, parent): + if not self.is_subpath(subpath, parent): + raise Exception( + 'Can only accept targets inside {}, given: {}'.format( + parent, + subpath + ) + ) + def get_elf_entry(self, elf_file_path): readelf_header = subprocess.check_output([ self.get_toolchain_tool('readelf'), @@ -1028,6 +1049,12 @@ lunch aosp_{}-eng if flush: sys.stdout.flush() + def is_subpath(self, subpath, parent): + ''' + https://stackoverflow.com/questions/3812849/how-to-check-whether-a-directory-is-a-sub-directory-of-another-directory + ''' + return os.path.abspath(subpath).startswith(os.path.abspath(parent)) + def main(self, *args, **kwargs): ''' Run timed_main across all selected archs and emulators. @@ -1049,10 +1076,6 @@ lunch aosp_{}-eng real_emulators = env['emulators'] return_value = 0 try: - ret = self.setup() - if ret is not None and ret != 0: - return_value = ret - raise ExitLoop() for emulator in real_emulators: for arch in real_archs: if arch in env['arch_short_to_long_dict']: @@ -1079,6 +1102,7 @@ lunch aosp_{}-eng dry_run=self.env['dry_run'], quiet=self.env['quiet'], ) + self.setup_one() ret = self.timed_main() if not env['dry_run']: end_time = time.time() @@ -1163,92 +1187,70 @@ lunch aosp_{}-eng ] ) - def resolve_source_tree(self, in_path, exts, source_tree_root, empty_ok=False): + def resolve_executable( + self, + in_path, + magic_in_dir, + magic_out_dir, + executable_ext + ): ''' - Convert a convenient shorthand user input string to paths of existing files - in the source tree. + Resolve the path of an userland or baremetal executable. - Ensure that the path lies inside source_tree_root. + If it is in tree, resolve source paths to their corresponding executables. - Multiple matches may happen if multiple multiple exts files exist. - E.g., after an in-tree build, in_path='hello' and exts=['.c', '.out'] - would match both: + If it is out of tree, return the same exact path as input. - - userland/hello.c - - userland/hello.out - - If you also want directories to be matched, just add an empty string - `''` to exts, which leads all of the following to match the arch directory: - - - arch - - arch.c - - userland/arch - - /full/path/to/userland/arch - - Note however that this potentially prevents differentiation between - files and directories: e.g. if you had both a file arch.c and a directory arch/, - and exts=['', '.c'], then both would get matched. + If the input path is a file, add the executable extension automatically. ''' - in_path = os.path.abspath(in_path) - source_tree_root = os.path.abspath(source_tree_root) - if not in_path.startswith(source_tree_root): - raise Exception( - 'The input path {} is not inside the source directory {}'.format( - in_path, - source_tree_root - ) + in_path_abs = os.path.abspath(in_path) + magic_in_dir_abs = os.path.abspath(magic_in_dir) + magic_out_dir_abs = os.path.abspath(magic_out_dir) + if self.is_subpath(in_path_abs, magic_in_dir_abs): + out = os.path.abspath(os.path.join( + magic_out_dir_abs, + os.path.relpath( + os.path.splitext(in_path_abs)[0], + os.path.abspath(magic_in_dir_abs) + )), ) - result = [] - name, ext = os.path.splitext(in_path) - for try_ext in exts: - try_path = name + try_ext - if os.path.exists(try_path): - result.append(try_path) - if not result and not empty_ok: - raise Exception('No file not found for input: ' + in_path) - return result - - def resolve_executable(self, in_path, magic_in_dir, magic_out_dir, out_exts): - if os.path.isabs(in_path): - return in_path + if os.path.isfile(in_path): + out += executable_ext + return out else: - paths = [ - os.path.join(magic_out_dir, in_path), - os.path.join( - magic_out_dir, - os.path.relpath(in_path, magic_in_dir), - ) - ] - for path in paths: - for out_ext in out_exts: - path = os.path.splitext(path)[0] + out_ext - if os.path.exists(path): - return path - if not self.env['dry_run']: - raise Exception('Executable file not found. Tried:\n' + '\n'.join(paths)) + return in_path_abs + + def resolve_targets(self, source_dir, targets): + if not targets: + targets = [source_dir] + new_targets = [] + for target in targets: + target = self.toplevel_to_source_dir(target, source_dir) + self.assert_is_subpath(target, source_dir) + new_targets.append(target) + return new_targets def resolve_userland_executable(self, path): - ''' - Convert an userland source path-like string to an - absolute userland build output path. - ''' return self.resolve_executable( path, self.env['userland_source_dir'], self.env['userland_build_dir'], - [self.env['userland_build_ext']], + self.env['userland_executable_ext'], ) - def setup(self): + def setup_one(self): ''' - Similar to timed_main, but gets run only once for all --arch and --emulator, - before timed_main. - - Different from __init__, since at this point env has already been calculated, - so variables that don't depend on --arch or --emulator can be used. + Run just before timed_main, after _init_env. ''' pass + def toplevel_to_source_dir(self, path, source_dir): + path = os.path.abspath(path) + if path == self.env['root_dir']: + return source_dir + else: + return path + def timed_main(self): ''' Main action of the derived class. @@ -1259,38 +1261,10 @@ lunch aosp_{}-eng def teardown(self): ''' - Similar to setup, but run after timed_main. + Similar to setup, but run once after all timed_main are called. ''' pass - def walk_source_targets(self, targets, exts, empty_ok=False): - ''' - Resolve userland or baremetal source tree targets, and walk them. - - Ignore the input extension of targets, and select only files - with the given extensions exts. - ''' - if targets: - targets = targets - else: - targets = [self.env['userland_source_dir']] - for target in targets: - resolved_targets = self.resolve_source_tree( - target, - exts + [''], - self.env['userland_source_dir'], - empty_ok=empty_ok, - ) - for resolved_target in resolved_targets: - for path, dirnames, filenames in self.sh.walk(resolved_target): - dirnames.sort() - filenames = [ - filename for filename in filenames - if os.path.splitext(filename)[1] in exts - ] - filenames.sort() - yield path, dirnames, filenames - class BuildCliFunction(LkmcCliFunction): ''' A CLI function with common facilities to build stuff, e.g.: diff --git a/path_properties.py b/path_properties.py index 1ea427d..72615ab 100644 --- a/path_properties.py +++ b/path_properties.py @@ -2,6 +2,8 @@ import os +from shell_helpers import LF + class PathProperties: ''' Encodes properties of userland and baremetal paths. @@ -12,30 +14,42 @@ class PathProperties: self, **kwargs ): - self.properties = { - 'allowed_archs': None, - 'exit_status': 0, - 'interactive': False, - 'more_than_1s': False, + property_keys = { + 'allowed_archs', + 'c_std', + 'cc_flags', + 'cc_pedantic', + 'cxx_std', + 'exit_status', + 'interactive', + 'skip_run_unclassified', + 'more_than_1s', # The path does not generate an executable in itself, e.g. # it only generates intermediate object files. - 'no_executable': False, + 'no_executable', + 'pedantic', # the test receives a signal. We skip those tests for now, # on userland because we are lazy to figure out the exact semantics # of how Python + QEMU + gem5 determine the exit status of signals. - 'receives_signal': False, - 'requires_kernel_modules': False, + 'receives_signal', + 'requires_kernel_modules', } for key in kwargs: - if not key in self.properties: + if not key in property_keys: raise ValueError('Unknown key: {}'.format(key)) - self.properties.update(kwargs) + self.properties = kwargs def __getitem__(self, key): return self.properties[key] + def __repr__(self): + return str(self.properties) + def update(self, other): - return self.properties.update(other.properties) + other_tmp_properties = other.properties.copy() + if 'cc_flags' in self.properties and 'cc_flags' in other_tmp_properties: + other_tmp_properties['cc_flags'] = self.properties['cc_flags'] + other_tmp_properties['cc_flags'] + return self.properties.update(other_tmp_properties) def should_be_tested(self, arch): return \ @@ -44,57 +58,138 @@ class PathProperties: not self['no_executable'] and \ not self['receives_signal'] and \ not self['requires_kernel_modules'] and \ + not self['skip_run_unclassified'] and \ ( self['allowed_archs'] is None or arch in self['allowed_archs'] ) class PrefixTree: - def __init__(self, children=None, value=None): - if children == None: + def __init__(self, path_properties_dict=None, children=None): + if children is None: children = {} + if path_properties_dict is None: + path_properties_dict = {} self.children = children - self.value = value + self.path_properties = PathProperties(**path_properties_dict) -path_properties_tree = PrefixTree({ - 'arch': PrefixTree({ - 'x86_64': PrefixTree( +default_c_std = 'c11' +default_cxx_std = 'c++17' +gnu_extensions = { + 'c_std': 'gnu11', + 'cc_pedantic': False, + 'cxx_std': 'gnu++17' +} +path_properties_tree = PrefixTree( + { + 'c_std': default_c_std, + 'cxx_std': default_cxx_std, + 'pedantic': True, + 'allowed_archs': None, + 'c_std': None, + 'cc_flags': [], + 'cc_pedantic': True, + 'cxx_std': None, + 'exit_status': 0, + 'interactive': False, + 'skip_run_unclassified': False, + 'more_than_1s': False, + # The path does not generate an executable in itself, e.g. + # it only generates intermediate object files. + 'no_executable': False, + 'pedantic': False, + # the test receives a signal. We skip those tests for now, + # on userland because we are lazy to figure out the exact semantics + # of how Python + QEMU + gem5 determine the exit status of signals. + 'receives_signal': False, + 'requires_kernel_modules': False, + }, + { + 'userland': PrefixTree( { - 'c': PrefixTree({ - 'ring0.c': PrefixTree(value=PathProperties(receives_signal=True)) - }) }, - PathProperties(allowed_archs={'x86_64'}), - ), - 'arm': PrefixTree(value=PathProperties(allowed_archs={'arm'})), - 'aarch64': PrefixTree(value=PathProperties(allowed_archs={'aarch64'})), - 'empty.S': PrefixTree(value=PathProperties(no_executable=True)), - 'fail.S': PrefixTree(value=PathProperties(no_executable=True)), - 'main.c': PrefixTree(value=PathProperties(no_executable=True)), - }), - 'c': PrefixTree({ - 'assert_fail.c': PrefixTree(value=PathProperties(exit_status=1)), - 'false.c': PrefixTree(value=PathProperties(exit_status=1)), - 'getchar.c': PrefixTree(value=PathProperties(interactive=True)), - 'infinite_loop.c': PrefixTree(value=PathProperties(more_than_1s=True)), - }), - 'kernel_modules': PrefixTree(value=PathProperties(requires_kernel_modules=True)), - 'linux': PrefixTree(value=PathProperties(requires_kernel_modules=True)), - 'posix': PrefixTree({ - 'count.c': PrefixTree(value=PathProperties(more_than_1s=True)), - 'sleep_forever.c': PrefixTree(value=PathProperties(more_than_1s=True)), - 'virt_to_phys_test.c': PrefixTree(value=PathProperties(more_than_1s=True)), - }) -}) + { + 'arch': PrefixTree( + { + 'cc_flags': [ + '-fno-pie', LF, + '-no-pie', LF, + ] + }, + { + 'arm': PrefixTree( + { + 'allowed_archs': {'arm'}, + 'cc_flags': [ + '-Xassembler', '-mcpu=cortex-a72', LF, + # To prevent: + # > vfp.S: Error: selected processor does not support in ARM mode + # https://stackoverflow.com/questions/41131432/cross-compiling-error-selected-processor-does-not-support-fmrx-r3-fpexc-in/52875732#52875732 + # We aim to take the most extended mode currently available that works on QEMU. + '-Xassembler', '-mfpu=crypto-neon-fp-armv8.1', LF, + '-Xassembler', '-meabi=5', LF, + # Treat inline assembly as arm instead of thumb + # The opposite of -mthumb. + '-marm', LF, + # Make gcc generate .syntax unified for inline assembly. + # However, it gets ignored if -marm is given, which a GCC bug that was recently fixed: + # https://stackoverflow.com/questions/54078112/how-to-write-syntax-unified-ual-armv7-inline-assembly-in-gcc/54132097#54132097 + # So we just write divided inline assembly for now. + '-masm-syntax-unified', LF, + ] + } + ), + 'aarch64': PrefixTree({'allowed_archs': {'aarch64'}}), + 'empty.S': PrefixTree({'no_executable': True}), + 'fail.S': PrefixTree({'no_executable': True}), + 'main.c': PrefixTree({'no_executable': True}), + 'x86_64': PrefixTree( + {'allowed_archs': {'x86_64'}}, + { + 'c': PrefixTree( + {}, + { + 'ring0.c': PrefixTree({'receives_signal': True}) + } + ), + } + ), + } + ), + 'c': PrefixTree( + {}, + { + 'assert_fail.c': PrefixTree({'exit_status': 1}), + 'false.c': PrefixTree({'exit_status': 1}), + 'getchar.c': PrefixTree({'interactive': True}), + 'infinite_loop.c': PrefixTree({'more_than_1s': True}), + } + ), + 'gcc': PrefixTree(gnu_extensions), + 'kernel_modules': PrefixTree({**gnu_extensions, **{'requires_kernel_modules': True}}), + 'linux': PrefixTree( + {**gnu_extensions, **{'skip_run_unclassified': True}}, + ), + 'posix': PrefixTree( + {}, + { + 'count.c': PrefixTree({'more_than_1s': True}), + 'sleep_forever.c': PrefixTree({'more_than_1s': True}), + 'virt_to_phys_test.c': PrefixTree({'more_than_1s': True}), + } + ) + } + ) + } +) def get(test_path): cur_node = path_properties_tree - path_properties = PathProperties() + path_properties = PathProperties(**cur_node.path_properties.properties) for path_component in test_path.split(os.sep): if path_component in cur_node.children: cur_node = cur_node.children[path_component] - if cur_node.value is not None: - path_properties.update(cur_node.value) + path_properties.update(cur_node.path_properties) else: break return path_properties diff --git a/run b/run index 8183441..200d22c 100755 --- a/run +++ b/run @@ -18,7 +18,8 @@ Run some content on an emulator. ''' ) self.add_argument( - '--background', default=False, + '--background', + default=False, help='''\ Send QEMU serial output to a file instead of the terminal so it does not require a terminal attached to run on the background. Interactive input cannot be given. @@ -342,10 +343,10 @@ Extra options to append at the end of the emulator command line. if not self.env['_args_given']['gdb_wait']: self.env['gdb_wait'] = True if not self.env['_args_given']['tmux_args']: - if self.env['userland'] is not None: - self.env['tmux_args'] = 'main' - else: + if self.env['userland'] is None and self.env['baremetal'] is None: self.env['tmux_args'] = 'start_kernel' + else: + self.env['tmux_args'] = 'main' if not self.env['_args_given']['tmux_program']: self.env['tmux_program'] = 'gdb' if self.env['tmux_args'] is not None or self.env['_args_given']['tmux_program']: @@ -621,7 +622,11 @@ Extra options to append at the end of the emulator command line. '-kernel', self.env['image'], LF, '-m', self.env['memory'], LF, '-monitor', 'telnet::{},server,nowait'.format(self.env['qemu_monitor_port']), LF, - '-netdev', 'user,hostfwd=tcp::{}-:{},hostfwd=tcp::{}-:22,id=net0'.format(self.env['qemu_hostfwd_generic_port'], self.env['qemu_hostfwd_generic_port'], self.env['qemu_hostfwd_ssh_port']), LF, + '-netdev', 'user,hostfwd=tcp::{}-:{},hostfwd=tcp::{}-:22,id=net0'.format( + self.env['qemu_hostfwd_generic_port'], + self.env['qemu_hostfwd_generic_port'], + self.env['qemu_hostfwd_ssh_port'] + ), LF, '-no-reboot', LF, '-smp', str(self.env['cpus']), LF, ] + @@ -657,7 +662,12 @@ Extra options to append at the end of the emulator command line. self.raw_to_qcow2(qemu_which=self.env['qemu_which']) extra_emulator_args.extend([ '-drive', - 'file={},format=qcow2,if={}{}{}'.format(self.env['disk_image'], driveif, snapshot, rrid), + 'file={},format=qcow2,if={}{}{}'.format( + self.env['disk_image'], + driveif, + snapshot, + rrid + ), LF, ]) if rr: @@ -668,7 +678,10 @@ Extra options to append at the end of the emulator command line. if rr: extra_emulator_args.extend([ '-object', 'filter-replay,id=replay,netdev=net0', - '-icount', 'shift=7,rr={},rrfile={}'.format('record' if self.env['record'] else 'replay', self.env['qemu_rrfile']), + '-icount', 'shift=7,rr={},rrfile={}'.format( + 'record' if self.env['record'] else 'replay', + self.env['qemu_rrfile'] + ), ]) virtio_gpu_pci = [] else: @@ -717,6 +730,8 @@ Extra options to append at the end of the emulator command line. tmux_args += " --baremetal '{}'".format(self.env['baremetal']) if self.env['userland']: tmux_args += " --userland '{}'".format(self.env['userland']) + if self.env['in_tree']: + tmux_args += ' --in-tree' if self.env['tmux_args'] is not None: tmux_args += ' {}'.format(self.env['tmux_args']) subprocess.Popen([ diff --git a/shell_helpers.py b/shell_helpers.py index a52b45b..b5101bf 100644 --- a/shell_helpers.py +++ b/shell_helpers.py @@ -136,8 +136,7 @@ class ShellHelpers: self.copy_dir_if_update_non_recursive(srcdir, destdir, filter_ext) srcdir_abs = os.path.abspath(srcdir) srcdir_abs_len = len(srcdir_abs) - for path, dirnames, filenames in os.walk(srcdir_abs): - dirnames.sort() + for path, dirnames, filenames in self.walk(srcdir_abs): for dirname in dirnames: dirpath = os.path.join(path, dirname) dirpath_relative_root = dirpath[srcdir_abs_len + 1:] @@ -329,6 +328,8 @@ class ShellHelpers: yield dirname, [], [basename] else: for path, dirnames, filenames in os.walk(root): + dirnames.sort() + filenames.sort() yield path, dirnames, filenames def wget(self, url, download_path): diff --git a/test-baremetal b/test-baremetal index bfb19fd..17e3e9f 100755 --- a/test-baremetal +++ b/test-baremetal @@ -43,7 +43,7 @@ If given, run only the given tests. Otherwise, run all tests. self.env['emulator'] == 'gem5' and os.path.basename(path).startswith('semihost_') or self.env['emulator'] == 'qemu' and os.path.basename(path).startswith('gem5_') ): - sources.append(os.path.relpath(path, self.env['baremetal_source_dir'])) + sources.append(os.path.relpath(path, self.env['root_dir'])) else: sources = self.env['tests'] for source in sources: diff --git a/test-build-userland b/test-build-userland index a248853..5e748f4 100755 --- a/test-build-userland +++ b/test-build-userland @@ -1,52 +1,71 @@ #!/usr/bin/env bash -# Quick sanity check that userland target resoltion at least does not blow up. +# Quick sanity check that userland target resolution works. set -eux -./build-userland -./build-userland --clean -./build-userland userland/c -./build-userland --clean userland/c -./build-userland userland/c/hello -./build-userland --clean userland/c/hello -./build-userland userland/c/hello. -./build-userland --clean userland/c/hello. -./build-userland userland/c/hello.c -./build-userland --clean userland/c/hello.c -./build-userland userland/c/hello.out -./build-userland --clean userland/c/hello.out -./build-userland "$(pwd)/userland/c/hello.out" -./build-userland --clean "$(pwd)/userland/c/hello.out" +for in_tree in '' --in-tree; do + userland_build_dir="$(./getvar $in_tree userland_build_dir)" + # Toplevel. + ./build-userland $in_tree + [ -f "${userland_build_dir}/c/hello.out" ] + ./build-userland $in_tree --clean + ! [ -f "${userland_build_dir}/c/hello.out" ] -./build-userland --in-tree -./build-userland --in-tree --clean -./build-userland --in-tree userland/c -./build-userland --in-tree --clean userland/c -./build-userland --in-tree userland/c/hello -./build-userland --in-tree --clean userland/c/hello -./build-userland --in-tree userland/c/hello. -./build-userland --in-tree --clean userland/c/hello. -./build-userland --in-tree userland/c/hello.c -./build-userland --in-tree --clean userland/c/hello.c -./build-userland --in-tree userland/c/hello.out -./build-userland --in-tree --clean userland/c/hello.out -./build-userland --in-tree "$(pwd)/userland/c/hello.out" -./build-userland --in-tree --clean "$(pwd)/userland/c/hello.out" -./build-userland --in-tree --clean + # Toplevel explicit. + ./build-userland $in_tree userland/ + [ -f "${userland_build_dir}/c/hello.out" ] + ./build-userland $in_tree --clean + ! [ -f "${userland_build_dir}/c/hello.out" ] + + # Toplevel root dir. + ./build-userland $in_tree . + [ -f "${userland_build_dir}/c/hello.out" ] + ./build-userland $in_tree --clean + ! [ -f "${userland_build_dir}/c/hello.out" ] + + # Subdirectory. + ./build-userland $in_tree userland/c + [ -f "${userland_build_dir}/c/hello.out" ] + ./build-userland $in_tree --clean userland/c + ! [ -f "${userland_build_dir}/c/hello.out" ] + + # One program. + ./build-userland $in_tree userland/c/hello.c + [ -f "${userland_build_dir}/c/hello.out" ] + ./build-userland $in_tree --clean userland/c/hello.c + ! [ -f "${userland_build_dir}/c/hello.out" ] + + # Things that don't work: building: + # - non-existent files + # - paths outside of tree + ! ./build-userland $in_tree userland/c/hello + ! ./build-userland $in_tree userland/c/hello. + ! ./build-userland $in_tree "${userland_build_dir}/c/hello.out" + tmpfile="$(mktemp)" + ! ./build-userland $in_tree "$tmpfile" + rm "$tmpfile" + ! ./build-userland $in_tree .. + ! ./build-userland $in_tree kernel_modules + ./build-userland --clean $in_tree + + # Clean is however more forgiving and accepts paths that don't exist. + ./build-userland --clean $in_tree userland/does_not_exist +done + +./build-userland-in-tree +[ -f userland/c/hello.out ] +./build-userland-in-tree --clean +! [ -f userland/c/hello.out ] cd userland ./build +[ -f c/hello.out ] ./build --clean +! [ -f c/hello.out ] ./build c +[ -f c/hello.out ] ./build --clean c -./build c/hello -./build --clean c/hello -./build c/hello. -./build --clean c/hello. -./build c/hello.c +! [ -f c/hello.out ] ./build --clean c/hello.c -./build c/hello.out -./build --clean c/hello.out -./build "$(pwd)/c/hello.out" -./build --clean "$(pwd)/c/hello.out" +! [ -f c/hello.out ] diff --git a/test-user-mode b/test-user-mode index c6d12d7..17f176c 100755 --- a/test-user-mode +++ b/test-user-mode @@ -27,6 +27,12 @@ If given, run only the given tests. Otherwise, run all tests. ''' ) + def setup_one(self): + self.env['tests'] = self.resolve_targets( + self.env['userland_source_dir'], + self.env['tests'] + ) + def timed_main(self): run_args = self.get_common_args() run_args['ctrl_c_host'] = True @@ -35,37 +41,36 @@ If given, run only the given tests. Otherwise, run all tests. if self.env['emulator'] == 'gem5': run_args['userland_build_id'] = 'static' had_failure = False - rootdir_abs_len = len(self.env['userland_source_dir']) + rootdir_abs_len = len(self.env['root_dir']) with ThreadPool( self.run_test, nthreads=self.env['nproc'], thread_id_arg='thread_id', ) as thread_pool: try: - for path, in_dirnames, in_filenames in self.walk_source_targets( - self.env['tests'], - self.env['userland_in_exts'] - ): - path_abs = os.path.abspath(path) - dirpath_relative_root = path_abs[rootdir_abs_len + 1:] - for in_filename in in_filenames: - path_relative_root = os.path.join(dirpath_relative_root, in_filename) - my_path_properties = path_properties.get(path_relative_root) - if my_path_properties.should_be_tested(self.env['arch']): - cur_run_args = run_args.copy() - cur_run_args.update({ - 'background': True, - 'userland': path_relative_root, - }) - error = thread_pool.submit({ - 'expected_exit_status': my_path_properties['exit_status'], - 'run_args': cur_run_args, - 'run_obj': self.import_path_main('run'), - 'test_id': path_relative_root, - }) - if error is not None: - if self.env['quit_on_fail']: - raise common.ExitLoop() + for test in self.env['tests']: + for path, in_dirnames, in_filenames in self.sh.walk(test): + path_abs = os.path.abspath(path) + dirpath_relative_root = path_abs[rootdir_abs_len + 1:] + for in_filename in in_filenames: + if os.path.splitext(in_filename)[1] in self.env['userland_in_exts']: + path_relative_root = os.path.join(dirpath_relative_root, in_filename) + my_path_properties = path_properties.get(path_relative_root) + if my_path_properties.should_be_tested(self.env['arch']): + cur_run_args = run_args.copy() + cur_run_args.update({ + 'background': True, + 'userland': os.path.relpath(os.path.join(path_abs, in_filename), os.getcwd()), + }) + error = thread_pool.submit({ + 'expected_exit_status': my_path_properties['exit_status'], + 'run_args': cur_run_args, + 'run_obj': self.import_path_main('run'), + 'test_id': path_relative_root, + }) + if error is not None: + if self.env['quit_on_fail']: + raise common.ExitLoop() except common.ExitLoop: pass error = thread_pool.get_error()