preparing test_user_mode, need to generalize stuff as usual

README.adoc

@@ -1011,6 +1011,7 @@ The `build` scripts inside link:userland/[] are just symlinks to link:build-user
 ./build-userland-in-tree
 ./build-userland-in-tree c
 ./build-userland-in-tree c/hello.c
+./build-userland-in-tree userland/c/hello.c
 ....
 
 which is in turn just a thin wrapper around link:build-userland[], so you can use any option supported by that script freely.
@@ -3641,7 +3642,7 @@ So programs that rely on those libraries might not compile as GCC can't find the
 For example, if we try to build <<blas>> statically:
 
 ....
-./build-userland --has-package openblas --static -- libs/openblas/hello
+./build-userland --has-package openblas --static -- userland/libs/openblas/hello.c
 ....
 
 it fails with:
@@ -8182,7 +8183,7 @@ DRM / DRI is the new interface that supersedes `fbdev`:
 
 ....
 ./build-buildroot --config 'BR2_PACKAGE_LIBDRM=y'
-./build-userland --has-package libdrm -- libs/libdrm/modeset
+./build-userland --has-package libdrm -- userland/libs/libdrm/modeset.c
 ./run --eval-after './libs/libdrm/modeset.out' --graphic
 ....
 
@@ -9931,7 +9932,7 @@ Buildroot supports it, which makes everything just trivial:
 
 ....
 ./build-buildroot --config 'BR2_PACKAGE_OPENBLAS=y'
-./build-userland --has-package openblas -- libs/openblas/hello
+./build-userland --has-package openblas -- userland/libs/openblas/hello.c
 ./run --eval-after './libs/openblas/hello.out; echo $?'
 ....
 
@@ -9971,7 +9972,7 @@ Header only linear algebra library with a mainline Buildroot package:
 
 ....
 ./build-buildroot --config 'BR2_PACKAGE_EIGEN=y'
-./build-userland --has-package eigen -- libs/eigen/hello
+./build-userland --has-package eigen -- userland/libs/eigen/hello.cpp
 ....
 
 Just create an array and print it:
@@ -11622,7 +11623,7 @@ TODO: review this section, make a more controlled userland experiment with <<m5o
 Let's have some fun and try to correlate the gem5 <<stats-txt>> `system.cpu.numCycles` cycle count with the link:https://en.wikipedia.org/wiki/Time_Stamp_Counter[x86 `rdtsc` instruction] that is supposed to do the same thing:
 
 ....
-./build-userland --static arch/x86_64/c/rdtsc
+./build-userland --static userland/arch/x86_64/c/rdtsc.c
 ./run --eval './arch/x86_64/c/rdtsc.out;m5 exit;' --emulator gem5
 ./gem5-stat
 ....

build-userland

@@ -47,7 +47,7 @@ building with the host toolchain.
             default=False,
             help='''\
 Treat targets as relative to the current working directory. If the current working
-directory is outside of userland/, use userland/ instead.
+directory is outside of userland/, this has no effect.
 ''',
         )
         self.add_argument(
@@ -141,17 +141,46 @@ Default: build all examples that have their package dependencies met, e.g.:
             return self.env['userland_source_dir']
 
     def _get_targets(self):
+        '''
+        Resolve target_relative_cwd and default targets.
+        '''
         if self.env['_args_given']['targets']:
             targets = self.env['targets']
             if self.env['target_relative_cwd']:
                 cwd = self._get_cwd()
-                targets = [os.path.join(cwd, target) for target in targets]
+                for target in targets:
-            return targets
+                    yield os.path.join(cwd, target)
+            else:
+                for target in targets:
+                    yield target
         else:
             if self.env['target_relative_cwd']:
-                return [self._get_cwd()]
+                yield self._get_cwd()
             else:
-                return [self.env['userland_source_dir']]
+                yield self.env['userland_source_dir']
+
+    def _walk_targets(self):
+        '''
+        Walk existing directories and files pointed to by the targets
+        from the command line from under the userland/ source tree.
+
+        This may include outputs of in-tree builds.
+
+        File extensions are ignored, e.g.:
+
+            c/hello
+
+        will find both:
+
+            c/hello.c
+            c/hello.out
+        '''
+        for target in self._get_targets():
+            target = self.resolve_userland_source(target)
+            noext, ext = os.path.splitext(filename)
+
+            for path, in_dirnames, in_filenames in self.sh.walk(target):
+                yield path, in_dirnames, in_filenames
 
     def build(self):
         build_dir = self.get_build_dir()
@@ -225,15 +254,13 @@ Default: build all examples that have their package dependencies met, e.g.:
             'openblas': {},
         }
         rootdir_abs_len = len(self.env['userland_source_dir'])
-        thread_pool = ThreadPool(
+        with ThreadPool(
             self._build_one,
             nthreads=self.env['nproc'],
-        )
+        ) as thread_pool:
-        class ExitLoop(Exception): pass
+            class ExitLoop(Exception): pass
-        try:
+            try:
-            for target in self._get_targets():
+                for path, in_dirnames, in_filenames in self._walk_targets():
-                target = self.resolve_userland_source(target)
-                for path, in_dirnames, in_filenames in self.sh.walk(target):
                     in_dirnames.sort()
                     in_filenames.sort()
                     path_abs = os.path.abspath(path)
@@ -348,9 +375,8 @@ Default: build all examples that have their package dependencies met, e.g.:
                         })
                         if error is not None:
                             raise ExitLoop()
-        except ExitLoop:
+            except ExitLoop:
-            pass
+                pass
-        error = thread_pool.join()
         if error is not None:
             print(error)
             return 1
@@ -364,16 +390,15 @@ Default: build all examples that have their package dependencies met, e.g.:
 
     def clean(self):
         if self.env['in_tree']:
-            for target in self._get_targets():
+            for path, dirnames, filenames in self._walk_targets():
-                if os.path.exists(target):
+                filenames.sort()
-                    for path, dirnames, filenames in os.walk(target):
+                dirnames.sort()
-                        filenames.sort()
+                for filename in filenames:
-                        dirnames.sort()
+                    noext, ext = os.path.splitext(filename)
-                        for filename in filenames:
+                    for out_ext in self.env['userland_out_exts']:
-                            if os.path.splitext(filename)[1] in self.env['userland_out_exts']:
+                        out_path = os.path.join(path, noext + out_ext)
-                                self.sh.rmrf(os.path.join(path, filename))
+                        if os.path.exists(out_path):
-                else:
+                            self.sh.rmrf(out_path)
-                    raise Exception('Path does not exist: ' + target)
         else:
             self.sh.rmrf(self.get_build_dir())
 

common.py

@@ -784,7 +784,7 @@ Valid emulators: {}
                     env['baremetal'],
                     env['baremetal_source_dir'],
                     env['baremetal_build_dir'],
-                    env['baremetal_build_ext'],
+                    [env['baremetal_build_ext']],
                 )
                 source_path_noext = os.path.splitext(join(
                     env['baremetal_source_dir'],
@@ -1121,26 +1121,46 @@ lunch aosp_{}-eng
             ]
         )
 
-    def resolve_source(self, in_path, magic_in_dir, in_exts):
+    def resolve_source(self, in_path, magic_in_dir, exts):
         '''
-        Convert a path-like string to a source file to the full source path,
+        Convert a convenient shorthand user input string to paths of existing files
-        e.g. all follogin work and to do the same:
+        in the source tree.
+
+        Input path file extensions are ignored.
+
+        All the following input paths would be equivalent for
+        magic_in_dir == '/full/path/to/userland'
 
         - hello
         - hello.
         - hello.c
+        - hello.out
         - userland/hello
         - userland/hello.
         - userland/hello.c
+        - userland/hello.out
         - /full/path/to/userland/hello
         - /full/path/to/userland/hello.
         - /full/path/to/userland/hello.c
 
-        Also works on directories:
+        Multiple matches may happen if multiple multiple exts files exist.
+        E.g., after an in-tree build, in_path='hello' and exts=['.c', '.out']
+        would match both:
+
+        - userland/hello.c
+        - userland/hello.out
+
+        If you also want directories to be matched, just add an empty string
+        `''` to exts, which leads all of the following to match the arch directory:
 
         - arch
+        - arch.c
         - userland/arch
         - /full/path/to/userland/arch
+
+        Note however that this potentially prevents differentiation between
+        files and directories: e.g. if you had both a file arch.c and a directory arch,
+        and exts=['', '.c'], then both would get matched.
         '''
         if os.path.isabs(in_path):
             return in_path
@@ -1154,18 +1174,14 @@ lunch aosp_{}-eng
             ]
             for path in paths:
                 name, ext = os.path.splitext(path)
-                if len(ext) > 1:
+                for try_ext in exts:
-                    try_exts = [ext]
+                    path = name + try_ext
-                else:
-                    try_exts = in_exts + ['']
-                for in_ext in try_exts:
-                    path = name + in_ext
                     if os.path.exists(path):
                         return path
             if not self.env['dry_run']:
-                raise Exception('Source file not found for input: ' + in_path)
+                raise Exception('No file not found for input: ' + in_path)
 
-    def resolve_executable(self, in_path, magic_in_dir, magic_out_dir, out_ext):
+    def resolve_executable(self, in_path, magic_in_dir, magic_out_dir, out_exts):
         if os.path.isabs(in_path):
             return in_path
         else:
@@ -1177,9 +1193,10 @@ lunch aosp_{}-eng
                 )
             ]
             for path in paths:
-                path = os.path.splitext(path)[0] + out_ext
+                for out_ext in out_exts:
-                if os.path.exists(path):
+                    path = os.path.splitext(path)[0] + out_ext
-                    return path
+                    if os.path.exists(path):
+                        return path
             if not self.env['dry_run']:
                 raise Exception('Executable file not found. Tried:\n' + '\n'.join(paths))
 
@@ -1192,7 +1209,7 @@ lunch aosp_{}-eng
             path,
             self.env['userland_source_dir'],
             self.env['userland_build_dir'],
-            self.env['userland_build_ext'],
+            [self.env['userland_build_ext']],
         )
 
     def resolve_userland_source(self, path):

example_properties.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python3
+
+class ExampleProperties:
+    '''
+    Encodes properties of userland and baremetal examples.
+    For directories, it applies to all files under the directory.
+    Used to determine how to build and test the examples.
+    '''
+    def __init__(
+        exit_status=0,
+        interactive=False,
+        more_than_1s=False,
+    ):
+        self.exit_status = exit_status
+        self.interactive = interactive
+        self.more_than_1s = more_than_1s
+
+    def should_be_tested(self):
+        return \
+            not self.interactive and \
+            not self.more_than_1s
+
+executable_properties = {
+    'userland/arch/x86_64/c/ring0.c': ExecutableProperties(exits_nonzero=True),
+}

file_properties.py

@@ -1,14 +0,0 @@
-#!/usr/bin/env python3
-
-class FileProperties:
-    def __init__(
-        more_than_1s=False,
-        exits_nonzero=False,
-        interactive=False,
-    ):
-        self.more_than_1s = more_than_1s
-        self.exits_nonzero = exits_nonzero
-
-executable_properties = {
-    'userland/arch/x86_64/c/ring0.c': ExecutableProperties(exits_nonzero=True),
-}

test-build-userland

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+set -eux
+
+./build-userland
+./build-userland --clean
+./build-userland c
+./build-userland --clean
+./build-userland userland/c
+./build-userland --clean
+./build-userland userland/c/hello
+./build-userland --clean
+./build-userland userland/c/hello.c
+./build-userland --clean
+./build-userland userland/c/hello.out
+./build-userland --clean
+
+./build-userland --in-tree
+./build-userland --in-tree --clean
+./build-userland --in-tree c
+./build-userland --in-tree --clean
+./build-userland --in-tree userland/c
+./build-userland --in-tree --clean
+./build-userland --in-tree userland/c/hello
+./build-userland --in-tree --clean
+./build-userland --in-tree userland/c/hello.c
+./build-userland --in-tree --clean
+./build-userland --in-tree userland/c/hello.out
+./build-userland --in-tree --clean
+
+cd userland
+
+./build-userland --in-tree
+./build-userland --in-tree --clean
+./build-userland --in-tree c
+./build-userland --in-tree --clean
+./build-userland --in-tree c/hello
+./build-userland --in-tree --clean
+./build-userland --in-tree c/hello.c
+./build-userland --in-tree --clean
+./build-userland --in-tree c/hello.out
+./build-userland --in-tree --clean

test-user-mode

@@ -27,7 +27,7 @@ If given, run only the given tests. Otherwise, run all tests.
         if self.env['emulator'] == 'gem5':
             run_args['userland_build_id'] = 'static'
         if self.env['tests'] == []:
-            sources = [
+            tests = [
                 'add.c',
                 'hello.c',
                 'hello_cpp.cpp',
@@ -48,12 +48,16 @@ If given, run only the given tests. Otherwise, run all tests.
                 for arch_source
                 in arch_sources
             ]
-            sources.extend(arch_sources)
+            tests.extend(arch_sources)
         else:
-            sources = self.env['tests']
+            tests = self.env['tests']
-        for source in sources:
+        for test_dir_or_file in tests:
-            run_args['userland'] = source
+            for test in self.sh.walk(self.resolve_userland_source(test_dir_or_file)):
-            self.run_test(run, run_args, source)
+
+                consts['userland_in_exts'] = [
+
+                run_args['userland'] = test
+                self.run_test(run, run_args)
 
 if __name__ == '__main__':
     Main().cli()

thread_pool.py

@@ -78,6 +78,12 @@ class ThreadPool:
             self.threads.append(thread)
             thread.start()
 
+    def __enter__(self):
+        return self
+
+    def __exit__(self, type, value, traceback):
+        self.join()
+
     def submit(self, work):
         '''
         Submit work. Block if there is already enough work scheduled (~nthreads).