userland: build really truly working now

userland test: start work, in a working state, but no features

README.adoc

@@ -13774,6 +13774,8 @@ Failure is detected by looking for the <<magic-failure-string>>
 
 Most userland programs that don't rely on kernel modules can also be tested in user mode simulation as explained at: <<user-mode-tests>>.
 
+TODO: we really need a mechanism to automatically generate the test list much like user-mode-tests, currently there are many tests missing, and we have to add everything manually which is very annoying.
+
 ===== Test GDB
 
 We have some link:https://github.com/pexpect/pexpect[pexpect] automated tests for the baremetal programs!

build-userland

@@ -125,36 +125,6 @@ Default: build all examples that have their package dependencies met, e.g.:
                 )
         return ret
 
-    def _walk_targets(self, exts):
-        '''
-        Resolve userland input targets, and walk them if directories.
-
-        Ignore the input extension of targets, and select only files
-        with the given extensions exts.
-
-        An empty extension indicates that directories will also be chosen.
-        '''
-        if self.env['targets']:
-            targets = self.env['targets']
-        else:
-            targets = [self.env['userland_source_dir']]
-        for target in targets:
-            resolved_targets = self.resolve_source_tree(
-                target,
-                exts + [''],
-                self.env['userland_source_dir']
-            )
-            for resolved_target in resolved_targets:
-                for path, dirnames, filenames in self.sh.walk(resolved_target):
-                    dirnames.sort()
-                    filenames = [
-                        filename for filename in filenames
-                        if os.path.splitext(filename)[1] in exts
-                    ]
-                    filenames.sort()
-                    for filename in filenames:
-                        yield path, dirnames, filenames
-
     def build(self):
         build_dir = self.get_build_dir()
         has_packages = set(self.env['has_package'])
@@ -231,9 +201,9 @@ Default: build all examples that have their package dependencies met, e.g.:
             self._build_one,
             nthreads=self.env['nproc'],
         ) as thread_pool:
-            class ExitLoop(Exception): pass
             try:
-                for path, in_dirnames, in_filenames in self._walk_targets(
+                for path, in_dirnames, in_filenames in self.walk_source_targets(
+                    self.env['targets'],
                     self.env['userland_in_exts']
                 ):
                     path_abs = os.path.abspath(path)
@@ -347,8 +317,8 @@ Default: build all examples that have their package dependencies met, e.g.:
                             'ccflags_after': ccflags_after,
                         })
                         if error is not None:
-                            raise ExitLoop()
+                            raise common.ExitLoop()
-            except ExitLoop:
+            except common.ExitLoop:
                 pass
         error = thread_pool.get_error()
         if error is not None:
@@ -364,8 +334,10 @@ Default: build all examples that have their package dependencies met, e.g.:
 
     def clean(self):
         if self.env['in_tree']:
-            for path, dirnames, filenames in self._walk_targets(
+            for path, dirnames, filenames in self.walk_source_targets(
-                self.env['userland_out_exts']
+                self.env['targets'],
+                self.env['userland_out_exts'],
+                empty_ok=True
             ):
                 for filename in filenames:
                     self.sh.rmrf(os.path.join(path, filename))

common.py

@@ -122,6 +122,9 @@ for key in consts['emulator_short_to_long_dict']:
     consts['emulator_choices'].add(consts['emulator_short_to_long_dict'][key])
 consts['host_arch'] = platform.processor()
 
+class ExitLoop(Exception):
+    pass
+
 class LkmcCliFunction(cli_function.CliFunction):
     '''
     Common functionality shared across our CLI functions:
@@ -1010,12 +1013,11 @@ lunch aosp_{}-eng
         else:
             real_emulators = env['emulators']
         return_value = 0
-        class GetOutOfLoop(Exception): pass
         try:
             ret = self.setup()
             if ret is not None and ret != 0:
                 return_value = ret
-                raise GetOutOfLoop()
+                raise ExitLoop()
             for emulator in real_emulators:
                 for arch in real_archs:
                     if arch in env['arch_short_to_long_dict']:
@@ -1045,11 +1047,10 @@ lunch aosp_{}-eng
                         if ret is not None and ret != 0:
                             return_value = ret
                             if self.env['quit_on_fail']:
-                                raise GetOutOfLoop()
+                                raise ExitLoop()
                     elif not real_all_archs:
                         raise Exception('Unsupported arch for this action: ' + arch)
-
+        except ExitLoop:
-        except GetOutOfLoop:
             pass
         ret = self.teardown()
         if ret is not None and ret != 0:
@@ -1122,7 +1123,7 @@ lunch aosp_{}-eng
             ]
         )
 
-    def resolve_source_tree(self, in_path, exts, source_tree_root):
+    def resolve_source_tree(self, in_path, exts, source_tree_root, empty_ok=False):
         '''
         Convert a convenient shorthand user input string to paths of existing files
         in the source tree.
@@ -1178,7 +1179,7 @@ lunch aosp_{}-eng
             try_path = name + try_ext
             if os.path.exists(try_path):
                 result.append(try_path)
-        if not result:
+        if not result and not empty_ok:
             raise Exception('No file not found for input: ' + in_path)
         return result
 
@@ -1237,6 +1238,34 @@ lunch aosp_{}-eng
         '''
         pass
 
+    def walk_source_targets(self, targets, exts, empty_ok=False):
+        '''
+        Resolve userland or baremetal source tree targets, and walk them.
+
+        Ignore the input extension of targets, and select only files
+        with the given extensions exts.
+        '''
+        if targets:
+            targets = targets
+        else:
+            targets = [self.env['userland_source_dir']]
+        for target in targets:
+            resolved_targets = self.resolve_source_tree(
+                target,
+                exts + [''],
+                self.env['userland_source_dir'],
+                empty_ok=empty_ok,
+            )
+            for resolved_target in resolved_targets:
+                for path, dirnames, filenames in self.sh.walk(resolved_target):
+                    dirnames.sort()
+                    filenames = [
+                        filename for filename in filenames
+                        if os.path.splitext(filename)[1] in exts
+                    ]
+                    filenames.sort()
+                    yield path, dirnames, filenames
+
 class BuildCliFunction(LkmcCliFunction):
     '''
     A CLI function with common facilities to build stuff, e.g.:
@@ -1364,7 +1393,13 @@ class TestCliFunction(LkmcCliFunction):
         super().__init__(*args, **kwargs)
         self.tests = []
 
-    def run_test(self, run_obj, run_args=None, test_id=None):
+    def run_test(
+        self,
+        run_obj,
+        run_args=None,
+        test_id=None,
+        expected_exit_status=None
+    ):
         '''
         This is a setup / run / teardown setup for simple tests that just do a single run.
 
@@ -1380,7 +1415,12 @@ class TestCliFunction(LkmcCliFunction):
                 run_args = {}
             test_id_string = self.test_setup(test_id)
             exit_status = run_obj(**run_args)
-            self.test_teardown(run_obj, exit_status, test_id_string)
+            return self.test_teardown(
+                run_obj,
+                exit_status,
+                test_id_string,
+                expected_exit_status=expected_exit_status
+            )
 
     def test_setup(self, test_id):
         test_id_string = '{} {}'.format(self.env['emulator'], self.env['arch'])
@@ -1389,15 +1429,20 @@ class TestCliFunction(LkmcCliFunction):
         self.log_info('test_id {}'.format(test_id_string), flush=True)
         return test_id_string
 
-    def test_teardown(self, run_obj, exit_status, test_id_string):
+    def test_teardown(
+        self,
+        run_obj,
+        exit_status,
+        test_id_string,
+        expected_exit_status=None
+    ):
+        if expected_exit_status is None:
+            expected_exit_status = 0
         if not self.env['dry_run']:
-            if exit_status == 0:
+            if exit_status == expected_exit_status:
                 test_result = TestResult.PASS
             else:
                 test_result = TestResult.FAIL
-                if self.env['quit_on_fail']:
-                    self.log_error('Test failed')
-                    sys.exit(1)
             self.log_info('test_result {}'.format(test_result.name))
             ellapsed_seconds = run_obj.ellapsed_seconds
         else:
@@ -1405,6 +1450,7 @@ class TestCliFunction(LkmcCliFunction):
             ellapsed_seconds = None
         self.log_info()
         self.tests.append(Test(test_id_string, test_result, ellapsed_seconds))
+        return test_result
 
     def teardown(self):
         '''

example_properties.py

@@ -1,12 +1,13 @@
 #!/usr/bin/env python3
 
-class ExampleProperties:
+class ExecutableProperties:
     '''
     Encodes properties of userland and baremetal examples.
     For directories, it applies to all files under the directory.
     Used to determine how to build and test the examples.
     '''
     def __init__(
+        self,
         exit_status=0,
         interactive=False,
         more_than_1s=False,
@@ -21,5 +22,12 @@ class ExampleProperties:
             not self.more_than_1s
 
 executable_properties = {
-    'userland/arch/x86_64/c/ring0.c': ExecutableProperties(exits_nonzero=True),
+    'c/assert_fail.c': ExecutableProperties(exit_status=0),
+    'c/false.c': ExecutableProperties(exit_status=0),
 }
+
+def get(test_path):
+    if test_path in executable_properties:
+        return executable_properties[test_path]
+    else:
+        return ExecutableProperties()

test-user-mode

@@ -4,6 +4,7 @@ import os
 import sys
 
 import common
+import example_properties
 
 class Main(common.TestCliFunction):
     def __init__(self):
@@ -27,37 +28,36 @@ If given, run only the given tests. Otherwise, run all tests.
         if self.env['emulator'] == 'gem5':
             run_args['userland_build_id'] = 'static'
         if self.env['tests'] == []:
-            tests = [
+            test_paths = [
-                'add.c',
+                'c/add.c',
-                'hello.c',
+                'c/false.c',
-                'hello_cpp.cpp',
+                'c/hello.c',
-                'print_argv.c',
+                'c/print_argv.c',
+                'cpp/hello.cpp',
             ]
-            if self.env['arch'] == 'x86_64':
-                arch_sources = [
-                    'asm_hello'
-                ]
-            elif self.env['arch'] == 'aarch64':
-                arch_sources = [
-                    'asm_hello'
-                ]
-            else:
-                arch_sources = []
-            arch_sources[:] = [
-                os.path.join('arch', self.env['arch'], arch_source)
-                for arch_source
-                in arch_sources
-            ]
-            tests.extend(arch_sources)
         else:
-            tests = self.env['tests']
+            test_paths = self.env['tests']
-        for test_dir_or_file in tests:
+        had_failure = False
-            for test in self.sh.walk(self.resolve_userland_source(test_dir_or_file)):
+        for test_path in test_paths:
-
+            test = example_properties.get(test_path)
-                consts['userland_in_exts'] = [
+            if test.should_be_tested():
-
+                # for test in self.sh.walk(self.resolve_userland_source(test_dir_or_file)):
-                run_args['userland'] = test
+                run_args['userland'] = test_path
-                self.run_test(run, run_args)
+                test_result = self.run_test(
+                    run,
+                    run_args,
+                    test_id=test_path,
+                    expected_exit_status=test.exit_status
+                )
+                if test_result != common.TestResult.PASS:
+                    if self.env['quit_on_fail']:
+                        return 1
+                    else:
+                        had_failure = True
+        if had_failure:
+            return 1
+        else:
+            return 0
 
 if __name__ == '__main__':
     Main().cli()