userland: allow selecting targets, including directories

build-userland

@@ -30,10 +30,12 @@ allows us to build examples that rely on it.
             'targets',
             default=[],
             help='''\
-Build only the given userland programs.
+Build only the given userland programs or all programs in the given directories.
-Default: build all examples that have their package dependencies met.
+
-For example, an OpenBLAS example can only be built if the target root filesystem
+Default: build all examples that have their package dependencies met, e.g.:
-has the OpenBLAS libraries and headers installed.
+-   userland/arch/ programs only build if the target arch matches
+-   an OpenBLAS example can only be built if the target root filesystem
+    has the OpenBLAS libraries and headers installed, which you must inform with --has-package
 ''',
             nargs='*',
         )
@@ -90,7 +92,7 @@ has the OpenBLAS libraries and headers installed.
                                 '-o', out_path, LF,
                                 in_path, LF,
                             ] +
-                            extra_objs +
+                            self.sh.add_newlines(extra_objs) +
                             [
                                 '-lm', LF,
                                 '-pthread', LF,
@@ -160,68 +162,74 @@ has the OpenBLAS libraries and headers installed.
         rootdir_abs_len = len(self.env['userland_source_dir'])
         thread_limiter = threading.BoundedSemaphore(self.env['nproc'])
         self.error = False
-        for path, in_dirnames, in_filenames in os.walk(self.env['userland_source_dir']):
+        if self.env['_args_given']['targets']:
-            in_dirnames.sort()
+            targets = self.env['targets']
-            dirpath_relative_root = path[rootdir_abs_len + 1:]
+        else:
-            dirpath_relative_root_components = dirpath_relative_root.split(os.sep)
+            targets = [self.env['userland_source_dir']]
-            if (
+        for target in targets:
-                len(dirpath_relative_root_components) < 2 or
+            for path, in_dirnames, in_filenames in self.sh.walk(target):
-                dirpath_relative_root_components[0] != 'arch' or
+                in_dirnames.sort()
-                dirpath_relative_root_components[1] == self.env['arch']
+                path_abs = os.path.abspath(path)
-            ):
+                dirpath_relative_root = path_abs[rootdir_abs_len + 1:]
-                out_dir = os.path.join(
+                dirpath_relative_root_components = dirpath_relative_root.split(os.sep)
-                    self.env['userland_build_dir'],
+                if (
-                    dirpath_relative_root
+                    len(dirpath_relative_root_components) < 2 or
-                )
+                    dirpath_relative_root_components[0] != 'arch' or
-                os.makedirs(out_dir, exist_ok=True)
+                    dirpath_relative_root_components[1] == self.env['arch']
-                for in_filename in in_filenames:
+                ):
-                    in_path = os.path.join(path, in_filename)
+                    out_dir = os.path.join(
-                    in_name, in_ext = os.path.splitext(in_filename)
+                        self.env['userland_build_dir'],
-                    out_path = os.path.join(
+                        dirpath_relative_root
-                        out_dir,
-                        in_name + self.env['userland_build_ext']
                     )
-                    pkg_key = in_name.split('_')[0]
+                    os.makedirs(out_dir, exist_ok=True)
-                    ccflags_file = ccflags.copy()
+                    for in_filename in in_filenames:
-                    ccflags_after = []
+                        in_path = os.path.join(path, in_filename)
-                    if pkg_key in pkgs:
+                        in_name, in_ext = os.path.splitext(in_filename)
-                        if pkg_key not in has_packages:
+                        out_path = os.path.join(
-                            continue
+                            out_dir,
-                        pkg = pkgs[pkg_key]
+                            in_name + self.env['userland_build_ext']
-                        if 'ccflags' in pkg:
+                        )
-                            ccflags_file.extend(pkg['ccflags'])
+                        pkg_key = in_name.split('_')[0]
-                        else:
+                        ccflags_file = ccflags.copy()
-                            pkg_config_output = subprocess.check_output([
+                        ccflags_after = []
-                                self.env['buildroot_pkg_config'],
+                        if pkg_key in pkgs:
-                                '--cflags',
+                            if pkg_key not in has_packages:
-                                pkg_key
+                                continue
-                            ]).decode()
+                            pkg = pkgs[pkg_key]
-                            ccflags_file.extend(self.sh.shlex_split(pkg_config_output))
+                            if 'ccflags' in pkg:
-                        if 'ccflags_after' in pkg:
+                                ccflags_file.extend(pkg['ccflags'])
-                            ccflags_file.extend(pkg['ccflags_after'])
+                            else:
-                        else:
+                                pkg_config_output = subprocess.check_output([
-                            pkg_config_output = subprocess.check_output([
+                                    self.env['buildroot_pkg_config'],
-                                self.env['buildroot_pkg_config'],
+                                    '--cflags',
-                                '--libs',
+                                    pkg_key
-                                pkg_key
+                                ]).decode()
-                            ]).decode()
+                                ccflags_file.extend(self.sh.shlex_split(pkg_config_output))
-                            ccflags_after.extend(self.sh.shlex_split(pkg_config_output))
+                            if 'ccflags_after' in pkg:
-                    thread_limiter.acquire()
+                                ccflags_file.extend(pkg['ccflags_after'])
-                    if self.error:
+                            else:
-                        return 1
+                                pkg_config_output = subprocess.check_output([
-                    thread = threading.Thread(
+                                    self.env['buildroot_pkg_config'],
-                        target=self._build_one,
+                                    '--libs',
-                        kwargs={
+                                    pkg_key
-                            'in_path':in_path,
+                                ]).decode()
-                            'out_path':out_path,
+                                ccflags_after.extend(self.sh.shlex_split(pkg_config_output))
-                            'ccflags':ccflags_file,
+                        thread_limiter.acquire()
-                            'extra_objs':[common_obj],
+                        if self.error:
-                            'ccflags_after':ccflags_after,
+                            return 1
-                            'raise_on_failure':False,
+                        thread = threading.Thread(
-                            'thread_limiter':thread_limiter,
+                            target=self._build_one,
-                        }
+                            kwargs={
-                    )
+                                'in_path':in_path,
-                    thread.start()
+                                'out_path':out_path,
+                                'ccflags':ccflags_file,
+                                'extra_objs':[common_obj],
+                                'ccflags_after':ccflags_after,
+                                'raise_on_failure':False,
+                                'thread_limiter':thread_limiter,
+                            }
+                        )
+                        thread.start()
         self.sh.copy_dir_if_update(
             srcdir=build_dir,
             destdir=self.env['out_rootfs_overlay_dir'],

common.py

@@ -1169,7 +1169,10 @@ class BuildCliFunction(LkmcCliFunction):
         self.add_argument(
             '--force-rebuild',
             default=False,
-            help="Force rebuild even if sources didn't chage",
+            help='''\
+Force rebuild even if sources didn't chage.
+TODO: not yet implemented on all scripts.
+'''
         )
         self.add_argument(
             '-j',

shell_helpers.py

@@ -310,6 +310,19 @@ class ShellHelpers:
             else:
                 os.unlink(path)
 
+    def walk(self, root):
+        '''
+        Extended walk that can take files or directories.
+        '''
+        if not os.path.exists(root):
+            raise Exception('Path does not exist: ' + root)
+        if os.path.isfile(root):
+            dirname, basename = os.path.split(root)
+            yield dirname, [], [basename]
+        else:
+            for path, dirnames, filenames in os.walk(root):
+                yield path, dirnames, filenames
+
     def wget(self, url, download_path):
         '''
         Append extra KEY=val configs into the given config file.

userland/arch/aarch64/Makefile

@@ -1 +0,0 @@
-../../Makefile

userland/arch/aarch64/params.mk

@@ -1 +0,0 @@
-COMMON_DIR = ../../..

userland/arch/x86_64/Makefile

@@ -1 +0,0 @@
-../../Makefile

userland/arch/x86_64/params.mk

@@ -1 +0,0 @@
-COMMON_DIR = ../../..