build-userland: multithreading

Looking ahead for when I'll move cpp-cheat C++ slowness in.

First tried with concurrent as in:
https://stackoverflow.com/questions/19369724/the-right-way-to-limit-maximum-number-of-threads-running-at-once/19370282#19370282
Implementation was easy, but I can't find out how to exit immediately on error,
so I came up with this setup instead.

build-userland

@@ -4,9 +4,12 @@ import os
 import shlex
 
 import common
+import threading
 import subprocess
 from shell_helpers import LF
 
+error = False
+
 class Main(common.BuildCliFunction):
     def __init__(self):
         super().__init__(
@@ -45,48 +48,64 @@ has the OpenBLAS libraries and headers installed.
         extra_objs=None,
         std=None,
         ccflags_after=None,
+        raise_on_failure=True,
+        thread_limiter=None,
     ):
-        if extra_deps is None:
+        try:
-            extra_deps = []
+            if extra_deps is None:
-        if extra_objs is None:
+                extra_deps = []
-            extra_objs = []
+            if extra_objs is None:
-        if ccflags_after is None:
+                extra_objs = []
-            ccflags_after = []
+            if ccflags_after is None:
-        if self.need_rebuild([in_path] + extra_objs + extra_deps, out_path):
+                ccflags_after = []
-            ccflags = ccflags.copy()
+            ret = 0
-            if not link:
+            if self.need_rebuild([in_path] + extra_objs + extra_deps, out_path):
-                ccflags.extend(['-c', LF])
+                ccflags = ccflags.copy()
-            in_ext = os.path.splitext(in_path)[1]
+                if not link:
-            if in_ext == self.env['c_ext']:
+                    ccflags.extend(['-c', LF])
-                cc = self.env['gcc']
+                in_ext = os.path.splitext(in_path)[1]
-                if std is None:
+                do_compile = True
-                    std = 'c11'
+                if in_ext == self.env['c_ext']:
-                ccflags.extend([
+                    cc = self.env['gcc']
-                    '-fopenmp', LF,
+                    if std is None:
-                ])
+                        std = 'c11'
-            elif in_ext == self.env['cxx_ext']:
+                    ccflags.extend([
-                cc = self.env['gxx']
+                        '-fopenmp', LF,
-                if std is None:
+                    ])
-                    std = 'c++17'
+                elif in_ext == self.env['cxx_ext']:
-            else:
+                    cc = self.env['gxx']
-                return
+                    if std is None:
-            self.sh.run_cmd(
+                        std = 'c++17'
-                [
+                else:
-                    cc, LF,
+                    do_compile = False
-                ] +
+                if do_compile:
-                ccflags +
+                    ret = self.sh.run_cmd(
-                [
+                        (
-                    '-std={}'.format(std), LF,
+                            [
-                    '-o', out_path, LF,
+                                cc, LF,
-                    in_path, LF,
+                            ] +
-                ] +
+                            ccflags +
-                extra_objs +
+                            [
-                [
+                                '-std={}'.format(std), LF,
-                    '-lm', LF,
+                                '-o', out_path, LF,
-                    '-pthread', LF,
+                                in_path, LF,
-                ] +
+                            ] +
-                ccflags_after
+                            extra_objs +
-            )
+                            [
+                                '-lm', LF,
+                                '-pthread', LF,
+                            ] +
+                            ccflags_after
+                        ),
+                        extra_paths=[self.env['ccache_dir']],
+                        raise_on_failure=raise_on_failure,
+                    )
+        finally:
+            if thread_limiter is not None:
+                thread_limiter.release()
+        if ret != 0:
+            self.error = True
+        return ret
 
     def build(self):
         build_dir = self.get_build_dir()
@@ -139,6 +158,8 @@ has the OpenBLAS libraries and headers installed.
             'openblas': {},
         }
         rootdir_abs_len = len(self.env['userland_source_dir'])
+        thread_limiter = threading.BoundedSemaphore(self.env['nproc'])
+        self.error = False
         for path, in_dirnames, in_filenames in os.walk(self.env['userland_source_dir']):
             in_dirnames.sort()
             dirpath_relative_root = path[rootdir_abs_len + 1:]
@@ -185,51 +206,28 @@ has the OpenBLAS libraries and headers installed.
                                 pkg_key
                             ]).decode()
                             ccflags_after.extend(self.sh.shlex_split(pkg_config_output))
-                    self._build_one(
+                    thread_limiter.acquire()
-                        in_path=in_path,
+                    if self.error:
-                        out_path=out_path,
+                        return 1
-                        ccflags=ccflags_file,
+                    thread = threading.Thread(
-                        extra_objs=[common_obj],
+                        target=self._build_one,
-                        ccflags_after=ccflags_after,
+                        kwargs={
+                            'in_path':in_path,
+                            'out_path':out_path,
+                            'ccflags':ccflags_file,
+                            'extra_objs':[common_obj],
+                            'ccflags_after':ccflags_after,
+                            'raise_on_failure':False,
+                            'thread_limiter':thread_limiter,
+                        }
                     )
-
+                    thread.start()
-        #self.sh.run_cmd(
-        #    (
-        #        [
-        #            'make', LF,
-        #            '-j', str(self.env['nproc']), LF,
-        #            'ARCH={}'.format(self.env['arch']), LF,
-        #            'CCFLAGS_SCRIPT={} {}'.format('-I', self.env['userland_source_dir']), LF,
-        #            'COMMON_DIR={}'.format(self.env['root_dir']), LF,
-        #            'CC={}'.format(self.env['gcc']), LF,
-        #            'CXX={}'.format(self.env['gxx']), LF,
-        #            'PKG_CONFIG={}'.format(self.env['buildroot_pkg_config']), LF,
-        #            'STAGING_DIR={}'.format(self.env['buildroot_staging_dir']), LF,
-        #            'OUT_DIR={}'.format(build_dir), LF,
-        #        ] +
-        #        self.sh.add_newlines([
-        #            'HAS_{}=y'.format(package.upper())
-        #            for package in
-        #            self.env['has_package']
-        #        ]) +
-        #        make_args +
-        #        self.sh.add_newlines([
-        #            os.path.join(
-        #                build_dir,
-        #                os.path.splitext(os.path.split(target)[1])[0]
-        #            ) + self.env['userland_build_ext']
-        #            for target in self.env['targets']
-        #        ])
-        #    ),
-        #    cwd=self.env['userland_source_dir'],
-        #    extra_paths=[self.env['ccache_dir']],
-        #)
-
         self.sh.copy_dir_if_update(
             srcdir=build_dir,
             destdir=self.env['out_rootfs_overlay_dir'],
             filter_ext=self.env['userland_build_ext'],
         )
+        return 0
 
     def get_build_dir(self):
         return self.env['userland_build_dir']

userland/Makefile

@@ -1,75 +0,0 @@
-.PHONY: all clean mkdir
-
-ARCH = $(shell uname -m)
-CCFLAGS = -ggdb3 -I$(COMMON_DIR) -O0 -Wall -Werror -Wextra -Wno-unused-function $(CCFLAGS_EXTRA) $(CCFLAGS_SCRIPT)
-CFLAGS = -fopenmp -std=c99 $(CCFLAGS) $(CFLAGS_EXTRA)
-CXXFLAGS = -std=c++17 $(CCFLAGS) $(CXXFLAGS_EXTRA)
-# -Wno-unused-function for function definitions on headers,
-#  because we are lazy to make a shared object. TODO.
-COMMON_DIR = $(CURDIR)/..
-COMMON_BASENAME = lkmc
-COMMON_OBJ = $(OUT_DIR)/$(COMMON_BASENAME)$(OBJ_EXT)
-IN_EXT_ASM = .S
-IN_EXT_C = .c
-IN_EXT_CXX = .cpp
-LIBS = -lm
-OBJ_EXT = .o
-OUT_EXT = .out
-OUT_DIR = $(CURDIR)
-
--include params.mk
-
-OUTS := $(foreach IN_EXT,$(IN_EXT_ASM) $(IN_EXT_C) $(IN_EXT_CXX),$(addsuffix $(OUT_EXT), $(basename $(wildcard *$(IN_EXT)))))
-ifeq ($(HAS_EIGEN),y)
-	CXXFLAGS_EXTRA += -I$(STAGING_DIR)/usr/include/eigen3
-	# TODO: was failing with:
-	# fatal error: Eigen/Dense: No such file or directory as of
-	# 975ce0723ee3fa1fea1766e6683e2f3acb8558d6
-	# http://lists.busybox.net/pipermail/buildroot/2018-June/222914.html
-	#CXXFLAGS_EXTRA += $(shell $(PKG_CONFIG) --cflags eigen3)
-else
-	OUTS := $(filter-out eigen_%$(OUT_EXT),$(OUTS))
-endif
-ifeq ($(HAS_LIBDRM),y)
-	LIBS += $(shell $(PKG_CONFIG) --libs libdrm)
-	CFLAGS_EXTRA += $(shell $(PKG_CONFIG) --cflags libdrm)
-else
-	OUTS := $(filter-out libdrm_%$(OUT_EXT),$(OUTS))
-endif
-ifeq ($(HAS_OPENBLAS),y)
-	LIBS += $(shell $(PKG_CONFIG) --libs openblas)
-	CFLAGS_EXTRA += $(shell $(PKG_CONFIG) --cflags openblas)
-else
-	OUTS := $(filter-out openblas_%$(OUT_EXT),$(OUTS))
-endif
-OUTS := $(addprefix $(OUT_DIR)/,$(OUTS))
-
-all: mkdir $(OUTS)
-	for subdir in $(SUBDIRS); do \
-	  if [ -d "$${subdir}" ]; then \
-	    $(MAKE) -C "$${subdir}" OUT_DIR="$(OUT_DIR)/$$subdir"; \
-	  fi \
-	done
-
-$(COMMON_OBJ): $(COMMON_DIR)/$(COMMON_BASENAME)$(IN_EXT_C)
-	$(CC) $(CFLAGS) -c -o '$@' '$<' $(LIBS)
-
-$(OUT_DIR)/%$(OUT_EXT): %$(IN_EXT_ASM) $(COMMON_OBJ)
-	$(CC) $(CFLAGS) $(COMMON_OBJ) -o '$@' '$<' $(LIBS)
-
-$(OUT_DIR)/%$(OUT_EXT): %$(IN_EXT_C) $(COMMON_OBJ)
-	$(CC) $(CFLAGS) $(COMMON_OBJ) -o '$@' '$<' $(LIBS)
-
-$(OUT_DIR)/%$(OUT_EXT): %$(IN_EXT_CXX) $(COMMON_OBJ)
-	$(CXX) $(CXXFLAGS) -o '$@' '$<' $(LIBS)
-
-clean:
-	rm -f *'$(OBJ_EXT)' *'$(OUT_EXT)'
-	for subdir in $(SUBDIRS); do \
-	  if [ -d "$${subdir}" ]; then \
-	    $(MAKE) -C $${subdir} clean; \
-	  fi \
-	done
-
-mkdir:
-	mkdir -p '$(OUT_DIR)'