diff --git a/build-userland b/build-userland index 7412c0b..d545c73 100755 --- a/build-userland +++ b/build-userland @@ -4,9 +4,12 @@ import os import shlex import common +import threading import subprocess from shell_helpers import LF +error = False + class Main(common.BuildCliFunction): def __init__(self): super().__init__( @@ -45,48 +48,64 @@ has the OpenBLAS libraries and headers installed. extra_objs=None, std=None, ccflags_after=None, + raise_on_failure=True, + thread_limiter=None, ): - if extra_deps is None: - extra_deps = [] - if extra_objs is None: - extra_objs = [] - if ccflags_after is None: - ccflags_after = [] - if self.need_rebuild([in_path] + extra_objs + extra_deps, out_path): - ccflags = ccflags.copy() - if not link: - ccflags.extend(['-c', LF]) - in_ext = os.path.splitext(in_path)[1] - if in_ext == self.env['c_ext']: - cc = self.env['gcc'] - if std is None: - std = 'c11' - ccflags.extend([ - '-fopenmp', LF, - ]) - elif in_ext == self.env['cxx_ext']: - cc = self.env['gxx'] - if std is None: - std = 'c++17' - else: - return - self.sh.run_cmd( - [ - cc, LF, - ] + - ccflags + - [ - '-std={}'.format(std), LF, - '-o', out_path, LF, - in_path, LF, - ] + - extra_objs + - [ - '-lm', LF, - '-pthread', LF, - ] + - ccflags_after - ) + try: + if extra_deps is None: + extra_deps = [] + if extra_objs is None: + extra_objs = [] + if ccflags_after is None: + ccflags_after = [] + ret = 0 + if self.need_rebuild([in_path] + extra_objs + extra_deps, out_path): + ccflags = ccflags.copy() + if not link: + ccflags.extend(['-c', LF]) + in_ext = os.path.splitext(in_path)[1] + do_compile = True + if in_ext == self.env['c_ext']: + cc = self.env['gcc'] + if std is None: + std = 'c11' + ccflags.extend([ + '-fopenmp', LF, + ]) + elif in_ext == self.env['cxx_ext']: + cc = self.env['gxx'] + if std is None: + std = 'c++17' + else: + do_compile = False + if do_compile: + ret = self.sh.run_cmd( + ( + [ + cc, LF, + ] + + ccflags + + [ + '-std={}'.format(std), LF, + '-o', out_path, LF, + in_path, LF, + ] + + extra_objs + + [ + '-lm', LF, + '-pthread', LF, + ] + + ccflags_after + ), + extra_paths=[self.env['ccache_dir']], + raise_on_failure=raise_on_failure, + ) + finally: + if thread_limiter is not None: + thread_limiter.release() + if ret != 0: + self.error = True + return ret def build(self): build_dir = self.get_build_dir() @@ -139,6 +158,8 @@ has the OpenBLAS libraries and headers installed. 'openblas': {}, } rootdir_abs_len = len(self.env['userland_source_dir']) + thread_limiter = threading.BoundedSemaphore(self.env['nproc']) + self.error = False for path, in_dirnames, in_filenames in os.walk(self.env['userland_source_dir']): in_dirnames.sort() dirpath_relative_root = path[rootdir_abs_len + 1:] @@ -185,51 +206,28 @@ has the OpenBLAS libraries and headers installed. pkg_key ]).decode() ccflags_after.extend(self.sh.shlex_split(pkg_config_output)) - self._build_one( - in_path=in_path, - out_path=out_path, - ccflags=ccflags_file, - extra_objs=[common_obj], - ccflags_after=ccflags_after, + thread_limiter.acquire() + if self.error: + return 1 + thread = threading.Thread( + target=self._build_one, + kwargs={ + 'in_path':in_path, + 'out_path':out_path, + 'ccflags':ccflags_file, + 'extra_objs':[common_obj], + 'ccflags_after':ccflags_after, + 'raise_on_failure':False, + 'thread_limiter':thread_limiter, + } ) - - #self.sh.run_cmd( - # ( - # [ - # 'make', LF, - # '-j', str(self.env['nproc']), LF, - # 'ARCH={}'.format(self.env['arch']), LF, - # 'CCFLAGS_SCRIPT={} {}'.format('-I', self.env['userland_source_dir']), LF, - # 'COMMON_DIR={}'.format(self.env['root_dir']), LF, - # 'CC={}'.format(self.env['gcc']), LF, - # 'CXX={}'.format(self.env['gxx']), LF, - # 'PKG_CONFIG={}'.format(self.env['buildroot_pkg_config']), LF, - # 'STAGING_DIR={}'.format(self.env['buildroot_staging_dir']), LF, - # 'OUT_DIR={}'.format(build_dir), LF, - # ] + - # self.sh.add_newlines([ - # 'HAS_{}=y'.format(package.upper()) - # for package in - # self.env['has_package'] - # ]) + - # make_args + - # self.sh.add_newlines([ - # os.path.join( - # build_dir, - # os.path.splitext(os.path.split(target)[1])[0] - # ) + self.env['userland_build_ext'] - # for target in self.env['targets'] - # ]) - # ), - # cwd=self.env['userland_source_dir'], - # extra_paths=[self.env['ccache_dir']], - #) - + thread.start() self.sh.copy_dir_if_update( srcdir=build_dir, destdir=self.env['out_rootfs_overlay_dir'], filter_ext=self.env['userland_build_ext'], ) + return 0 def get_build_dir(self): return self.env['userland_build_dir'] diff --git a/userland/Makefile b/userland/Makefile deleted file mode 100644 index c8510af..0000000 --- a/userland/Makefile +++ /dev/null @@ -1,75 +0,0 @@ -.PHONY: all clean mkdir - -ARCH = $(shell uname -m) -CCFLAGS = -ggdb3 -I$(COMMON_DIR) -O0 -Wall -Werror -Wextra -Wno-unused-function $(CCFLAGS_EXTRA) $(CCFLAGS_SCRIPT) -CFLAGS = -fopenmp -std=c99 $(CCFLAGS) $(CFLAGS_EXTRA) -CXXFLAGS = -std=c++17 $(CCFLAGS) $(CXXFLAGS_EXTRA) -# -Wno-unused-function for function definitions on headers, -# because we are lazy to make a shared object. TODO. -COMMON_DIR = $(CURDIR)/.. -COMMON_BASENAME = lkmc -COMMON_OBJ = $(OUT_DIR)/$(COMMON_BASENAME)$(OBJ_EXT) -IN_EXT_ASM = .S -IN_EXT_C = .c -IN_EXT_CXX = .cpp -LIBS = -lm -OBJ_EXT = .o -OUT_EXT = .out -OUT_DIR = $(CURDIR) - --include params.mk - -OUTS := $(foreach IN_EXT,$(IN_EXT_ASM) $(IN_EXT_C) $(IN_EXT_CXX),$(addsuffix $(OUT_EXT), $(basename $(wildcard *$(IN_EXT))))) -ifeq ($(HAS_EIGEN),y) - CXXFLAGS_EXTRA += -I$(STAGING_DIR)/usr/include/eigen3 - # TODO: was failing with: - # fatal error: Eigen/Dense: No such file or directory as of - # 975ce0723ee3fa1fea1766e6683e2f3acb8558d6 - # http://lists.busybox.net/pipermail/buildroot/2018-June/222914.html - #CXXFLAGS_EXTRA += $(shell $(PKG_CONFIG) --cflags eigen3) -else - OUTS := $(filter-out eigen_%$(OUT_EXT),$(OUTS)) -endif -ifeq ($(HAS_LIBDRM),y) - LIBS += $(shell $(PKG_CONFIG) --libs libdrm) - CFLAGS_EXTRA += $(shell $(PKG_CONFIG) --cflags libdrm) -else - OUTS := $(filter-out libdrm_%$(OUT_EXT),$(OUTS)) -endif -ifeq ($(HAS_OPENBLAS),y) - LIBS += $(shell $(PKG_CONFIG) --libs openblas) - CFLAGS_EXTRA += $(shell $(PKG_CONFIG) --cflags openblas) -else - OUTS := $(filter-out openblas_%$(OUT_EXT),$(OUTS)) -endif -OUTS := $(addprefix $(OUT_DIR)/,$(OUTS)) - -all: mkdir $(OUTS) - for subdir in $(SUBDIRS); do \ - if [ -d "$${subdir}" ]; then \ - $(MAKE) -C "$${subdir}" OUT_DIR="$(OUT_DIR)/$$subdir"; \ - fi \ - done - -$(COMMON_OBJ): $(COMMON_DIR)/$(COMMON_BASENAME)$(IN_EXT_C) - $(CC) $(CFLAGS) -c -o '$@' '$<' $(LIBS) - -$(OUT_DIR)/%$(OUT_EXT): %$(IN_EXT_ASM) $(COMMON_OBJ) - $(CC) $(CFLAGS) $(COMMON_OBJ) -o '$@' '$<' $(LIBS) - -$(OUT_DIR)/%$(OUT_EXT): %$(IN_EXT_C) $(COMMON_OBJ) - $(CC) $(CFLAGS) $(COMMON_OBJ) -o '$@' '$<' $(LIBS) - -$(OUT_DIR)/%$(OUT_EXT): %$(IN_EXT_CXX) $(COMMON_OBJ) - $(CXX) $(CXXFLAGS) -o '$@' '$<' $(LIBS) - -clean: - rm -f *'$(OBJ_EXT)' *'$(OUT_EXT)' - for subdir in $(SUBDIRS); do \ - if [ -d "$${subdir}" ]; then \ - $(MAKE) -C $${subdir} clean; \ - fi \ - done - -mkdir: - mkdir -p '$(OUT_DIR)'