#!/usr/bin/env python3 import os import shlex import common import threading import subprocess from shell_helpers import LF error = False class Main(common.BuildCliFunction): def __init__(self): super().__init__( description='''\ Build our compiled userland examples. ''' ) self.default_cstd = 'c11' self.default_cxxstd = 'c++17' self.add_argument( '--has-package', action='append', default=[], help='''\ Indicate that a given package is present in the root filesystem, which allows us to build examples that rely on it. ''', ) self.add_argument( 'targets', default=[], help='''\ Build only the given userland programs or all programs in the given directories. Default: build all examples that have their package dependencies met, e.g.: - userland/arch/ programs only build if the target arch matches - an OpenBLAS example can only be built if the target root filesystem has the OpenBLAS libraries and headers installed, which you must inform with --has-package ''', nargs='*', ) def _build_one( self, in_path, out_path, ccflags, ccflags_after=None, cstd=None, cxxstd=None, extra_deps=None, extra_objs=None, link=True, raise_on_failure=True, thread_limiter=None, ): try: if extra_deps is None: extra_deps = [] if extra_objs is None: extra_objs = [] if ccflags_after is None: ccflags_after = [] ret = 0 if self.need_rebuild([in_path] + extra_objs + extra_deps, out_path): ccflags = ccflags.copy() if not link: ccflags.extend(['-c', LF]) in_ext = os.path.splitext(in_path)[1] do_compile = True if in_ext == self.env['c_ext']: cc = self.env['gcc'] if cstd is None: std = self.default_cstd else: std = cstd ccflags.extend([ '-fopenmp', LF, ]) elif in_ext == self.env['cxx_ext']: cc = self.env['gxx'] if cxxstd is None: std = self.default_cxxstd else: std = cxxstd else: do_compile = False if do_compile: ret = self.sh.run_cmd( ( [ cc, LF, ] + ccflags + [ '-std={}'.format(std), LF, '-o', out_path, LF, in_path, LF, ] + self.sh.add_newlines(extra_objs) + [ '-lm', LF, '-pthread', LF, ] + ccflags_after ), extra_paths=[self.env['ccache_dir']], raise_on_failure=raise_on_failure, ) finally: if thread_limiter is not None: thread_limiter.release() if ret != 0: self.error = True return ret def build(self): build_dir = self.get_build_dir() os.makedirs(build_dir, exist_ok=True) has_packages = set(self.env['has_package']) ccflags = [ '-I', self.env['root_dir'], LF, '-I', self.env['userland_source_dir'], LF, '-O0', LF, '-Wall', LF, '-Werror', LF, '-Wextra', LF, '-Wno-unused-function', LF, '-ggdb3', LF, ] if self.env['static']: ccflags.extend(['-static', LF]) common_obj = os.path.join( build_dir, self.env['common_basename_noext'] + self.env['obj_ext'] ) self._build_one( in_path=self.env['common_c'], out_path=common_obj, ccflags=ccflags, extra_deps=[self.env['common_h']], link=False, ) pkgs = { 'eigen': { # TODO: was failing with: # fatal error: Eigen/Dense: No such file or directory as of # 975ce0723ee3fa1fea1766e6683e2f3acb8558d6 # http://lists.busybox.net/pipermail/buildroot/2018-June/222914.html 'ccflags': [ '-I', os.path.join( self.env['buildroot_staging_dir'], 'usr', 'include', 'eigen3' ), LF ], # Header only. 'ccflags_after': [], }, 'libdrm': {}, 'openblas': {}, } rootdir_abs_len = len(self.env['userland_source_dir']) thread_limiter = threading.BoundedSemaphore(self.env['nproc']) self.error = False if self.env['_args_given']['targets']: targets = self.env['targets'] else: targets = [self.env['userland_source_dir']] for target in targets: target = self.resolve_userland_source(target) for path, in_dirnames, in_filenames in self.sh.walk(target): in_dirnames.sort() path_abs = os.path.abspath(path) dirpath_relative_root = path_abs[rootdir_abs_len + 1:] dirpath_relative_root_components = dirpath_relative_root.split(os.sep) if ( len(dirpath_relative_root_components) < 2 or dirpath_relative_root_components[0] != 'arch' or dirpath_relative_root_components[1] == self.env['arch'] ): out_dir = os.path.join( self.env['userland_build_dir'], dirpath_relative_root ) os.makedirs(out_dir, exist_ok=True) ccflags_dir = ccflags.copy() if dirpath_relative_root_components == ['gcc']: cstd = 'gnu11' cxxstd = 'gnu++17' else: cstd = self.default_cstd cxxstd = self.default_cxxstd # -pedantic complains even if we use -std=gnu11. ccflags_dir.extend(['-pedantic', LF]) for in_filename in in_filenames: in_path = os.path.join(path, in_filename) in_name, in_ext = os.path.splitext(in_filename) out_path = os.path.join( out_dir, in_name + self.env['userland_build_ext'] ) pkg_key = in_name.split('_')[0] ccflags_file = ccflags_dir.copy() ccflags_after = [] if pkg_key in pkgs: if pkg_key not in has_packages: continue pkg = pkgs[pkg_key] if 'ccflags' in pkg: ccflags_file.extend(pkg['ccflags']) else: pkg_config_output = subprocess.check_output([ self.env['buildroot_pkg_config'], '--cflags', pkg_key ]).decode() ccflags_file.extend(self.sh.shlex_split(pkg_config_output)) if 'ccflags_after' in pkg: ccflags_file.extend(pkg['ccflags_after']) else: pkg_config_output = subprocess.check_output([ self.env['buildroot_pkg_config'], '--libs', pkg_key ]).decode() ccflags_after.extend(self.sh.shlex_split(pkg_config_output)) thread_limiter.acquire() if self.error: return 1 thread = threading.Thread( target=self._build_one, kwargs={ 'in_path': in_path, 'out_path': out_path, 'ccflags': ccflags_file, 'cstd': cstd, 'cxxstd': cxxstd, 'extra_objs': [common_obj], 'ccflags_after': ccflags_after, 'raise_on_failure': False, 'thread_limiter': thread_limiter, } ) thread.start() self.sh.copy_dir_if_update( srcdir=build_dir, destdir=self.env['out_rootfs_overlay_dir'], filter_ext=self.env['userland_build_ext'], ) return 0 def get_build_dir(self): return self.env['userland_build_dir'] if __name__ == '__main__': Main().cli()