#!/usr/bin/env python3 import os import shlex import subprocess import threading from shell_helpers import LF import common from thread_pool import ThreadPool import path_properties class Main(common.BuildCliFunction): def __init__(self, *args, **kwargs): if not 'description' in kwargs: kwargs['description'] = '''\ Build our compiled userland examples. ''' super().__init__(*args, **kwargs) self.add_argument( 'targets', default=[], help='''\ Select to build only the given userland programs, or all programs under the given directories. Default: build all. Must point to either sources or directories under userland/, or to LKMC toplevel which is a synonym for userland/. Default: build all examples that have their package dependencies met, e.g.: - userland/arch/ programs only build if the target arch matches - an OpenBLAS example can only be built if the target root filesystem has the OpenBLAS libraries and headers installed, which you must inform with --package ''', nargs='*', ) self._add_argument('--ccflags') self._add_argument('--force-rebuild') self._add_argument('--optimization-level') def _build_one( self, in_path, out_path, cc_flags, cc_flags_after=None, c_std=None, cxx_std=None, extra_deps=None, extra_objs=None, link=True, ): if extra_deps is None: extra_deps = [] if extra_objs is None: extra_objs = [] if cc_flags_after is None: cc_flags_after = [] ret = 0 if self.need_rebuild([in_path] + extra_objs + extra_deps, out_path): cc_flags = cc_flags.copy() if not link: cc_flags.extend(['-c', LF]) in_ext = os.path.splitext(in_path)[1] if in_ext in (self.env['c_ext'], self.env['asm_ext']): cc = self.env['gcc'] if c_std is None: std = path_properties.default_c_std else: std = c_std elif in_ext == self.env['cxx_ext']: cc = self.env['gxx'] if cxx_std is None: std = path_properties.default_cxx_std else: std = cxx_std os.makedirs(os.path.dirname(out_path), exist_ok=True) ret = self.sh.run_cmd( ( [ cc, LF, ] + cc_flags + [ '-std={}'.format(std), LF, '-o', out_path, LF, in_path, LF, ] + self.sh.add_newlines(extra_objs) + cc_flags_after ), extra_paths=[self.env['ccache_dir']], ) return ret def build(self): build_dir = self.get_build_dir() cc_flags = [ '-I', self.env['root_dir'], LF, '-O{}'.format(self.env['optimization_level']), LF, ] + self.sh.shlex_split(self.env['ccflags']) if self.env['static']: cc_flags.extend(['-static', LF]) common_obj = os.path.join( build_dir, self.env['common_basename_noext'] + self.env['obj_ext'] ) self._build_one( in_path=self.env['common_c'], out_path=common_obj, cc_flags=cc_flags, extra_deps=[self.env['common_h']], link=False, ) common_obj_asm = os.path.join( build_dir, 'arch', 'main' + self.env['obj_ext'] ) common_obj_asm_relpath = os.path.join( 'arch', 'main' + self.env['c_ext'] ) self._build_one( in_path=os.path.join( self.env['userland_source_dir'], common_obj_asm_relpath ), out_path=common_obj_asm, cc_flags=cc_flags, extra_deps=[self.env['common_h']], link=False, ) if self.env['gcc_which'] == 'host': eigen_root = '/' else: eigen_root = self.env['buildroot_staging_dir'] packages = { 'eigen': { # TODO: was failing with: # fatal error: Eigen/Dense: No such file or directory as of # 975ce0723ee3fa1fea1766e6683e2f3acb8558d6 # http://lists.busybox.net/pipermail/buildroot/2018-June/222914.html 'cc_flags': [ '-I', os.path.join( eigen_root, 'usr', 'include', 'eigen3' ), LF ], # Header only. 'cc_flags_after': [], }, } rootdir_abs_len = len(self.env['userland_source_dir']) with ThreadPool( self._build_one, nthreads=self.env['nproc'], ) as thread_pool: try: for target in self.env['targets']: for path, in_dirnames, in_filenames in self.sh.walk(target): path_abs = os.path.abspath(path) dirpath_relative_root = path_abs[rootdir_abs_len + 1:] dirpath_relative_root_components = dirpath_relative_root.split(os.sep) dirpath_relative_root_components_len = len(dirpath_relative_root_components) out_dir = os.path.join( build_dir, dirpath_relative_root ) for in_filename in in_filenames: in_path = os.path.join(path, in_filename) cc_flags_file = cc_flags.copy() cc_flags_after = [] in_ext = os.path.splitext(in_filename)[1] if not in_ext in self.env['userland_in_exts']: continue my_path_properties = path_properties.get(os.path.join( self.env['userland_subdir'], dirpath_relative_root, in_filename )) if my_path_properties.should_be_built(self.env): if dirpath_relative_root_components_len > 0: if dirpath_relative_root_components[0] == 'arch': cc_flags_file.extend([ '-I', os.path.join(self.env['userland_source_arch_arch_dir']), LF, '-I', os.path.join(self.env['userland_source_arch_dir']), LF, ]) elif dirpath_relative_root_components[0] == 'libs': if dirpath_relative_root_components_len > 1: package_key = dirpath_relative_root_components[1] if package_key in packages: package = packages[package_key] else: package = {} if 'cc_flags' in package: cc_flags_file.extend(package['cc_flags']) else: pkg_config_output = subprocess.check_output([ self.env['pkg_config'], '--cflags', package_key ]).decode() cc_flags_file.extend(self.sh.shlex_split(pkg_config_output)) if 'cc_flags_after' in package: cc_flags_file.extend(package['cc_flags_after']) else: pkg_config_output = subprocess.check_output([ self.env['pkg_config'], '--libs', package_key ]).decode() cc_flags_after.extend(self.sh.shlex_split(pkg_config_output)) if my_path_properties['cc_pedantic']: cc_flags_file.extend(['-pedantic', LF]) common_objs_file = [] if my_path_properties['extra_objs_lkmc_common']: common_objs_file.append(common_obj) if my_path_properties['extra_objs_userland_asm']: common_objs_file.append(common_obj_asm) error = thread_pool.submit({ 'c_std': my_path_properties['c_std'], 'cc_flags': cc_flags_file + my_path_properties['cc_flags'], 'cc_flags_after': cc_flags_after + my_path_properties['cc_flags_after'], 'cxx_std': my_path_properties['cxx_std'], 'extra_objs': common_objs_file, 'in_path': in_path, 'out_path': self.resolve_userland_executable(in_path), }) if error is not None: raise common.ExitLoop() except common.ExitLoop: pass error = thread_pool.get_error() if error is not None: print(error) return 1 if not self.env['in_tree']: self.sh.copy_dir_if_update( srcdir=build_dir, destdir=self.env['out_rootfs_overlay_lkmc_dir'], filter_ext=self.env['userland_executable_ext'], ) return 0 def clean(self): if self.env['in_tree']: for target in self.env['targets']: if os.path.exists(target): if os.path.isfile(target): self.sh.rmrf(self.resolve_userland_executable(target)) else: for path, dirnames, filenames in self.sh.walk(target): for filename in filenames: if os.path.splitext(filename)[1] in self.env['userland_out_exts']: self.sh.rmrf(os.path.join(path, filename)) else: for target in self.env['targets']: self.sh.rmrf(self.resolve_userland_executable(target)) def get_build_dir(self): return self.env['userland_build_dir'] def setup_one(self): self.env['targets'] = self.resolve_targets( self.env['userland_source_dir'], self.env['targets'] ) if __name__ == '__main__': Main().cli()