#!/usr/bin/env python3 import os import shlex import subprocess import threading from shell_helpers import LF import common from thread_pool import ThreadPool import path_properties class Main(common.BuildCliFunction): def __init__(self, *args, **kwargs): if not 'description' in kwargs: kwargs['description'] = '''\ Build our compiled userland examples. ''' super().__init__(*args, **kwargs) self.add_argument( '--has-package', action='append', help='''\ Indicate that a given package is present in the root filesystem, which allows us to build examples that rely on it. ''', ) self.add_argument( '--has-all-packages', action='store_true', help='''\ Indicate that all packages from --has-package are available. ''', ) self.add_argument( 'targets', default=[], help='''\ Select to build only the given userland programs, or all programs under the given directories. Default: build all. Must point to either sources or directories under userland/, or to LKMC toplevel which is a synonym for userland/. Default: build all examples that have their package dependencies met, e.g.: - userland/arch/ programs only build if the target arch matches - an OpenBLAS example can only be built if the target root filesystem has the OpenBLAS libraries and headers installed, which you must inform with --has-package ''', nargs='*', ) self._add_argument('--ccflags') self._add_argument('--force-rebuild') self._add_argument('--optimization-level') def _build_one( self, in_path, out_path, cc_flags, cc_flags_after=None, c_std=None, cxx_std=None, extra_deps=None, extra_objs=None, link=True, ): if extra_deps is None: extra_deps = [] if extra_objs is None: extra_objs = [] if cc_flags_after is None: cc_flags_after = [] ret = 0 if self.need_rebuild([in_path] + extra_objs + extra_deps, out_path): cc_flags = cc_flags.copy() if not link: cc_flags.extend(['-c', LF]) in_ext = os.path.splitext(in_path)[1] if in_ext in (self.env['c_ext'], self.env['asm_ext']): cc = self.env['gcc'] if c_std is None: std = path_properties.default_c_std else: std = c_std cc_flags.extend([ '-fopenmp', LF, ]) elif in_ext == self.env['cxx_ext']: cc = self.env['gxx'] if cxx_std is None: std = path_properties.default_cxx_std else: std = cxx_std os.makedirs(os.path.dirname(out_path), exist_ok=True) ret = self.sh.run_cmd( ( [ cc, LF, ] + cc_flags + [ '-std={}'.format(std), LF, '-o', out_path, LF, in_path, LF, ] + self.sh.add_newlines(extra_objs) + [ '-lm', LF, '-pthread', LF, ] + cc_flags_after ), extra_paths=[self.env['ccache_dir']], ) return ret def build(self): build_dir = self.get_build_dir() has_packages = set(self.env['has_package']) has_all_packages = self.env['has_all_packages'] cc_flags = [ '-I', self.env['root_dir'], LF, '-O{}'.format(self.env['optimization_level']), LF, '-Wall', LF, '-Werror', LF, '-Wextra', LF, '-Wno-unused-function', LF, '-ggdb3', LF, ] + self.sh.shlex_split(self.env['ccflags']) if self.env['static']: cc_flags.extend(['-static', LF]) common_obj = os.path.join( build_dir, self.env['common_basename_noext'] + self.env['obj_ext'] ) self._build_one( in_path=self.env['common_c'], out_path=common_obj, cc_flags=cc_flags, extra_deps=[self.env['common_h']], link=False, ) common_obj_asm = os.path.join( build_dir, 'arch', 'main' + self.env['obj_ext'] ) common_obj_asm_relpath = os.path.join( 'arch', 'main' + self.env['c_ext'] ) self._build_one( in_path=os.path.join( self.env['userland_source_dir'], common_obj_asm_relpath ), out_path=common_obj_asm, cc_flags=cc_flags, extra_deps=[self.env['common_h']], link=False, ) if self.env['gcc_which'] == 'host': eigen_root = '/' else: eigen_root = self.env['buildroot_staging_dir'] pkgs = { 'eigen': { # TODO: was failing with: # fatal error: Eigen/Dense: No such file or directory as of # 975ce0723ee3fa1fea1766e6683e2f3acb8558d6 # http://lists.busybox.net/pipermail/buildroot/2018-June/222914.html 'cc_flags': [ '-I', os.path.join( eigen_root, 'usr', 'include', 'eigen3' ), LF ], # Header only. 'cc_flags_after': [], }, 'libdrm': {}, 'openblas': {}, } rootdir_abs_len = len(self.env['userland_source_dir']) with ThreadPool( self._build_one, nthreads=self.env['nproc'], ) as thread_pool: try: for target in self.env['targets']: for path, in_dirnames, in_filenames in self.sh.walk(target): path_abs = os.path.abspath(path) dirpath_relative_root = path_abs[rootdir_abs_len + 1:] dirpath_relative_root_components = dirpath_relative_root.split(os.sep) dirpath_relative_root_components_len = len(dirpath_relative_root_components) out_dir = os.path.join( build_dir, dirpath_relative_root ) common_objs_dir = [] cc_flags_after = [] cc_flags_dir = cc_flags.copy() if dirpath_relative_root_components_len > 0: if dirpath_relative_root_components[0] == 'arch': if dirpath_relative_root_components_len > 1: if dirpath_relative_root_components[1] == self.env['arch']: cc_flags_dir.extend([ '-I', os.path.join(self.env['userland_source_arch_arch_dir']), LF, '-I', os.path.join(self.env['userland_source_arch_dir']), LF, ]) if 'freestanding' in dirpath_relative_root_components: common_objs_dir = [] cc_flags_dir.extend([ '-ffreestanding', LF, '-nostdlib', LF, '-static', LF, ]) else: if 'c' in dirpath_relative_root_components: common_objs_dir = [] else: common_objs_dir = [common_obj_asm] else: continue else: continue elif dirpath_relative_root_components[0] == 'libs': if dirpath_relative_root_components_len > 1: pkg_key = dirpath_relative_root_components[1] if not (has_all_packages or pkg_key in has_packages): continue pkg = pkgs[pkg_key] if 'cc_flags' in pkg: cc_flags_dir.extend(pkg['cc_flags']) else: pkg_config_output = subprocess.check_output([ self.env['pkg_config'], '--cflags', pkg_key ]).decode() cc_flags_dir.extend(self.sh.shlex_split(pkg_config_output)) if 'cc_flags_after' in pkg: cc_flags_dir.extend(pkg['cc_flags_after']) else: pkg_config_output = subprocess.check_output([ self.env['pkg_config'], '--libs', pkg_key ]).decode() cc_flags_after.extend(self.sh.shlex_split(pkg_config_output)) for in_filename in in_filenames: in_path = os.path.join(path, in_filename) cc_flags_file = cc_flags_dir.copy() in_ext = os.path.splitext(in_filename)[1] if not in_ext in self.env['userland_in_exts']: continue my_path_properties = path_properties.get(os.path.join( self.env['userland_subdir'], dirpath_relative_root, in_filename )) if my_path_properties['pedantic']: cc_flags_file.extend(['-pedantic', LF]) common_objs_file = common_objs_dir.copy() if my_path_properties['lkmc_common_obj']: common_objs_file.append(common_obj) error = thread_pool.submit({ 'c_std': my_path_properties['c_std'], 'cc_flags': cc_flags_file + my_path_properties['cc_flags'], 'cc_flags_after': cc_flags_after, 'cxx_std': my_path_properties['cxx_std'], 'extra_objs': common_objs_file, 'in_path': in_path, 'out_path': self.resolve_userland_executable(in_path), }) if error is not None: raise common.ExitLoop() except common.ExitLoop: pass error = thread_pool.get_error() if error is not None: print(error) return 1 if not self.env['in_tree']: self.sh.copy_dir_if_update( srcdir=build_dir, destdir=self.env['out_rootfs_overlay_lkmc_dir'], filter_ext=self.env['userland_executable_ext'], ) return 0 def clean(self): if self.env['in_tree']: for target in self.env['targets']: if os.path.exists(target): if os.path.isfile(target): self.sh.rmrf(self.resolve_userland_executable(target)) else: for path, dirnames, filenames in self.sh.walk(target): for filename in filenames: if os.path.splitext(filename)[1] in self.env['userland_out_exts']: self.sh.rmrf(os.path.join(path, filename)) else: for target in self.env['targets']: self.sh.rmrf(self.resolve_userland_executable(target)) def get_build_dir(self): return self.env['userland_build_dir'] def setup_one(self): self.env['targets'] = self.resolve_targets( self.env['userland_source_dir'], self.env['targets'] ) if __name__ == '__main__': Main().cli()