#!/usr/bin/env python3 import os import shlex import subprocess import threading from shell_helpers import LF import common from thread_pool import ThreadPool class Main(common.BuildCliFunction): def __init__(self, *args, **kwargs): if not 'description' in kwargs: kwargs['description'] = '''\ Build our compiled userland examples. ''' super().__init__(*args, **kwargs) self.default_cstd = 'c11' self.default_cxxstd = 'c++17' self.add_argument( '--has-package', action='append', default=[], help='''\ Indicate that a given package is present in the root filesystem, which allows us to build examples that rely on it. ''', ) self.add_argument( '--has-all-packages', action='store_true', help='''\ Indicate that all packages from --has-package are available. ''', ) self.add_argument( '--in-tree', default=False, help='''\ Place build output inside soure tree to conveniently run it, especially when building with the host toolchain. ''', ) self.add_argument( 'targets', default=[], help='''\ Build only the given userland programs or all programs in the given directories. Default: build all examples that have their package dependencies met, e.g.: - userland/arch/ programs only build if the target arch matches - an OpenBLAS example can only be built if the target root filesystem has the OpenBLAS libraries and headers installed, which you must inform with --has-package ''', nargs='*', ) self._add_argument('--ccflags') self._add_argument('--force-rebuild') self._add_argument('--optimization-level') def _build_one( self, in_path, out_path, ccflags, ccflags_after=None, cstd=None, cxxstd=None, extra_deps=None, extra_objs=None, link=True, ): if extra_deps is None: extra_deps = [] if extra_objs is None: extra_objs = [] if ccflags_after is None: ccflags_after = [] ret = 0 if self.need_rebuild([in_path] + extra_objs + extra_deps, out_path): ccflags = ccflags.copy() if not link: ccflags.extend(['-c', LF]) in_ext = os.path.splitext(in_path)[1] do_compile = True if in_ext in (self.env['c_ext'], self.env['asm_ext']): cc = self.env['gcc'] if cstd is None: std = self.default_cstd else: std = cstd ccflags.extend([ '-fopenmp', LF, ]) elif in_ext == self.env['cxx_ext']: cc = self.env['gxx'] if cxxstd is None: std = self.default_cxxstd else: std = cxxstd else: do_compile = False if do_compile: os.makedirs(os.path.dirname(out_path), exist_ok=True) ret = self.sh.run_cmd( ( [ cc, LF, ] + ccflags + [ '-std={}'.format(std), LF, '-o', out_path, LF, in_path, LF, ] + self.sh.add_newlines(extra_objs) + [ '-lm', LF, '-pthread', LF, ] + ccflags_after ), extra_paths=[self.env['ccache_dir']], ) return ret def _walk_targets(self, exts): ''' Resolve userland input targets, and walk them if directories. Ignore the input extension of targets, and select only files with the given extensions exts. An empty extension indicates that directories will also be chosen. ''' if self.env['targets']: targets = self.env['targets'] else: targets = [self.env['userland_source_dir']] for target in targets: resolved_targets = self.resolve_source_tree( target, exts + [''], self.env['userland_source_dir'] ) for resolved_target in resolved_targets: for path, dirnames, filenames in self.sh.walk(resolved_target): dirnames.sort() filenames = [ filename for filename in filenames if os.path.splitext(filename)[1] in exts ] filenames.sort() for filename in filenames: yield path, dirnames, filenames def build(self): build_dir = self.get_build_dir() has_packages = set(self.env['has_package']) has_all_packages = self.env['has_all_packages'] ccflags = [ '-I', self.env['root_dir'], LF, '-O{}'.format(self.env['optimization_level']), LF, '-Wall', LF, '-Werror', LF, '-Wextra', LF, '-Wno-unused-function', LF, '-ggdb3', LF, ] + self.sh.shlex_split(self.env['ccflags']) if self.env['static']: ccflags.extend(['-static', LF]) common_obj = os.path.join( build_dir, self.env['common_basename_noext'] + self.env['obj_ext'] ) self._build_one( in_path=self.env['common_c'], out_path=common_obj, ccflags=ccflags, extra_deps=[self.env['common_h']], link=False, ) common_obj_asm = os.path.join( build_dir, 'arch', 'main' + self.env['obj_ext'] ) common_obj_asm_relpath = os.path.join( 'arch', 'main' + self.env['c_ext'] ) self._build_one( in_path=os.path.join( self.env['userland_source_dir'], common_obj_asm_relpath ), out_path=common_obj_asm, ccflags=ccflags, extra_deps=[self.env['common_h']], link=False, ) if self.env['gcc_which'] == 'host': eigen_root = '/' else: eigen_root = self.env['buildroot_staging_dir'] pkgs = { 'eigen': { # TODO: was failing with: # fatal error: Eigen/Dense: No such file or directory as of # 975ce0723ee3fa1fea1766e6683e2f3acb8558d6 # http://lists.busybox.net/pipermail/buildroot/2018-June/222914.html 'ccflags': [ '-I', os.path.join( eigen_root, 'usr', 'include', 'eigen3' ), LF ], # Header only. 'ccflags_after': [], }, 'libdrm': {}, 'openblas': {}, } rootdir_abs_len = len(self.env['userland_source_dir']) with ThreadPool( self._build_one, nthreads=self.env['nproc'], ) as thread_pool: class ExitLoop(Exception): pass try: for path, in_dirnames, in_filenames in self._walk_targets( self.env['userland_in_exts'] ): path_abs = os.path.abspath(path) dirpath_relative_root = path_abs[rootdir_abs_len + 1:] dirpath_relative_root_components = dirpath_relative_root.split(os.sep) dirpath_relative_root_components_len = len(dirpath_relative_root_components) out_dir = os.path.join( build_dir, dirpath_relative_root ) common_objs_dir = [common_obj] ccflags_after = [] ccflags_dir = ccflags.copy() if dirpath_relative_root_components_len > 0: if dirpath_relative_root_components[0] in ( 'gcc', 'kernel_modules', 'linux', ): cstd = 'gnu11' cxxstd = 'gnu++17' else: cstd = self.default_cstd cxxstd = self.default_cxxstd # -pedantic complains even if we use -std=gnu11. ccflags_dir.extend(['-pedantic', LF]) if dirpath_relative_root_components[0] == 'arch': if dirpath_relative_root_components_len > 1: if dirpath_relative_root_components[1] == self.env['arch']: ccflags_dir.extend([ '-I', os.path.join(self.env['userland_source_arch_arch_dir']), LF, '-I', os.path.join(self.env['userland_source_arch_dir']), LF, '-fno-pie', LF, '-no-pie', LF, ]) if 'freestanding' in dirpath_relative_root_components: common_objs_dir = [] ccflags_dir.extend([ '-ffreestanding', LF, '-nostdlib', LF, '-static', LF, ]) else: if 'c' in dirpath_relative_root_components: common_objs_dir = [] else: common_objs_dir = [common_obj_asm] if self.env['arch'] == 'arm': ccflags_dir.extend([ '-Xassembler', '-mcpu=cortex-a72', LF, # To prevent: # > vfp.S: Error: selected processor does not support in ARM mode # https://stackoverflow.com/questions/41131432/cross-compiling-error-selected-processor-does-not-support-fmrx-r3-fpexc-in/52875732#52875732 # We aim to take the most extended mode currently available that works on QEMU. '-Xassembler', '-mfpu=crypto-neon-fp-armv8.1', LF, '-Xassembler', '-meabi=5', LF, # Treat inline assembly as arm instead of thumb # The opposite of -mthumb. '-marm', LF, # Make gcc generate .syntax unified for inline assembly. # However, it gets ignored if -marm is given, which a GCC bug that was recently fixed: # https://stackoverflow.com/questions/54078112/how-to-write-syntax-unified-ual-armv7-inline-assembly-in-gcc/54132097#54132097 # So we just write divided inline assembly for now. '-masm-syntax-unified', LF, ]) else: continue else: continue elif dirpath_relative_root_components[0] == 'libs': if dirpath_relative_root_components_len > 1: pkg_key = dirpath_relative_root_components[1] if not (has_all_packages or pkg_key in has_packages): continue pkg = pkgs[pkg_key] if 'ccflags' in pkg: ccflags_dir.extend(pkg['ccflags']) else: pkg_config_output = subprocess.check_output([ self.env['pkg_config'], '--cflags', pkg_key ]).decode() ccflags_dir.extend(self.sh.shlex_split(pkg_config_output)) if 'ccflags_after' in pkg: ccflags_dir.extend(pkg['ccflags_after']) else: pkg_config_output = subprocess.check_output([ self.env['pkg_config'], '--libs', pkg_key ]).decode() ccflags_after.extend(self.sh.shlex_split(pkg_config_output)) for in_filename in in_filenames: path_relative_root = os.path.join(dirpath_relative_root, in_filename) if path_relative_root == common_obj_asm_relpath: continue in_path = os.path.join(path, in_filename) in_name, in_ext = os.path.splitext(in_filename) out_path = os.path.join( out_dir, in_name + self.env['userland_build_ext'] ) error = thread_pool.submit({ 'in_path': in_path, 'out_path': out_path, 'ccflags': ccflags_dir, 'cstd': cstd, 'cxxstd': cxxstd, 'extra_objs': common_objs_dir, 'ccflags_after': ccflags_after, }) if error is not None: raise ExitLoop() except ExitLoop: pass error = thread_pool.get_error() if error is not None: print(error) return 1 if not self.env['in_tree']: self.sh.copy_dir_if_update( srcdir=build_dir, destdir=self.env['out_rootfs_overlay_lkmc_dir'], filter_ext=self.env['userland_build_ext'], ) return 0 def clean(self): if self.env['in_tree']: for path, dirnames, filenames in self._walk_targets( self.env['userland_out_exts'] ): for filename in filenames: self.sh.rmrf(os.path.join(path, filename)) else: self.sh.rmrf(self.get_build_dir()) def get_build_dir(self): if self.env['in_tree']: return self.env['userland_source_dir'] else: return self.env['userland_build_dir'] if __name__ == '__main__': Main().cli()