From 4f6051af1defe93e113e5028d83b5f7cb7a1b9bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ciro=20Santilli=20=E5=85=AD=E5=9B=9B=E4=BA=8B=E4=BB=B6=20?= =?UTF-8?q?=E6=B3=95=E8=BD=AE=E5=8A=9F?= Date: Wed, 23 Jan 2019 00:00:03 +0000 Subject: [PATCH] build android sketch --- README.adoc | 26 ++++++++++++++ build-android | 76 +++++++++++++++++++++++++++++++++++++++++ cli_function.py | 26 +++++++++++--- common.py | 51 +++++++++++++++++++-------- copy-overlay | 1 + release-download-latest | 1 + run | 4 +-- shell_helpers.py | 39 +++++++++++++++++---- 8 files changed, 197 insertions(+), 27 deletions(-) create mode 100755 build-android diff --git a/README.adoc b/README.adoc index 24ce12c..654f837 100644 --- a/README.adoc +++ b/README.adoc @@ -11353,6 +11353,32 @@ gem5: ** https://stackoverflow.com/questions/47997565/gem5-system-requirements-for-decent-performance/48941793#48941793 ** https://github.com/gem5/gem5/issues/25 +== WIP + +Big new features that are not yet working. + +=== Android + +Remember: Android AOSP is a huge undocumented piece of bloatware. It's integration into this repo will likely never be super good. + +https://stackoverflow.com/questions/1809774/how-to-compile-the-android-aosp-kernel-and-test-it-with-the-android-emulator/48310014#48310014 + +.... +./build-android \ + --android-base-dir /path/to/your/hd \ + --android-version 8.1.0_r60 \ + download \ + build \ +; +./run \ + --android-base-dir /path/to/your/hd \ + --android-version 8.1.0_r60 \ + --kvm \ +; +.... + +TODO hack the kernel and rebuild, hack userland and see message. + == About this repo === Supported hosts diff --git a/build-android b/build-android new file mode 100755 index 0000000..c70d13f --- /dev/null +++ b/build-android @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 + +import os +import subprocess + +import common +import shutil +from shell_helpers import LF + +class Main(common.BuildCliFunction): + def __init__(self): + super().__init__( + description='''\ +Download and build Android AOSP. + +https://github.com/cirosantilli/linux-kernel-module-cheat#android +''' + ) + self.add_argument( + 'targets', + default=['build'], + nargs='*', + ) + + def build(self): + if 'download' in self.env['targets']: + os.makedirs(self.env['android_dir'], exist_ok=True) + # Can only download base64. I kid you not: + # https://github.com/google/gitiles/issues/7 + self.sh.wget( + 'https://android.googlesource.com/tools/repo/+/v1.13.2/repo?format=TEXT', + self.env['repo_path_base64'], + ) + with open(self.env['repo_path_base64'], 'r') as input, \ + open(self.env['repo_path'], 'w') as output: + output.write(self.sh.base64_decode(input.read())) + self.sh.chmod(self.env['repo_path']) + self.sh.run_cmd( + [ + self.env['repo_path'], LF, + 'init', LF, + '-b', 'android-{}'.format(self.env['android_version']), LF, + '--depth', '1', LF, + '-u', 'https://android.googlesource.com/platform/manifest', LF, + ], + cwd=self.env['android_dir'], + ) + self.sh.run_cmd( + [ + self.env['repo_path'], LF, + 'sync', LF, + '-c', LF, + '-j', str(self.env['nproc']), LF, + '--no-tags', LF, + '--no-clone-bundle', LF, + ], + cwd=self.env['android_dir'], + ) + if 'build' in self.env['targets']: + # The crappy android build system requires + # https://stackoverflow.com/questions/7040592/calling-the-source-command-from-subprocess-popen + self.sh.run_cmd('''\ +. build/envsetup.sh +lunch aosp_{}-eng +USE_CCACHE=1 make -j {} +'''.format(self.env['android_arch'], self.env['nproc']), + cwd=self.env['android_dir'], + executable=shutil.which('bash'), + shell=True, + ) + + def get_build_dir(self): + return self.env['android_build_dir'] + +if __name__ == '__main__': + Main().cli() diff --git a/cli_function.py b/cli_function.py index a321983..7a96efe 100755 --- a/cli_function.py +++ b/cli_function.py @@ -199,7 +199,9 @@ class CliFunction: # Add missing args from hard-coded defaults. for key in self._arguments: argument = self._arguments[key] - if (not key in args_with_defaults) or args_with_defaults[key] is None: + # TODO: in (None, []) is ugly, and will probably go wrong at some point, + # there must be a better way to do it, but I'm lazy now to think. + if (not key in args_with_defaults) or args_with_defaults[key] in (None, []): if argument.optional: args_with_defaults[key] = argument.default else: @@ -231,6 +233,9 @@ class CliFunction: for key in self._arguments: argument = self._arguments[key] parser.add_argument(*argument.args, **argument.kwargs) + # print(key) + # print(argument.args) + # print(argument.kwargs) if argument.is_bool: new_longname = '--no' + argument.longname[1:] kwargs = argument.kwargs.copy() @@ -245,10 +250,10 @@ class CliFunction: def cli(self, *args, **kwargs): ''' - Same as cli, but also exit the program with status equal to the return value of main. - main must return an integer for this to be used. + Same as cli_noxit, but also exit the program with status equal to the + return value of main. main must return an integer for this to be used. - None is considered 0. + None is considered as 0. ''' exit_status = self.cli_noexit(*args, **kwargs) if exit_status is None: @@ -453,6 +458,19 @@ amazing function! assert one_cli_function.get_cli(pos_mandatory=1, pos_optional=2, args_star=['asdf', 'qwer']) == [('--bool-cli',), ('1',), ('2',), ('asdf',), ('qwer',)] assert one_cli_function.get_cli(pos_mandatory=1, append=['2', '3']) == [('--append', '2'), ('--append', '3',), ('--bool-cli',), ('1',)] + class NargsWithDefault(CliFunction): + def __init__(self): + super().__init__() + self.add_argument('args-star', default=['1', '2'], nargs='*'), + def main(self, **kwargs): + return kwargs + nargs_with_default = NargsWithDefault() + default = nargs_with_default() + assert default['args_star'] == ['1', '2'] + default_cli = nargs_with_default.cli_noexit([]) + assert default_cli['args_star'] == ['1', '2'] + assert nargs_with_default.cli_noexit(['1', '2', '3', '4'])['args_star'] == ['1', '2', '3', '4'] + if len(sys.argv) > 1: # CLI call with argv command line arguments. print(one_cli_function.cli()) diff --git a/common.py b/common.py index e597cef..21c0e28 100644 --- a/common.py +++ b/common.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 import argparse -import base64 import collections import copy import datetime @@ -295,6 +294,24 @@ inside baremetal/ and then try to use corresponding executable. help='Boot with the Buildroot Linux kernel instead of our custom built one. Mostly for sanity checks.' ) + # Android. + self.add_argument( + '--rootfs-type', default='buildroot', choices=('buildroot', 'android'), + help='Which rootfs to use.' + ) + self.add_argument( + '--android-version', default='8.1.0_r60', + help='Which android version to use. implies --rootfs-type android' + ) + self.add_argument( + '--android-base-dir', + help='''\ +If given, place all android sources and build files into the given directory. +One application of this is to put those large directories in your HD instead +of SSD. +''' + ) + # crosstool-ng self.add_argument( '--crosstool-ng-build-id', default=consts['default_build_id'], @@ -307,20 +324,19 @@ Use the docker download Ubuntu root filesystem instead of the default Buildroot ''' ) - self.add_argument( - '--machine', - help='''Machine type. -QEMU default: virt -gem5 default: VExpress_GEM5_V1 -See the documentation for other values known to work. -''' - ) - # QEMU. self.add_argument( '-Q', '--qemu-build-id', default=consts['default_build_id'], help='QEMU build ID. Allows you to keep multiple separate QEMU builds.' ) + self.add_argument( + '--machine', + help='''\ +Machine type: +* QEMU default: virt +* gem5 default: VExpress_GEM5_V1 +''' + ) # Userland. self.add_argument( @@ -491,6 +507,14 @@ Valid emulators: {} common.extract_vmlinux = os.path.join(env['linux_source_dir'], 'scripts', 'extract-vmlinux') env['linux_buildroot_build_dir'] = join(env['buildroot_build_build_dir'], 'linux-custom') + # Android + if not env['_args_given']['android_base_dir']: + env['android_base_dir'] = join(env['out_dir'], 'android') + env['android_dir'] = join(env['android_base_dir'], env['android_version']) + env['android_build_dir'] = join(env['android_dir'], 'out') + env['repo_path'] = join(env['android_base_dir'], 'repo') + env['repo_path_base64'] = env['repo_path'] + '.base64' + # QEMU env['qemu_build_dir'] = join(env['out_dir'], 'qemu', env['qemu_build_id']) env['qemu_executable_basename'] = 'qemu-system-{}'.format(env['arch']) @@ -586,12 +610,15 @@ Valid emulators: {} env['linux_build_dir'] = join(env['out_dir'], 'linux', env['linux_build_id'], env['arch']) env['lkmc_vmlinux'] = join(env['linux_build_dir'], 'vmlinux') if env['arch'] == 'arm': + env['android_arch'] = 'arm' env['linux_arch'] = 'arm' env['linux_image_prefix'] = join('arch', env['linux_arch'], 'boot', 'zImage') elif env['arch'] == 'aarch64': + env['android_arch'] = 'arm64' env['linux_arch'] = 'arm64' env['linux_image_prefix'] = join('arch', env['linux_arch'], 'boot', 'Image') elif env['arch'] == 'x86_64': + env['android_arch'] = 'x86_64' env['linux_arch'] = 'x86' env['linux_image_prefix'] = join('arch', env['linux_arch'], 'boot', 'bzImage') env['lkmc_linux_image'] = join(env['linux_build_dir'], env['linux_image_prefix']) @@ -695,10 +722,6 @@ Valid emulators: {} self._common_args.add(key) super().add_argument(*args, **kwargs) - @staticmethod - def base64_encode(string): - return base64.b64encode(string.encode()).decode() - def get_elf_entry(self, elf_file_path): readelf_header = subprocess.check_output([ self.get_toolchain_tool('readelf'), diff --git a/copy-overlay b/copy-overlay index 53bae55..130d46d 100755 --- a/copy-overlay +++ b/copy-overlay @@ -13,6 +13,7 @@ class Main(common.BuildCliFunction): description='''\ https://github.com/cirosantilli/linux-kernel-module-cheat#rootfs_overlay ''') + def build(self): # TODO: print rsync equivalent, move into shell_helpers. distutils.dir_util.copy_tree( diff --git a/release-download-latest b/release-download-latest index 81fa163..60e6154 100755 --- a/release-download-latest +++ b/release-download-latest @@ -15,6 +15,7 @@ https://stackoverflow.com/questions/24987542/is-there-a-link-to-github-for-downl ) def timed_main(self): + self.log_info('Downloading the release, this may take several seconds / a few minutes.') _json = self.github_make_request(path='/releases') asset = _json[0]['assets'][0] self.sh.wget(asset['browser_download_url'], asset['name']) diff --git a/run b/run index 0a55788..c7c33b7 100755 --- a/run +++ b/run @@ -233,7 +233,7 @@ Run QEMU with VNC instead of the default SDL. Connect to it with: if self.env['wait_gdb']: extra_qemu_args.extend(['-S', LF]) if self.env['eval_after'] is not None: - kernel_cli_after_dash += ' lkmc_eval_base64="{}"'.format(self.base64_encode(self.env['eval_after'])) + kernel_cli_after_dash += ' lkmc_eval_base64="{}"'.format(self.sh.base64_encode(self.env['eval_after'])) if self.env['kernel_cli_after_dash'] is not None: kernel_cli_after_dash += ' {}'.format(self.env['kernel_cli_after_dash']) if self.env['vnc']: @@ -242,7 +242,7 @@ Run QEMU with VNC instead of the default SDL. Connect to it with: vnc = [] if self.env['eval'] is not None: kernel_cli += ' {}=/eval_base64.sh'.format(self.env['initarg']) - kernel_cli_after_dash += ' lkmc_eval="{}"'.format(self.base64_encode(self.env['eval'])) + kernel_cli_after_dash += ' lkmc_eval="{}"'.format(self.sh.base64_encode(self.env['eval'])) if not self.env['graphic']: extra_qemu_args.extend(['-nographic', LF]) console = None diff --git a/shell_helpers.py b/shell_helpers.py index a60d495..3d7283f 100644 --- a/shell_helpers.py +++ b/shell_helpers.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import base64 import distutils.file_util import itertools import os @@ -62,10 +63,27 @@ class ShellHelpers: out.extend([arg, LF]) return out - def cp(self, src, dest, **kwargs): - self.print_cmd(['cp', src, dest]) - if not self.dry_run: - shutil.copy2(src, dest) + def base64_encode(self, string): + ''' + TODO deal with redirection and print nicely. + ''' + return base64.b64encode(string.encode()).decode() + + def base64_decode(self, string): + return base64.b64decode(string.encode()).decode() + + def chmod(self, path, add_rm_abs='+', mode_delta=stat.S_IXUSR): + ''' + TODO extend further, shell print equivalent. + ''' + old_mode = os.stat(path).st_mode + if add_rm_abs == '+': + new_mode = old_mode | mode_delta + elif add_rm_abs == '': + new_mode = mode_delta + elif add_rm_abs == '-': + new_mode = old_mode & ~mode_delta + os.chmod(path, new_mode) @staticmethod def cmd_to_string(cmd, cwd=None, extra_env=None, extra_paths=None): @@ -115,6 +133,11 @@ class ShellHelpers: update=1, ) + def cp(self, src, dest, **kwargs): + self.print_cmd(['cp', src, dest]) + if not self.dry_run: + shutil.copy2(src, dest) + def print_cmd(self, cmd, cwd=None, cmd_file=None, extra_env=None, extra_paths=None): ''' Print cmd_to_string to stdout. @@ -135,8 +158,7 @@ class ShellHelpers: with open(cmd_file, 'w') as f: f.write('#!/usr/bin/env bash\n') f.write(cmd_string) - st = os.stat(cmd_file) - os.chmod(cmd_file, st.st_mode | stat.S_IXUSR) + self.chmod(cmd_file) def run_cmd( self, @@ -260,7 +282,10 @@ class ShellHelpers: return self.add_newlines(shlex.split(string)) def strip_newlines(self, cmd): - return [x for x in cmd if x != LF] + if type(cmd) is str: + return cmd + else: + return [x for x in cmd if x != LF] def rmrf(self, path): self.print_cmd(['rm', '-r', '-f', path, LF])