build android sketch

README.adoc

@@ -11353,6 +11353,32 @@ gem5:
 ** https://stackoverflow.com/questions/47997565/gem5-system-requirements-for-decent-performance/48941793#48941793
 ** https://github.com/gem5/gem5/issues/25
 
+== WIP
+
+Big new features that are not yet working.
+
+=== Android
+
+Remember: Android AOSP is a huge undocumented piece of bloatware. It's integration into this repo will likely never be super good.
+
+https://stackoverflow.com/questions/1809774/how-to-compile-the-android-aosp-kernel-and-test-it-with-the-android-emulator/48310014#48310014
+
+....
+./build-android \
+  --android-base-dir /path/to/your/hd \
+  --android-version 8.1.0_r60 \
+  download \
+  build \
+;
+./run \
+  --android-base-dir /path/to/your/hd \
+  --android-version 8.1.0_r60 \
+  --kvm \
+;
+....
+
+TODO hack the kernel and rebuild, hack userland and see message.
+
 == About this repo
 
 === Supported hosts

build-android

@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+
+import os
+import subprocess
+
+import common
+import shutil
+from shell_helpers import LF
+
+class Main(common.BuildCliFunction):
+    def __init__(self):
+        super().__init__(
+            description='''\
+Download and build Android AOSP.
+
+https://github.com/cirosantilli/linux-kernel-module-cheat#android
+'''
+        )
+        self.add_argument(
+            'targets',
+            default=['build'],
+            nargs='*',
+        )
+
+    def build(self):
+        if 'download' in self.env['targets']:
+            os.makedirs(self.env['android_dir'], exist_ok=True)
+            # Can only download base64. I kid you not:
+            # https://github.com/google/gitiles/issues/7
+            self.sh.wget(
+                'https://android.googlesource.com/tools/repo/+/v1.13.2/repo?format=TEXT', 
+                self.env['repo_path_base64'],
+            )
+            with open(self.env['repo_path_base64'], 'r') as input, \
+                open(self.env['repo_path'], 'w') as output:
+                output.write(self.sh.base64_decode(input.read()))
+            self.sh.chmod(self.env['repo_path'])
+            self.sh.run_cmd(
+                [
+                    self.env['repo_path'], LF,
+                    'init', LF,
+                    '-b', 'android-{}'.format(self.env['android_version']), LF,
+                    '--depth', '1', LF,
+                    '-u', 'https://android.googlesource.com/platform/manifest', LF,
+                ],
+                cwd=self.env['android_dir'],
+            )
+            self.sh.run_cmd(
+                [
+                    self.env['repo_path'], LF,
+                    'sync', LF,
+                    '-c', LF,
+                    '-j', str(self.env['nproc']), LF,
+                    '--no-tags', LF,
+                    '--no-clone-bundle', LF,
+                ],
+                cwd=self.env['android_dir'],
+            )
+        if 'build' in self.env['targets']:
+            # The crappy android build system requires
+            # https://stackoverflow.com/questions/7040592/calling-the-source-command-from-subprocess-popen
+            self.sh.run_cmd('''\
+. build/envsetup.sh
+lunch aosp_{}-eng
+USE_CCACHE=1 make -j {}
+'''.format(self.env['android_arch'], self.env['nproc']),
+                cwd=self.env['android_dir'],
+                executable=shutil.which('bash'),
+                shell=True,
+            )
+
+    def get_build_dir(self):
+        return self.env['android_build_dir']
+
+if __name__ == '__main__':
+    Main().cli()

cli_function.py

@@ -199,7 +199,9 @@ class CliFunction:
         # Add missing args from hard-coded defaults.
         for key in self._arguments:
             argument = self._arguments[key]
-            if (not key in args_with_defaults) or args_with_defaults[key] is None:
+            # TODO: in (None, []) is ugly, and will probably go wrong at some point,
+            # there must be a better way to do it, but I'm lazy now to think.
+            if (not key in args_with_defaults) or args_with_defaults[key] in (None, []):
                 if argument.optional:
                     args_with_defaults[key] = argument.default
                 else:
@@ -231,6 +233,9 @@ class CliFunction:
         for key in self._arguments:
             argument = self._arguments[key]
             parser.add_argument(*argument.args, **argument.kwargs)
+            # print(key)
+            # print(argument.args)
+            # print(argument.kwargs)
             if argument.is_bool:
                 new_longname = '--no' + argument.longname[1:]
                 kwargs = argument.kwargs.copy()
@@ -245,10 +250,10 @@ class CliFunction:
 
     def cli(self, *args, **kwargs):
         '''
-        Same as cli, but also exit the program with status equal to the return value of main.
-        main must return an integer for this to be used.
+        Same as cli_noxit, but also exit the program with status equal to the
+        return value of main. main must return an integer for this to be used.
 
-        None is considered 0.
+        None is considered as 0.
         '''
         exit_status = self.cli_noexit(*args, **kwargs)
         if exit_status is None:
@@ -453,6 +458,19 @@ amazing function!
     assert one_cli_function.get_cli(pos_mandatory=1, pos_optional=2, args_star=['asdf', 'qwer']) == [('--bool-cli',), ('1',), ('2',), ('asdf',), ('qwer',)]
     assert one_cli_function.get_cli(pos_mandatory=1, append=['2', '3']) == [('--append', '2'), ('--append', '3',), ('--bool-cli',), ('1',)]
 
+    class NargsWithDefault(CliFunction):
+        def __init__(self):
+            super().__init__()
+            self.add_argument('args-star', default=['1', '2'], nargs='*'),
+        def main(self, **kwargs):
+            return kwargs
+    nargs_with_default = NargsWithDefault()
+    default = nargs_with_default()
+    assert default['args_star'] == ['1', '2']
+    default_cli = nargs_with_default.cli_noexit([])
+    assert default_cli['args_star'] == ['1', '2']
+    assert nargs_with_default.cli_noexit(['1', '2', '3', '4'])['args_star'] == ['1', '2', '3', '4']
+
     if len(sys.argv) > 1:
         # CLI call with argv command line arguments.
         print(one_cli_function.cli())

common.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python3
 
 import argparse
-import base64
 import collections
 import copy
 import datetime
@@ -295,6 +294,24 @@ inside baremetal/ and then try to use corresponding executable.
             help='Boot with the Buildroot Linux kernel instead of our custom built one. Mostly for sanity checks.'
         )
 
+        # Android.
+        self.add_argument(
+            '--rootfs-type', default='buildroot', choices=('buildroot', 'android'),
+            help='Which rootfs to use.'
+        )
+        self.add_argument(
+            '--android-version', default='8.1.0_r60',
+            help='Which android version to use. implies --rootfs-type android'
+        )
+        self.add_argument(
+            '--android-base-dir',
+            help='''\
+If given, place all android sources and build files into the given directory.
+One application of this is to put those large directories in your HD instead
+of SSD.
+'''
+        )
+
         # crosstool-ng
         self.add_argument(
             '--crosstool-ng-build-id', default=consts['default_build_id'],
@@ -307,20 +324,19 @@ Use the docker download Ubuntu root filesystem instead of the default Buildroot
 '''
         )
 
-        self.add_argument(
-            '--machine',
-            help='''Machine type.
-QEMU default: virt
-gem5 default: VExpress_GEM5_V1
-See the documentation for other values known to work.
-'''
-        )
-
         # QEMU.
         self.add_argument(
             '-Q', '--qemu-build-id', default=consts['default_build_id'],
             help='QEMU build ID. Allows you to keep multiple separate QEMU builds.'
         )
+        self.add_argument(
+            '--machine',
+            help='''\
+Machine type:
+* QEMU default: virt
+* gem5 default: VExpress_GEM5_V1
+'''
+        )
 
         # Userland.
         self.add_argument(
@@ -491,6 +507,14 @@ Valid emulators: {}
         common.extract_vmlinux = os.path.join(env['linux_source_dir'], 'scripts', 'extract-vmlinux')
         env['linux_buildroot_build_dir'] = join(env['buildroot_build_build_dir'], 'linux-custom')
 
+        # Android
+        if not env['_args_given']['android_base_dir']:
+            env['android_base_dir'] = join(env['out_dir'], 'android')
+        env['android_dir'] = join(env['android_base_dir'], env['android_version'])
+        env['android_build_dir'] = join(env['android_dir'], 'out')
+        env['repo_path'] = join(env['android_base_dir'], 'repo')
+        env['repo_path_base64'] = env['repo_path'] + '.base64'
+
         # QEMU
         env['qemu_build_dir'] = join(env['out_dir'], 'qemu', env['qemu_build_id'])
         env['qemu_executable_basename'] = 'qemu-system-{}'.format(env['arch'])
@@ -586,12 +610,15 @@ Valid emulators: {}
             env['linux_build_dir'] = join(env['out_dir'], 'linux', env['linux_build_id'], env['arch'])
         env['lkmc_vmlinux'] = join(env['linux_build_dir'], 'vmlinux')
         if env['arch'] == 'arm':
+            env['android_arch'] = 'arm'
             env['linux_arch'] = 'arm'
             env['linux_image_prefix'] = join('arch', env['linux_arch'], 'boot', 'zImage')
         elif env['arch'] == 'aarch64':
+            env['android_arch'] = 'arm64'
             env['linux_arch'] = 'arm64'
             env['linux_image_prefix'] = join('arch', env['linux_arch'], 'boot', 'Image')
         elif env['arch'] == 'x86_64':
+            env['android_arch'] = 'x86_64'
             env['linux_arch'] = 'x86'
             env['linux_image_prefix'] = join('arch', env['linux_arch'], 'boot', 'bzImage')
         env['lkmc_linux_image'] = join(env['linux_build_dir'], env['linux_image_prefix'])
@@ -695,10 +722,6 @@ Valid emulators: {}
             self._common_args.add(key)
         super().add_argument(*args, **kwargs)
 
-    @staticmethod
-    def base64_encode(string):
-        return base64.b64encode(string.encode()).decode()
-
     def get_elf_entry(self, elf_file_path):
         readelf_header = subprocess.check_output([
             self.get_toolchain_tool('readelf'),

copy-overlay

@@ -13,6 +13,7 @@ class Main(common.BuildCliFunction):
             description='''\
 https://github.com/cirosantilli/linux-kernel-module-cheat#rootfs_overlay
 ''')
+
     def build(self):
         # TODO: print rsync equivalent, move into shell_helpers.
         distutils.dir_util.copy_tree(

release-download-latest

@@ -15,6 +15,7 @@ https://stackoverflow.com/questions/24987542/is-there-a-link-to-github-for-downl
         )
 
     def timed_main(self):
+        self.log_info('Downloading the release, this may take several seconds / a few minutes.')
         _json = self.github_make_request(path='/releases')
         asset = _json[0]['assets'][0]
         self.sh.wget(asset['browser_download_url'], asset['name'])

run

@@ -233,7 +233,7 @@ Run QEMU with VNC instead of the default SDL. Connect to it with:
         if self.env['wait_gdb']:
             extra_qemu_args.extend(['-S', LF])
         if self.env['eval_after'] is not None:
-            kernel_cli_after_dash += ' lkmc_eval_base64="{}"'.format(self.base64_encode(self.env['eval_after']))
+            kernel_cli_after_dash += ' lkmc_eval_base64="{}"'.format(self.sh.base64_encode(self.env['eval_after']))
         if self.env['kernel_cli_after_dash'] is not None:
             kernel_cli_after_dash += ' {}'.format(self.env['kernel_cli_after_dash'])
         if self.env['vnc']:
@@ -242,7 +242,7 @@ Run QEMU with VNC instead of the default SDL. Connect to it with:
             vnc = []
         if self.env['eval'] is not None:
             kernel_cli += ' {}=/eval_base64.sh'.format(self.env['initarg'])
-            kernel_cli_after_dash += ' lkmc_eval="{}"'.format(self.base64_encode(self.env['eval']))
+            kernel_cli_after_dash += ' lkmc_eval="{}"'.format(self.sh.base64_encode(self.env['eval']))
         if not self.env['graphic']:
             extra_qemu_args.extend(['-nographic', LF])
         console = None

shell_helpers.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 
+import base64
 import distutils.file_util
 import itertools
 import os
@@ -62,10 +63,27 @@ class ShellHelpers:
             out.extend([arg, LF])
         return out
 
-    def cp(self, src, dest, **kwargs):
-        self.print_cmd(['cp', src, dest])
-        if not self.dry_run:
-            shutil.copy2(src, dest)
+    def base64_encode(self, string):
+        '''
+        TODO deal with redirection and print nicely.
+        '''
+        return base64.b64encode(string.encode()).decode()
+
+    def base64_decode(self, string):
+        return base64.b64decode(string.encode()).decode()
+
+    def chmod(self, path, add_rm_abs='+', mode_delta=stat.S_IXUSR):
+        '''
+        TODO extend further, shell print equivalent.
+        '''
+        old_mode = os.stat(path).st_mode
+        if add_rm_abs == '+':
+            new_mode = old_mode | mode_delta
+        elif add_rm_abs == '':
+            new_mode = mode_delta
+        elif add_rm_abs == '-':
+            new_mode = old_mode & ~mode_delta
+        os.chmod(path, new_mode)
 
     @staticmethod
     def cmd_to_string(cmd, cwd=None, extra_env=None, extra_paths=None):
@@ -115,6 +133,11 @@ class ShellHelpers:
                         update=1,
                     )
 
+    def cp(self, src, dest, **kwargs):
+        self.print_cmd(['cp', src, dest])
+        if not self.dry_run:
+            shutil.copy2(src, dest)
+
     def print_cmd(self, cmd, cwd=None, cmd_file=None, extra_env=None, extra_paths=None):
         '''
         Print cmd_to_string to stdout.
@@ -135,8 +158,7 @@ class ShellHelpers:
             with open(cmd_file, 'w') as f:
                 f.write('#!/usr/bin/env bash\n')
                 f.write(cmd_string)
-            st = os.stat(cmd_file)
-            os.chmod(cmd_file, st.st_mode | stat.S_IXUSR)
+            self.chmod(cmd_file)
 
     def run_cmd(
             self,
@@ -260,6 +282,9 @@ class ShellHelpers:
         return self.add_newlines(shlex.split(string))
 
     def strip_newlines(self, cmd):
+        if type(cmd) is str:
+            return cmd
+        else:
             return [x for x in cmd if x != LF]
 
     def rmrf(self, path):