#!/usr/bin/env python3

import argparse
import collections
import re
import os

import common

class Component:
    '''
    Yes, we are re-inventing a crappy dependency resolution system.
    I can't believe it.

    The hard part is that we have optional dependencies as well...
    e.g. buildroot optionally depends on m5 to put m5 in the root filesystem,
    and buildroot optionally depends on qemu to build the qcow2 version
    of the image.
    '''
    def __init__(
        self,
        build_callback=None,
        supported_archs=None,
        dependencies=None,
        apt_get_pkgs=None,
        apt_build_deps=None,
        submodules=None,
        submodules_shallow=None,
        python2_pkgs=None,
        python3_pkgs=None,
    ):
        self.build_callback = build_callback
        self.supported_archs = supported_archs
        self.dependencies = dependencies or set()
        self.apt_get_pkgs = apt_get_pkgs or set()
        self.apt_build_deps = apt_build_deps or set()
        self.submodules = submodules or set()
        self.submodules_shallow = submodules_shallow or set()
        self.python2_pkgs = python2_pkgs or set()
        self.python3_pkgs = python3_pkgs or set()
    def build(self, arch):
        if (
            (self.build_callback is not None) and
            (self.supported_archs is None or arch in self.supported_archs)
        ):
            self.build_callback(arch)

def run_cmd(cmd, arch):
    global args
    cmd_abs = cmd.copy()
    cmd_abs[0] = os.path.join(common.root_dir, cmd[0])
    cmd_abs.extend(['--arch', arch])
    if args.extra_args:
        cmd_abs.append(args.extra_args)
    common.run_cmd(cmd_abs, dry_run=args.dry_run)

buildroot_component = Component(
    lambda arch: run_cmd(['build-buildroot'], arch),
    submodules = {'buildroot'},
)

name_to_component_map = {
    # Leaves without dependencies.
    'baremetal-qemu': Component(
        lambda arch: run_cmd(['build-baremetal'], arch),
        supported_archs=common.crosstool_ng_supported_archs,
    ),
    'baremetal-gem5': Component(
        lambda arch: run_cmd(['build-baremetal', '--gem5'], arch),
        supported_archs=common.crosstool_ng_supported_archs,
    ),
    'baremetal-gem5-pbx': Component(
        lambda arch: run_cmd(['build-baremetal', '--gem5', '--machine', 'RealViewPBX'], arch),
        supported_archs=common.crosstool_ng_supported_archs,
    ),
    'buildroot': buildroot_component,
    'buildroot-gcc': buildroot_component,
    'copy-overlay': Component(
        lambda arch: run_cmd(['copy-overlay'], arch),
    ),
    'crosstool-ng': Component(
        lambda arch: run_cmd(['build-crosstool-ng'], arch),
        supported_archs=common.crosstool_ng_supported_archs,
        # http://crosstool-ng.github.io/docs/os-setup/
        apt_get_pkgs={
            'bison',
            'docbook2x',
            'flex',
            'gcc',
            'gperf',
            'help2man',
            'libncurses5-dev',
            'libtool-bin',
            'make',
            'python-dev',
            'texinfo',
        },
        submodules={'crosstool-ng'},
    ),
    'gem5': Component(
        lambda arch: run_cmd(['build-gem5'], arch),
        # TODO test it out on Docker and answer that question properly:
        # https://askubuntu.com/questions/350475/how-can-i-install-gem5
        apt_get_pkgs={
            'diod',
            'libgoogle-perftools-dev',
            'protobuf-compiler',
            'python-dev',
            'python-pip',
            'scons',
        },
        python2_pkgs={
            # Generate graphs of config.ini under m5out.
            'pydot',
        },
        submodules={'gem5'},
    ),
    'gem5-debug': Component(
        lambda arch: run_cmd(['build-gem5', '--gem5-build-type', 'debug'], arch),
    ),
    'gem5-fast': Component(
        lambda arch: run_cmd(['build-gem5', '--gem5-build-type', 'fast'], arch),
    ),
    'linux': Component(
        lambda arch: run_cmd(['build-linux'], arch),
        submodules_shallow={'linux'},
    ),
    'modules': Component(
        lambda arch: run_cmd(['build-modules'], arch),
    ),
    'm5': Component(
        lambda arch: run_cmd(['build-m5'], arch),
        submodules={'gem5'},
    ),
    'qemu': Component(
        lambda arch: run_cmd(['build-qemu'], arch),
        apt_build_deps={'qemu'},
        apt_get_pkgs={'libsdl2-dev'},
        submodules={'qemu'},
    ),
    'qemu-user': Component(
        lambda arch: run_cmd(['build-qemu', '--userland'], arch),
        apt_build_deps = {'qemu'},
        apt_get_pkgs={'libsdl2-dev'},
        submodules = {'qemu'},
    ),
    'parsec-benchmark': Component(
        submodules = {'parsec-benchmark'},
    ),
    'userland': Component(
        lambda arch: run_cmd(['build-userland'], arch),
    ),

    # Dependency only nodes.
    'all': Component(dependencies=[
        'all-linux',
        'all-baremetal',
    ]),
    'all-baremetal': Component(dependencies=[
            'qemu-baremetal',
            'gem5-baremetal',
            'baremetal-gem5-pbx',
        ],
        supported_archs=common.crosstool_ng_supported_archs,
    ),
    'all-linux': Component(dependencies=[
        'qemu-gem5-buildroot',
        'gem5-debug',
        'gem5-fast',
        'qemu-user',
    ]),
    'baremetal': Component(dependencies=[
        'baremetal-gem5',
        'baremetal-qemu',
    ]),
    'gem5-buildroot': Component(dependencies=[
        'buildroot-gcc',
        'linux',
        'm5',
        'overlay',
        'gem5',
    ]),
    'gem5-baremetal': Component(dependencies=[
        'gem5',
        'crosstool-ng',
        'baremetal-gem5',
    ]),
    'overlay': Component(dependencies=[
        'copy-overlay',
        'modules',
        'userland',
        'buildroot',
    ]),
    'qemu-baremetal': Component(dependencies=[
        'qemu',
        'crosstool-ng',
        'baremetal-qemu',
    ]),
    'qemu-buildroot': Component(dependencies=[
        'qemu',
        'buildroot-gcc',
        'overlay',
        'linux',
    ]),
    'qemu-gem5-buildroot': Component(dependencies=[
        'qemu',
        'gem5-buildroot',
    ]),
    'release': Component(dependencies=[
        'qemu-buildroot',
    ]),
}
parser = argparse.ArgumentParser(
    description= '''\
Shallow helper to build everything, or a subset of everything conveniently.

Our build-* scripts don't build any dependencies to make iterative
development fast and more predictable.

While modifying a specific component however, you will likely want to just run the
individual build-* commands which:

* build no dependencies, and so are fast and predictable
* can take multiple options to custumize the build

Without any args, build only what is necessary for
https://github.com/cirosantilli/linux-kernel-module-cheat#qemu-buildroot-setup
for x86_64:

....
./%(prog)s
....

This is equivalent to:

....
./%(prog)s --arch x86_64 qemu-buildroot
....

If `--arch` is given, build just for the given archs:

....
./%(prog)s --arch arm --arch aarch64
....

This will build `qemu-buildroot` for arm and aarch64 only, but not `x86_64`.

Clean all Linux kernel builds:

....
./build --all-archs --extra-args=--clean buildroot
....
''',
    formatter_class=argparse.RawTextHelpFormatter,
)
parser.add_argument('--all', default=False, action='store_true', help='''\
Build absolutely everything for all archs.
''')
group = parser.add_mutually_exclusive_group(required=False)
group.add_argument('-A', '--all-archs', default=False, action='store_true', help='''\
Build the selected components for all archs.
''')
group.add_argument('-a', '--arch', choices=common.arch_choices, default=[], action='append', help='''\
Build the selected components for this arch. Select multiple archs by
passing this option multiple times. Default: [{}]
'''.format(common.default_arch))
parser.add_argument('-D', '--download-dependencies', default=False, action='store_true', help='''\
Also download all dependencies required for a given build: Ubuntu packages,
Python packages and git submodules.
''')
parser.add_argument('--extra-args', default='', help='''\
Extra args to pass to all scripts.
'''
)
parser.add_argument('--travis', default=False, action='store_true', help='''\
Extra args to pass to all scripts.
'''
)
parser.add_argument('components', choices=list(name_to_component_map.keys()) + [[]], default=[], nargs='*', help='''\
Which components to build.
'''.format(common.default_arch))
common.add_dry_run_argument(parser)
args = parser.parse_args()
common.setup_dry_run_arguments(args)

# Decide archs.
if args.arch == []:
    if args.all or args.all_archs:
        archs = common.all_archs.copy()
    else:
        archs = set([common.default_arch])
else:
    archs = set()
    for arch in args.arch:
        if arch in common.arch_short_to_long_dict:
            arch = common.arch_short_to_long_dict[arch]
        archs.add(arch)

# Decide components.
components = args.components
if args.all:
    components = ['all']
elif components == []:
    components = ['qemu-buildroot']
selected_components = []
selected_component_name_set = set()
for component_name in components:
    todo = [component_name]
    while todo:
        current_name = todo.pop(0)
        if current_name not in selected_component_name_set:
            selected_component_name_set.add(current_name)
            component = name_to_component_map[current_name]
            selected_components.append(component)
            todo.extend(component.dependencies)


if args.download_dependencies:
    apt_get_pkgs = {
        # TODO: figure out what needs those exactly.
        'automake',
        'build-essential',
        'coreutils',
        'cpio',
        'libguestfs-tools',
        'moreutils', # ts
        'rsync',
        'unzip',
        'wget',

        # Linux kernel build dependencies.
        'bison',
        'flex',
        # Without this started failing in kernel 4.15 with:
        # Makefile:932: *** "Cannot generate ORC metadata for CONFIG_UNWINDER_ORC=y, please install libelf-dev, libelf-devel or elfutils-libelf-devel".  Stop.
        'libelf-dev',

        # Our misc stuff.
        'git',
        'python3-pip',
        'tmux',
        'vinagre',

        # Userland.
        'gcc-aarch64-linux-gnu',
        'gcc-arm-linux-gnueabihf',
        'g++-aarch64-linux-gnu',
        'g++-arm-linux-gnueabihf',
    }
    apt_build_deps = set()
    submodules = set()
    submodules_shallow = set()
    python2_pkgs = set()
    python3_pkgs = {
        'pexpect==4.6.0',
    }
    for component in selected_components:
        apt_get_pkgs.update(component.apt_get_pkgs)
        apt_build_deps.update(component.apt_build_deps)
        submodules.update(component.submodules)
        submodules_shallow.update(component.submodules_shallow)
        python2_pkgs.update(component.python2_pkgs)
        python3_pkgs.update(component.python3_pkgs)
    if apt_get_pkgs or apt_build_deps:
        if args.travis:
            interacive_pkgs = {
                'libsdl2-dev',
            }
            apt_get_pkgs.difference_update(interacive_pkgs)
        if common.in_docker:
            sudo = []
            # https://askubuntu.com/questions/909277/avoiding-user-interaction-with-tzdata-when-installing-certbot-in-a-docker-contai
            os.environ['DEBIAN_FRONTEND'] = 'noninteractive'
            # https://askubuntu.com/questions/496549/error-you-must-put-some-source-uris-in-your-sources-list
            sources_path = os.path.join('/etc', 'apt', 'sources.list')
            with open(sources_path, 'r') as f:
                sources_txt = f.read()
            sources_txt = re.sub('^# deb-src ', 'deb-src ', sources_txt, flags=re.MULTILINE)
            with open(sources_path, 'w') as f:
                f.write(sources_txt)
        else:
            sudo = ['sudo']
        if common.in_docker or args.travis:
            y = ['-y']
        else:
            y = []
        common.run_cmd(
            sudo + ['apt-get', 'update', common.Newline]
        )
        if apt_get_pkgs:
            common.run_cmd(
                sudo + ['apt-get', 'install'] + y + [common.Newline] +
                common.add_newlines(sorted(apt_get_pkgs))
            )
        if apt_build_deps:
            common.run_cmd(
                sudo +
                ['apt-get', 'build-dep'] + y + [common.Newline] +
                common.add_newlines(sorted(apt_build_deps))
            )
    if python2_pkgs:
        common.run_cmd(
            ['python', '-m', 'pip', 'install', '--user', common.Newline] +
            common.add_newlines(sorted(python2_pkgs))
        )
    if python3_pkgs:
        # Not with pip executable directly:
        # https://stackoverflow.com/questions/49836676/error-after-upgrading-pip-cannot-import-name-main/51846054#51846054
        common.run_cmd(
            ['python3', '-m', 'pip', 'install', '--user', common.Newline] +
            common.add_newlines(sorted(python3_pkgs))
        )
    git_cmd_common = ['git', 'submodule', 'update', '--init', '--recursive']
    if submodules:
        # == Other nice git options for when distros move to newer Git
        #
        # Currently not on Ubuntu 16.04:
        #
        # `--progress`: added on Git 2.10:
        #
        # * https://stackoverflow.com/questions/32944468/how-to-show-progress-for-submodule-fetching
        # * https://stackoverflow.com/questions/4640020/progress-indicator-for-git-clone
        #
        # `--jobs"`: https://stackoverflow.com/questions/26957237/how-to-make-git-clone-faster-with-multiple-threads/52327638#52327638
        common.run_cmd(
            git_cmd_common + ['--', common.Newline] +
            common.add_newlines([os.path.join(common.submodules_dir, x) for x in sorted(submodules)])
        )
    if submodules_shallow:
        # == Shallow cloning.
        #
        # TODO Ideally we should shallow clone --depth 1 all of them.
        #
        # However, most git servers out there are crap or craply configured
        # and don't allow shallow cloning except for branches.
        #
        # So for now, let's shallow clone only the Linux kernel, which has by far
        # the largest .git repo history, and full clone the others.
        #
        # Then we will maintain a GitHub Linux kernel mirror / fork that always has a
        # lkmc branch, and point to it, so that it will always succeed.
        #
        # See also:
        #
        # * https://stackoverflow.com/questions/3489173/how-to-clone-git-repository-with-specific-revision-changeset
        # * https://stackoverflow.com/questions/2144406/git-shallow-submodules/47374702#47374702
        # * https://unix.stackexchange.com/questions/338578/why-is-the-git-clone-of-the-linux-kernel-source-code-much-larger-than-the-extrac
        #
        common.run_cmd(
            git_cmd_common + ['--depth', '1', '--', common.Newline] +
            common.add_newlines([os.path.join(common.submodules_dir, x) for x in sorted(submodules_shallow)])
        )

# Do the build.
for arch in archs:
    for component in selected_components:
        component.build(arch)