Build the Linux kernel independently from Buildroot

This will allow for other types of root filesystems that don't rely on Buildroot
to be added and used in the future.

Propagate --verbose on all build scripts to see full GCC commands.

build-all: allow for neat subsets

also 9p share rootfs_overlay. TODO document.

common.py

@@ -32,6 +32,8 @@ submodules_dir = os.path.join(root_dir, 'submodules')
 buildroot_src_dir = os.path.join(submodules_dir, 'buildroot')
 crosstool_ng_src_dir = os.path.join(submodules_dir, 'crosstool-ng')
 linux_src_dir = os.path.join(submodules_dir, 'linux')
+linux_config_dir = os.path.join(this.root_dir, 'kernel_config')
+rootfs_overlay_dir = os.path.join(this.root_dir, 'rootfs_overlay')
 extract_vmlinux = os.path.join(linux_src_dir, 'scripts', 'extract-vmlinux')
 qemu_src_dir = os.path.join(submodules_dir, 'qemu')
 parsec_benchmark_src_dir = os.path.join(submodules_dir, 'parsec-benchmark')
@@ -47,6 +49,10 @@ sha = subprocess.check_output(['git', '-C', root_dir, 'log', '-1', '--format=%H'
 release_dir = os.path.join(this.out_dir, 'release')
 release_zip_file = os.path.join(this.release_dir, 'lkmc-{}.zip'.format(this.sha))
 github_repo_id = 'cirosantilli/linux-kernel-module-cheat'
+asm_ext = '.S'
+c_ext = '.c'
+kernel_module_ext = '.ko'
+obj_ext = '.o'
 config_file = os.path.join(data_dir, 'config')
 if os.path.exists(config_file):
     config = imp.load_source('config', config_file)
@@ -101,6 +107,11 @@ def get_argparse(default_args=None, argparse_args=None):
         '--baremetal',
         help='Use Baremetal examples instead of Linux kernel ones'
     )
+    parser.add_argument(
+        '--buildroot-build-id',
+        default=default_build_id,
+        help='Buildroot build ID. Allows you to keep multiple separate gem5 builds. Default: %(default)s'
+    )
     parser.add_argument(
         '--crosstool-ng-build-id', default=default_build_id,
         help='Crosstool-NG build ID. Allows you to keep multiple separate crosstool-NG builds. Default: %(default)s'
@@ -109,6 +120,12 @@ def get_argparse(default_args=None, argparse_args=None):
         '-g', '--gem5', default=False, action='store_true',
         help='Use gem5 instead of QEMU'
     )
+    parser.add_argument(
+        '--gem5-src',
+        help='''\
+Use the given directory as the gem5 source tree. Ignore `--gem5-worktree`.
+'''
+    )
     parser.add_argument(
         '-L', '--linux-build-id', default=default_build_id,
         help='Linux build ID. Allows you to keep multiple separate Linux builds. Default: %(default)s'
@@ -173,15 +190,14 @@ Default: the run ID (-n) if that is an integer, otherwise 0.
         '-Q', '--qemu-build-id', default=default_build_id,
         help='QEMU build ID. Allows you to keep multiple separate QEMU builds. Default: %(default)s'
     )
-    parser.add_argument(
-        '--buildroot-build-id',
-        default=default_build_id,
-        help='Buildroot build ID. Allows you to keep multiple separate gem5 builds. Default: %(default)s'
-    )
     parser.add_argument(
         '-t', '--gem5-build-type', default='opt',
         help='gem5 build type, most often used for "debug" builds. Default: %(default)s'
     )
+    parser.add_argument(
+        '-v', '--verbose', default=False, action='store_true',
+        help='Show full compilation commands when they are not shown by default.'
+    )
     if hasattr(this, 'configs'):
         defaults = this.configs.copy()
     else:
@@ -224,12 +240,39 @@ def get_stats(stat_re=None, stats_file=None):
                     ret.append(cols[1])
     return ret
 
-def get_toolchain_tool(tool):
-    global this
-    if this.baremetal is None:
-        return glob.glob(os.path.join(this.host_bin_dir, '*-buildroot-*-{}'.format(tool)))[0]
+def get_toolchain_prefix(tool, allowed_toolchains=None):
+    buildroot_full_prefix = os.path.join(this.host_bin_dir, this.buildroot_toolchain_prefix)
+    buildroot_exists = os.path.exists('{}-{}'.format(buildroot_full_prefix, tool))
+    crosstool_ng_full_prefix = os.path.join(this.crosstool_ng_bin_dir, this.crosstool_ng_toolchain_prefix)
+    crosstool_ng_exists = os.path.exists('{}-{}'.format(crosstool_ng_full_prefix, tool))
+    host_tool = '{}-{}'.format(this.ubuntu_toolchain_prefix, tool)
+    host_path = shutil.which(host_tool)
+    if host_path is not None:
+        host_exists = True
+        host_full_prefix = host_path[:-(len(tool)+1)]
     else:
-        return os.path.join(this.crosstool_ng_bin_dir, '{}-{}'.format(this.crosstool_ng_prefix, tool))
+        host_exists = False
+        host_full_prefix = None
+    known_toolchains = {
+        'crosstool-ng': (crosstool_ng_exists, crosstool_ng_full_prefix),
+        'buildroot': (buildroot_exists, buildroot_full_prefix),
+        'host': (host_exists, host_full_prefix),
+    }
+    if allowed_toolchains is None:
+        if this.baremetal is None:
+            allowed_toolchains = ['buildroot', 'crosstool-ng', 'host']
+        else:
+            allowed_toolchains = ['crosstool-ng', 'buildroot', 'host']
+    tried = []
+    for toolchain in allowed_toolchains:
+        exists, prefix = known_toolchains[toolchain]
+        tried.append('{}-{}'.format(prefix, tool))
+        if exists:
+            return prefix
+    raise Exception('Tool not found. Tried:\n' + '\n'.join(tried))
+
+def get_toolchain_tool(tool, allowed_toolchains=None):
+    return '{}-{}'.format(this.get_toolchain_prefix(tool, allowed_toolchains), tool)
 
 def github_make_request(
         authenticate=False,
@@ -273,7 +316,7 @@ def mkdir():
     os.makedirs(this.qemu_run_dir, exist_ok=True)
     os.makedirs(this.p9_dir, exist_ok=True)
 
-def print_cmd(cmd, cwd, cmd_file=None, extra_env=None, extra_paths=None):
+def print_cmd(cmd, cwd=None, cmd_file=None, extra_env=None, extra_paths=None):
     '''
     Format a command given as a list of strings so that it can
     be viewed nicely and executed by bash directly and print it to stdout.
@@ -283,7 +326,8 @@ def print_cmd(cmd, cwd, cmd_file=None, extra_env=None, extra_paths=None):
     '''
     newline_separator = ' \\\n'
     out = []
-    out.append('cd {} &&{}'.format(shlex.quote(cwd), newline_separator))
+    if cwd is not None:
+        out.append('cd {} &&{}'.format(shlex.quote(cwd), newline_separator))
     if extra_paths is not None:
         out.append('PATH="{}:${{PATH}}"'.format(':'.join(extra_paths)) + newline_separator)
     for key in extra_env:
@@ -395,7 +439,7 @@ def run_cmd(
     if 'cwd' in kwargs:
         cwd = kwargs['cwd']
     else:
-        cwd = os.getcwd()
+        cwd = None
     env = os.environ.copy()
     env.update(extra_env)
     if extra_paths is not None:
@@ -458,7 +502,9 @@ def setup(parser):
         this.armv = 7
         this.gem5_arch = 'ARM'
         this.mcpu = 'cortex-a15'
-        this.crosstool_ng_prefix = 'arm-unknown-eabi'
+        this.buildroot_toolchain_prefix = 'arm-buildroot-linux-uclibcgnueabihf'
+        this.crosstool_ng_toolchain_prefix = 'arm-unknown-eabi'
+        this.ubuntu_toolchain_prefix = 'arm-linux-gnueabihf'
         if args.gem5:
             if this.machine is None:
                 this.machine = 'VExpress_GEM5_V1'
@@ -469,7 +515,9 @@ def setup(parser):
         this.armv = 8
         this.gem5_arch = 'ARM'
         this.mcpu = 'cortex-a57'
-        this.crosstool_ng_prefix = 'aarch64-unknown-elf'
+        this.buildroot_toolchain_prefix = 'aarch64-buildroot-linux-uclibc'
+        this.crosstool_ng_toolchain_prefix = 'aarch64-unknown-elf'
+        this.ubuntu_toolchain_prefix = 'aarch64-linux-gnu'
         if args.gem5:
             if this.machine is None:
                 this.machine = 'VExpress_GEM5_V1'
@@ -477,8 +525,10 @@ def setup(parser):
             if this.machine is None:
                 this.machine = 'virt'
     elif args.arch == 'x86_64':
-        this.crosstool_ng_prefix = 'TODO'
+        this.crosstool_ng_toolchain_prefix = 'x86_64-unknown-elf'
         this.gem5_arch = 'X86'
+        this.buildroot_toolchain_prefix = 'x86_64-buildroot-linux-uclibc'
+        this.ubuntu_toolchain_prefix = 'x86_64-linux-gnu'
         if args.gem5:
             if this.machine is None:
                 this.machine = 'TODO'
@@ -490,9 +540,6 @@ def setup(parser):
     this.buildroot_download_dir = os.path.join(this.buildroot_out_dir, 'download')
     this.buildroot_config_file = os.path.join(this.buildroot_build_dir, '.config')
     this.build_dir = os.path.join(this.buildroot_build_dir, 'build')
-    this.linux_build_dir = os.path.join(this.build_dir, 'linux-custom')
-    this.linux_variant_dir = '{}.{}'.format(this.linux_build_dir, args.linux_build_id)
-    this.vmlinux = os.path.join(this.linux_variant_dir, "vmlinux")
     this.qemu_build_dir = os.path.join(this.out_dir, 'qemu', args.qemu_build_id)
     this.qemu_executable_basename = 'qemu-system-{}'.format(args.arch)
     this.qemu_executable = os.path.join(this.qemu_build_dir, '{}-softmmu'.format(args.arch), this.qemu_executable_basename)
@@ -560,15 +607,28 @@ def setup(parser):
     this.gem5_se_file = os.path.join(this.gem5_config_dir, 'example', 'se.py')
     this.gem5_fs_file = os.path.join(this.gem5_config_dir, 'example', 'fs.py')
     this.run_cmd_file = os.path.join(this.run_dir, 'run.sh')
-    if args.arch == 'arm':
-        this.linux_image = os.path.join('arch', 'arm', 'boot', 'zImage')
-    elif args.arch == 'aarch64':
-        this.linux_image = os.path.join('arch', 'arm64', 'boot', 'Image')
-    elif args.arch == 'x86_64':
-        this.linux_image = os.path.join('arch', 'x86', 'boot', 'bzImage')
-    this.linux_image = os.path.join(this.linux_variant_dir, linux_image)
 
-    # Ports.
+    # Linux
+    this.linux_buildroot_build_dir = os.path.join(this.build_dir, 'linux-custom')
+    this.linux_build_dir = os.path.join(this.out_dir, 'linux', args.linux_build_id, args.arch)
+    this.vmlinux = os.path.join(this.linux_build_dir, "vmlinux")
+    if args.arch == 'arm':
+        this.linux_arch = 'arm'
+        this.linux_image = os.path.join('arch', this.linux_arch, 'boot', 'zImage')
+    elif args.arch == 'aarch64':
+        this.linux_arch = 'arm64'
+        this.linux_image = os.path.join('arch', this.linux_arch, 'boot', 'Image')
+    elif args.arch == 'x86_64':
+        this.linux_arch = 'x86'
+        this.linux_image = os.path.join('arch', this.linux_arch, 'boot', 'bzImage')
+    this.linux_image = os.path.join(this.linux_build_dir, linux_image)
+
+    # Kernel modules.
+    this.kernel_modules_build_base_dir = os.path.join(this.out_dir, 'kernel_modules')
+    this.kernel_modules_build_dir = os.path.join(this.kernel_modules_build_base_dir, args.arch)
+    this.kernel_modules_build_host_dir = os.path.join(this.kernel_modules_build_base_dir, 'host')
+
+    # Ports
     if args.port_offset is None:
         try:
             args.port_offset = int(args.run_id)
@@ -590,9 +650,6 @@ def setup(parser):
     this.baremetal_lib_basename = 'lib'
     this.baremetal_src_dir = os.path.join(this.root_dir, 'baremetal')
     this.baremetal_src_lib_dir = os.path.join(this.baremetal_src_dir, this.baremetal_lib_basename)
-    this.c_ext = '.c'
-    this.asm_ext = '.S'
-    this.obj_ext = '.o'
     if args.gem5:
         this.simulator_name = 'gem5'
     else: