gem5 basic build and boot

build

@@ -28,7 +28,7 @@ defaults = {
 }
 
 def path_relative_to_buildroot(abspath):
-    return os.path.relpath(abspath, common.buildroot_dir)
+    return os.path.relpath(abspath, common.buildroot_src_dir)
 
 def main(args, extra_args=None):
     global defaults
@@ -55,7 +55,7 @@ def main(args, extra_args=None):
     # Configure.
     if not args.skip_configure:
         # Initial make configure.
-        #cd "${common_buildroot_dir}"
+        #cd "${common_buildroot_src_dir}"
         #for p in $(find "${common_root_dir}/patches/buildroot/" -maxdepth 1 -name '*.patch' -print); do
         #  patch -N -r - -p 1 < "$p" || :
         #done
@@ -73,7 +73,7 @@ def main(args, extra_args=None):
                 'BR2_EXTERNAL={}'.format(br2_external_str),
                 defconfig,
             ],
-            cwd=common.buildroot_dir,
+            cwd=common.buildroot_src_dir,
         )
 
         # Extra buildroot configs.
@@ -175,9 +175,8 @@ def main(args, extra_args=None):
                 'O={}'.format(common.buildroot_out_dir),
                 'olddefconfig',
             ],
-            cwd=common.buildroot_dir,
+            cwd=common.buildroot_src_dir,
         )
-
     common.mkdir()
 
     # Manage Linux kernel and QEMU variants.
@@ -198,6 +197,7 @@ def main(args, extra_args=None):
     #  fi
     #fi
 
+    # Do the actual build.
     assert common.run_cmd(
         [
             'make',
@@ -209,7 +209,7 @@ def main(args, extra_args=None):
         ,
         out_file=os.path.join(common.out_arch_dir, 'buildroot.log'),
         delete_env=['LD_LIBRARY_PATH'],
-        cwd=common.buildroot_dir,
+        cwd=common.buildroot_src_dir,
     ) == 0
 
     # Create the qcow2 from ext2.

build-gem5

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+
+import glob
+import multiprocessing
+import os
+import pathlib
+import shutil
+import subprocess
+
+import common
+
+parser = common.get_argparse()
+parser.add_argument(
+    'extra_scons_args',
+    default=[],
+    metavar='extra-scons-args',
+    nargs='*'
+)
+args = common.setup(parser)
+binaries_dir = os.path.join(common.gem5_system_dir, 'binaries')
+disks_dir = os.path.join(common.gem5_system_dir, 'disks')
+os.makedirs(binaries_dir, exist_ok=True)
+os.makedirs(disks_dir, exist_ok=True)
+if args.arch == 'x86_64':
+    dummy_img_path = os.path.join(disks_dir, 'linux-bigswap2.img')
+    with open(dummy_img_path, 'wb') as dummy_img_file:
+        zeroes = b'\x00' * (2 ** 16)
+        for i in range(2 ** 10):
+            dummy_img_file.write(zeroes)
+    subprocess.check_call(['mkswap', dummy_img_path])
+    with open(os.path.join(binaries_dir, 'x86_64-vmlinux-2.6.22.9'), 'w'):
+        # This file must always be present, despite --kernel overriding that default and selecting the kernel.
+        # I'm not even joking. No one has ever built x86 gem5 without the magic dist dir present.
+        pass
+elif args.arch == 'arm' or args.arch == 'aarch64':
+    gem5_system_src_dir = os.path.join(common.gem5_src_dir, 'system')
+
+    # dtb
+    dt_src_dir = os.path.join(gem5_system_src_dir, 'arm', 'dt')
+    dt_build_dir = os.path.join(common.gem5_system_dir, 'arm', 'dt')
+    subprocess.check_call(['make', '-C', dt_src_dir])
+    os.makedirs(dt_build_dir, exist_ok=True)
+    for dt in glob.glob(os.path.join(dt_src_dir, '*.dtb')):
+        shutil.copy2(dt, dt_build_dir)
+
+    # Bootloader 32.
+    bootloader32_dir = os.path.join(gem5_system_src_dir, 'arm', 'simple_bootloader')
+    # TODO use the buildroot cross compiler here, and remove the dependencies from configure.
+    subprocess.check_call(['make', '-C', bootloader32_dir])
+    # bootloader
+    shutil.copy2(os.path.join(bootloader32_dir, 'boot_emm.arm'), binaries_dir)
+
+    # Bootloader 64.
+    bootloader64_dir = os.path.join(gem5_system_src_dir, 'arm', 'aarch64_bootloader')
+    # TODO cross_compile is ignored because the make does not use CC...
+    subprocess.check_call(['make', '-C', bootloader64_dir])
+    shutil.copy2(os.path.join(bootloader64_dir, 'boot_emm.arm64'), binaries_dir)
+assert common.run_cmd([
+        'scons',
+        # TODO factor with build.
+        '-j', str(multiprocessing.cpu_count()),
+        '--ignore-style',
+        common.gem5_executable
+    ] +
+    args.extra_scons_args,
+    cwd=common.gem5_src_dir,
+    extra_env={'PATH': '/usr/lib/ccache:' + os.environ['PATH']},
+) == 0
+term_src_dir = os.path.join(common.gem5_src_dir, 'util/term')
+subprocess.check_call(['make', '-C', term_src_dir])
+shutil.copy2(os.path.join(term_src_dir, 'm5term'), common.gem5_m5term)

build-qemu

@@ -6,13 +6,10 @@ import subprocess
 
 import common
 
-parser = common.get_argparse(argparse_args={
+parser = common.get_argparse()
-    'description': 'Build QEMU'
-})
 parser.add_argument(
     'extra_config_args',
     default=[],
-    help='Extra arguments for the tool.',
     metavar='extra-config-args',
     nargs='*'
 )
@@ -33,9 +30,8 @@ subprocess.check_call(
 subprocess.check_call(
     [
         'make',
-        '-j',
         # TODO factor with build.
-        str(multiprocessing.cpu_count()),
+        '-j', str(multiprocessing.cpu_count()),
     ],
     cwd=common.qemu_build_dir
 )

common.py

@@ -377,12 +377,12 @@ data_dir = os.path.join(root_dir, 'data')
 p9_dir = os.path.join(data_dir, '9p')
 gem5_non_default_src_root_dir = os.path.join(data_dir, 'gem5')
 gem5_readfile_file = os.path.join(data_dir, 'readfile')
-gem5_default_src_dir = os.path.join(root_dir, 'gem5', 'gem5')
 out_dir = os.path.join(root_dir, 'out')
 bench_boot = os.path.join(out_dir, 'bench-boot.txt')
 common_dir = os.path.join(out_dir, 'common')
 submodules_dir = os.path.join(root_dir, 'submodules')
-buildroot_dir = os.path.join(submodules_dir, 'buildroot')
+buildroot_src_dir = os.path.join(submodules_dir, 'buildroot')
+gem5_default_src_dir = os.path.join(submodules_dir, 'gem5')
 qemu_src_dir = os.path.join(submodules_dir, 'qemu')
 
 # Other default variables.

packages/gem5/build

@@ -1,61 +0,0 @@
-#!/usr/bin/env bash
-set -eux
-arch=x86_64
-build_type=opt
-cross_compile=
-j=
-outdir="$(pwd)"
-while getopts a:c:j:o:t: OPT; do
-  case "$OPT" in
-    a)
-      arch="$OPTARG"
-      ;;
-    c)
-      cross_compile="CROSS_COMPILE=$OPTARG"
-      ;;
-    j)
-      j="$OPTARG"
-      ;;
-    o)
-      outdir="$OPTARG"
-      ;;
-    t)
-      build_type="$OPTARG"
-      ;;
-    ?)
-      exit 2
-      ;;
-  esac
-done
-shift "$(($OPTIND - 1))"
-if [ -z "$j" ]; then
-  j="$(nproc)"
-fi
-system_dir="${outdir}/system"
-binaries_dir="${system_dir}/binaries"
-disks_dir="${system_dir}/disks"
-mkdir -p "$binaries_dir" "$disks_dir"
-export PATH="/usr/lib/ccache:${PATH}"
-if [ "$arch" = x86_64 ]; then
-  scons -j "$j" --ignore-style "${outdir}/build/X86/gem5.${build_type}"
-  f="${disks_dir}/linux-bigswap2.img"
-  dd if=/dev/zero of="$f" bs=1024 count=65536
-  mkswap "$f"
-  # This file must always be present, despite --kernel overriding that default and selecting the kernel.
-  # I'm not even joking. No one has ever built x86 gem5 without the magic dist dir present.
-  touch "${binaries_dir}/x86_64-vmlinux-2.6.22.9"
-elif [ "$arch" = arm ] || [ "$arch" = aarch64 ]; then
-  scons -j "$j" --ignore-style "${outdir}/build/ARM/gem5.${build_type}"
-  make -C ./system/arm/dt/
-  mkdir -p "${system_dir}/arm/dt"
-  # || true in case they are the same directory.
-  cp ./system/arm/dt/*.dtb "${system_dir}/arm/dt" || true
-  # TODO use the buildroot cross compiler here, and remove the dependencies from configure.
-  make -C ./system/arm/simple_bootloader/ $cross_compile
-  cp ./system/arm/simple_bootloader/boot_emm.arm "$binaries_dir"
-  # TODO cross_compile is ignored because the make does not use CC...
-  make -C ./system/arm/aarch64_bootloader/ $cross_compile
-  cp ./system/arm/aarch64_bootloader/boot_emm.arm64 "$binaries_dir"
-fi
-make -C util/term
-cp util/term/m5term "${outdir}"

packages/gem5/external.mk

@@ -15,9 +15,6 @@ ARCH_MAKE = $(ARCH)
 endif
 
 define GEM5_BUILD_CMDS
-	# TODO cannot pass "-c '$(TARGET_CROSS)'" here because the ARM build uses aarch64 for the bootloader...
-	cd '$(GEM5_LKMC_SRCDIR)' && '$(GEM5_SITE)/build' -a '$(ARCH)' -j '$(BR2_JLEVEL)' -o '$(GEM5_LKMC_OUTDIR)' -t '$(GEM5_LKMC_GEM5_BUILD_TYPE)'
-
 	# TODO cannot use TARGET_CONFIGURE_OPTS here because it overrides the CFLAGS on m5,
 	# which have an include. We should patch gem5 to add a += instead of = there.
 	cd '$(@D)/gem5/util/m5' && $(MAKE) -f 'Makefile.$(ARCH_MAKE)' CC='$(TARGET_CC)' LD='$(TARGET_LD)'