diff --git a/build b/build
index d6be4b9..ba956cd 100755
--- a/build
+++ b/build
@@ -100,13 +100,13 @@ BR2_JLEVEL=$j
 BR2_ROOTFS_POST_SCRIPT_ARGS=\"$post_script_args\"
 " >> "${config_file}"
   if "$gem5"; then
+    printf "BR2_PACKAGE_GEM5=y\n" >> "${config_file}"
     if [ -z "$linux_kernel_custom_config_file" ]; then
-      printf "BR2_LINUX_KERNEL_CUSTOM_CONFIG_FILE=\"../kernel_config_${arch_dir}\"" >> "${config_file}"
+      printf "BR2_LINUX_KERNEL_CUSTOM_CONFIG_FILE=\"../kernel_config_${arch_dir}\"\n" >> "${config_file}"
     fi
-    printf "BR2_PACKAGE_GEM5=\"../kernel_config_${arch_dir}\"" >> "${config_file}"
   else
     if [ -z "$linux_kernel_custom_config_file" ]; then
-      printf "BR2_LINUX_KERNEL_CONFIG_FRAGMENT_FILES=\"../kernel_config_fragment\"" >> "${config_file}"
+      printf "BR2_LINUX_KERNEL_CONFIG_FRAGMENT_FILES=\"../kernel_config_fragment\"\n" >> "${config_file}"
     fi
   fi
   if [ "$arch" = 'mips64' ]; then
@@ -114,11 +114,8 @@ BR2_ROOTFS_POST_SCRIPT_ARGS=\"$post_script_args\"
     # http://lists.busybox.net/pipermail/buildroot/2017-August/201053.html
     sed -Ei 's/^BR2_PACKAGE_LINUX_TOOLS_GPIO/BR2_PACKAGE_LINUX_TOOLS_GPIO=n/' "${config_file}"
   fi
-  # TODO why is this needed. Otherwise kernel .config does not change, even though Buildroot
-  # .config did due to -K. even if we are running linux-kernel-reconfigure!
-  rm -f "${out_dir}/build/linux-custom/.config"
   if [ -n "$linux_kernel_custom_config_file" ]; then
-    printf "BR2_LINUX_KERNEL_CUSTOM_CONFIG_FILE=\"../${linux_kernel_custom_config_file}\"" >> "${config_file}"
+    printf "BR2_LINUX_KERNEL_CUSTOM_CONFIG_FILE=\"../${linux_kernel_custom_config_file}\"\n" >> "${config_file}"
   fi
   make O="$out_dir" olddefconfig
 fi
@@ -147,7 +144,3 @@ cmd="time \
 "
 echo "$cmd" | tee "${root_dir}/build.log"
 eval "$cmd"
-if "$gem5"; then
-  cd "${root_dir}"
-  ./build-gem5 -a "$arch"
-fi
diff --git a/gem5/.gitignore b/gem5/.gitignore
index 645010b..cbe1b02 100644
--- a/gem5/.gitignore
+++ b/gem5/.gitignore
@@ -1 +1 @@
-/gem5-system
+/system
diff --git a/gem5/README.adoc b/gem5/README.adoc
new file mode 100644
index 0000000..82e0c86
--- /dev/null
+++ b/gem5/README.adoc
@@ -0,0 +1,10 @@
+= gem5
+
+We Build gem5 as a Buildroot package for the following reasons:
+
+* if we build in tree without Buildroot, we still want to build the `m5` tool through Buildroot. But then Buildroot rsyncs the build for all archs into `output/build`, which takes quite some disk space. There seems to be no way to exclude some packages from the sync.
+* it allows us to reuse Buildroot's:
+** configuration system, which reduces duplication: set the gem5 package option, everything just gets built
+** timestamping system, which could save a few seconds on rebuilds, since we then don't ever run `scons` when gem5 is not modified
+
+What is preventing upstreaming to Buildroot: a minimal kernel config like the ones QEMU has.
diff --git a/build-gem5 b/gem5/build
similarity index 66%
rename from build-gem5
rename to gem5/build
index eb23f8e..7e7285d 100755
--- a/build-gem5
+++ b/gem5/build
@@ -1,22 +1,29 @@
 #/usr/bin/env bash
 set -eu
 arch=x86_64
-while getopts 'a:' OPT; do
+j=''
+while getopts a:j: OPT; do
   case "$OPT" in
     a)
       arch="$OPTARG"
       ;;
+    j)
+      j="$OPTARG"
+      ;;
   esac
 done
 shift "$(($OPTIND - 1))"
-top="$(pwd)/gem5"
-system_dir="${top}/gem5-system"
+if [ -z "$j" ]; then
+  j="$(nproc)"
+fi
+top="$(dirname "$0")"
+system_dir="${top}/system"
 binaries_dir="${system_dir}/binaries"
 disks_dir="${system_dir}/disks"
 mkdir -p "$binaries_dir" "$disks_dir"
 cd "${top}/gem5"
 if [ "$arch" = x86_64 ]; then
-  CC=gcc-6 CXX=g++-6 scons -j"$(nproc)" --ignore-style build/X86/gem5.opt
+  CC=gcc-6 CXX=g++-6 scons -j"$j" --ignore-style build/X86/gem5.opt
   f="${disks_dir}/linux-bigswap2.img"
   dd if=/dev/zero of="$f" bs=1024 count=65536
   mkswap "$f"
@@ -26,10 +33,11 @@ if [ "$arch" = x86_64 ]; then
 elif [ "$arch" = arm ] || [ "$arch" = aarch64 ]; then
   # Compilation fails with gcc 7 on that commit.
   # There were some recent portability patches, so it will likely get there soon.
-  CC=gcc-6 CXX=g++-6 scons -j"$(nproc)" --ignore-style build/ARM/gem5.opt
-  make -C system/arm/dt
-  make -C system/arm/simple_bootloader
-  make -C system/arm/aarch64_bootloader/
+  CC=gcc-6 CXX=g++-6 scons -j"$j" --ignore-style build/ARM/gem5.opt
+  make -C ./system/arm/dt
+  # TODO use the buildroot cross compiler here, and remove the dependencies from configure.
+  make -C ./system/arm/simple_bootloader
+  make -C ./system/arm/aarch64_bootloader/
   cp ./system/arm/simple_bootloader/boot_emm.arm "$binaries_dir"
   cp ./system/arm/aarch64_bootloader/boot_emm.arm64 "$binaries_dir"
 fi
diff --git a/gem5/external.mk b/gem5/external.mk
index 2c7a826..6da5bd3 100644
--- a/gem5/external.mk
+++ b/gem5/external.mk
@@ -15,6 +15,7 @@ ARCH_MAKE = $(ARCH)
 endif
 
 define GEM5_BUILD_CMDS
+	cd '$(@D)' && ./build -a '$(ARCH)' -j '$(BR2_JLEVEL)'
 	cd '$(@D)/gem5/util/m5' && $(MAKE) -f 'Makefile.$(ARCH_MAKE)' CC='$(TARGET_CC)' LD='$(TARGET_LD)'
 endef
 
diff --git a/run b/run
index 884223b..6f8f7bc 100755
--- a/run
+++ b/run
@@ -74,13 +74,15 @@ buildroot_dir="${root_dir}/buildroot"
 out_dir="${root_dir}/buildroot/output.${arch_dir}~"
 
 if "$gem5"; then
-  gem5_dir="${root_dir}/gem5/gem5"
+  build_dir="${out_dir}/build/gem5-1.0"
+  gem5_dir="${build_dir}/gem5"
   if [ "$arch" = x86_64 ]; then
     if "$kvm"; then
       extra_flags="$extra_flags --cpu-type=X86KvmCPU"
     fi
     cmd="\
-M5_PATH='${root_dir}/gem5/gem5-system' \
+M5_PATH='${build_dir}/system' \
+$debug_vm \
 '${gem5_dir}/build/X86/gem5.opt' \
 ${gem5opts} \
 '${gem5_dir}/configs/example/fs.py' \
@@ -92,7 +94,7 @@ $extra_flags \
 "
   elif [ "$arch" = arm ] || [ "$arch" = aarch64 ]; then
     cmd="\
-M5_PATH='${root_dir}/gem5/gem5-system' \
+M5_PATH='${build_dir}/system' \
 $debug_vm \
 '${gem5_dir}/build/ARM/gem5.opt' \
 ${gem5opts} \
@@ -102,7 +104,7 @@ ${gem5opts} \
 --dtb-file='${gem5_dir}/system/arm/dt/$([ "$arch" = arm ] && echo "armv7_gem5_v1_${cpus}cpu" || echo "armv8_gem5_v1_${cpus}cpu").dtb' \
 --kernel='${out_dir}/build/linux-custom/vmlinux' \
 --machine-type=VExpress_GEM5_V1 \
---num-cpus=${cpus} \
+--num-cpus='${cpus}' \
 $extra_flags \
 "
   fi
@@ -113,8 +115,8 @@ else
   extra_flags="${extra_flags_qemu} ${extra_flags}"
   images_dir="${out_dir}/images"
   qemu_common="\
-${debug_vm} \
-${out_dir}/host/usr/bin/qemu-system-${arch} \
+'${debug_vm}' \
+'${out_dir}/host/usr/bin/qemu-system-${arch}' \
 -m 128M \
 -monitor telnet::45454,server,nowait \
 -netdev user,hostfwd=tcp::45455-:45455,id=net0 \
@@ -141,7 +143,7 @@ ${out_dir}/host/usr/bin/qemu-system-${arch} \
 -device edu \
 -device lkmc_pci_min \
 -device virtio-net-pci,netdev=net0 \
--kernel ${images_dir}/bzImage \
+-kernel '${images_dir}/bzImage' \
 ${extra_flags} \
 "
     ;;
@@ -157,8 +159,8 @@ ${extra_flags} \
 -M versatilepb \
 -append '$root $extra_append' \
 -device rtl8139,netdev=net0 \
--dtb ${images_dir}/versatile-pb.dtb \
--kernel ${images_dir}/zImage \
+-dtb '${images_dir}/versatile-pb.dtb' \
+-kernel '${images_dir}/zImage' \
 -serial stdio \
 $extra_flags \
 "
@@ -172,7 +174,7 @@ $extra_flags \
 -append 'root=/dev/sda ${extra_append}' \
 -cpu cortex-a57 \
 -device virtio-net-device,netdev=net0 \
--kernel ${images_dir}/Image \
+-kernel '${images_dir}/Image' \
 -nographic \
 -serial stdio \
 ${extra_flags} \
@@ -188,7 +190,7 @@ ${extra_flags} \
 -append 'root=/dev/hda ${extra_append}' \
 -cpu I6400 \
 -device pcnet \
--kernel ${images_dir}/vmlinux \
+-kernel '${images_dir}/vmlinux' \
 -nographic \
 ${extra_flags} \
 "