From 9e5173f324c3aeda707afcafb9a165e572eae990 Mon Sep 17 00:00:00 2001
From: Ciro Santilli <ciro.santilli@gmail.com>
Date: Wed, 21 Feb 2018 00:44:17 +0000
Subject: [PATCH] GEM5 checkpoint

Add gem5 as buildroot package to cross compile m5.

Add gem5 as a submodule.

Split gem5 from arch on CLI with "-a arm -g" instead of "-a arm-gem5".
---
 .gitignore         |  2 --
 .gitmodules        |  3 +++
 README.adoc        | 56 ++++++++++++++++++++++++++++++++++++++++++----
 build              | 45 +++++++++++++++++++++----------------
 build-gem5         | 17 ++++++--------
 gem5/.gitignore    |  1 +
 gem5/Config.in     |  4 ++++
 gem5/external.desc |  1 +
 gem5/external.mk   | 19 ++++++++++++++++
 gem5/gem5          |  1 +
 rungem5            | 38 +++++++++++++++----------------
 11 files changed, 133 insertions(+), 54 deletions(-)
 create mode 100644 gem5/.gitignore
 create mode 100644 gem5/Config.in
 create mode 100644 gem5/external.desc
 create mode 100644 gem5/external.mk
 create mode 160000 gem5/gem5

diff --git a/.gitignore b/.gitignore
index 84f14e7..c2ce50a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,6 +13,4 @@ modules.order
 trace*
 
 # GEM5
-/gem5
-/gem5-system
 /m5out
diff --git a/.gitmodules b/.gitmodules
index eb2bb23..8dd0a80 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -8,3 +8,6 @@
 [submodule "linux"]
 	path = linux
 	url = https://github.com/cirosantilli/linux
+[submodule "gem5/gem5"]
+	path = gem5/gem5
+	url = https://gem5.googlesource.com/public/gem5
diff --git a/README.adoc b/README.adoc
index c7de190..fca92ea 100644
--- a/README.adoc
+++ b/README.adoc
@@ -1208,8 +1208,8 @@ Instead, we have only chip makers, who keep everything that really works closed,
 ==== GEM5 ARM
 
 ....
-./configure && ./build -a arm-gem5
-./rungem5 -a arm-gem5
+./configure && ./build -a arm -g
+./rungem5 -a arm
 ....
 
 On another shell:
@@ -1223,7 +1223,7 @@ On another shell:
 E.g., to add `printk.time=y`, run:
 
 ....
-./rungem5 -a arm-gem5 -- --command-line='earlyprintk=pl011,0x1c090000 console=ttyAMA0 lpj=19988480 norandmaps rw loglevel=8 mem=512MB root=/dev/sda printk.time=y'
+./rungem5 -a arm -- --command-line='earlyprintk=pl011,0x1c090000 console=ttyAMA0 lpj=19988480 norandmaps rw loglevel=8 mem=512MB root=/dev/sda printk.time=y'
 ....
 
 When you use `--command-line=`, it overrides default command lines, which are required to boot properly.
@@ -1233,7 +1233,7 @@ So if you pass just `--command-line='printk.time=y'`, it removes the required op
 An easy way to find the other options is to to an initial boot:
 
 ....
-./rungem5 -a arm-gem5
+./rungem5 -a arm
 ....
 
 and then look at the line of the linux kernel that starts with
@@ -1295,6 +1295,54 @@ Escape character is '^]'.
 
 I have also tried to copy the exact same kernel command line options used by QEMU, but nothing changed.
 
+===== GEM5 checkpoint
+
+Analogous to QEMU's <<snapshot>>.
+
+Documentation: http://gem5.org/Checkpoints
+
+....
+./rungem5 -a arm
+....
+
+In guest, wait for the boot to end and run:
+
+....
+/m5 checkpoint
+....
+
+To restore the checkpoint, kill the VM and run:
+
+....
+./rungem5 -a arm -- -r 1
+....
+
+Let's create a second checkpoint to see how it works, in guest:
+
+....
+date >f
+/m5 checkpoint
+....
+
+Kill the VM, and try it out:
+
+....
+./rungem5 -a arm -- -r 2
+....
+
+and now in the guest:
+
+....
+cat f
+....
+
+contains the `date`. The file `f` wouldn't exist had we used the first checkpoint with `-r 1`.
+
+Internals:
+
+- the checkpoints are stored under `m5out/cpt.*`
+- `m5` is a guest utility present inside the GEM5 tree which we cross-compiled and installed into the guest
+
 ==== GEM5 x86
 
 TODO didn't get it working yet.
diff --git a/build b/build
index 26d0b36..308d244 100755
--- a/build
+++ b/build
@@ -2,16 +2,20 @@
 set -e
 arch=x86_64
 extra_targets=''
-qemu_sdl='--enable-sdl --with-sdlabi=2.0'
+gem5=false
 j="$(($(nproc) - 2))"
 post_script_args=''
+qemu_sdl='--enable-sdl --with-sdlabi=2.0'
 x11=false
 v=0
-while getopts 'a:j:lp:qSt:v' OPT; do
+while getopts 'a:gj:lp:qSt:v' OPT; do
   case "$OPT" in
     a)
       arch="$OPTARG"
     ;;
+    g)
+      gem5=true
+    ;;
     j)
       j="$OPTARG"
     ;;
@@ -43,19 +47,17 @@ case "$arch" in
   x86_64)
     defconfig=qemu_x86_64_defconfig
   ;;
-  x86_64-gem5)
-    defconfig=qemu_x86_64_defconfig
-  ;;
   arm)
-    # qemu_arm_vexpress_defconfig required a newer QEMU than 2.0.0 on a Ubuntu host.
-    # so let's stick to versatile for now.
-    defconfig=qemu_arm_versatile_defconfig
-  ;;
-  arm-gem5)
-    # Ideally we should use a custom clean config here.
-    # But let's just use this one as a starting point for now.
-    defconfig=qemu_arm_versatile_defconfig
-    post_script_args="$post_script_args -n"
+    if "$gem5"; then
+      # Ideally we should use a custom clean config here.
+      # But let's just use this one as a starting point for now.
+      defconfig=qemu_arm_versatile_defconfig
+      post_script_args="$post_script_args -n"
+    else
+      # qemu_arm_vexpress_defconfig required a newer QEMU than 2.0.0 on a Ubuntu host.
+      # so let's stick to versatile for now.
+      defconfig=qemu_arm_versatile_defconfig
+    fi
   ;;
   aarch64)
     defconfig=qemu_aarch64_virt_defconfig
@@ -64,6 +66,10 @@ case "$arch" in
     defconfig=qemu_mips64r6_malta_defconfig
   ;;
 esac
+arch_dir="$arch"
+if "$gem5"; then
+  arch_dir="${arch}-gem5"
+fi
 
 cd kernel_module
 ./make-host.sh -j "$j" clean
@@ -71,14 +77,14 @@ cd ../buildroot
 for p in $(find '../buildroot_patches/' -maxdepth 1 -name '*.patch' -print); do
     patch -N -r - -p 1 <"$p" || :
 done
-outdir="output.${arch}~"
-make O="$outdir" BR2_EXTERNAL="$(pwd)/../kernel_module" "$defconfig"
+outdir="output.${arch_dir}~"
+make O="$outdir" BR2_EXTERNAL="$(pwd)/../kernel_module:$(pwd)/../gem5" "$defconfig"
 # TODO Can't get rid of this for now.
 # http://stackoverflow.com/questions/44078245/is-it-possible-to-use-config-fragments-with-buildroots-config
 cat ../buildroot_config_fragment >> "${outdir}/.config"
-if echo "$arch" | grep -Eq -- '-gem5$'; then
+if "$gem5"; then
   echo "\
-BR2_LINUX_KERNEL_CUSTOM_CONFIG_FILE=\"../kernel_config_${arch}\"
+BR2_LINUX_KERNEL_CUSTOM_CONFIG_FILE=\"../kernel_config_${arch_dir}\"
 " >> "${outdir}/.config"
 else
   echo "\
@@ -105,6 +111,7 @@ time \
   make \
   O="$outdir" \
   BR2_JLEVEL="$j" \
+  BR2_PACKAGE_GEM5="$("$gem5" && echo y || echo n)" \
   BR2_ROOTFS_POST_SCRIPT_ARGS="$post_script_args" \
   HOST_QEMU_OPTS="--enable-debug --extra-cflags='-DDEBUG_PL061=1' --enable-trace-backends=simple $qemu_sdl" \
   V="$v" \
@@ -113,6 +120,6 @@ time \
   all \
 ;
 cd ..
-if echo "$arch" | grep -Eq -- '-gem5$'; then
+if "$gem5"; then
   ./build-gem5 -a "$arch"
 fi
diff --git a/build-gem5 b/build-gem5
index a0b474a..f25b6dd 100755
--- a/build-gem5
+++ b/build-gem5
@@ -1,6 +1,6 @@
 #/usr/bin/env bash
 set -eu
-arch='arm-gem5'
+arch=arm
 while getopts 'a:' OPT; do
   case "$OPT" in
     a)
@@ -9,16 +9,13 @@ while getopts 'a:' OPT; do
   esac
 done
 shift "$(($OPTIND - 1))"
-if [ ! -d 'gem5' ]; then
-  git clone https://gem5.googlesource.com/public/gem5
-fi
-cd gem5
-git checkout da79d6c6cde0fbe5473ce868c9be4771160a003b
-system_dir='../gem5-system'
+top="$(pwd)/gem5"
+system_dir="${top}/gem5-system"
 binaries_dir="${system_dir}/binaries"
 disks_dir="${system_dir}/disks"
 mkdir -p "$binaries_dir" "$disks_dir"
-if [ "$arch" = 'x86_64-gem5' ]; then
+cd "${top}/gem5"
+if [ "$arch" = x86_64 ]; then
   CC=gcc-6 CXX=g++-6 scons -j"$(nproc)" --ignore-style build/X86/gem5.opt
   f="${disks_dir}/linux-bigswap2.img"
   dd if=/dev/zero of="$f" bs=1024 count=65536
@@ -26,7 +23,7 @@ if [ "$arch" = 'x86_64-gem5' ]; then
   # This file must always be present, despite --kernel overriding that default and selecting the kernel.
   # I'm not even joking. No one has ever built x86 gem5 without the magic dist dir present.
   touch "${binaries_dir}/x86_64-vmlinux-2.6.22.9"
-elif [ "$arch" = 'arm-gem5' ]; then
+elif [ "$arch" = arm ]; then
   # Compilation fails with gcc 7 on that commit.
   # There were some recent portability patches, so it will likely get there soon.
   CC=gcc-6 CXX=g++-6 scons -j"$(nproc)" --ignore-style build/ARM/gem5.opt
@@ -37,4 +34,4 @@ elif [ "$arch" = 'arm-gem5' ]; then
   cp ./system/arm/aarch64_bootloader/boot_emm.arm64 "$binaries_dir"
 fi
 # TODO vs telnet?
-make -C gem5/util/term
+make -C util/term
diff --git a/gem5/.gitignore b/gem5/.gitignore
new file mode 100644
index 0000000..645010b
--- /dev/null
+++ b/gem5/.gitignore
@@ -0,0 +1 @@
+/gem5-system
diff --git a/gem5/Config.in b/gem5/Config.in
new file mode 100644
index 0000000..7cc3c3b
--- /dev/null
+++ b/gem5/Config.in
@@ -0,0 +1,4 @@
+config BR2_PACKAGE_GEM5
+	bool "gem5"
+	help
+		GEM5
diff --git a/gem5/external.desc b/gem5/external.desc
new file mode 100644
index 0000000..f59470f
--- /dev/null
+++ b/gem5/external.desc
@@ -0,0 +1 @@
+name: GEM5
diff --git a/gem5/external.mk b/gem5/external.mk
new file mode 100644
index 0000000..81f4482
--- /dev/null
+++ b/gem5/external.mk
@@ -0,0 +1,19 @@
+################################################################################
+#
+# GEM5
+#
+################################################################################
+
+GEM5_VERSION = 1.0
+GEM5_SITE = $(BR2_EXTERNAL_GEM5_PATH)
+GEM5_SITE_METHOD = local
+
+define GEM5_BUILD_CMDS
+	cd '$(@D)/gem5/util/m5' && $(MAKE) -f 'Makefile.$(ARCH)' CC='$(TARGET_CC)' LD='$(TARGET_LD)'
+endef
+
+define GEM5_INSTALL_TARGET_CMDS
+	$(INSTALL) -D -m 0755 '$(@D)/gem5/util/m5/m5' '$(TARGET_DIR)'
+endef
+
+$(eval $(generic-package))
diff --git a/gem5/gem5 b/gem5/gem5
new file mode 160000
index 0000000..da79d6c
--- /dev/null
+++ b/gem5/gem5
@@ -0,0 +1 @@
+Subproject commit da79d6c6cde0fbe5473ce868c9be4771160a003b
diff --git a/rungem5 b/rungem5
index fc65081..fc70c99 100755
--- a/rungem5
+++ b/rungem5
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-arch='arm-gem5'
+arch=arm
 while getopts 'a:' OPT; do
   case "$OPT" in
     a)
@@ -8,27 +8,27 @@ while getopts 'a:' OPT; do
   esac
 done
 shift "$(($OPTIND - 1))"
-outdir="$(pwd)/buildroot/output.${arch}~"
-echo $arch
-if echo "$arch" | grep -Eq '^x86_64(-|$)'; then
+outdir="$(pwd)/buildroot/output.${arch}-gem5~"
+gem5_dir="$(pwd)/gem5/gem5"
+if [ "$arch" = x86_64 ]; then
   cmd="\
-M5_PATH=\"$(pwd)/gem5-system\" \
-./gem5/build/X86/gem5.opt \
-gem5/configs/example/fs.py \
---disk-image=\"${outdir}/images/rootfs.ext2\" \
---kernel=\"${outdir}/build/linux-custom/vmlinux\" \
---root-device='/dev/sda' \
+M5_PATH='$(pwd)/gem5/gem5-system' \
+'${gem5_dir}/build/X86/gem5.opt' \
+'${gem5_dir}/configs/example/fs.py' \
+--disk-image='${outdir}/images/rootfs.ext2' \
+--kernel='${outdir}/build/linux-custom/vmlinux' \
+--root-device=/dev/sda \
 "
-elif echo "$arch" | grep -Eq '^arm(-|$)'; then
+elif [ "$arch" = arm ]; then
   cmd="\
-M5_PATH=\"$(pwd)/gem5-system\" \
-./gem5/build/ARM/gem5.opt \
-gem5/configs/example/fs.py \
---disk-image=\"${outdir}/images/rootfs.ext2\" \
---dtb-file=\"$(pwd)/gem5/system/arm/dt/armv7_gem5_v1_1cpu.dtb\" \
---kernel=\"${outdir}/build/linux-custom/vmlinux\" \
---machine-type='VExpress_GEM5_V1' \
---root-device='/dev/sda' \
+M5_PATH='$(pwd)/gem5/gem5-system' \
+'${gem5_dir}/build/ARM/gem5.opt' \
+'${gem5_dir}/configs/example/fs.py' \
+--disk-image='${outdir}/images/rootfs.ext2' \
+--dtb-file='${gem5_dir}/system/arm/dt/armv7_gem5_v1_1cpu.dtb' \
+--kernel='${outdir}/build/linux-custom/vmlinux' \
+--machine-type=VExpress_GEM5_V1 \
+--root-device=/dev/sda \
 "
 fi
 cmd="$cmd \"\$@\""