gem5: support variants

README.adoc

@@ -6517,10 +6517,36 @@ The `-L` option should be passed to all scripts that support it, much like `-a`
 ./rungdb -L v4.16
 .....
 
-It should also be easy to extend this method to all `-custom` `_OVERRIDE_SRCDIR` packages, which luckily tend to be the ones that we care the most about.
-
 This technique is implemented semi-hackishly by moving symlinks around inside the Buildroot build dir at build time, and selecting the right build directory at runtime.
 
+==== gem5 build variants
+
+Analogous to the <<linux-kernel-build-variants>> but with the `-M` option instead:
+
+....
+./build -g
+git -C gem5/gem5 checkout some-branch
+./build -M some-branch -g
+git -C gem5/gem5 checkout -
+./run -g
+./run -M some-branch -g
+....
+
+Since we control the gem5 build however, unlike Linux which uses Buildroot, we make it awesomer, and use `git worktree` instead of a mere `rsync` for the copy.
+
+It works like this:
+
+* when you don't pass the `-M` option, which is the same as the `-M default` variant, we use the source code from under the `gem5/gem5` submodule for the build
+* otherwise, if you pass `-M some-branch`, we generate a git worktree checkout under `data/gem5/some-branch`.
++
+The initial revision for that worktree is whatever `gem5/gem5` is currently points to.
++
+However, if the worktree already exists, we leave it untouched.
++
+Therefore, you can safely go to that directory and edit the source there without fear that it will get deleted.
+
+All build outputs end up at: `out/common/gem5/<variant>` regardless.
+
 ==== Generic package build variants
 
 This hack-ish technique allows us to rebuild just one package at a time:
@@ -7044,12 +7070,22 @@ The action seems to be happening at: `hw/arm/virt.c`.
 === Directory structure
 
 * `data`: gitignored user created data. Deleting this might lead to loss of data. Of course, if something there becomes is important enough to you, git track it.
+** `data/readfile`: see <<m5-readfile>>
+** `data/9p`: see <<9p>>
+** `data/gem5/<variant>`: see: <<gem5-build-variants>>
 * `out`: gitignored Build outputs. You won't lose data by deleting this folder since everything there can be re-generated, only time.
-* `out/<arch>`: arch specific outputs
+** `out/<arch>`: arch specific outputs
-* `out/<arch>/buildroot`: Buildroot output
+*** `out/<arch>/buildroot`: standard Buildroot output
-* `out/<arch>/qemu`: QEMU runtime outputs
+**** `out/<arch>/buildroot/build/linux-custom`: symlink to a variant, custom madness that we do on top of Buildroot: <<linux-kernel-build-variants>>
-* `out/<arch>/gem5`: GEM5 runtime outputs
+**** `out/<arch>/buildroot/build/linux-custom.<variant>`: what `linux-custom` points to
-* `out/common`: cross arch outputs. Notably gem5, which has the same build for `arm` and `aarch64`.
+*** `out/<arch>/qemu`: QEMU runtime outputs
+** `out/common`: cross arch outputs, for when we can gain a lot of time and space by sharing things that are common across different archs.
+*** `out/common/dl/`: Buildroot caches downloaded source there due to `BR2_DL_DIR`
+*** `out/common/gem5/`: `arm` and `aarch64` have the same build.
+**** `out/common/gem5/<variant>/`
+***** `out/common/gem5/<variant>/build/`
+***** `out/common/gem5/<variant>/m5out`
+***** `out/common/gem5/<variant>/system`
 
 :leveloffset: +3
 

bench-all

@@ -95,7 +95,7 @@ if "$bench_gem5_build"; then
     cd "${root_dir}/gem5/gem5"
     git clean -xdf
     cd "${root_dir}/gem5"
-    results_file="${gem5_run_out_dir}/bench-build.txt"
+    results_file="${common_gem5_out_dir}/bench-build.txt"
     gem5_outdir="${out_dir}/bench_build"
     rm -fr "$results_file" "${gem5_outdir}"
     # TODO understand better: --foreground required otherwise we cannot

build

@@ -20,7 +20,7 @@ post_script_args=
 qemu_sdl='--enable-sdl --with-sdlabi=2.0'
 suffix=
 v=0
-while getopts 'a:B:b:C:c:fGgj:hIiK:kL:lp:qSs:v' OPT; do
+while getopts 'a:B:b:C:c:fGgj:hIiK:kL:lM:p:qSs:v' OPT; do
   case "$OPT" in
     a)
       arch="$OPTARG"
@@ -81,6 +81,9 @@ BR2_TARGET_ROOTFS_INITRAMFS=n
       extra_make_args="${extra_make_args} linux-reconfigure \\
 "
       ;;
+    M)
+      common_gem5_variant="$OPTARG"
+      ;;
     p)
       post_script_args="$OPTARG"
       ;;
@@ -104,7 +107,7 @@ BR2_TARGET_ROOTFS_INITRAMFS=n
 done
 shift $(($OPTIND - 1))
 extra_make_args="${extra_make_args} $@"
-set_common_vars -L "$common_linux_variant" "$arch" "$gem5" "$suffix"
+set_common_vars -L "$common_linux_variant" -M "$common_gem5_variant" "$arch" "$gem5" "$suffix"
 config_file="${buildroot_out_dir}/.config"
 case "$arch" in
   x86_64)
@@ -170,6 +173,8 @@ BR2_ROOTFS_POST_SCRIPT_ARGS=\"${post_script_args}\"
 fi
 
 common_mkdir
+
+# Manage Linux kernel variants.
 if [ -h "$common_linux_custom_dir" ]; then
   rm "$common_linux_custom_dir"
 elif [ -d "$common_linux_custom_dir" ]; then
@@ -178,6 +183,14 @@ elif [ -d "$common_linux_custom_dir" ]; then
 fi
 mkdir -p "$common_linux_variant_dir"
 ln -s "$common_linux_variant_dir" "$common_linux_custom_dir"
+
+# Manage gem5 variants.
+if "$gem5"; then
+  if [ ! -e "${common_gem5_src_dir}/.git" ]; then
+    git -C "$common_gem5_default_src_dir" worktree add -b "${common_gem5_variant}" "${common_gem5_src_dir}"
+  fi
+fi
+
 cd "$buildroot_dir"
 # HOST_QEMU_OPTS is a hack that happens to work because the QEMU package luckly uses += at all times.
 # It shouldn't be necessary in the first place: https://bugs.busybox.net/show_bug.cgi?id=9936
@@ -193,6 +206,8 @@ env \\
 make \\
 O='${buildroot_out_dir}' \\
 HOST_QEMU_OPTS='--enable-debug --enable-trace-backends=simple ${qemu_sdl}' \\
+GEM5_LKMC_SRCDIR="$common_gem5_src_dir" \\
+GEM5_LKMC_OUTDIR="$common_gem5_out_dir" \\
 V='${v}' \\
 ${extra_make_args} \
 all \\

common

@@ -21,15 +21,18 @@ common_bench_cmd() (
 )
 set_common_vars() {
   linux_variant=
+  gem5_variant=
   OPTIND=1
-  while getopts L: OPT; do
+  while getopts L:M: OPT; do
     case "$OPT" in
       L)
         linux_variant="$OPTARG"
         ;;
+      M)
+        gem5_variant="$OPTARG"
+        ;;
     esac
   done
-  echo $OPTIND
   shift "$(($OPTIND - 1))"
   arch="$1"
   gem5="${2:-false}"
@@ -60,21 +63,28 @@ set_common_vars() {
   out_arch_dir="${out_dir}/${arch_dir}"
   buildroot_out_dir="${out_arch_dir}/buildroot"
   build_dir="${buildroot_out_dir}/build"
-  images_dir="${buildroot_out_dir}/images"
+  common_images_dir="${buildroot_out_dir}/images"
   host_dir="${buildroot_out_dir}/host"
   qemu_out_dir="${out_arch_dir}/qemu"
-  gem5_run_out_dir="${out_arch_dir}/gem5"
+  common_linux_custom_dir="${build_dir}/linux-custom"
-  m5out_dir="${gem5_run_out_dir}/m5out"
+  common_linux_variant_dir="${common_linux_custom_dir}.${linux_variant}"
+  common_vmlinux="${common_linux_variant_dir}/vmlinux"
+  if [ "$gem5_variant" = default ]; then
+    common_gem5_src_dir="${root_dir}/gem5/gem5"
+  else
+    common_gem5_src_dir="${common_gem5_non_default_src_root_dir}/${gem5_variant}"
+  fi
+  common_gem5_out_dir="${common_dir}/gem5/${gem5_variant}"
+  common_gem5_build_dir="${common_gem5_out_dir}/build"
+  common_gem5_system_dir="${common_gem5_out_dir}/system"
+  common_m5out_dir="${common_gem5_out_dir}/m5out"
   if "$gem5"; then
-    common_out_run_dir="$gem5_run_out_dir"
+    common_out_run_dir="$common_gem5_out_dir"
-    common_trace_txt_file="${m5out_dir}/trace.txt"
+    common_trace_txt_file="${common_m5out_dir}/trace.txt"
   else
     common_out_run_dir="$qemu_out_dir"
     common_trace_txt_file="${common_out_run_dir}/trace.txt"
   fi
-  common_linux_custom_dir="${build_dir}/linux-custom"
-  common_linux_variant_dir="${common_linux_custom_dir}.${linux_variant}"
-  common_vmlinux="${common_linux_variant_dir}/vmlinux"
   case "$arch" in
     arm)
       common_linux_image=arch/arm/boot/zImage
@@ -94,7 +104,7 @@ set_common_vars() {
 common_mkdir() (
   mkdir -p \
     "$build_dir" \
-    "$gem5_out_dir" \
+    "$common_gem5_out_dir" \
     "$qemu_out_dir" \
     "$p9_dir" \
   ;
@@ -107,7 +117,9 @@ data_dir="${root_dir}/data"
 p9_dir="${data_dir}/9p"
 readfile_file="${data_dir}/readfile"
 common_dir="${out_dir}/common"
-gem5_out_dir="${common_dir}/gem5"
+common_gem5_default_src_dir="${root_dir}/gem5/gem5"
+common_gem5_non_default_src_root_dir="${data_dir}/gem5"
+common_gem5_variant=default
 f="${data_dir}/cli"
 if [ -f "$f" ]; then
   . "$f"

fs-biglittle

@@ -1,13 +1,13 @@
 #!/usr/bin/env bash
-gem5_out_dir="$(pwd)/out/common/gem5"
+. common
-buildroot_out_dir="$(pwd)/out/aarch64/buildroot"
+set_common_vars -L "${common_linux_variant}" -M "${common_gem5_variant}" aarch64 true
-M5_PATH="${gem5_out_dir}/system" \
+M5_PATH="${common_gem5_system_dir}" \
-  "${gem5_out_dir}/build/ARM/gem5.opt" \
+  "${common_gem5_build_dir}/ARM/gem5.opt" \
-  ./gem5/gem5/configs/example/arm/fs_bigLITTLE.py \
+  "${common_gem5_default_src_dir}/configs/example/arm/fs_bigLITTLE.py" \
   --big-cpus=2 \
   --caches \
-  --disk="${buildroot_out_dir}/images/rootfs.ext2" \
+  --disk="${common_images_dir}/rootfs.ext2" \
-  --dtb "${gem5_out_dir}/system/arm/dt/armv8_gem5_v1_big_little_2_2.dtb" \
+  --dtb "${common_gem5_system_dir}/arm/dt/armv8_gem5_v1_big_little_2_2.dtb" \
-  --kernel="${buildroot_out_dir}/build/linux-custom/vmlinux" \
+  --kernel="${common_vmlinux}" \
   --little-cpus=2 \
 ;

gem5-bench-cache

@@ -19,7 +19,7 @@ set_common_vars "$arch" true
 cmd="./run -a $arch -g"
 cache_small='--caches --l2cache --l1d_size=1024   --l1i_size=1024   --l2_size=1024   --l3_size=1024  '
 cache_large='--caches --l2cache --l1d_size=1024kB --l1i_size=1024kB --l2_size=1024kB --l3_size=1024kB'
-results_file="${gem5_run_out_dir}/bench-cache.txt"
+results_file="${common_gem5_out_dir}/bench-cache.txt"
 
 bench() (
   common_bench_cmd "$1" "$results_file"
@@ -66,7 +66,7 @@ bench-all() (
 
 if "$generate_checkpoints"; then
   # Create the checkpoints after the kernel boot.
-  rm -rf "${m5out_dir}"/cpt.*;
+  rm -rf "${common_m5out_dir}"/cpt.*;
   printf 'm5 exit' > "${readfile_file}"
   cpt_cmd="-E 'm5 checkpoint;m5 readfile > a.sh;sh a.sh'"
   # 1

gem5-stat

@@ -22,4 +22,4 @@ else
   stat=system.cpu[0-9]*.numCycles
 fi
 set_common_vars "$arch" true
-awk "/^$stat /{ print \$2 }" "${m5out_dir}/stats.txt"
+awk "/^$stat /{ print \$2 }" "${common_m5out_dir}/stats.txt"

gem5/build

@@ -1,9 +1,9 @@
 #!/usr/bin/env bash
-set -eu
+set -eux
 arch=x86_64
 cross_compile=
 j=
-outdir="$(pwd)/out"
+outdir="$(pwd)"
 while getopts a:c:j:o: OPT; do
   case "$OPT" in
     a)
@@ -28,7 +28,6 @@ system_dir="${outdir}/system"
 binaries_dir="${system_dir}/binaries"
 disks_dir="${system_dir}/disks"
 mkdir -p "$binaries_dir" "$disks_dir"
-cd "gem5"
 if [ "$arch" = x86_64 ]; then
   scons -j "$j" --ignore-style "${outdir}/build/X86/gem5.opt"
   f="${disks_dir}/linux-bigswap2.img"
@@ -41,7 +40,8 @@ elif [ "$arch" = arm ] || [ "$arch" = aarch64 ]; then
   scons -j "$j" --ignore-style "${outdir}/build/ARM/gem5.opt"
   make -C ./system/arm/dt/
   mkdir -p "${system_dir}/arm/dt"
-  cp ./system/arm/dt/*.dtb "${system_dir}/arm/dt"
+  # || true in case they are the same directory.
+  cp ./system/arm/dt/*.dtb "${system_dir}/arm/dt" || true
   # TODO use the buildroot cross compiler here, and remove the dependencies from configure.
   make -C ./system/arm/simple_bootloader/ $cross_compile
   cp ./system/arm/simple_bootloader/boot_emm.arm "$binaries_dir"

gem5/external.mk

@@ -15,10 +15,8 @@ ARCH_MAKE = $(ARCH)
 endif
 
 define GEM5_BUILD_CMDS
-	# Cannot pass "-c '$(TARGET_CROSS)'" here because the ARM build uses aarch64 for the bootloader...
+	# TODO cannot pass "-c '$(TARGET_CROSS)'" here because the ARM build uses aarch64 for the bootloader...
-	cd '$(GEM5_SITE)' && ./build -a '$(ARCH)' -j '$(BR2_JLEVEL)' -o '$(GEM5_SITE)/../out/common/gem5'
+	cd '$(GEM5_LKMC_SRCDIR)' && '$(GEM5_SITE)/build' -a '$(ARCH)' -j '$(BR2_JLEVEL)' -o '$(GEM5_LKMC_OUTDIR)'
-	# This would build inside the buildroot directory as a more normal package.
-	#cd '$(@D)' && ./build -a '$(ARCH)' -j '$(BR2_JLEVEL)'
 
 	# TODO cannot use TARGET_CONFIGURE_OPTS here because it overrides the CFLAGS on m5,
 	# which have an include. We should patch gem5 to add a += instead of = there.

run

@@ -34,7 +34,7 @@ trace_enabled=false
 # just to prevent QEMU from emitting a warning that '' is not valid.
 trace_type=pr_manager_run
 vnc=
-while getopts a:c:DdE:e:F:f:G:ghIiKkL:m:PT:U:uVx OPT; do
+while getopts a:c:DdE:e:F:f:G:ghIiKkL:M:m:PT:U:uVx OPT; do
   case "$OPT" in
     a)
       arch="$OPTARG"
@@ -94,6 +94,9 @@ while getopts a:c:DdE:e:F:f:G:ghIiKkL:m:PT:U:uVx OPT; do
     L)
       common_linux_variant="$OPTARG"
       ;;
+    M)
+      common_gem5_variant="$OPTARG"
+      ;;
     m)
       memory="$OPTARG"
       ;;
@@ -124,8 +127,7 @@ while getopts a:c:DdE:e:F:f:G:ghIiKkL:m:PT:U:uVx OPT; do
 done
 shift "$(($OPTIND - 1))"
 extra_flags="$extra_flags $@"
-set_common_vars -L "$common_linux_variant" "$arch" "$gem5"
+set_common_vars -L "$common_linux_variant" -M "$common_gem5_variant" "$arch" "$gem5"
-images_dir="${buildroot_out_dir}/images"
 if "$debug" && "$kvm"; then
   echo 'error: -d and -K are incompatible' 1>&2
   exit 1
@@ -156,9 +158,6 @@ if [ -n "$extra_append_after_dash" ]; then
 fi
 
 if "$gem5"; then
-  gem5_build_dir="${buildroot_out_dir}/build/gem5-1.0"
-  gem5_src_dir="${root_dir}/gem5/gem5"
-  gem5_system_dir="${gem5_out_dir}/system"
   memory="${memory}B"
   if [ "$arch" = x86_64 ]; then
     gem5_arch=X86
@@ -170,14 +169,14 @@ if "$gem5"; then
 "
   fi
   gem5_common="\
-M5_PATH='${gem5_system_dir}' \\
+M5_PATH='${common_gem5_system_dir}' \\
 ${debug_vm} \
-'${gem5_out_dir}/build/${gem5_arch}/gem5.opt' \\
+'${common_gem5_build_dir}/${gem5_arch}/gem5.opt' \\
 --debug-file=trace.txt \\
 ${gem5opts} \
--d '${m5out_dir}' \\
+-d '${common_m5out_dir}' \\
-'${gem5_src_dir}/configs/example/fs.py' \\
+'${common_gem5_src_dir}/configs/example/fs.py' \\
---disk-image='${images_dir}/rootfs.ext2' \\
+--disk-image='${common_images_dir}/rootfs.ext2' \\
 --kernel='${common_vmlinux}' \\
 --mem-size='${memory}' \\
 --num-cpus='${cpus}' \\
@@ -197,7 +196,7 @@ ${extra_flags} \
     # Anything smaller than physical blows up as expected, but why can't it auto-detect the right value?
     cmd="${gem5_common} \
 --command-line='earlyprintk=pl011,0x1c090000 console=ttyAMA0 lpj=19988480 rw loglevel=8 mem=${memory} root=/dev/sda ${extra_append}' \\
---dtb-file='${gem5_system_dir}/arm/dt/$([ "$arch" = arm ] && echo "armv7_gem5_v1_${cpus}cpu" || echo "armv8_gem5_v1_${cpus}cpu").dtb' \\
+--dtb-file='${common_gem5_system_dir}/arm/dt/$([ "$arch" = arm ] && echo "armv7_gem5_v1_${cpus}cpu" || echo "armv8_gem5_v1_${cpus}cpu").dtb' \\
 --machine-type=VExpress_GEM5_V1 \\
 ${extra_flags} \
 "
@@ -235,7 +234,7 @@ ${serial_monitor} \
 -virtfs 'local,path=${buildroot_out_dir}/build,mount_tag=host_out,security_model=mapped,id=host_out' \\
 ${vnc}"
   if "$initrd"; then
-    extra_flags="${extra_flags} -initrd '${images_dir}/rootfs.cpio' \\
+    extra_flags="${extra_flags} -initrd '${common_images_dir}/rootfs.cpio' \\
 "
   fi
   if "$ramfs"; then
@@ -243,7 +242,7 @@ ${vnc}"
     root='root=/dev/anything'
   else
     if [ ! "$arch" = mips64 ]; then
-      extra_flags="${extra_flags} -drive 'file=${images_dir}/rootfs.ext2.qcow2,format=qcow2,if=virtio,snapshot' \\
+      extra_flags="${extra_flags} -drive 'file=${common_images_dir}/rootfs.ext2.qcow2,format=qcow2,if=virtio,snapshot' \\
 "
       root='root=/dev/vda'
     fi
@@ -291,14 +290,14 @@ ${qemu_common} \
 -append '${root} ${extra_append}' \\
 -cpu cortex-a57 \\
 -device virtio-gpu-pci \\
--kernel '${images_dir}/Image' \\
+-kernel '${common_images_dir}/Image' \\
 ${extra_flags} \
 "
     ;;
     mips64)
       if ! "$ramfs"; then
         root='root=/dev/hda'
-        extra_flags="${extra_flags} -drive 'file=${images_dir}/rootfs.ext2.qcow2,format=qcow2,snapshot' \\
+        extra_flags="${extra_flags} -drive 'file=${common_images_dir}/rootfs.ext2.qcow2,format=qcow2,snapshot' \\
 "
       fi
       cmd="\