gem5: possible to edit python in tree, factor arm and aarch64 build

readme: stress kills gem5, some gem5 builds much slower than others

README.adoc

@@ -2993,6 +2993,8 @@ It just runs forever, so kill it when you get tired:
 kill %1
 ....
 
+`stress -c 1 -t 1` makes gem5 irresponsive for a very long time.
+
 == QEMU
 
 Some QEMU specific features to play with and limitations to cry over.
@@ -4544,19 +4546,22 @@ Unfortunately-we didn't manage to find an ARM analogue: link:kernel_module/pmccn
 
 === gem5 Python scripts without rebuild
 
-TODO
+We have made a crazy setup that allows you to just `cd` into `gem5/gem5`, and edit Python scripts directly there.
 
-Currently, when you modify the python scripts under `gem5/gem5`, you still have to run `./build` to copy the updated scripts to the `out/<arch>/buildroot` directory, which kind of defeats the purpose of having an interpreted language as Python.
+This is not normally possible with Buildroot, since normal Buildroot packages first copy files to the output directory (`out/<arch>/buildroot/build/<pkg>`), and then build there.
 
-We would like to solve that, but are blocked on either of the following:
+So if you modified the Python scripts with this setup, you would still need to `./build` to copy the modified files over.
 
-* build in-tree and skip the Buildroot sync step. If sync is not skipped, it would copy all builds to all archs, which is huge.
-+
-But then we have to deal specially with the `m5` tool, which has to be cross compiled
-* gem5 out of tree build. Is there support? Looks like SCons does not have anything generic as usual and leaves the burden on project developers:
-** https://www.mail-archive.com/gem5-users@gem5.org/msg15421.html
-** https://stackoverflow.com/questions/3720142/how-to-force-scons-output-exe-obj-lib-dll-to-specific-build-directory
-** https://stackoverflow.com/questions/1762044/how-to-do-an-out-of-source-build-with-scons
+For gem5 specifically however, we have hacked up the build so that we `cd` into the `gem5/gem5` tree, and then do an link:https://www.mail-archive.com/gem5-users@gem5.org/msg15421.html[out of tree] build to `out/common/gem5`.
+
+Another advantage of this method is the we factor out the `arm` and `aarch64` gem5 builds which are identical and large, as well as the smaller arch generic pieces.
+
+Using Buildroot for gem5 is still convenient because we use it to:
+
+* to cross build `m5` for us
+* check timestamps and skip the gem5 build when it is not requested
+
+The out of build tree is required, because otherwise Buildroot would copy the output build of all archs to each arch directory, resulting in `arch^2` build copies, which is significant.
 
 === gem5 fs_bigLITTLE
 
@@ -5125,6 +5130,8 @@ Get results with:
 tail -n+1 ../linux-kernel-module-cheat-regression/*/gem5-bench-build-*.txt
 ....
 
+However, I have noticed that for some builds, with the exact same commands, it just take way longer sometimes, but I haven't been able to pin it down: https://github.com/cirosantilli-work/gem5-issues/issues/10
+
 === Benchmark machines
 
 ==== P51

bench-all

@@ -95,7 +95,7 @@ if "$bench_gem5_build"; then
     cd "${root_dir}/gem5/gem5"
     git clean -xdf
     cd "${root_dir}/gem5"
-    results_file="${gem5_out_dir}/bench-build.txt"
+    results_file="${gem5_run_out_dir}/bench-build.txt"
     rm "$results_file"
     common_bench_cmd "timeout 900 ./build -a '$arch'" "$results_file"
     cp "$results_file" "${new_dir}/gem5-bench-build-${arch}.txt"

common

@@ -33,12 +33,12 @@ set_common_vars() {
   build_dir="${buildroot_out_dir}/build"
   images_dir="${buildroot_out_dir}/images"
   host_dir="${buildroot_out_dir}/host"
-  gem5_out_dir="${out_arch_dir}/gem5"
-  m5out_dir="${gem5_out_dir}/m5out"
   qemu_out_dir="${out_arch_dir}/qemu"
+  gem5_run_out_dir="${out_arch_dir}/gem5"
+  m5out_dir="${gem5_run_out_dir}/m5out"
   if "$gem5"; then
-    common_out_run_dir="$gem5_out_dir"
-    common_trace_txt_file="${common_out_run_dir}/m5out/trace.txt"
+    common_out_run_dir="$gem5_run_out_dir"
+    common_trace_txt_file="${m5out_dir}/trace.txt"
   else
     common_out_run_dir="$qemu_out_dir"
     common_trace_txt_file="${common_out_run_dir}/trace.txt"
@@ -51,6 +51,7 @@ data_dir="${root_dir}/data"
 p9_dir="${data_dir}/9p"
 readfile_file="${data_dir}/readfile"
 common_dir="${out_dir}/common"
+gem5_out_dir="${common_dir}/gem5"
 f="${data_dir}/cli"
 if [ -f "$f" ]; then
   . "$f"

fs-biglittle

@@ -1,11 +1,13 @@
 #!/usr/bin/env bash
-M5_PATH="$(pwd)/out/aarch64/buildroot/build/gem5-1.0/system" \
-  ./out/aarch64/buildroot/build/gem5-1.0/gem5/build/ARM/gem5.opt \
-  ./out/aarch64/buildroot/build/gem5-1.0/gem5/configs/example/arm/fs_bigLITTLE.py \
+gem5_out_dir="$(pwd)/out/common/gem5"
+buildroot_out_dir="$(pwd)/out/aarch64/buildroot"
+M5_PATH="${gem5_out_dir}/system" \
+  "${gem5_out_dir}/build/ARM/gem5.opt" \
+  ./gem5/gem5/configs/example/arm/fs_bigLITTLE.py \
   --big-cpus=2 \
   --caches \
-  --disk="$(pwd)/out/aarch64/buildroot/images/rootfs.ext2" \
-  --dtb "$(pwd)/out/aarch64/buildroot/build/gem5-1.0/gem5/system/arm/dt/armv8_gem5_v1_big_little_2_2.dtb" \
-  --kernel="$(pwd)/out/aarch64/buildroot/build/linux-custom/vmlinux" \
+  --disk="${buildroot_out_dir}/images/rootfs.ext2" \
+  --dtb "${gem5_out_dir}/system/arm/dt/armv8_gem5_v1_big_little_2_2.dtb" \
+  --kernel="${buildroot_out_dir}/build/linux-custom/vmlinux" \
   --little-cpus=2 \
 ;

gem5-bench-cache

@@ -19,7 +19,7 @@ set_common_vars "$arch" true
 cmd="./run -a $arch -g"
 cache_small='--caches --l2cache --l1d_size=1024   --l1i_size=1024   --l2_size=1024   --l3_size=1024  '
 cache_large='--caches --l2cache --l1d_size=1024kB --l1i_size=1024kB --l2_size=1024kB --l3_size=1024kB'
-results_file="${gem5_out_dir}/bench-cache.txt"
+results_file="${gem5_run_out_dir}/bench-cache.txt"
 
 bench() (
   common_bench_cmd "$1" "$results_file"

gem5/build

@@ -3,7 +3,8 @@ set -eu
 arch=x86_64
 cross_compile=
 j=
-while getopts a:c:j: OPT; do
+outdir=./
+while getopts a:c:j:o: OPT; do
   case "$OPT" in
     a)
       arch="$OPTARG"
@@ -14,20 +15,22 @@ while getopts a:c:j: OPT; do
     j)
       j="$OPTARG"
       ;;
+    o)
+      outdir="$OPTARG"
+      ;;
   esac
 done
 shift "$(($OPTIND - 1))"
 if [ -z "$j" ]; then
   j="$(nproc)"
 fi
-top="$(cd $(dirname "$0") && pwd)"
-system_dir="${top}/system"
+system_dir="${outdir}/system"
 binaries_dir="${system_dir}/binaries"
 disks_dir="${system_dir}/disks"
 mkdir -p "$binaries_dir" "$disks_dir"
-cd "${top}/gem5"
+cd "gem5"
 if [ "$arch" = x86_64 ]; then
-  scons -j "$j" --ignore-style build/X86/gem5.opt
+  scons -j "$j" --ignore-style "${outdir}/build/X86/gem5.opt"
   f="${disks_dir}/linux-bigswap2.img"
   dd if=/dev/zero of="$f" bs=1024 count=65536
   mkswap "$f"
@@ -35,8 +38,10 @@ if [ "$arch" = x86_64 ]; then
   # I'm not even joking. No one has ever built x86 gem5 without the magic dist dir present.
   touch "${binaries_dir}/x86_64-vmlinux-2.6.22.9"
 elif [ "$arch" = arm ] || [ "$arch" = aarch64 ]; then
-  scons -j "$j" --ignore-style build/ARM/gem5.opt
+  scons -j "$j" --ignore-style "${outdir}/build/ARM/gem5.opt"
   make -C ./system/arm/dt/
+  mkdir -p "${outdir}/system/arm/dt"
+  cp ./system/arm/dt/*.dtb "${outdir}/system/arm/dt"
   # TODO use the buildroot cross compiler here, and remove the dependencies from configure.
   make -C ./system/arm/simple_bootloader/ $cross_compile
   cp ./system/arm/simple_bootloader/boot_emm.arm "$binaries_dir"

gem5/external.mk

@@ -16,7 +16,10 @@ endif
 
 define GEM5_BUILD_CMDS
 	# Cannot pass "-c '$(TARGET_CROSS)'" here because the ARM build uses aarch64 for the bootloader...
-	cd '$(@D)' && ./build -a '$(ARCH)' -j '$(BR2_JLEVEL)'
+	cd '$(GEM5_SITE)' && ./build -a '$(ARCH)' -j '$(BR2_JLEVEL)' -o '$(GEM5_SITE)/../out/common/gem5'
+	# This would build inside the buildroot directory as a more normal package.
+	#cd '$(@D)' && ./build -a '$(ARCH)' -j '$(BR2_JLEVEL)'
+
 	# TODO cannot use TARGET_CONFIGURE_OPTS here because it overrides the CFLAGS on m5,
 	# which have an include. We should patch gem5 to add a += instead of = there.
 	cd '$(@D)/gem5/util/m5' && $(MAKE) -f 'Makefile.$(ARCH_MAKE)' CC='$(TARGET_CC)' LD='$(TARGET_LD)'

run

@@ -150,7 +150,8 @@ fi
 
 if "$gem5"; then
   gem5_build_dir="${buildroot_out_dir}/build/gem5-1.0"
-  gem5_src_dir="${gem5_build_dir}/gem5"
+  gem5_src_dir="${root_dir}/gem5/gem5"
+  gem5_system_dir="${gem5_out_dir}/system"
   memory="${memory}B"
   if [ "$arch" = x86_64 ]; then
     gem5_arch=X86
@@ -162,9 +163,9 @@ if "$gem5"; then
 "
   fi
   gem5_common="\
-M5_PATH='${gem5_build_dir}/system' \\
+M5_PATH='${gem5_system_dir}' \\
 ${debug_vm} \
-'${gem5_src_dir}/build/${gem5_arch}/gem5.opt' \\
+'${gem5_out_dir}/build/${gem5_arch}/gem5.opt' \\
 --debug-file=trace.txt \\
 ${gem5opts} \
 -d '${m5out_dir}' \\
@@ -189,7 +190,7 @@ ${extra_flags} \
     # Anything smaller than physical blows up as expected, but why can't it auto-detect the right value?
     cmd="${gem5_common} \
 --command-line='earlyprintk=pl011,0x1c090000 console=ttyAMA0 lpj=19988480 rw loglevel=8 mem=${memory} root=/dev/sda ${extra_append}' \\
---dtb-file='${gem5_src_dir}/system/arm/dt/$([ "$arch" = arm ] && echo "armv7_gem5_v1_${cpus}cpu" || echo "armv8_gem5_v1_${cpus}cpu").dtb' \\
+--dtb-file='${gem5_system_dir}/arm/dt/$([ "$arch" = arm ] && echo "armv7_gem5_v1_${cpus}cpu" || echo "armv8_gem5_v1_${cpus}cpu").dtb' \\
 --machine-type=VExpress_GEM5_V1 \\
 ${extra_flags} \
 "
@@ -294,7 +295,7 @@ ${extra_flags} \
 fi
 if "$tmux"; then
   if "$gem5"; then
-    eval "./tmu 'sleep 2;./gem5-shell'"
+    eval "./tmu 'sleep 2;./gem5-shell ${tmux_args};'"
   elif  "$debug"; then
     eval "./tmu ./rungdb -a '${arch}' ${tmux_args}"
   fi