From b562d8dfe7d350345e0e3c42a4b0b8025675e485 Mon Sep 17 00:00:00 2001 From: Ciro Santilli Date: Fri, 8 Jun 2018 17:50:26 +0100 Subject: [PATCH] gem5: support variants --- README.adoc | 50 +++++++++++++++++++++++++++++++++++++++++------- bench-all | 2 +- build | 19 ++++++++++++++++-- common | 36 ++++++++++++++++++++++------------ fs-biglittle | 16 ++++++++-------- gem5-bench-cache | 4 ++-- gem5-stat | 2 +- gem5/build | 8 ++++---- gem5/external.mk | 6 ++---- run | 31 +++++++++++++++--------------- 10 files changed, 117 insertions(+), 57 deletions(-) diff --git a/README.adoc b/README.adoc index d19622a..be20d06 100644 --- a/README.adoc +++ b/README.adoc @@ -6517,10 +6517,36 @@ The `-L` option should be passed to all scripts that support it, much like `-a` ./rungdb -L v4.16 ..... -It should also be easy to extend this method to all `-custom` `_OVERRIDE_SRCDIR` packages, which luckily tend to be the ones that we care the most about. - This technique is implemented semi-hackishly by moving symlinks around inside the Buildroot build dir at build time, and selecting the right build directory at runtime. +==== gem5 build variants + +Analogous to the <> but with the `-M` option instead: + +.... +./build -g +git -C gem5/gem5 checkout some-branch +./build -M some-branch -g +git -C gem5/gem5 checkout - +./run -g +./run -M some-branch -g +.... + +Since we control the gem5 build however, unlike Linux which uses Buildroot, we make it awesomer, and use `git worktree` instead of a mere `rsync` for the copy. + +It works like this: + +* when you don't pass the `-M` option, which is the same as the `-M default` variant, we use the source code from under the `gem5/gem5` submodule for the build +* otherwise, if you pass `-M some-branch`, we generate a git worktree checkout under `data/gem5/some-branch`. ++ +The initial revision for that worktree is whatever `gem5/gem5` is currently points to. ++ +However, if the worktree already exists, we leave it untouched. ++ +Therefore, you can safely go to that directory and edit the source there without fear that it will get deleted. + +All build outputs end up at: `out/common/gem5/` regardless. + ==== Generic package build variants This hack-ish technique allows us to rebuild just one package at a time: @@ -7044,12 +7070,22 @@ The action seems to be happening at: `hw/arm/virt.c`. === Directory structure * `data`: gitignored user created data. Deleting this might lead to loss of data. Of course, if something there becomes is important enough to you, git track it. +** `data/readfile`: see <> +** `data/9p`: see <<9p>> +** `data/gem5/`: see: <> * `out`: gitignored Build outputs. You won't lose data by deleting this folder since everything there can be re-generated, only time. -* `out/`: arch specific outputs -* `out//buildroot`: Buildroot output -* `out//qemu`: QEMU runtime outputs -* `out//gem5`: GEM5 runtime outputs -* `out/common`: cross arch outputs. Notably gem5, which has the same build for `arm` and `aarch64`. +** `out/`: arch specific outputs +*** `out//buildroot`: standard Buildroot output +**** `out//buildroot/build/linux-custom`: symlink to a variant, custom madness that we do on top of Buildroot: <> +**** `out//buildroot/build/linux-custom.`: what `linux-custom` points to +*** `out//qemu`: QEMU runtime outputs +** `out/common`: cross arch outputs, for when we can gain a lot of time and space by sharing things that are common across different archs. +*** `out/common/dl/`: Buildroot caches downloaded source there due to `BR2_DL_DIR` +*** `out/common/gem5/`: `arm` and `aarch64` have the same build. +**** `out/common/gem5//` +***** `out/common/gem5//build/` +***** `out/common/gem5//m5out` +***** `out/common/gem5//system` :leveloffset: +3 diff --git a/bench-all b/bench-all index a9b5429..674d963 100755 --- a/bench-all +++ b/bench-all @@ -95,7 +95,7 @@ if "$bench_gem5_build"; then cd "${root_dir}/gem5/gem5" git clean -xdf cd "${root_dir}/gem5" - results_file="${gem5_run_out_dir}/bench-build.txt" + results_file="${common_gem5_out_dir}/bench-build.txt" gem5_outdir="${out_dir}/bench_build" rm -fr "$results_file" "${gem5_outdir}" # TODO understand better: --foreground required otherwise we cannot diff --git a/build b/build index bef5ae7..3f13c51 100755 --- a/build +++ b/build @@ -20,7 +20,7 @@ post_script_args= qemu_sdl='--enable-sdl --with-sdlabi=2.0' suffix= v=0 -while getopts 'a:B:b:C:c:fGgj:hIiK:kL:lp:qSs:v' OPT; do +while getopts 'a:B:b:C:c:fGgj:hIiK:kL:lM:p:qSs:v' OPT; do case "$OPT" in a) arch="$OPTARG" @@ -81,6 +81,9 @@ BR2_TARGET_ROOTFS_INITRAMFS=n extra_make_args="${extra_make_args} linux-reconfigure \\ " ;; + M) + common_gem5_variant="$OPTARG" + ;; p) post_script_args="$OPTARG" ;; @@ -104,7 +107,7 @@ BR2_TARGET_ROOTFS_INITRAMFS=n done shift $(($OPTIND - 1)) extra_make_args="${extra_make_args} $@" -set_common_vars -L "$common_linux_variant" "$arch" "$gem5" "$suffix" +set_common_vars -L "$common_linux_variant" -M "$common_gem5_variant" "$arch" "$gem5" "$suffix" config_file="${buildroot_out_dir}/.config" case "$arch" in x86_64) @@ -170,6 +173,8 @@ BR2_ROOTFS_POST_SCRIPT_ARGS=\"${post_script_args}\" fi common_mkdir + +# Manage Linux kernel variants. if [ -h "$common_linux_custom_dir" ]; then rm "$common_linux_custom_dir" elif [ -d "$common_linux_custom_dir" ]; then @@ -178,6 +183,14 @@ elif [ -d "$common_linux_custom_dir" ]; then fi mkdir -p "$common_linux_variant_dir" ln -s "$common_linux_variant_dir" "$common_linux_custom_dir" + +# Manage gem5 variants. +if "$gem5"; then + if [ ! -e "${common_gem5_src_dir}/.git" ]; then + git -C "$common_gem5_default_src_dir" worktree add -b "${common_gem5_variant}" "${common_gem5_src_dir}" + fi +fi + cd "$buildroot_dir" # HOST_QEMU_OPTS is a hack that happens to work because the QEMU package luckly uses += at all times. # It shouldn't be necessary in the first place: https://bugs.busybox.net/show_bug.cgi?id=9936 @@ -193,6 +206,8 @@ env \\ make \\ O='${buildroot_out_dir}' \\ HOST_QEMU_OPTS='--enable-debug --enable-trace-backends=simple ${qemu_sdl}' \\ +GEM5_LKMC_SRCDIR="$common_gem5_src_dir" \\ +GEM5_LKMC_OUTDIR="$common_gem5_out_dir" \\ V='${v}' \\ ${extra_make_args} \ all \\ diff --git a/common b/common index ab65394..f6b9b3a 100644 --- a/common +++ b/common @@ -21,15 +21,18 @@ common_bench_cmd() ( ) set_common_vars() { linux_variant= + gem5_variant= OPTIND=1 - while getopts L: OPT; do + while getopts L:M: OPT; do case "$OPT" in L) linux_variant="$OPTARG" ;; + M) + gem5_variant="$OPTARG" + ;; esac done - echo $OPTIND shift "$(($OPTIND - 1))" arch="$1" gem5="${2:-false}" @@ -60,21 +63,28 @@ set_common_vars() { out_arch_dir="${out_dir}/${arch_dir}" buildroot_out_dir="${out_arch_dir}/buildroot" build_dir="${buildroot_out_dir}/build" - images_dir="${buildroot_out_dir}/images" + common_images_dir="${buildroot_out_dir}/images" host_dir="${buildroot_out_dir}/host" qemu_out_dir="${out_arch_dir}/qemu" - gem5_run_out_dir="${out_arch_dir}/gem5" - m5out_dir="${gem5_run_out_dir}/m5out" + common_linux_custom_dir="${build_dir}/linux-custom" + common_linux_variant_dir="${common_linux_custom_dir}.${linux_variant}" + common_vmlinux="${common_linux_variant_dir}/vmlinux" + if [ "$gem5_variant" = default ]; then + common_gem5_src_dir="${root_dir}/gem5/gem5" + else + common_gem5_src_dir="${common_gem5_non_default_src_root_dir}/${gem5_variant}" + fi + common_gem5_out_dir="${common_dir}/gem5/${gem5_variant}" + common_gem5_build_dir="${common_gem5_out_dir}/build" + common_gem5_system_dir="${common_gem5_out_dir}/system" + common_m5out_dir="${common_gem5_out_dir}/m5out" if "$gem5"; then - common_out_run_dir="$gem5_run_out_dir" - common_trace_txt_file="${m5out_dir}/trace.txt" + common_out_run_dir="$common_gem5_out_dir" + common_trace_txt_file="${common_m5out_dir}/trace.txt" else common_out_run_dir="$qemu_out_dir" common_trace_txt_file="${common_out_run_dir}/trace.txt" fi - common_linux_custom_dir="${build_dir}/linux-custom" - common_linux_variant_dir="${common_linux_custom_dir}.${linux_variant}" - common_vmlinux="${common_linux_variant_dir}/vmlinux" case "$arch" in arm) common_linux_image=arch/arm/boot/zImage @@ -94,7 +104,7 @@ set_common_vars() { common_mkdir() ( mkdir -p \ "$build_dir" \ - "$gem5_out_dir" \ + "$common_gem5_out_dir" \ "$qemu_out_dir" \ "$p9_dir" \ ; @@ -107,7 +117,9 @@ data_dir="${root_dir}/data" p9_dir="${data_dir}/9p" readfile_file="${data_dir}/readfile" common_dir="${out_dir}/common" -gem5_out_dir="${common_dir}/gem5" +common_gem5_default_src_dir="${root_dir}/gem5/gem5" +common_gem5_non_default_src_root_dir="${data_dir}/gem5" +common_gem5_variant=default f="${data_dir}/cli" if [ -f "$f" ]; then . "$f" diff --git a/fs-biglittle b/fs-biglittle index a0d7379..6ecbfc9 100755 --- a/fs-biglittle +++ b/fs-biglittle @@ -1,13 +1,13 @@ #!/usr/bin/env bash -gem5_out_dir="$(pwd)/out/common/gem5" -buildroot_out_dir="$(pwd)/out/aarch64/buildroot" -M5_PATH="${gem5_out_dir}/system" \ - "${gem5_out_dir}/build/ARM/gem5.opt" \ - ./gem5/gem5/configs/example/arm/fs_bigLITTLE.py \ +. common +set_common_vars -L "${common_linux_variant}" -M "${common_gem5_variant}" aarch64 true +M5_PATH="${common_gem5_system_dir}" \ + "${common_gem5_build_dir}/ARM/gem5.opt" \ + "${common_gem5_default_src_dir}/configs/example/arm/fs_bigLITTLE.py" \ --big-cpus=2 \ --caches \ - --disk="${buildroot_out_dir}/images/rootfs.ext2" \ - --dtb "${gem5_out_dir}/system/arm/dt/armv8_gem5_v1_big_little_2_2.dtb" \ - --kernel="${buildroot_out_dir}/build/linux-custom/vmlinux" \ + --disk="${common_images_dir}/rootfs.ext2" \ + --dtb "${common_gem5_system_dir}/arm/dt/armv8_gem5_v1_big_little_2_2.dtb" \ + --kernel="${common_vmlinux}" \ --little-cpus=2 \ ; diff --git a/gem5-bench-cache b/gem5-bench-cache index 962ccb6..51dcaf7 100755 --- a/gem5-bench-cache +++ b/gem5-bench-cache @@ -19,7 +19,7 @@ set_common_vars "$arch" true cmd="./run -a $arch -g" cache_small='--caches --l2cache --l1d_size=1024 --l1i_size=1024 --l2_size=1024 --l3_size=1024 ' cache_large='--caches --l2cache --l1d_size=1024kB --l1i_size=1024kB --l2_size=1024kB --l3_size=1024kB' -results_file="${gem5_run_out_dir}/bench-cache.txt" +results_file="${common_gem5_out_dir}/bench-cache.txt" bench() ( common_bench_cmd "$1" "$results_file" @@ -66,7 +66,7 @@ bench-all() ( if "$generate_checkpoints"; then # Create the checkpoints after the kernel boot. - rm -rf "${m5out_dir}"/cpt.*; + rm -rf "${common_m5out_dir}"/cpt.*; printf 'm5 exit' > "${readfile_file}" cpt_cmd="-E 'm5 checkpoint;m5 readfile > a.sh;sh a.sh'" # 1 diff --git a/gem5-stat b/gem5-stat index b1eb2fd..8b17688 100755 --- a/gem5-stat +++ b/gem5-stat @@ -22,4 +22,4 @@ else stat=system.cpu[0-9]*.numCycles fi set_common_vars "$arch" true -awk "/^$stat /{ print \$2 }" "${m5out_dir}/stats.txt" +awk "/^$stat /{ print \$2 }" "${common_m5out_dir}/stats.txt" diff --git a/gem5/build b/gem5/build index f5ee553..6b5fa78 100755 --- a/gem5/build +++ b/gem5/build @@ -1,9 +1,9 @@ #!/usr/bin/env bash -set -eu +set -eux arch=x86_64 cross_compile= j= -outdir="$(pwd)/out" +outdir="$(pwd)" while getopts a:c:j:o: OPT; do case "$OPT" in a) @@ -28,7 +28,6 @@ system_dir="${outdir}/system" binaries_dir="${system_dir}/binaries" disks_dir="${system_dir}/disks" mkdir -p "$binaries_dir" "$disks_dir" -cd "gem5" if [ "$arch" = x86_64 ]; then scons -j "$j" --ignore-style "${outdir}/build/X86/gem5.opt" f="${disks_dir}/linux-bigswap2.img" @@ -41,7 +40,8 @@ elif [ "$arch" = arm ] || [ "$arch" = aarch64 ]; then scons -j "$j" --ignore-style "${outdir}/build/ARM/gem5.opt" make -C ./system/arm/dt/ mkdir -p "${system_dir}/arm/dt" - cp ./system/arm/dt/*.dtb "${system_dir}/arm/dt" + # || true in case they are the same directory. + cp ./system/arm/dt/*.dtb "${system_dir}/arm/dt" || true # TODO use the buildroot cross compiler here, and remove the dependencies from configure. make -C ./system/arm/simple_bootloader/ $cross_compile cp ./system/arm/simple_bootloader/boot_emm.arm "$binaries_dir" diff --git a/gem5/external.mk b/gem5/external.mk index 94f002a..cdf518b 100644 --- a/gem5/external.mk +++ b/gem5/external.mk @@ -15,10 +15,8 @@ ARCH_MAKE = $(ARCH) endif define GEM5_BUILD_CMDS - # Cannot pass "-c '$(TARGET_CROSS)'" here because the ARM build uses aarch64 for the bootloader... - cd '$(GEM5_SITE)' && ./build -a '$(ARCH)' -j '$(BR2_JLEVEL)' -o '$(GEM5_SITE)/../out/common/gem5' - # This would build inside the buildroot directory as a more normal package. - #cd '$(@D)' && ./build -a '$(ARCH)' -j '$(BR2_JLEVEL)' + # TODO cannot pass "-c '$(TARGET_CROSS)'" here because the ARM build uses aarch64 for the bootloader... + cd '$(GEM5_LKMC_SRCDIR)' && '$(GEM5_SITE)/build' -a '$(ARCH)' -j '$(BR2_JLEVEL)' -o '$(GEM5_LKMC_OUTDIR)' # TODO cannot use TARGET_CONFIGURE_OPTS here because it overrides the CFLAGS on m5, # which have an include. We should patch gem5 to add a += instead of = there. diff --git a/run b/run index 26169ca..7a80345 100755 --- a/run +++ b/run @@ -34,7 +34,7 @@ trace_enabled=false # just to prevent QEMU from emitting a warning that '' is not valid. trace_type=pr_manager_run vnc= -while getopts a:c:DdE:e:F:f:G:ghIiKkL:m:PT:U:uVx OPT; do +while getopts a:c:DdE:e:F:f:G:ghIiKkL:M:m:PT:U:uVx OPT; do case "$OPT" in a) arch="$OPTARG" @@ -94,6 +94,9 @@ while getopts a:c:DdE:e:F:f:G:ghIiKkL:m:PT:U:uVx OPT; do L) common_linux_variant="$OPTARG" ;; + M) + common_gem5_variant="$OPTARG" + ;; m) memory="$OPTARG" ;; @@ -124,8 +127,7 @@ while getopts a:c:DdE:e:F:f:G:ghIiKkL:m:PT:U:uVx OPT; do done shift "$(($OPTIND - 1))" extra_flags="$extra_flags $@" -set_common_vars -L "$common_linux_variant" "$arch" "$gem5" -images_dir="${buildroot_out_dir}/images" +set_common_vars -L "$common_linux_variant" -M "$common_gem5_variant" "$arch" "$gem5" if "$debug" && "$kvm"; then echo 'error: -d and -K are incompatible' 1>&2 exit 1 @@ -156,9 +158,6 @@ if [ -n "$extra_append_after_dash" ]; then fi if "$gem5"; then - gem5_build_dir="${buildroot_out_dir}/build/gem5-1.0" - gem5_src_dir="${root_dir}/gem5/gem5" - gem5_system_dir="${gem5_out_dir}/system" memory="${memory}B" if [ "$arch" = x86_64 ]; then gem5_arch=X86 @@ -170,14 +169,14 @@ if "$gem5"; then " fi gem5_common="\ -M5_PATH='${gem5_system_dir}' \\ +M5_PATH='${common_gem5_system_dir}' \\ ${debug_vm} \ -'${gem5_out_dir}/build/${gem5_arch}/gem5.opt' \\ +'${common_gem5_build_dir}/${gem5_arch}/gem5.opt' \\ --debug-file=trace.txt \\ ${gem5opts} \ --d '${m5out_dir}' \\ -'${gem5_src_dir}/configs/example/fs.py' \\ ---disk-image='${images_dir}/rootfs.ext2' \\ +-d '${common_m5out_dir}' \\ +'${common_gem5_src_dir}/configs/example/fs.py' \\ +--disk-image='${common_images_dir}/rootfs.ext2' \\ --kernel='${common_vmlinux}' \\ --mem-size='${memory}' \\ --num-cpus='${cpus}' \\ @@ -197,7 +196,7 @@ ${extra_flags} \ # Anything smaller than physical blows up as expected, but why can't it auto-detect the right value? cmd="${gem5_common} \ --command-line='earlyprintk=pl011,0x1c090000 console=ttyAMA0 lpj=19988480 rw loglevel=8 mem=${memory} root=/dev/sda ${extra_append}' \\ ---dtb-file='${gem5_system_dir}/arm/dt/$([ "$arch" = arm ] && echo "armv7_gem5_v1_${cpus}cpu" || echo "armv8_gem5_v1_${cpus}cpu").dtb' \\ +--dtb-file='${common_gem5_system_dir}/arm/dt/$([ "$arch" = arm ] && echo "armv7_gem5_v1_${cpus}cpu" || echo "armv8_gem5_v1_${cpus}cpu").dtb' \\ --machine-type=VExpress_GEM5_V1 \\ ${extra_flags} \ " @@ -235,7 +234,7 @@ ${serial_monitor} \ -virtfs 'local,path=${buildroot_out_dir}/build,mount_tag=host_out,security_model=mapped,id=host_out' \\ ${vnc}" if "$initrd"; then - extra_flags="${extra_flags} -initrd '${images_dir}/rootfs.cpio' \\ + extra_flags="${extra_flags} -initrd '${common_images_dir}/rootfs.cpio' \\ " fi if "$ramfs"; then @@ -243,7 +242,7 @@ ${vnc}" root='root=/dev/anything' else if [ ! "$arch" = mips64 ]; then - extra_flags="${extra_flags} -drive 'file=${images_dir}/rootfs.ext2.qcow2,format=qcow2,if=virtio,snapshot' \\ + extra_flags="${extra_flags} -drive 'file=${common_images_dir}/rootfs.ext2.qcow2,format=qcow2,if=virtio,snapshot' \\ " root='root=/dev/vda' fi @@ -291,14 +290,14 @@ ${qemu_common} \ -append '${root} ${extra_append}' \\ -cpu cortex-a57 \\ -device virtio-gpu-pci \\ --kernel '${images_dir}/Image' \\ +-kernel '${common_images_dir}/Image' \\ ${extra_flags} \ " ;; mips64) if ! "$ramfs"; then root='root=/dev/hda' - extra_flags="${extra_flags} -drive 'file=${images_dir}/rootfs.ext2.qcow2,format=qcow2,snapshot' \\ + extra_flags="${extra_flags} -drive 'file=${common_images_dir}/rootfs.ext2.qcow2,format=qcow2,snapshot' \\ " fi cmd="\