From ad3c48dfeee3a2866be47d4ecd7c256beb447419 Mon Sep 17 00:00:00 2001 From: Ciro Santilli Date: Sat, 3 Mar 2018 16:36:24 +0000 Subject: [PATCH] Rename parsec to parsec_benchmark --- .gitignore | 1 + README.adoc | 163 ++++++++++++++++++++++++++++-- build | 117 +++++++++++---------- buildroot_config_fragment | 5 +- buildroot_config_fragment_parsec | 8 ++ buildroot_override | 2 +- parsec-benchmark/Config.in | 23 ++++- parsec-benchmark/external.desc | 2 +- parsec-benchmark/external.mk | 55 +++++++--- parsec-benchmark/parsec-benchmark | 2 +- 10 files changed, 295 insertions(+), 83 deletions(-) create mode 100644 buildroot_config_fragment_parsec diff --git a/.gitignore b/.gitignore index 75aa743..0de9d0e 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ *~ .tmp_versions /rootfs_overlay/etc/init.d/S99 +/rootfs_overlay/ignore.sh Module.symvers README.html modules.order diff --git a/README.adoc b/README.adoc index 1db8582..df4f9d1 100644 --- a/README.adoc +++ b/README.adoc @@ -87,7 +87,7 @@ For example, you you modify the kernel modules, you must rebuild with: which is just an alias for: .... -./build -t kernel_module-reconfigure +./build -- kernel_module-reconfigure .... where `kernel_module` is the name of out Buildroot package that contains the kernel modules. @@ -95,7 +95,7 @@ where `kernel_module` is the name of out Buildroot package that contains the ker Other important targets are: .... -./build -t linux-reconfigure -t host-qemu-reconfigure +./build -- linux-reconfigure host-qemu-reconfigure .... which are aliased respectively to: @@ -249,13 +249,23 @@ Instead, you can either run them from a minimal init: ./run -e 'init=/eval.sh - lkmc_eval="insmod /hello.ko;/poweroff.out"' -n .... +or if the script is large, add it to a gitignored file that will go into the guest: + +.... +echo ' +insmod /hello.ko +/poweroff.out +' > rootfs_overlay/ignore.sh +./run -e 'init=/ignore.sh' -n +.... + or run them at the end of the BusyBox init, which does things like setting up networking: .... ./run -e '- lkmc_eval="insmod /hello.ko;wget -S google.com;poweroff.out;"' .... -or add them to a new `init.d` entry: +or add them to a new `init.d` entry to run at the end o the BusyBox init: .... cp rootfs_overlay/etc/init.d/S98 rootfs_overlay/etc/init.d/S99 @@ -977,7 +987,7 @@ To disable networking, use: To restore it, run: .... -./build -t initscripts-reconfigure +./build -- initscripts-reconfigure .... === The init environment @@ -1047,7 +1057,7 @@ Only tested successfully in `x86_64`. Build: .... -./build -x +./build -i buildroot_config_fragment_x11 ./run .... @@ -1257,7 +1267,7 @@ One obvious use case is having an encrypted root filesystem: you keep the initrd I think GRUB then knows read common disk formats, and then loads that initrd to memory with a `/boot/grub/grub.cfg` directive of type: - initrd /initrd.img-4.4.0-108-generic + initrd /initrd.img-4.4.0-108-generic Related: https://stackoverflow.com/questions/6405083/initrd-and-booting-the-linux-kernel @@ -1830,6 +1840,147 @@ External open source benchmarks. We will try to create Buildroot packages for th * http://parsec.cs.princeton.edu/ Mentioned on docs: http://gem5.org/PARSEC_benchmarks * http://www.m5sim.org/Splash_benchmarks +===== PARSEC benchmark + +We have ported the PARSEC benchmark http://parsec.cs.princeton.edu for cross compilation at: https://github.com/cirosantilli/parsec-benchmark + +This repo makes it trivial to get started with it: + +.... +./build -a arm -g -i buildroot_config_fragment_parsec +./run -a arm -g +.... + +As mentioned at link:https://github.com/cirosantilli/parsec-benchmark[], only SPLASH2 was currently ported. + +Keep in mind that activating PARSEC makes images huge at a few Gigs, specially so since we make multiple images for each configuration for different purposes, e.g. `.ext2`, `.cpio`, etc. This is why we don't build PARSEC by default. + +Once inside the guest, we could in theory launch PARSEC exactly as you would launch it on the host: + +.... +cd /parsec/ +bash +. env.sh +parsecmgmt -a run -p splash2x.fmm -i test +.... + +TODO: `splash2x.barnes` segfaults on `arsecmgmt -a run -p splash2x.fmm -i simsmall` inside QEMU. Why? Other benchmarks ran fine. + +.... +[PARSEC] [---------- Beginning of output ----------] +Generating input file input_1... +Running /parsec/ext/splash2x/apps/barnes/inst/arm-linux.gcc/bin/barnes 1 < input_1: +reading input file : + +Segmentation fault +.... + +However, while this is fine inside QEMU, it is not practical in gem5, since the `parsecmgmt` Bash scripts just takes too long to run in that case! + +So instead, you must find out the raw executable command, and run it manually yourself. + +This command can be found from the `Running` line that `parsecmgmt` outputs when running the programs. + +"Luckily", we run the run scripts while creating the image to extract the inputs, so you can just do a find in your shell history to find the run command and find a line of type: + +.... +Running /parsec/ext/splash2x/apps/fmm/inst/arm-linux.gcc/bin/fmm 1 < input_1: +.... + +which teaches you that you can run `fmm` as: + +.... +cd /parsec/ext/splash2x/apps/fmm/run +/parsec/ext/splash2x/apps/fmm/inst/arm-linux.gcc/bin/fmm 1 < input_1 +.... + +And so inside of `gem5`, you likely want to do: + +.... +cd /parsec/ext/splash2x/apps/fmm/run +m5 checkpoint +m5 resetstats && /parsec/ext/splash2x/apps/fmm/inst/arm-linux.gcc/bin/fmm 1 < input_1 && m5 dumpstats +.... + +You will always want to `cd` into the `run` directory first, which is where the input is located. + +====== PARSEC change the input size + +One limitation is that only one input size is available on the guest for a given build. + +To change that, edit link:buildroot_config_fragment_parsec[] to contain for example: + +.... +BR2_PACKAGE_PARSEC_BENCHMARK_INPUT_SIZE=simsmall +.... + +and then rebuild with: + +.... +./build -a arm -g -i buildroot_config_fragment_parsec parsec-benchmark-reconfigure +.... + +This limitation exists because `parsecmgmt` generates the input files just before running, but we can't run on gem5 as it is too slow! + +One option would be to do that inside the guest with QEMU, but this would required a full rebuild due to <>. + +Also, we can't generate all input sizes at once, because many of them have the same name and would overwrite one another... Parsec clearly needs a redesign for embedded, maybe we will patch it later. + +====== PARSEC uninstall + +If you want to remove PARSEC later, Buildroot doesn't provide an automated package removal mechanism as documented at: link:https://github.com/buildroot/buildroot/blob/2017.08/docs/manual/rebuilding-packages.txt#L90[], but the following procedure should be satisfactory: + +.... +rm -rf \ + ./buildroot/dl/parsec-* \ + ./buildroot/output.arm-gem5~/build/parsec-* \ + ./buildroot/output.arm-gem5~/images/rootfs.* \ + ./buildroot/output.arm-gem5~/target/parsec-* \ +; +./build -a arm -g +.... + +====== PARSEC benchmark hacking + +If you end up going inside link:parsec/parsec[] to hack up the benchmark (you will!), these tips will be helpful. + +Buildroot was not designed to deal with large images, and currently cross rebuilds are a bit slow, due to some image generation and validation steps. + +A few workarounds are: + +* develop in host first as much as you can. Our PARSEC fork supports it. ++ +If you do this, don't forget to do a: ++ +.... +cd parsec-benchmark/parsec-benchmark +git clean -xdf . +.... +before going for the cross compile build. ++ +* patch Buildroot to work well, and keep cross compiling all the way. This should be totally viable, and we should do it. ++ +Don't forget to explicitly rebuild PARSEC with: +.... +./build -a arm -g -i buildroot_config_fragment_parsec parsec-benchmark-reconfigure +.... ++ +You may also want to test if your patches are still functionally correct inside of QEMU first, which is a faster emulator. ++ +* sell your soul, and compile natively inside the guest. We won't do this, not only because it is evil, but also because Buildroot explicitly does not support it: https://buildroot.org/downloads/manual/manual.html#faq-no-compiler-on-target ARM employees have been known to do this: https://github.com/arm-university/arm-gem5-rsk/blob/aa3b51b175a0f3b6e75c9c856092ae0c8f2a7cdc/parsec_patches/qemu-patch.diff + +TODO Buildroot is slow because of the `pkg-generic` `GLOBAL_INSTRUMENTATION_HOOKS` sanitation which go over the entire tree doing complex operations... I no like, in particular `check_bin_arch` and `check_host_rpath`. + +The pause is followed by: + +.... +buildroot/output.arm~/build/parsec-benchmark-custom/.stamp_target_installed +.... + +so which shows that the whole delay is inside our install itself. + +I put an `echo f` in `check_bin_arch`, and it just loops forever, does not stop on a particular package. + === gem5 kernel command line parameters Analogous <>: diff --git a/build b/build index 8dd2797..d22092e 100755 --- a/build +++ b/build @@ -1,51 +1,53 @@ #!/usr/bin/env bash -set -e +set -eu arch=x86_64 -extra_targets='' +configure=true +config_fragments=buildroot_config_fragment +extra_make_args='' gem5=false j="$(($(nproc) - 2))" post_script_args='' qemu_sdl='--enable-sdl --with-sdlabi=2.0' -x11=false v=0 -while getopts 'a:gj:lp:qSt:v' OPT; do +while getopts 'a:Cgj:i:klp:qS:v' OPT; do case "$OPT" in a) arch="$OPTARG" ;; + C) + configure=false + ;; g) gem5=true ;; j) j="$OPTARG" ;; - l) - extra_targets="$extra_args kernel_module-reconfigure" + i) + config_fragments="$config_fragments $OPTARG" + ;; + k) + extra_make_args="$extra_make_args kernel_module-reconfigure" ;; l) - extra_targets="$extra_args linux-reconfigure" + extra_make_args="$extra_make_args linux-reconfigure" ;; p) post_script_args="$OPTARG" ;; q) - extra_targets="$extra_args host-qemu-reconfigure" + extra_make_args="$extra_make_args host-qemu-reconfigure" ;; S) qemu_sdl='' ;; - t) - extra_targets="$extra_args $OPTARG" - ;; - x) - x11=true - ;; v) v=1 ;; esac done shift $(($OPTIND - 1)) +extra_make_args="$extra_make_args $@" case "$arch" in x86_64) defconfig=qemu_x86_64_defconfig @@ -65,62 +67,71 @@ if "$gem5"; then arch_dir="${arch}-gem5" # Networking was not working, so disable it to speed things up. post_script_args="$post_script_args -n" +else + config_fragments="$config_fragments buildroot_config_fragment_qemu" +fi +root_dir="$(pwd)" +buildroot_dir="${root_dir}/buildroot" +out_dir="${buildroot_dir}/output.${arch_dir}~" +config_file="${out_dir}/.config" + +if "$configure"; then + cd "${buildroot_dir}" + for p in $(find "${root_dir}/buildroot_patches/" -maxdepth 1 -name '*.patch' -print); do + patch -N -r - -p 1 <"$p" || : + done + make O="$out_dir" BR2_EXTERNAL="${root_dir}/kernel_module:${root_dir}/gem5:${root_dir}/parsec-benchmark" "$defconfig" + # TODO Can't get rid of these for now. + # http://stackoverflow.com/questions/44078245/is-it-possible-to-use-config-fragments-with-buildroots-config + for config_fragment in $config_fragments; do + cat "../$config_fragment" >> "${config_file}" + done + printf " +BR2_JLEVEL=$j +BR2_ROOTFS_POST_SCRIPT_ARGS=\"$post_script_args\" +" >> "${config_file}" + if "$gem5"; then + printf "\ +BR2_LINUX_KERNEL_CUSTOM_CONFIG_FILE=\"../kernel_config_${arch_dir}\" +BR2_PACKAGE_GEM5=y +" >> "${config_file}" + else + printf "\ +BR2_LINUX_KERNEL_CONFIG_FRAGMENT_FILES=\"../kernel_config_fragment\" +" >> "${config_file}" + fi + if [ "$arch" = 'mips64' ]; then + # Workaround for: + # http://lists.busybox.net/pipermail/buildroot/2017-August/201053.html + sed -Ei 's/^BR2_PACKAGE_LINUX_TOOLS_GPIO/BR2_PACKAGE_LINUX_TOOLS_GPIO=n/' "${config_file}" + fi + make O="$out_dir" olddefconfig fi -rootdir="$(pwd)" -cd kernel_module +cd "${root_dir}/kernel_module" ./make-host.sh -j "$j" clean -cd "$rootdir/buildroot" -for p in $(find "${rootdir}/buildroot_patches/" -maxdepth 1 -name '*.patch' -print); do - patch -N -r - -p 1 <"$p" || : -done -outdir="output.${arch_dir}~" -make O="$outdir" BR2_EXTERNAL="${rootdir}/kernel_module:${rootdir}/gem5:${rootdir}/parsec" "$defconfig" -# TODO Can't get rid of this for now. -# http://stackoverflow.com/questions/44078245/is-it-possible-to-use-config-fragments-with-buildroots-config -cat ../buildroot_config_fragment >> "${outdir}/.config" -if "$gem5"; then - echo "\ -BR2_LINUX_KERNEL_CUSTOM_CONFIG_FILE=\"../kernel_config_${arch_dir}\" -" >> "${outdir}/.config" -else - echo "\ -BR2_LINUX_KERNEL_CONFIG_FRAGMENT_FILES=\"../kernel_config_fragment\" -" >> "${outdir}/.config" -fi -if "$x11"; then - cat ../buildroot_config_fragment_x11 >> "${outdir}/.config" -fi -if ! "$gem5"; then - cat ../buildroot_config_fragment_qemu >> "${outdir}/.config" -fi -if [ "$arch" = 'mips64' ]; then - # Workaround for: - # http://lists.busybox.net/pipermail/buildroot/2017-August/201053.html - sed -Ei 's/^BR2_PACKAGE_LINUX_TOOLS_GPIO/BR2_PACKAGE_LINUX_TOOLS_GPIO=n/' "${outdir}/.config" -fi -make O="$outdir" olddefconfig +cd "${buildroot_dir}" # HOST_QEMU_OPTS is a hack that happens to work because the QEMU package luckly uses += at all times. # It shouldn't be necessary in the first place: https://bugs.busybox.net/show_bug.cgi?id=9936 # # Even if were an autotools package, there is no general way currently to pass extra configs to it: # https://stackoverflow.com/questions/44341188/how-to-pass-extra-custom-configure-autotools-options-to-a-buildroot-package/44341225#44341225 +# +# BR2_ options may be given on the command line here, and they do have direct "define" effects. +# But this is generally bad, as it skips the Kconfig mechanism, e.g. it does not set defaults properly. cmd="time \ env \ -u LD_LIBRARY_PATH \ make \ - O='$outdir' \ - BR2_JLEVEL='$j' \ - BR2_PACKAGE_GEM5='$("$gem5" && echo y || echo n)' \ - BR2_ROOTFS_POST_SCRIPT_ARGS='$post_script_args' \ + O='$out_dir' \ HOST_QEMU_OPTS='--enable-debug --extra-cflags=-DDEBUG_PL061=1 --enable-trace-backends=simple $qemu_sdl' \ V='$v' \ - $extra_targets \ + $extra_make_args \ all \ " -echo "$cmd" | tee "${rootdir}/build.log" +echo "$cmd" | tee "${root_dir}/build.log" eval "$cmd" -cd .. if "$gem5"; then + cd "${root_dir}" ./build-gem5 -a "$arch" fi diff --git a/buildroot_config_fragment b/buildroot_config_fragment index efe0020..67009ba 100644 --- a/buildroot_config_fragment +++ b/buildroot_config_fragment @@ -1,9 +1,5 @@ # Custom packages BR2_PACKAGE_KERNEL_MODULE=y -BR2_PACKAGE_PARSEC=y -BR2_PACKAGE_PARSEC_BUILD_LIST="splash2" -# Required for parsec. -BR2_TARGET_ROOTFS_EXT2_SIZE="1G" BR2_GCC_ENABLE_GRAPHITE=y BR2_GCC_ENABLE_LTO=y @@ -11,6 +7,7 @@ BR2_GCC_ENABLE_OPENMP=y BR2_GLOBAL_PATCH_DIR="../global_patch_dir" BR2_PACKAGE_BUSYBOX_CONFIG_FRAGMENT_FILES="../busybox_config_fragment" BR2_PACKAGE_DHRYSTONE=y +BR2_PACKAGE_FILE=y BR2_PACKAGE_OVERRIDE_FILE="../buildroot_override" # For qemu-ga on guest. TODO: do something with it, and document it. BR2_PACKAGE_QEMU=y diff --git a/buildroot_config_fragment_parsec b/buildroot_config_fragment_parsec new file mode 100644 index 0000000..896066e --- /dev/null +++ b/buildroot_config_fragment_parsec @@ -0,0 +1,8 @@ +BR2_PACKAGE_PARSEC_BENCHMARK=y +#BR2_PACKAGE_PARSEC_BENCHMARK_BUILD_LIST="splash2x" +#BR2_PACKAGE_PARSEC_BENCHMARK_INPUT_SIZE="simsmall" +# Because PARSEC + its data are huge. TODO: can't we automate calculating the size? +# Problems will arise if someone tries to use two such benchmarks. +# Cannot be selected automatically from Kconfig: +# https://stackoverflow.com/questions/40309054/how-to-select-the-value-of-a-string-option-from-another-option-in-kbuild-kconfig/49096538#49096538 +BR2_TARGET_ROOTFS_EXT2_SIZE="128M" diff --git a/buildroot_override b/buildroot_override index 2d6e4e0..2a8fdaf 100644 --- a/buildroot_override +++ b/buildroot_override @@ -1,4 +1,4 @@ HOST_QEMU_OVERRIDE_SRCDIR = ../qemu LINUX_OVERRIDE_SRCDIR = ../linux -PARSEC_OVERRIDE_SRCDIR = ../parsec-benchmark/parsec-benchmark +PARSEC_BENCHMARK_OVERRIDE_SRCDIR = ../parsec-benchmark/parsec-benchmark QEMU_OVERRIDE_SRCDIR = ../qemu diff --git a/parsec-benchmark/Config.in b/parsec-benchmark/Config.in index 6e47215..139c4c2 100644 --- a/parsec-benchmark/Config.in +++ b/parsec-benchmark/Config.in @@ -1,16 +1,29 @@ -config BR2_PACKAGE_PARSEC - bool "PARSEC" +config BR2_PACKAGE_PARSEC_BENCHMARK + bool "PARSEC_BENCHMARK" +# Parsec shell scripts use a hardcoded /bin/bash +# One option would be to try and use /bin/sh. +# But symlinking fails because of BusyBox' symlink mechanism. +# The other option would be to patch Parsec to use /bin/sh and be POSIX compliant. +# But let's take the path of smallest resistance for now. + select BR2_PACKAGE_BUSYBOX_SHOW_OTHERS + select BR2_PACKAGE_BASH help Parsec system benchmark. http://parsec.cs.princeton.edu/ -if BR2_PACKAGE_PARSEC +if BR2_PACKAGE_PARSEC_BENCHMARK -config BR2_PACKAGE_PARSEC_BUILD_LIST +config BR2_PACKAGE_PARSEC_BENCHMARK_BUILD_LIST string "build_list" - default all + default splash2x help Space separated list of parsec packages to build. +config BR2_PACKAGE_PARSEC_BENCHMARK_INPUT_SIZE + string "input_size" + default test + help + Which input size to generate on the host for the guest. + endif diff --git a/parsec-benchmark/external.desc b/parsec-benchmark/external.desc index 38ffebd..248d333 100644 --- a/parsec-benchmark/external.desc +++ b/parsec-benchmark/external.desc @@ -1 +1 @@ -name: PARSEC +name: PARSEC_BENCHMARK diff --git a/parsec-benchmark/external.mk b/parsec-benchmark/external.mk index b150747..6a153b8 100644 --- a/parsec-benchmark/external.mk +++ b/parsec-benchmark/external.mk @@ -1,23 +1,54 @@ ################################################################################ # -# PARSEC +# PARSEC_BENCHMARK # ################################################################################ -PARSEC_VERSION = master -PARSEC_SITE = git@github.com:cirosantilli/parsec-benchmark.git -PARSEC_SITE_METHOD = git +PARSEC_BENCHMARK_VERSION = master +PARSEC_BENCHMARK_SITE = git@github.com:cirosantilli/parsec-benchmark.git +PARSEC_BENCHMARK_SITE_METHOD = git -define PARSEC_BUILD_CMDS - cd $(@D) && . env.sh && for pkg in $(BR2_PACKAGE_PARSEC_BUILD_LIST); do parsecmgmt -a build -p $$pkg; done - #cd '$(@D)/gem5/util/m5' && $(MAKE) -f 'Makefile.$(ARCH_MAKE)' CC='$(TARGET_CC)' LD='$(TARGET_LD)' +define PARSEC_BENCHMARK_BUILD_CMDS + # TODO make this nicer, only untar when extract step is done. + # EXTRACT_CMDS and EXTRA_DOWNLOADS would be good candidates, + # but they don't run with OVERRIDE_SRCDIR. + '$(PARSEC_BENCHMARK_PKGDIR)/parsec-benchmark/get-inputs' $(if $(filter $(V),1),-v,) '$(DL_DIR)' '$(@D)/' + # We run the benchmarks with the wrong arch here to generate the inputs on the host. + # This is because on gem5 this takes too long to do. + cd $(@D) && . env.sh && for pkg in $(BR2_PACKAGE_PARSEC_BENCHMARK_BUILD_LIST); do \ + export HOSTCC='$(HOSTCC)'; \ + export M4='$(HOST_DIR)/usr/bin/m4'; \ + export MAKE='$(MAKE)'; \ + export OSTYPE=linux; \ + export TARGET_CROSS='$(TARGET_CROSS)'; \ + export HOSTTYPE='$(BR2_ARCH)'; \ + parsecmgmt -a build -p $$pkg; \ + parsecmgmt -a run -p $$pkg -i $(BR2_PACKAGE_PARSEC_BENCHMARK_INPUT_SIZE); \ + done endef -define PARSEC_INSTALL_TARGET_CMDS - # This is a bit coarse and makes the image huge with useless source code, - # and input files, but I don't feel like creating per-package installs. - # And it doesn't matter much for simulators anyways. - rsync -av '$(@D)/' '$(TARGET_DIR)/parsec' +define PARSEC_BENCHMARK_INSTALL_TARGET_CMDS + # This is a bit coarse and makes the image larger with useless source code. + # + # But according to du, the source accounts for only 1/5 of the total size, + # so benchmarks dominate, and it doesn't matter much. + # + # Also it is not so critical for simulators anyways unlike real embedded systems. + # + # One possibility to make this better may be to install only the 'inst/' and 'input/' + # folders for each package + toplevel '/bin/' and '/config/', but of course we won't + # know if this works until time consuming testing is done :-) + mkdir -p '$(TARGET_DIR)/parsec/' + + rsync -am $(if $(filter $(V),1),-v,) --include '*/' \ + --include '/bin/***' \ + --include '/config/***' \ + --include '/env.sh' \ + --include 'inst/***' \ + --include 'run/***' \ + --exclude '*' '$(@D)/' '$(TARGET_DIR)/parsec/' \ + ; + # rsync finished. endef $(eval $(generic-package)) diff --git a/parsec-benchmark/parsec-benchmark b/parsec-benchmark/parsec-benchmark index 7ce7fde..b7d37ec 160000 --- a/parsec-benchmark/parsec-benchmark +++ b/parsec-benchmark/parsec-benchmark @@ -1 +1 @@ -Subproject commit 7ce7fde92365248beafdbe6b94468a04a17b9c2d +Subproject commit b7d37ec654e652be6eb75f44dddd18493b1212f2