From c482a682d2264844175c19d43d40d9a8ec407652 Mon Sep 17 00:00:00 2001 From: Ciro Santilli Date: Fri, 2 Mar 2018 07:34:31 +0000 Subject: [PATCH 01/17] Persistency lost on initrd --- README.adoc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.adoc b/README.adoc index f730efc..6a8236b 100644 --- a/README.adoc +++ b/README.adoc @@ -158,6 +158,8 @@ the disk image gets overwritten by a fresh filesystem and you lose all changes. Remember that if you forcibly turn QEMU off without `sync` or `poweroff` from inside the VM, e.g. by closing the QEMU window, disk changes may not be saved. +When booting from <> however without a disk, persistency is lost. + === Message control We use `printk` a lot, and it shows on the QEMU terminal by default. If that annoys you (e.g. you want to see stdout separately), do: From 6e15dcb60e8d0ecd5196ba3f28babd8795149a81 Mon Sep 17 00:00:00 2001 From: Ciro Santilli Date: Fri, 2 Mar 2018 08:26:38 +0000 Subject: [PATCH 02/17] Start sketching parsec buildroot package --- .gitmodules | 3 +++ README.adoc | 23 ++++++++++++++-------- build | 29 ++++++++++++++++------------ buildroot_config_fragment | 9 +++++++-- buildroot_override | 1 + configure | 9 +++++---- gem5/Config.in | 5 ++++- parsec-benchmark/Config.in | 16 +++++++++++++++ parsec-benchmark/external.desc | 1 + parsec-benchmark/external.mk | 23 ++++++++++++++++++++++ parsec-benchmark/parsec-benchmark | 1 + parsec => parsec-benchmark/parsec.sh | 0 12 files changed, 93 insertions(+), 27 deletions(-) create mode 100644 parsec-benchmark/Config.in create mode 100644 parsec-benchmark/external.desc create mode 100644 parsec-benchmark/external.mk create mode 160000 parsec-benchmark/parsec-benchmark rename parsec => parsec-benchmark/parsec.sh (100%) diff --git a/.gitmodules b/.gitmodules index 8dd0a80..5da02ec 100644 --- a/.gitmodules +++ b/.gitmodules @@ -11,3 +11,6 @@ [submodule "gem5/gem5"] path = gem5/gem5 url = https://gem5.googlesource.com/public/gem5 +[submodule "parsec-benchmark/parsec-benchmark"] + path = parsec-benchmark/parsec-benchmark + url = https://github.com/cirosantilli/parsec-benchmark diff --git a/README.adoc b/README.adoc index 6a8236b..e0c594b 100644 --- a/README.adoc +++ b/README.adoc @@ -76,29 +76,36 @@ git ls-files | grep modulename === Rebuild -If you make changes to the kernel modules or most configurations tracked on this repository, you can just use again: +After making changes to a package, you must explicitly tell it to be rebuilt. + +For example, you you modify the kernel modules, you must rebuild with: .... -./build -./run +./build -k .... -and the modified files will be rebuilt. +which is just an alias for: -If you change any package besides `kernel_module`, you must also request those packages to be reconfigured or rebuilt with extra targets, e.g.: +.... +./build -t kernel_module-reconfigure +.... + +where `kernel_module` is the name of out Buildroot package that contains the kernel modules. + +Other important targets are: .... ./build -t linux-reconfigure -t host-qemu-reconfigure .... -Those aren't turned on by default because they take quite a few seconds. - -Linux and QEMU rebuilds are so common that we have dedicated shortcut flags for them: +which are aliased respectively to: .... ./build -l -q .... +We don't rebuild by default because, even with `make` incremental rebuilds, the timestamp check takes a few annoying seconds. + === Clean the build You did something crazy, and nothing seems to work anymore? diff --git a/build b/build index 13e9d0f..8dd2797 100755 --- a/build +++ b/build @@ -19,6 +19,9 @@ while getopts 'a:gj:lp:qSt:v' OPT; do j) j="$OPTARG" ;; + l) + extra_targets="$extra_args kernel_module-reconfigure" + ;; l) extra_targets="$extra_args linux-reconfigure" ;; @@ -64,14 +67,15 @@ if "$gem5"; then post_script_args="$post_script_args -n" fi +rootdir="$(pwd)" cd kernel_module ./make-host.sh -j "$j" clean -cd ../buildroot -for p in $(find '../buildroot_patches/' -maxdepth 1 -name '*.patch' -print); do +cd "$rootdir/buildroot" +for p in $(find "${rootdir}/buildroot_patches/" -maxdepth 1 -name '*.patch' -print); do patch -N -r - -p 1 <"$p" || : done outdir="output.${arch_dir}~" -make O="$outdir" BR2_EXTERNAL="$(pwd)/../kernel_module:$(pwd)/../gem5" "$defconfig" +make O="$outdir" BR2_EXTERNAL="${rootdir}/kernel_module:${rootdir}/gem5:${rootdir}/parsec" "$defconfig" # TODO Can't get rid of this for now. # http://stackoverflow.com/questions/44078245/is-it-possible-to-use-config-fragments-with-buildroots-config cat ../buildroot_config_fragment >> "${outdir}/.config" @@ -101,20 +105,21 @@ make O="$outdir" olddefconfig # # Even if were an autotools package, there is no general way currently to pass extra configs to it: # https://stackoverflow.com/questions/44341188/how-to-pass-extra-custom-configure-autotools-options-to-a-buildroot-package/44341225#44341225 -time \ +cmd="time \ env \ -u LD_LIBRARY_PATH \ make \ - O="$outdir" \ - BR2_JLEVEL="$j" \ - BR2_PACKAGE_GEM5="$("$gem5" && echo y || echo n)" \ - BR2_ROOTFS_POST_SCRIPT_ARGS="$post_script_args" \ - HOST_QEMU_OPTS="--enable-debug --extra-cflags='-DDEBUG_PL061=1' --enable-trace-backends=simple $qemu_sdl" \ - V="$v" \ - kernel_module-rebuild \ + O='$outdir' \ + BR2_JLEVEL='$j' \ + BR2_PACKAGE_GEM5='$("$gem5" && echo y || echo n)' \ + BR2_ROOTFS_POST_SCRIPT_ARGS='$post_script_args' \ + HOST_QEMU_OPTS='--enable-debug --extra-cflags=-DDEBUG_PL061=1 --enable-trace-backends=simple $qemu_sdl' \ + V='$v' \ $extra_targets \ all \ -; +" +echo "$cmd" | tee "${rootdir}/build.log" +eval "$cmd" cd .. if "$gem5"; then ./build-gem5 -a "$arch" diff --git a/buildroot_config_fragment b/buildroot_config_fragment index 1107574..efe0020 100644 --- a/buildroot_config_fragment +++ b/buildroot_config_fragment @@ -1,11 +1,16 @@ -BR2_ENABLE_LOCALE=y +# Custom packages +BR2_PACKAGE_KERNEL_MODULE=y +BR2_PACKAGE_PARSEC=y +BR2_PACKAGE_PARSEC_BUILD_LIST="splash2" +# Required for parsec. +BR2_TARGET_ROOTFS_EXT2_SIZE="1G" + BR2_GCC_ENABLE_GRAPHITE=y BR2_GCC_ENABLE_LTO=y BR2_GCC_ENABLE_OPENMP=y BR2_GLOBAL_PATCH_DIR="../global_patch_dir" BR2_PACKAGE_BUSYBOX_CONFIG_FRAGMENT_FILES="../busybox_config_fragment" BR2_PACKAGE_DHRYSTONE=y -BR2_PACKAGE_KERNEL_MODULE=y BR2_PACKAGE_OVERRIDE_FILE="../buildroot_override" # For qemu-ga on guest. TODO: do something with it, and document it. BR2_PACKAGE_QEMU=y diff --git a/buildroot_override b/buildroot_override index cf57077..2d6e4e0 100644 --- a/buildroot_override +++ b/buildroot_override @@ -1,3 +1,4 @@ HOST_QEMU_OVERRIDE_SRCDIR = ../qemu LINUX_OVERRIDE_SRCDIR = ../linux +PARSEC_OVERRIDE_SRCDIR = ../parsec-benchmark/parsec-benchmark QEMU_OVERRIDE_SRCDIR = ../qemu diff --git a/configure b/configure index 85a0e27..bac177d 100755 --- a/configure +++ b/configure @@ -12,19 +12,20 @@ while getopts t OPT; do done shift $(($OPTIND - 1)) +submodules='buildroot linux qemu gem5/gem5' ( set -e - # Shallow clonning saves a considerable ammount of time, specially because of the linux kernel. - # However, git submodules are buggy as usual, and this is the best way i've found to get it done: + # Shallow cloning saves a considerable amount of time, specially because of the linux kernel. + # However, git submodules are buggy as usual, and this is the best way I've found to get it done: # https://stackoverflow.com/questions/2144406/git-shallow-submodules/47374702#47374702 # In particular: # - `shallow = true` on the submodule has no effect for the non default educational branches of our submodules # - QEMU's submodules point to commits that are neither under branches nor tags, and so `--shallow-submodules` fails - git submodule update --depth 1 --jobs 4 --init + git submodule update --depth 1 --jobs 4 --init -- $submodules cd qemu git submodule update --init ) & -wait $! || git submodule update --init +wait $! || git submodule update --init -- $submodules pkgs="\ automake \ diff --git a/gem5/Config.in b/gem5/Config.in index 7cc3c3b..0508e89 100644 --- a/gem5/Config.in +++ b/gem5/Config.in @@ -1,4 +1,7 @@ config BR2_PACKAGE_GEM5 bool "gem5" help - GEM5 + gem5 system simulator. Only builds the m5 guest instrumentation + tool for now, not the simulator itself. + + http://gem5.org diff --git a/parsec-benchmark/Config.in b/parsec-benchmark/Config.in new file mode 100644 index 0000000..6e47215 --- /dev/null +++ b/parsec-benchmark/Config.in @@ -0,0 +1,16 @@ +config BR2_PACKAGE_PARSEC + bool "PARSEC" + help + Parsec system benchmark. + + http://parsec.cs.princeton.edu/ + +if BR2_PACKAGE_PARSEC + +config BR2_PACKAGE_PARSEC_BUILD_LIST + string "build_list" + default all + help + Space separated list of parsec packages to build. + +endif diff --git a/parsec-benchmark/external.desc b/parsec-benchmark/external.desc new file mode 100644 index 0000000..38ffebd --- /dev/null +++ b/parsec-benchmark/external.desc @@ -0,0 +1 @@ +name: PARSEC diff --git a/parsec-benchmark/external.mk b/parsec-benchmark/external.mk new file mode 100644 index 0000000..b150747 --- /dev/null +++ b/parsec-benchmark/external.mk @@ -0,0 +1,23 @@ +################################################################################ +# +# PARSEC +# +################################################################################ + +PARSEC_VERSION = master +PARSEC_SITE = git@github.com:cirosantilli/parsec-benchmark.git +PARSEC_SITE_METHOD = git + +define PARSEC_BUILD_CMDS + cd $(@D) && . env.sh && for pkg in $(BR2_PACKAGE_PARSEC_BUILD_LIST); do parsecmgmt -a build -p $$pkg; done + #cd '$(@D)/gem5/util/m5' && $(MAKE) -f 'Makefile.$(ARCH_MAKE)' CC='$(TARGET_CC)' LD='$(TARGET_LD)' +endef + +define PARSEC_INSTALL_TARGET_CMDS + # This is a bit coarse and makes the image huge with useless source code, + # and input files, but I don't feel like creating per-package installs. + # And it doesn't matter much for simulators anyways. + rsync -av '$(@D)/' '$(TARGET_DIR)/parsec' +endef + +$(eval $(generic-package)) diff --git a/parsec-benchmark/parsec-benchmark b/parsec-benchmark/parsec-benchmark new file mode 160000 index 0000000..7ce7fde --- /dev/null +++ b/parsec-benchmark/parsec-benchmark @@ -0,0 +1 @@ +Subproject commit 7ce7fde92365248beafdbe6b94468a04a17b9c2d diff --git a/parsec b/parsec-benchmark/parsec.sh similarity index 100% rename from parsec rename to parsec-benchmark/parsec.sh From 1b6ab614261ac4bcbb3e7b68fd3f87838dbb75c1 Mon Sep 17 00:00:00 2001 From: Ciro Santilli Date: Sat, 3 Mar 2018 16:36:00 +0000 Subject: [PATCH 03/17] Link to gem5 instrumentation on ARM --- README.adoc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.adoc b/README.adoc index e0c594b..1db8582 100644 --- a/README.adoc +++ b/README.adoc @@ -23,7 +23,7 @@ cd linux-kernel-module-cheat ./configure && ./build && ./run .... -The first build will take a while (https://stackoverflow.com/questions/10833672/buildroot-environment-with-host-toolchain[GCC], Linux kernel), e.g.: +The first build will take a while (link:https://stackoverflow.com/questions/10833672/buildroot-environment-with-host-toolchain[GCC], Linux kernel), e.g.: * 2 hours on a mid end 2012 laptop * 30 minutes on a high end 2017 desktop @@ -1608,6 +1608,8 @@ It works like this: * the first commond boots linux with the default simplified `AtomicSimpleCPU`, and generates a <> after the kernel boots and before running the benchmark * the second command restores the checkpoint with the more detailed `HPI` CPU model, and runs the benchmark. We don't boot with it because that is much slower. +ARM employees have just been modifying benchmarking code with instrumentation directly: https://github.com/arm-university/arm-gem5-rsk/blob/aa3b51b175a0f3b6e75c9c856092ae0c8f2a7cdc/parsec_patches/xcompile-patch.diff#L230 + A few imperfections of our benchmarking method are: * when we do `m5 resetstats` and `m5 exit`, there is some time passed before the `exec` system call returns and the actual benchmark starts and ends From ad3c48dfeee3a2866be47d4ecd7c256beb447419 Mon Sep 17 00:00:00 2001 From: Ciro Santilli Date: Sat, 3 Mar 2018 16:36:24 +0000 Subject: [PATCH 04/17] Rename parsec to parsec_benchmark --- .gitignore | 1 + README.adoc | 163 ++++++++++++++++++++++++++++-- build | 117 +++++++++++---------- buildroot_config_fragment | 5 +- buildroot_config_fragment_parsec | 8 ++ buildroot_override | 2 +- parsec-benchmark/Config.in | 23 ++++- parsec-benchmark/external.desc | 2 +- parsec-benchmark/external.mk | 55 +++++++--- parsec-benchmark/parsec-benchmark | 2 +- 10 files changed, 295 insertions(+), 83 deletions(-) create mode 100644 buildroot_config_fragment_parsec diff --git a/.gitignore b/.gitignore index 75aa743..0de9d0e 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ *~ .tmp_versions /rootfs_overlay/etc/init.d/S99 +/rootfs_overlay/ignore.sh Module.symvers README.html modules.order diff --git a/README.adoc b/README.adoc index 1db8582..df4f9d1 100644 --- a/README.adoc +++ b/README.adoc @@ -87,7 +87,7 @@ For example, you you modify the kernel modules, you must rebuild with: which is just an alias for: .... -./build -t kernel_module-reconfigure +./build -- kernel_module-reconfigure .... where `kernel_module` is the name of out Buildroot package that contains the kernel modules. @@ -95,7 +95,7 @@ where `kernel_module` is the name of out Buildroot package that contains the ker Other important targets are: .... -./build -t linux-reconfigure -t host-qemu-reconfigure +./build -- linux-reconfigure host-qemu-reconfigure .... which are aliased respectively to: @@ -249,13 +249,23 @@ Instead, you can either run them from a minimal init: ./run -e 'init=/eval.sh - lkmc_eval="insmod /hello.ko;/poweroff.out"' -n .... +or if the script is large, add it to a gitignored file that will go into the guest: + +.... +echo ' +insmod /hello.ko +/poweroff.out +' > rootfs_overlay/ignore.sh +./run -e 'init=/ignore.sh' -n +.... + or run them at the end of the BusyBox init, which does things like setting up networking: .... ./run -e '- lkmc_eval="insmod /hello.ko;wget -S google.com;poweroff.out;"' .... -or add them to a new `init.d` entry: +or add them to a new `init.d` entry to run at the end o the BusyBox init: .... cp rootfs_overlay/etc/init.d/S98 rootfs_overlay/etc/init.d/S99 @@ -977,7 +987,7 @@ To disable networking, use: To restore it, run: .... -./build -t initscripts-reconfigure +./build -- initscripts-reconfigure .... === The init environment @@ -1047,7 +1057,7 @@ Only tested successfully in `x86_64`. Build: .... -./build -x +./build -i buildroot_config_fragment_x11 ./run .... @@ -1257,7 +1267,7 @@ One obvious use case is having an encrypted root filesystem: you keep the initrd I think GRUB then knows read common disk formats, and then loads that initrd to memory with a `/boot/grub/grub.cfg` directive of type: - initrd /initrd.img-4.4.0-108-generic + initrd /initrd.img-4.4.0-108-generic Related: https://stackoverflow.com/questions/6405083/initrd-and-booting-the-linux-kernel @@ -1830,6 +1840,147 @@ External open source benchmarks. We will try to create Buildroot packages for th * http://parsec.cs.princeton.edu/ Mentioned on docs: http://gem5.org/PARSEC_benchmarks * http://www.m5sim.org/Splash_benchmarks +===== PARSEC benchmark + +We have ported the PARSEC benchmark http://parsec.cs.princeton.edu for cross compilation at: https://github.com/cirosantilli/parsec-benchmark + +This repo makes it trivial to get started with it: + +.... +./build -a arm -g -i buildroot_config_fragment_parsec +./run -a arm -g +.... + +As mentioned at link:https://github.com/cirosantilli/parsec-benchmark[], only SPLASH2 was currently ported. + +Keep in mind that activating PARSEC makes images huge at a few Gigs, specially so since we make multiple images for each configuration for different purposes, e.g. `.ext2`, `.cpio`, etc. This is why we don't build PARSEC by default. + +Once inside the guest, we could in theory launch PARSEC exactly as you would launch it on the host: + +.... +cd /parsec/ +bash +. env.sh +parsecmgmt -a run -p splash2x.fmm -i test +.... + +TODO: `splash2x.barnes` segfaults on `arsecmgmt -a run -p splash2x.fmm -i simsmall` inside QEMU. Why? Other benchmarks ran fine. + +.... +[PARSEC] [---------- Beginning of output ----------] +Generating input file input_1... +Running /parsec/ext/splash2x/apps/barnes/inst/arm-linux.gcc/bin/barnes 1 < input_1: +reading input file : + +Segmentation fault +.... + +However, while this is fine inside QEMU, it is not practical in gem5, since the `parsecmgmt` Bash scripts just takes too long to run in that case! + +So instead, you must find out the raw executable command, and run it manually yourself. + +This command can be found from the `Running` line that `parsecmgmt` outputs when running the programs. + +"Luckily", we run the run scripts while creating the image to extract the inputs, so you can just do a find in your shell history to find the run command and find a line of type: + +.... +Running /parsec/ext/splash2x/apps/fmm/inst/arm-linux.gcc/bin/fmm 1 < input_1: +.... + +which teaches you that you can run `fmm` as: + +.... +cd /parsec/ext/splash2x/apps/fmm/run +/parsec/ext/splash2x/apps/fmm/inst/arm-linux.gcc/bin/fmm 1 < input_1 +.... + +And so inside of `gem5`, you likely want to do: + +.... +cd /parsec/ext/splash2x/apps/fmm/run +m5 checkpoint +m5 resetstats && /parsec/ext/splash2x/apps/fmm/inst/arm-linux.gcc/bin/fmm 1 < input_1 && m5 dumpstats +.... + +You will always want to `cd` into the `run` directory first, which is where the input is located. + +====== PARSEC change the input size + +One limitation is that only one input size is available on the guest for a given build. + +To change that, edit link:buildroot_config_fragment_parsec[] to contain for example: + +.... +BR2_PACKAGE_PARSEC_BENCHMARK_INPUT_SIZE=simsmall +.... + +and then rebuild with: + +.... +./build -a arm -g -i buildroot_config_fragment_parsec parsec-benchmark-reconfigure +.... + +This limitation exists because `parsecmgmt` generates the input files just before running, but we can't run on gem5 as it is too slow! + +One option would be to do that inside the guest with QEMU, but this would required a full rebuild due to <>. + +Also, we can't generate all input sizes at once, because many of them have the same name and would overwrite one another... Parsec clearly needs a redesign for embedded, maybe we will patch it later. + +====== PARSEC uninstall + +If you want to remove PARSEC later, Buildroot doesn't provide an automated package removal mechanism as documented at: link:https://github.com/buildroot/buildroot/blob/2017.08/docs/manual/rebuilding-packages.txt#L90[], but the following procedure should be satisfactory: + +.... +rm -rf \ + ./buildroot/dl/parsec-* \ + ./buildroot/output.arm-gem5~/build/parsec-* \ + ./buildroot/output.arm-gem5~/images/rootfs.* \ + ./buildroot/output.arm-gem5~/target/parsec-* \ +; +./build -a arm -g +.... + +====== PARSEC benchmark hacking + +If you end up going inside link:parsec/parsec[] to hack up the benchmark (you will!), these tips will be helpful. + +Buildroot was not designed to deal with large images, and currently cross rebuilds are a bit slow, due to some image generation and validation steps. + +A few workarounds are: + +* develop in host first as much as you can. Our PARSEC fork supports it. ++ +If you do this, don't forget to do a: ++ +.... +cd parsec-benchmark/parsec-benchmark +git clean -xdf . +.... +before going for the cross compile build. ++ +* patch Buildroot to work well, and keep cross compiling all the way. This should be totally viable, and we should do it. ++ +Don't forget to explicitly rebuild PARSEC with: +.... +./build -a arm -g -i buildroot_config_fragment_parsec parsec-benchmark-reconfigure +.... ++ +You may also want to test if your patches are still functionally correct inside of QEMU first, which is a faster emulator. ++ +* sell your soul, and compile natively inside the guest. We won't do this, not only because it is evil, but also because Buildroot explicitly does not support it: https://buildroot.org/downloads/manual/manual.html#faq-no-compiler-on-target ARM employees have been known to do this: https://github.com/arm-university/arm-gem5-rsk/blob/aa3b51b175a0f3b6e75c9c856092ae0c8f2a7cdc/parsec_patches/qemu-patch.diff + +TODO Buildroot is slow because of the `pkg-generic` `GLOBAL_INSTRUMENTATION_HOOKS` sanitation which go over the entire tree doing complex operations... I no like, in particular `check_bin_arch` and `check_host_rpath`. + +The pause is followed by: + +.... +buildroot/output.arm~/build/parsec-benchmark-custom/.stamp_target_installed +.... + +so which shows that the whole delay is inside our install itself. + +I put an `echo f` in `check_bin_arch`, and it just loops forever, does not stop on a particular package. + === gem5 kernel command line parameters Analogous <>: diff --git a/build b/build index 8dd2797..d22092e 100755 --- a/build +++ b/build @@ -1,51 +1,53 @@ #!/usr/bin/env bash -set -e +set -eu arch=x86_64 -extra_targets='' +configure=true +config_fragments=buildroot_config_fragment +extra_make_args='' gem5=false j="$(($(nproc) - 2))" post_script_args='' qemu_sdl='--enable-sdl --with-sdlabi=2.0' -x11=false v=0 -while getopts 'a:gj:lp:qSt:v' OPT; do +while getopts 'a:Cgj:i:klp:qS:v' OPT; do case "$OPT" in a) arch="$OPTARG" ;; + C) + configure=false + ;; g) gem5=true ;; j) j="$OPTARG" ;; - l) - extra_targets="$extra_args kernel_module-reconfigure" + i) + config_fragments="$config_fragments $OPTARG" + ;; + k) + extra_make_args="$extra_make_args kernel_module-reconfigure" ;; l) - extra_targets="$extra_args linux-reconfigure" + extra_make_args="$extra_make_args linux-reconfigure" ;; p) post_script_args="$OPTARG" ;; q) - extra_targets="$extra_args host-qemu-reconfigure" + extra_make_args="$extra_make_args host-qemu-reconfigure" ;; S) qemu_sdl='' ;; - t) - extra_targets="$extra_args $OPTARG" - ;; - x) - x11=true - ;; v) v=1 ;; esac done shift $(($OPTIND - 1)) +extra_make_args="$extra_make_args $@" case "$arch" in x86_64) defconfig=qemu_x86_64_defconfig @@ -65,62 +67,71 @@ if "$gem5"; then arch_dir="${arch}-gem5" # Networking was not working, so disable it to speed things up. post_script_args="$post_script_args -n" +else + config_fragments="$config_fragments buildroot_config_fragment_qemu" +fi +root_dir="$(pwd)" +buildroot_dir="${root_dir}/buildroot" +out_dir="${buildroot_dir}/output.${arch_dir}~" +config_file="${out_dir}/.config" + +if "$configure"; then + cd "${buildroot_dir}" + for p in $(find "${root_dir}/buildroot_patches/" -maxdepth 1 -name '*.patch' -print); do + patch -N -r - -p 1 <"$p" || : + done + make O="$out_dir" BR2_EXTERNAL="${root_dir}/kernel_module:${root_dir}/gem5:${root_dir}/parsec-benchmark" "$defconfig" + # TODO Can't get rid of these for now. + # http://stackoverflow.com/questions/44078245/is-it-possible-to-use-config-fragments-with-buildroots-config + for config_fragment in $config_fragments; do + cat "../$config_fragment" >> "${config_file}" + done + printf " +BR2_JLEVEL=$j +BR2_ROOTFS_POST_SCRIPT_ARGS=\"$post_script_args\" +" >> "${config_file}" + if "$gem5"; then + printf "\ +BR2_LINUX_KERNEL_CUSTOM_CONFIG_FILE=\"../kernel_config_${arch_dir}\" +BR2_PACKAGE_GEM5=y +" >> "${config_file}" + else + printf "\ +BR2_LINUX_KERNEL_CONFIG_FRAGMENT_FILES=\"../kernel_config_fragment\" +" >> "${config_file}" + fi + if [ "$arch" = 'mips64' ]; then + # Workaround for: + # http://lists.busybox.net/pipermail/buildroot/2017-August/201053.html + sed -Ei 's/^BR2_PACKAGE_LINUX_TOOLS_GPIO/BR2_PACKAGE_LINUX_TOOLS_GPIO=n/' "${config_file}" + fi + make O="$out_dir" olddefconfig fi -rootdir="$(pwd)" -cd kernel_module +cd "${root_dir}/kernel_module" ./make-host.sh -j "$j" clean -cd "$rootdir/buildroot" -for p in $(find "${rootdir}/buildroot_patches/" -maxdepth 1 -name '*.patch' -print); do - patch -N -r - -p 1 <"$p" || : -done -outdir="output.${arch_dir}~" -make O="$outdir" BR2_EXTERNAL="${rootdir}/kernel_module:${rootdir}/gem5:${rootdir}/parsec" "$defconfig" -# TODO Can't get rid of this for now. -# http://stackoverflow.com/questions/44078245/is-it-possible-to-use-config-fragments-with-buildroots-config -cat ../buildroot_config_fragment >> "${outdir}/.config" -if "$gem5"; then - echo "\ -BR2_LINUX_KERNEL_CUSTOM_CONFIG_FILE=\"../kernel_config_${arch_dir}\" -" >> "${outdir}/.config" -else - echo "\ -BR2_LINUX_KERNEL_CONFIG_FRAGMENT_FILES=\"../kernel_config_fragment\" -" >> "${outdir}/.config" -fi -if "$x11"; then - cat ../buildroot_config_fragment_x11 >> "${outdir}/.config" -fi -if ! "$gem5"; then - cat ../buildroot_config_fragment_qemu >> "${outdir}/.config" -fi -if [ "$arch" = 'mips64' ]; then - # Workaround for: - # http://lists.busybox.net/pipermail/buildroot/2017-August/201053.html - sed -Ei 's/^BR2_PACKAGE_LINUX_TOOLS_GPIO/BR2_PACKAGE_LINUX_TOOLS_GPIO=n/' "${outdir}/.config" -fi -make O="$outdir" olddefconfig +cd "${buildroot_dir}" # HOST_QEMU_OPTS is a hack that happens to work because the QEMU package luckly uses += at all times. # It shouldn't be necessary in the first place: https://bugs.busybox.net/show_bug.cgi?id=9936 # # Even if were an autotools package, there is no general way currently to pass extra configs to it: # https://stackoverflow.com/questions/44341188/how-to-pass-extra-custom-configure-autotools-options-to-a-buildroot-package/44341225#44341225 +# +# BR2_ options may be given on the command line here, and they do have direct "define" effects. +# But this is generally bad, as it skips the Kconfig mechanism, e.g. it does not set defaults properly. cmd="time \ env \ -u LD_LIBRARY_PATH \ make \ - O='$outdir' \ - BR2_JLEVEL='$j' \ - BR2_PACKAGE_GEM5='$("$gem5" && echo y || echo n)' \ - BR2_ROOTFS_POST_SCRIPT_ARGS='$post_script_args' \ + O='$out_dir' \ HOST_QEMU_OPTS='--enable-debug --extra-cflags=-DDEBUG_PL061=1 --enable-trace-backends=simple $qemu_sdl' \ V='$v' \ - $extra_targets \ + $extra_make_args \ all \ " -echo "$cmd" | tee "${rootdir}/build.log" +echo "$cmd" | tee "${root_dir}/build.log" eval "$cmd" -cd .. if "$gem5"; then + cd "${root_dir}" ./build-gem5 -a "$arch" fi diff --git a/buildroot_config_fragment b/buildroot_config_fragment index efe0020..67009ba 100644 --- a/buildroot_config_fragment +++ b/buildroot_config_fragment @@ -1,9 +1,5 @@ # Custom packages BR2_PACKAGE_KERNEL_MODULE=y -BR2_PACKAGE_PARSEC=y -BR2_PACKAGE_PARSEC_BUILD_LIST="splash2" -# Required for parsec. -BR2_TARGET_ROOTFS_EXT2_SIZE="1G" BR2_GCC_ENABLE_GRAPHITE=y BR2_GCC_ENABLE_LTO=y @@ -11,6 +7,7 @@ BR2_GCC_ENABLE_OPENMP=y BR2_GLOBAL_PATCH_DIR="../global_patch_dir" BR2_PACKAGE_BUSYBOX_CONFIG_FRAGMENT_FILES="../busybox_config_fragment" BR2_PACKAGE_DHRYSTONE=y +BR2_PACKAGE_FILE=y BR2_PACKAGE_OVERRIDE_FILE="../buildroot_override" # For qemu-ga on guest. TODO: do something with it, and document it. BR2_PACKAGE_QEMU=y diff --git a/buildroot_config_fragment_parsec b/buildroot_config_fragment_parsec new file mode 100644 index 0000000..896066e --- /dev/null +++ b/buildroot_config_fragment_parsec @@ -0,0 +1,8 @@ +BR2_PACKAGE_PARSEC_BENCHMARK=y +#BR2_PACKAGE_PARSEC_BENCHMARK_BUILD_LIST="splash2x" +#BR2_PACKAGE_PARSEC_BENCHMARK_INPUT_SIZE="simsmall" +# Because PARSEC + its data are huge. TODO: can't we automate calculating the size? +# Problems will arise if someone tries to use two such benchmarks. +# Cannot be selected automatically from Kconfig: +# https://stackoverflow.com/questions/40309054/how-to-select-the-value-of-a-string-option-from-another-option-in-kbuild-kconfig/49096538#49096538 +BR2_TARGET_ROOTFS_EXT2_SIZE="128M" diff --git a/buildroot_override b/buildroot_override index 2d6e4e0..2a8fdaf 100644 --- a/buildroot_override +++ b/buildroot_override @@ -1,4 +1,4 @@ HOST_QEMU_OVERRIDE_SRCDIR = ../qemu LINUX_OVERRIDE_SRCDIR = ../linux -PARSEC_OVERRIDE_SRCDIR = ../parsec-benchmark/parsec-benchmark +PARSEC_BENCHMARK_OVERRIDE_SRCDIR = ../parsec-benchmark/parsec-benchmark QEMU_OVERRIDE_SRCDIR = ../qemu diff --git a/parsec-benchmark/Config.in b/parsec-benchmark/Config.in index 6e47215..139c4c2 100644 --- a/parsec-benchmark/Config.in +++ b/parsec-benchmark/Config.in @@ -1,16 +1,29 @@ -config BR2_PACKAGE_PARSEC - bool "PARSEC" +config BR2_PACKAGE_PARSEC_BENCHMARK + bool "PARSEC_BENCHMARK" +# Parsec shell scripts use a hardcoded /bin/bash +# One option would be to try and use /bin/sh. +# But symlinking fails because of BusyBox' symlink mechanism. +# The other option would be to patch Parsec to use /bin/sh and be POSIX compliant. +# But let's take the path of smallest resistance for now. + select BR2_PACKAGE_BUSYBOX_SHOW_OTHERS + select BR2_PACKAGE_BASH help Parsec system benchmark. http://parsec.cs.princeton.edu/ -if BR2_PACKAGE_PARSEC +if BR2_PACKAGE_PARSEC_BENCHMARK -config BR2_PACKAGE_PARSEC_BUILD_LIST +config BR2_PACKAGE_PARSEC_BENCHMARK_BUILD_LIST string "build_list" - default all + default splash2x help Space separated list of parsec packages to build. +config BR2_PACKAGE_PARSEC_BENCHMARK_INPUT_SIZE + string "input_size" + default test + help + Which input size to generate on the host for the guest. + endif diff --git a/parsec-benchmark/external.desc b/parsec-benchmark/external.desc index 38ffebd..248d333 100644 --- a/parsec-benchmark/external.desc +++ b/parsec-benchmark/external.desc @@ -1 +1 @@ -name: PARSEC +name: PARSEC_BENCHMARK diff --git a/parsec-benchmark/external.mk b/parsec-benchmark/external.mk index b150747..6a153b8 100644 --- a/parsec-benchmark/external.mk +++ b/parsec-benchmark/external.mk @@ -1,23 +1,54 @@ ################################################################################ # -# PARSEC +# PARSEC_BENCHMARK # ################################################################################ -PARSEC_VERSION = master -PARSEC_SITE = git@github.com:cirosantilli/parsec-benchmark.git -PARSEC_SITE_METHOD = git +PARSEC_BENCHMARK_VERSION = master +PARSEC_BENCHMARK_SITE = git@github.com:cirosantilli/parsec-benchmark.git +PARSEC_BENCHMARK_SITE_METHOD = git -define PARSEC_BUILD_CMDS - cd $(@D) && . env.sh && for pkg in $(BR2_PACKAGE_PARSEC_BUILD_LIST); do parsecmgmt -a build -p $$pkg; done - #cd '$(@D)/gem5/util/m5' && $(MAKE) -f 'Makefile.$(ARCH_MAKE)' CC='$(TARGET_CC)' LD='$(TARGET_LD)' +define PARSEC_BENCHMARK_BUILD_CMDS + # TODO make this nicer, only untar when extract step is done. + # EXTRACT_CMDS and EXTRA_DOWNLOADS would be good candidates, + # but they don't run with OVERRIDE_SRCDIR. + '$(PARSEC_BENCHMARK_PKGDIR)/parsec-benchmark/get-inputs' $(if $(filter $(V),1),-v,) '$(DL_DIR)' '$(@D)/' + # We run the benchmarks with the wrong arch here to generate the inputs on the host. + # This is because on gem5 this takes too long to do. + cd $(@D) && . env.sh && for pkg in $(BR2_PACKAGE_PARSEC_BENCHMARK_BUILD_LIST); do \ + export HOSTCC='$(HOSTCC)'; \ + export M4='$(HOST_DIR)/usr/bin/m4'; \ + export MAKE='$(MAKE)'; \ + export OSTYPE=linux; \ + export TARGET_CROSS='$(TARGET_CROSS)'; \ + export HOSTTYPE='$(BR2_ARCH)'; \ + parsecmgmt -a build -p $$pkg; \ + parsecmgmt -a run -p $$pkg -i $(BR2_PACKAGE_PARSEC_BENCHMARK_INPUT_SIZE); \ + done endef -define PARSEC_INSTALL_TARGET_CMDS - # This is a bit coarse and makes the image huge with useless source code, - # and input files, but I don't feel like creating per-package installs. - # And it doesn't matter much for simulators anyways. - rsync -av '$(@D)/' '$(TARGET_DIR)/parsec' +define PARSEC_BENCHMARK_INSTALL_TARGET_CMDS + # This is a bit coarse and makes the image larger with useless source code. + # + # But according to du, the source accounts for only 1/5 of the total size, + # so benchmarks dominate, and it doesn't matter much. + # + # Also it is not so critical for simulators anyways unlike real embedded systems. + # + # One possibility to make this better may be to install only the 'inst/' and 'input/' + # folders for each package + toplevel '/bin/' and '/config/', but of course we won't + # know if this works until time consuming testing is done :-) + mkdir -p '$(TARGET_DIR)/parsec/' + + rsync -am $(if $(filter $(V),1),-v,) --include '*/' \ + --include '/bin/***' \ + --include '/config/***' \ + --include '/env.sh' \ + --include 'inst/***' \ + --include 'run/***' \ + --exclude '*' '$(@D)/' '$(TARGET_DIR)/parsec/' \ + ; + # rsync finished. endef $(eval $(generic-package)) diff --git a/parsec-benchmark/parsec-benchmark b/parsec-benchmark/parsec-benchmark index 7ce7fde..b7d37ec 160000 --- a/parsec-benchmark/parsec-benchmark +++ b/parsec-benchmark/parsec-benchmark @@ -1 +1 @@ -Subproject commit 7ce7fde92365248beafdbe6b94468a04a17b9c2d +Subproject commit b7d37ec654e652be6eb75f44dddd18493b1212f2 From 8ddf179eee010e4b8c6294526669e2efe43b76b7 Mon Sep 17 00:00:00 2001 From: Ciro Santilli Date: Thu, 8 Mar 2018 00:04:00 +0000 Subject: [PATCH 05/17] Nope, parsec is small now :-) --- README.adoc | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.adoc b/README.adoc index df4f9d1..0220b1c 100644 --- a/README.adoc +++ b/README.adoc @@ -1853,8 +1853,6 @@ This repo makes it trivial to get started with it: As mentioned at link:https://github.com/cirosantilli/parsec-benchmark[], only SPLASH2 was currently ported. -Keep in mind that activating PARSEC makes images huge at a few Gigs, specially so since we make multiple images for each configuration for different purposes, e.g. `.ext2`, `.cpio`, etc. This is why we don't build PARSEC by default. - Once inside the guest, we could in theory launch PARSEC exactly as you would launch it on the host: .... From 157f13571b6917ba465710d41b2e967b45a6b605 Mon Sep 17 00:00:00 2001 From: Ciro Santilli Date: Thu, 8 Mar 2018 00:06:04 +0000 Subject: [PATCH 06/17] Clarify input size Bash is slow --- README.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.adoc b/README.adoc index 0220b1c..8a08f5d 100644 --- a/README.adoc +++ b/README.adoc @@ -1918,7 +1918,7 @@ and then rebuild with: ./build -a arm -g -i buildroot_config_fragment_parsec parsec-benchmark-reconfigure .... -This limitation exists because `parsecmgmt` generates the input files just before running, but we can't run on gem5 as it is too slow! +This limitation exists because `parsecmgmt` generates the input files just before running via the Bash scripts, but we can't run `parsecmgmt` on gem5 as it is too slow! One option would be to do that inside the guest with QEMU, but this would required a full rebuild due to <>. From 2c12b21b304178a81c9912817b782ead0286d282 Mon Sep 17 00:00:00 2001 From: Ciro Santilli Date: Thu, 8 Mar 2018 00:08:33 +0000 Subject: [PATCH 07/17] Minor readme fixes --- README.adoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.adoc b/README.adoc index 8a08f5d..caf5ae2 100644 --- a/README.adoc +++ b/README.adoc @@ -1940,7 +1940,7 @@ rm -rf \ ====== PARSEC benchmark hacking -If you end up going inside link:parsec/parsec[] to hack up the benchmark (you will!), these tips will be helpful. +If you end up going inside link:parsec-benchmark/parsec-benchmark[] to hack up the benchmark (you will!), these tips will be helpful. Buildroot was not designed to deal with large images, and currently cross rebuilds are a bit slow, due to some image generation and validation steps. @@ -1959,12 +1959,12 @@ before going for the cross compile build. * patch Buildroot to work well, and keep cross compiling all the way. This should be totally viable, and we should do it. + Don't forget to explicitly rebuild PARSEC with: ++ .... ./build -a arm -g -i buildroot_config_fragment_parsec parsec-benchmark-reconfigure .... + You may also want to test if your patches are still functionally correct inside of QEMU first, which is a faster emulator. -+ * sell your soul, and compile natively inside the guest. We won't do this, not only because it is evil, but also because Buildroot explicitly does not support it: https://buildroot.org/downloads/manual/manual.html#faq-no-compiler-on-target ARM employees have been known to do this: https://github.com/arm-university/arm-gem5-rsk/blob/aa3b51b175a0f3b6e75c9c856092ae0c8f2a7cdc/parsec_patches/qemu-patch.diff TODO Buildroot is slow because of the `pkg-generic` `GLOBAL_INSTRUMENTATION_HOOKS` sanitation which go over the entire tree doing complex operations... I no like, in particular `check_bin_arch` and `check_host_rpath`. From 16560c896343dac8f71dfa93e0e6474b3b86763c Mon Sep 17 00:00:00 2001 From: Ciro Santilli Date: Thu, 8 Mar 2018 07:56:57 +0000 Subject: [PATCH 08/17] configure add optional parsec install --- README.adoc | 2 +- configure | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/README.adoc b/README.adoc index df4f9d1..1c6e21a 100644 --- a/README.adoc +++ b/README.adoc @@ -1847,7 +1847,7 @@ We have ported the PARSEC benchmark http://parsec.cs.princeton.edu for cross com This repo makes it trivial to get started with it: .... -./build -a arm -g -i buildroot_config_fragment_parsec +configure -p && ./build -a arm -g -i buildroot_config_fragment_parsec ./run -a arm -g .... diff --git a/configure b/configure index bac177d..edf8e99 100755 --- a/configure +++ b/configure @@ -1,9 +1,13 @@ #!/usr/bin/env bash set -e interactive_pkgs=libsdl2-dev +submodules='buildroot linux qemu gem5/gem5' y='' -while getopts t OPT; do +while getopts pt OPT; do case "$OPT" in + p) + submodules="$submodules parsec-benchmark/parsec-benchmark" + ;; t) interactive_pkgs='' y='-y' @@ -12,7 +16,6 @@ while getopts t OPT; do done shift $(($OPTIND - 1)) -submodules='buildroot linux qemu gem5/gem5' ( set -e # Shallow cloning saves a considerable amount of time, specially because of the linux kernel. From a5eda22ac1a4ef4060c380b4201904bb2e039da7 Mon Sep 17 00:00:00 2001 From: Ciro Santilli Date: Thu, 8 Mar 2018 08:14:35 +0000 Subject: [PATCH 09/17] Organize benchmarks for this repo --- README.adoc | 54 +++++++++++++++++++++++-------- parsec-benchmark/parsec-benchmark | 2 +- 2 files changed, 42 insertions(+), 14 deletions(-) diff --git a/README.adoc b/README.adoc index caf5ae2..1452214 100644 --- a/README.adoc +++ b/README.adoc @@ -23,10 +23,7 @@ cd linux-kernel-module-cheat ./configure && ./build && ./run .... -The first build will take a while (link:https://stackoverflow.com/questions/10833672/buildroot-environment-with-host-toolchain[GCC], Linux kernel), e.g.: - -* 2 hours on a mid end 2012 laptop -* 30 minutes on a high end 2017 desktop +The first configure will take a while (30 minutes to 2 hours) to clone and build, see <> for more details. If you don't want to wait, you could also try to compile the examples and run them on your host computer as explained on at <>, but as explained on that section, that is dangerous, limited, and will likely not work. @@ -1591,12 +1588,7 @@ and the results were: |gem5 X86_64 |5 minutes 30 seconds| 82 |=== -on a Lenovo P51 laptop with: - -* Intel Core i7-7820HQ Processor (8MB Cache, up to 3.90GHz) (4 cores 8 threads) -* 32GB(16+16) DDR4 2400MHz SODIMM -* 512GB SSD PCIe TLC OPAL2 -* Ubuntu 17.10 +tested on the <>. === gem5 run benchmark @@ -1915,7 +1907,7 @@ BR2_PACKAGE_PARSEC_BENCHMARK_INPUT_SIZE=simsmall and then rebuild with: .... -./build -a arm -g -i buildroot_config_fragment_parsec parsec-benchmark-reconfigure +./build -a arm -g -i buildroot_config_fragment_parsec -- parsec-benchmark-reconfigure .... This limitation exists because `parsecmgmt` generates the input files just before running via the Bash scripts, but we can't run `parsecmgmt` on gem5 as it is too slow! @@ -2210,7 +2202,7 @@ info: Entering event queue @ 0. Starting simulation... and the `telnet` at: .... -2017-12-28-11-59-51@ciro@ciro-p51$ ./gem5-shell +$ ./gem5-shell Trying 127.0.0.1... Connected to localhost. Escape character is '^]'. @@ -2497,7 +2489,13 @@ diff .config.olg .config Copy and paste the diff additions to `buildroot_config_fragment`. -==== What is making my build so slow? +==== Benchmarking this repo + +In this section document how fast the build and clone are, and how to investigate them. + +Send a pull request if you try it out on something significantly different. + +===== What is making my build so slow? .... cd buildroot/output.x86_64~ @@ -2510,6 +2508,36 @@ Our philosophy is: * if something adds little to the build time, build it in by default * otherwise, make it optional +The biggest time hog is always GCC, can we use a precompiled one? https://stackoverflow.com/questions/10833672/buildroot-environment-with-host-toolchain + +===== Benchmark machines + +====== P51 + +Build time at 2c12b21b304178a81c9912817b782ead0286d282: 30 minutes + +Lenovo link:https://www3.lenovo.com/gb/en/laptops/thinkpad/p-series/P51/p/22TP2WPWP51[P51 laptop]: + +* 2500 USD in 2018 (high end) +* Intel Core i7-7820HQ Processor (8MB Cache, up to 3.90GHz) (4 cores 8 threads) +* 32GB(16+16) DDR4 2400MHz SODIMM +* 512GB SSD PCIe TLC OPAL2 +* Ubuntu 17.10 + +====== T430 + +Build time: 2 hours. + +TODO specs, SHA. + +===== Benchmark internets + +====== 38Mbps + +Shallow clone of all submodules at 2c12b21b304178a81c9912817b782ead0286d282: 4 minutes. + +Google M-lab speed test 36.4Mbps + === About This project is for people who want to learn and modify low level system components: diff --git a/parsec-benchmark/parsec-benchmark b/parsec-benchmark/parsec-benchmark index b7d37ec..762545a 160000 --- a/parsec-benchmark/parsec-benchmark +++ b/parsec-benchmark/parsec-benchmark @@ -1 +1 @@ -Subproject commit b7d37ec654e652be6eb75f44dddd18493b1212f2 +Subproject commit 762545a1b7bf447e51aec105fad3878db3529a77 From ef287e985e2720ae9d94f975b06bf08d3ed50b2f Mon Sep 17 00:00:00 2001 From: Ciro Santilli Date: Thu, 8 Mar 2018 08:26:58 +0000 Subject: [PATCH 10/17] Benchamrks for this repo: split make source time --- README.adoc | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/README.adoc b/README.adoc index 2539328..054ae9a 100644 --- a/README.adoc +++ b/README.adoc @@ -2512,11 +2512,13 @@ The biggest time hog is always GCC, can we use a precompiled one? https://stacko ===== Benchmark machines +The build times are calculated after doing link:https://buildroot.org/downloads/manual/manual.html#_offline_builds[`make source`], which downloads the sources, and basically benchmarks the Internet. + ====== P51 -Build time at 2c12b21b304178a81c9912817b782ead0286d282: 30 minutes +Build time at 2c12b21b304178a81c9912817b782ead0286d282: 28 minutes -Lenovo link:https://www3.lenovo.com/gb/en/laptops/thinkpad/p-series/P51/p/22TP2WPWP51[P51 laptop]: +Lenovo ThinkPad link:https://www3.lenovo.com/gb/en/laptops/thinkpad/p-series/P51/p/22TP2WPWP51[P51 laptop]: * 2500 USD in 2018 (high end) * Intel Core i7-7820HQ Processor (8MB Cache, up to 3.90GHz) (4 cores 8 threads) @@ -2534,9 +2536,12 @@ TODO specs, SHA. ====== 38Mbps -Shallow clone of all submodules at 2c12b21b304178a81c9912817b782ead0286d282: 4 minutes. +2c12b21b304178a81c9912817b782ead0286d282: -Google M-lab speed test 36.4Mbps +* shallow clone of all submodules: 4 minutes. +* `make source`: 2 minutes + +Google M-lab speed test: 36.4Mbps === About From 5d5c6b954c4a91e388de71ff55ab28a900752010 Mon Sep 17 00:00:00 2001 From: Ciro Santilli Date: Thu, 8 Mar 2018 08:41:00 +0000 Subject: [PATCH 11/17] configure: make qemu and gem5 steps optional --- README.adoc | 6 ++++-- configure | 57 ++++++++++++++++++++++++++++++++++++----------------- 2 files changed, 43 insertions(+), 20 deletions(-) diff --git a/README.adoc b/README.adoc index 054ae9a..49fb1d2 100644 --- a/README.adoc +++ b/README.adoc @@ -1501,12 +1501,14 @@ To get started, have a look at the "Hardware device drivers" secion under link:k == gem5 +=== gem5 getting started + gem5 is a system simulator, much <>: http://gem5.org/ For the most part, just add the `-g` option to the QEMU commands and everything should magically work: .... -./configure && ./build -a arm -g +./configure -gq && ./build -a arm -g ./run -a arm -g .... @@ -1839,7 +1841,7 @@ We have ported the PARSEC benchmark http://parsec.cs.princeton.edu for cross com This repo makes it trivial to get started with it: .... -configure -p && ./build -a arm -g -i buildroot_config_fragment_parsec +configure -gpq && ./build -a arm -g -i buildroot_config_fragment_parsec ./run -a arm -g .... diff --git a/configure b/configure index edf8e99..65c5816 100755 --- a/configure +++ b/configure @@ -1,10 +1,18 @@ #!/usr/bin/env bash set -e interactive_pkgs=libsdl2-dev -submodules='buildroot linux qemu gem5/gem5' +gem5=false +qemu=true +submodules='buildroot linux' y='' -while getopts pt OPT; do +while getopts gqpt OPT; do case "$OPT" in + g) + gem5=true + ;; + q) + qemu=false + ;; p) submodules="$submodules parsec-benchmark/parsec-benchmark" ;; @@ -16,6 +24,14 @@ while getopts pt OPT; do done shift $(($OPTIND - 1)) +## Submodules + +if "$qemu"; then + submodules="$submodules qemu" +fi +if "$gem5"; then + submodules="$submodules gem5/gem5" +fi ( set -e # Shallow cloning saves a considerable amount of time, specially because of the linux kernel. @@ -25,36 +41,39 @@ shift $(($OPTIND - 1)) # - `shallow = true` on the submodule has no effect for the non default educational branches of our submodules # - QEMU's submodules point to commits that are neither under branches nor tags, and so `--shallow-submodules` fails git submodule update --depth 1 --jobs 4 --init -- $submodules - cd qemu - git submodule update --init + if "$qemu"; then + cd qemu + git submodule update --init + fi ) & wait $! || git submodule update --init -- $submodules +## apt-get + pkgs="\ automake \ build-essential \ coreutils \ " - -# GEM5 -pkgs="$pkgs \ -g++-6 \ -gcc-6 \ -gcc-aarch64-linux-gnu \ -gcc-arm-linux-gnueabi \ -libgoogle-perftools-dev \ -protobuf-compiler \ -" - +if "$gem5"; then + pkgs="$pkgs \ + g++-6 \ + gcc-6 \ + gcc-aarch64-linux-gnu \ + gcc-arm-linux-gnueabi \ + libgoogle-perftools-dev \ + protobuf-compiler \ + " +fi command -v apt-get >/dev/null 2>&1 || { - cat < Date: Thu, 8 Mar 2018 08:48:30 +0000 Subject: [PATCH 12/17] parsec: remove parsec.sh leftover --- parsec-benchmark/parsec.sh | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100755 parsec-benchmark/parsec.sh diff --git a/parsec-benchmark/parsec.sh b/parsec-benchmark/parsec.sh deleted file mode 100755 index ac5de14..0000000 --- a/parsec-benchmark/parsec.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash -wget http://parsec.cs.princeton.edu/download/3.0/parsec-3.0.tar.gz -tar -xvzf parsec-3.0.tar.gz -cd parsec-3.0 -. env.sh -parsecmgmt -a build -p streamcluster -parsecmgmt -a run -p streamcluster - -parsecmgmt -a build -p all - -wget http://parsec.cs.princeton.edu/download/3.0/parsec-3.0-input-sim.tar.gz -tar -xzf parsec-3.0-input-sim.tar.gz -parsecmgmt -a run -p streamcluster -i simsmall -parsecmgmt -a run -p streamcluster -i simlarge -n 2 From 03b1a720c47352a7b8d0a13eb325f7a079b953c9 Mon Sep 17 00:00:00 2001 From: Ciro Santilli Date: Thu, 8 Mar 2018 09:04:09 +0000 Subject: [PATCH 13/17] parsec: test.sh with many raw run commands --- README.adoc | 4 +++- parsec-benchmark/external.mk | 2 ++ parsec-benchmark/test.sh | 24 ++++++++++++++++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) create mode 100755 parsec-benchmark/test.sh diff --git a/README.adoc b/README.adoc index 49fb1d2..d5bb0f0 100644 --- a/README.adoc +++ b/README.adoc @@ -1883,9 +1883,11 @@ which teaches you that you can run `fmm` as: .... cd /parsec/ext/splash2x/apps/fmm/run -/parsec/ext/splash2x/apps/fmm/inst/arm-linux.gcc/bin/fmm 1 < input_1 +../inst/arm-linux.gcc/bin/fmm 1 < input_1 .... +We are also collecting more raw commands for testing at: link:parsec-benchmark/test.sh[] + And so inside of `gem5`, you likely want to do: .... diff --git a/parsec-benchmark/external.mk b/parsec-benchmark/external.mk index 6a153b8..30dd9f4 100644 --- a/parsec-benchmark/external.mk +++ b/parsec-benchmark/external.mk @@ -40,6 +40,8 @@ define PARSEC_BENCHMARK_INSTALL_TARGET_CMDS # know if this works until time consuming testing is done :-) mkdir -p '$(TARGET_DIR)/parsec/' + $(INSTALL) -D -m 0755 '$(PARSEC_BENCHMARK_PKGDIR)/test.sh' '$(TARGET_DIR)/parsec/' + rsync -am $(if $(filter $(V),1),-v,) --include '*/' \ --include '/bin/***' \ --include '/config/***' \ diff --git a/parsec-benchmark/test.sh b/parsec-benchmark/test.sh new file mode 100755 index 0000000..6fa108f --- /dev/null +++ b/parsec-benchmark/test.sh @@ -0,0 +1,24 @@ +#!/bin/sh + +# Since Parsec does not stop on errors and we need some raw commands for gem5, +# let's do some of our own unit tests here. Thses should be run on QEMU +# to be reasonably fast. + +set -ex + +ncpus=1 + +cd /parsec/ext/splash2x/apps/fmm/run +../inst/*/bin/fmm "$ncpus" < input_1 + +cd /parsec/ext/splash2x/apps/raytrace/run +../inst/*/bin/raytrace -s -p"$ncpus" -a4 teapot.env + +cd /parsec/ext/splash2x/apps/volrend/run +../inst/*/bin/volrend "$ncpus" head-scaleddown4 4 < input_1 + +cd /parsec/ext/splash2x/apps/water_nsquared/run +../inst/*/bin/water_nsquared "$ncpus" < input_1 + +cd /parsec/ext/splash2x/apps/water_spatial/run +../inst/*/bin/water_spatial "$ncpus" < input_1 From cb08ab2e0163ac8b209a18a062bba1dba61cdac4 Mon Sep 17 00:00:00 2001 From: Ciro Santilli Date: Fri, 9 Mar 2018 03:57:42 +0000 Subject: [PATCH 14/17] Document kvm --- README.adoc | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/README.adoc b/README.adoc index d5bb0f0..7b5ad66 100644 --- a/README.adoc +++ b/README.adoc @@ -835,7 +835,7 @@ continue This is of least reliable setup as there might be other processes that use the given virtual address. -== Architecture +== Architectures The portability of the kernel and toolchains is amazing: change an option and most things magically work on completely different hardware. @@ -856,6 +856,23 @@ Debug: Known quirks of the supported architectures are documented in this section. +=== x86 + +You can make QEMU <> by passing it the `-enable-kvm` option as in: + +.... +./run - -enable-kvm +.... + +This uses the link:https://en.wikipedia.org/wiki/Kernel-based_Virtual_Machine[KVM Linux kernel feature] of the host to run most instructions natively, and therefore. + +There are several downsides to this however: + +* only works for x86 guest +* limits visibility, since more things are running natively + +So for those reasons we don't turn it on by default. + === arm TODOs: From e88dacf7a6ad0ce7d6dc805f89e3b20ad46c45ff Mon Sep 17 00:00:00 2001 From: Ciro Santilli Date: Fri, 9 Mar 2018 04:04:05 +0000 Subject: [PATCH 15/17] How to get the command that was run --- README.adoc | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/README.adoc b/README.adoc index 7b5ad66..3c351f5 100644 --- a/README.adoc +++ b/README.adoc @@ -385,6 +385,32 @@ Just make sure that you never click inside the QEMU window when doing that, othe You can still send key presses to QEMU however even without the mouse capture, just either click on the title bar, or alt tab to give it focus. +=== What command was actually run? + +When asking for help on upstream repositories outside of this repository, you will need to provide the commands that you are running in detail without referencing our scripts. + +For example, QEMU developers will only want to see the final QEMU command that you are running. + +We make that easy by building commands as strings, and then echoing them before evaling. + +So for example when you run: + +.... +./run -a arm +.... + +Stdout shows a line with the full command of type: + +.... +./buildroot/output.arm~/host/usr/bin/qemu-system-arm -m 128M -monitor telnet::45454,server,nowait -netdev user,hostfwd=tcp::45455-:45455,id=net0 -smp 1 -M versatilepb -append 'root=/dev/sda nokaslr norandmaps printk.devkmsg=on printk.time=y' -device rtl8139,netdev=net0 -dtb ./buildroot/output.arm~/images/versatile-pb.dtb -kernel ./buildroot/output.arm~/images/zImage -serial stdio -drive file='./buildroot/output.arm~/images/rootfs.ext2.qcow2,if=scsi,format=qcow2' +.... + +This line is also saved to a file for convenience: + +.... +cat ./run.log +.... + [[gdb]] == GDB step debugging From 32a543c5dbffc49e7cf8d1ac63c5c49332227e24 Mon Sep 17 00:00:00 2001 From: Ciro Santilli Date: Fri, 9 Mar 2018 04:22:30 +0000 Subject: [PATCH 16/17] parsec: test must be run with test input size --- parsec-benchmark/test.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/parsec-benchmark/test.sh b/parsec-benchmark/test.sh index 6fa108f..a6483d7 100755 --- a/parsec-benchmark/test.sh +++ b/parsec-benchmark/test.sh @@ -3,6 +3,8 @@ # Since Parsec does not stop on errors and we need some raw commands for gem5, # let's do some of our own unit tests here. Thses should be run on QEMU # to be reasonably fast. +# +# Must be run with BR2_PACKAGE_PARSEC_BENCHMARK_INPUT_SIZE=test set -ex From c8003c3389c3e48a26caa3eb707c6c1c3215d587 Mon Sep 17 00:00:00 2001 From: Ciro Santilli Date: Fri, 9 Mar 2018 07:20:25 +0000 Subject: [PATCH 17/17] kvm --- README.adoc | 50 +++++++++++++++++++++++++++++--------------------- run | 12 +++++++++++- 2 files changed, 40 insertions(+), 22 deletions(-) diff --git a/README.adoc b/README.adoc index 3c351f5..0052b24 100644 --- a/README.adoc +++ b/README.adoc @@ -882,23 +882,6 @@ Debug: Known quirks of the supported architectures are documented in this section. -=== x86 - -You can make QEMU <> by passing it the `-enable-kvm` option as in: - -.... -./run - -enable-kvm -.... - -This uses the link:https://en.wikipedia.org/wiki/Kernel-based_Virtual_Machine[KVM Linux kernel feature] of the host to run most instructions natively, and therefore. - -There are several downsides to this however: - -* only works for x86 guest -* limits visibility, since more things are running natively - -So for those reasons we don't turn it on by default. - === arm TODOs: @@ -1090,6 +1073,32 @@ Kernel modules built from the Linux mainline tree with `CONFIG_SOME_MOD=m`, are modprobe dummy-irq .... +== KVM + +You can make QEMU or gem5 <> by passing enabling KVM with: + +.... +./run -K +.... + +but it was broken in gem5 with pending patches: https://www.mail-archive.com/gem5-users@gem5.org/msg15046.html + +KVM uses the link:https://en.wikipedia.org/wiki/Kernel-based_Virtual_Machine[KVM Linux kernel feature] of the host to run most instructions natively. + +We don't enable KVM by default because: + +* only works if the architecture of the guest equals that of the host. ++ +We have only tested / supported it on x86, but it is rumoured that QEMU and gem5 also have ARM KVM support if you are link:https://www.youtube.com/watch?v=8ItXpmLsINs[running an ARM desktop for some weird reason] :-) +* limits visibility, since more things are running natively: +** can't use GDB +** can't do instruction tracing +* kernel boots are already fast enough without `-enable-kvm` + +The main use case for `-enable-kvm` in this repository is to test if something that takes a long time to run is functionally correct. + +For example, when porting a benchmark to Buildroot, you can first use QEMU's KVM to test that benchmarks is producing the correct results, before analysing them more deeply in gem5, which runs much slower. + == X11 Only tested successfully in `x86_64`. @@ -1616,7 +1625,7 @@ time ./run -a arm -e 'init=/poweroff.out' time ./run -a arm -e 'm5 exit' -g time ./run -a arm -e 'm5 exit' -g -- --caches --cpu-type=HPI time ./run -a x86_64 -e 'init=/poweroff.out' -time ./run -a x86_64 -e 'init=/poweroff.out' - -enable-kvm +time ./run -a x86_64 -e 'init=/poweroff.out' -- -enable-kvm time ./run -a x86_64 -e 'init=/poweroff.out' -g .... @@ -1879,7 +1888,7 @@ External open source benchmarks. We will try to create Buildroot packages for th ===== PARSEC benchmark -We have ported the PARSEC benchmark http://parsec.cs.princeton.edu for cross compilation at: https://github.com/cirosantilli/parsec-benchmark +We have ported parts of the link:http://parsec.cs.princeton.edu[PARSEC benchmark] for cross compilation at: https://github.com/cirosantilli/parsec-benchmark See the documentation on that repo to find out which benchmarks have been ported. This repo makes it trivial to get started with it: @@ -1888,8 +1897,6 @@ configure -gpq && ./build -a arm -g -i buildroot_config_fragment_parsec ./run -a arm -g .... -As mentioned at link:https://github.com/cirosantilli/parsec-benchmark[], only SPLASH2 was currently ported. - Once inside the guest, we could in theory launch PARSEC exactly as you would launch it on the host: .... @@ -1971,6 +1978,7 @@ If you want to remove PARSEC later, Buildroot doesn't provide an automated packa rm -rf \ ./buildroot/dl/parsec-* \ ./buildroot/output.arm-gem5~/build/parsec-* \ + ./buildroot/output.arm-gem5~/build/packages-file-list.txt \ ./buildroot/output.arm-gem5~/images/rootfs.* \ ./buildroot/output.arm-gem5~/target/parsec-* \ ; diff --git a/run b/run index 5349aff..7adc97c 100755 --- a/run +++ b/run @@ -7,6 +7,7 @@ arch=x86_64 cpus=1 debug_vm='' kgdb=false +kvm=false nographic=false # norandmaps: Don't use address space randomization. Equivalent to echo 0 > /proc/sys/kernel/randomize_va_space. # printk.time=y: log in format: "[time ] msg" for all printk messages. @@ -19,7 +20,7 @@ gem5=false gem5opts='' initrd=false root='' -while getopts a:c:Dde:G:giknt:x OPT; do +while getopts a:c:Dde:G:giKknt:x OPT; do case "$OPT" in a) arch="$OPTARG" @@ -36,6 +37,9 @@ while getopts a:c:Dde:G:giknt:x OPT; do e) extra_append="$extra_append $OPTARG" ;; + K) + kvm=true + ;; k) extra_append="$extra_append kgdbwait" # For those who want to try KDB. @@ -66,6 +70,9 @@ if $gem5; then outdir="$(pwd)/buildroot/output.${arch}-gem5~" gem5_dir="$(pwd)/gem5/gem5" if [ "$arch" = x86_64 ]; then + if "$kvm"; then + extra_flags="$extra_flags --cpu-type=X86KvmCPU" + fi cmd="\ M5_PATH='$(pwd)/gem5/gem5-system' \ '${gem5_dir}/build/X86/gem5.opt' \ @@ -95,6 +102,9 @@ $extra_flags \ fi else buildroot_out_dir="./buildroot/output.${arch}~" + if "$kvm"; then + extra_flags="$extra_flags -enable-kvm" + fi extra_flags="$extra_flags_qemu $extra_flags" images_dir="$buildroot_out_dir/images" qemu_common="\