From 2092180ac841fc639e6fb1fed6c6feb13cf6eaff Mon Sep 17 00:00:00 2001 From: Ciro Santilli Date: Sun, 25 Feb 2018 20:27:35 +0000 Subject: [PATCH] Update GEM5 to master to fix x86 boot via 3feeb994ae613fd6b3734c1a991285b2ecbd1946 See also: http://gem5-users.gem5.narkive.com/CQh9J6fU/fail-to-run-fs-mode-with-linux-kernel-v4-8-13-and-ubuntu-image-16-04-1 Start sketching how to tweak system parameters, number of CPUs works, caches don't. --- README.adoc | 258 ++++++++++------------------------------------------ build | 13 +-- build-gem5 | 2 +- gem5-shell | 2 +- gem5/gem5 | 2 +- run | 14 ++- rungdb | 2 +- 7 files changed, 66 insertions(+), 227 deletions(-) diff --git a/README.adoc b/README.adoc index b39ff89..6f30f8d 100644 --- a/README.adoc +++ b/README.adoc @@ -938,7 +938,7 @@ no module object found for '' ./build -a mips64 .... -Keep in mind that MIPS has the worst support our architectures due to the smaller community. Patches welcome as usual. +Keep in mind that MIPS has the worst support compared to our other architectures due to the smaller community. Patches welcome as usual. TODOs: @@ -1651,6 +1651,54 @@ External open source benchmarks. We will try to create Buildroot packages for th * http://parsec.cs.princeton.edu/ Mentioned on docs: http://gem5.org/PARSEC_benchmarks * http://www.m5sim.org/Splash_benchmarks +====== GEM5 change system parameters + +Besides optimizing a program for a given CPU setup, chip developers can also do the inverse, and optimize the chip for a given benchmark! + +* Number of CPUs: ++ +.... +./run -a arm -c 2 -g +.... ++ +Check with: ++ +.... +cat /proc/cpuinfo +getconf _NPROCESSORS_CONF +.... +* Cache sizes: ++ +.... +./run -a arm -c 2 -g -- --caches --l2cache +.... ++ +Check with: ++ +.... +getconf -a | grep CACHE +cat /sys/devices/system/cpu/cpu0/cache/index2/level +cat /sys/devices/system/cpu/cpu0/cache/index2/size +.... ++ +Checking `level` is needed, for example `level0` and `level1` represented the same level on Linux 4.15. ++ +TODO: check not working. `/sys` files don't exist, and `getconf` values empty. Ticket inbound. Related: +** http://gem5-users.gem5.narkive.com/4xVBlf3c/verify-cache-configuration +* Memory latency: TODO +* Disk and network latency: TODO +* Clock frequency: TODO. Check with: + +.... +m5 resetstats && sleep 10 && m5 dumpstats +.... + +and then: + +.... +grep numCycles m5out/stats.txt +.... + ===== GEM5 kernel command line parameters Analogous <>: @@ -1886,219 +1934,11 @@ I have also tried to copy the exact same kernel command line options used by QEM ==== GEM5 x86 -TODO didn't get it working yet. - -Related threads: - -* https://www.mail-archive.com/gem5-users@gem5.org/msg11384.html -* https://stackoverflow.com/questions/37906425/booting-gem5-x86-ubuntu-full-system-simulation -* http://www.lowepower.com/jason/creating-disk-images-for-gem5.html claims to have a working config for x86_64 kernel 4.8.13 - -===== GEM5 x86 best attempt - .... ./configure && ./build -a x86_64 -g ./run -a x86_64 -g .... -telnet: - -.... -i8042: PNP: No PS/2 controller found. -i8042: Probing ports directly. -Connection closed by foreign host. -.... - -stdout: - -.... -panic: Data written for unrecognized command 0xd1 -Memory Usage: 1235908 KBytes -Program aborted at tick 427627410500 -.... - -The same failure happens if we use the working QEMU Linux kernel, and / or if we use the kernel 4.8.13 as proposed in lowepower's post.. - -If we look a bit into the source, the panic message comes from `i8042.cc`, and on the header we see that the missing command is: - -.... - WriteOutputPort = 0xD1, -.... - -The kernel was compiled with `CONFIG_SERIO_I8042=y`, I didn't dare disable it yet. The Linux kernel driver has no `grep` hits for either of `0xd1` nor `output.?port`, it must be using some random bitmask to build it then. - -This byte is documented at http://wiki.osdev.org/%228042%22_PS/2_Controller, as usual :-) - -There are also a bunch of `i8042` kernel CLI options, I tweaked all of them but nothing. - -===== GDM5 x86 working baseline with magic image - -Working x86 with the pre-built magic image with an ancient 2.6.22.9 kernel starting point: - -.... -sudo mkdir -p /dist/m5/system -sudo chmod 777 /dist/m5/system -cd /dist/m5/system -# Backed up at: -# https://github.com/cirosantilli/media/releases/tag/gem5 -wget http://www.gem5.org/dist/current/x86/x86-system.tar.bz2 -tar xvf x86-system.tar.bz2 -cd x86-system -dd if=/dev/zero of=disks/linux-bigswap2.img bs=1024 count=65536 -mkswap disks/linux-bigswap2.img -cd .. - -git clone https://gem5.googlesource.com/public/gem5 -cd gem5 -git checkout da79d6c6cde0fbe5473ce868c9be4771160a003b -scons -j$(nproc) build/X86/gem5.opt -# That old blob has wrong filenames. -./build/X86/gem5.opt \ - -d /tmp/output \ - --disk-image=/dist/m5/system/disks/linux-x86.img \ - --kernel=/dist/m5/system/binaries/x86_64-vmlinux-2.6.22.9 \ - configs/example/fs.py -.... - -On another shell: - -.... -telnet localhost 3456 -.... - -===== GEM5 unmodified Buildroot images 2 - -bzImage fails, so we always try with vmlinux obtained from inside build/. - -rootfs.ext2 and vmlinux from 670366caaded57d318b6dbef34e863e3b30f7f29ails as: - -Fails as: - -.... -Global frequency set at 1000000000000 ticks per second -warn: DRAM device capacity (8192 Mbytes) does not match the address range assigned (512 Mbytes) -info: kernel located at: /data/git/linux-kernel-module-cheat/buildroot/output.x86_64~/build/linux-custom/vmlinux -Listening for com_1 connection on port 3456 - 0: rtc: Real-time clock set to Sun Jan 1 00:00:00 2012 -0: system.remote_gdb.listener: listening for remote gdb #0 on port 7000 -warn: Reading current count from inactive timer. -**** REAL SIMULATION **** -info: Entering event queue @ 0. Starting simulation... -warn: instruction 'fninit' unimplemented -warn: Don't know what interrupt to clear for console. -12516923000: system.pc.com_1.terminal: attach terminal 0 -warn: i8042 "Write output port" command not implemented. -warn: i8042 "Write keyboard output buffer" command not implemented. -warn: Write to unknown i8042 (keyboard controller) command port. -hack: Assuming logical destinations are 1 << id. -panic: Resetting mouse wrap mode unimplemented. -Memory Usage: 1003456 KBytes -Program aborted at tick 632745027500 ---- BEGIN LIBC BACKTRACE --- -./build/X86/gem5.opt(_Z15print_backtracev+0x15)[0x12b8165] -./build/X86/gem5.opt(_Z12abortHandleri+0x39)[0x12c32f9] -/lib/x86_64-linux-gnu/libpthread.so.0(+0x11390)[0x7fe047a71390] -/lib/x86_64-linux-gnu/libc.so.6(gsignal+0x38)[0x7fe046601428] -/lib/x86_64-linux-gnu/libc.so.6(abort+0x16a)[0x7fe04660302a] -./build/X86/gem5.opt(_ZN6X86ISA8PS2Mouse11processDataEh+0xf5)[0x1391095] -./build/X86/gem5.opt(_ZN6X86ISA5I80425writeEP6Packet+0x51c)[0x13927ec] -./build/X86/gem5.opt(_ZN7PioPort10recvAtomicEP6Packet+0x66)[0x139f7b6] -./build/X86/gem5.opt(_ZN15NoncoherentXBar10recvAtomicEP6Packets+0x200)[0x1434af0] -./build/X86/gem5.opt(_ZN6Bridge15BridgeSlavePort10recvAtomicEP6Packet+0x5d)[0x140ee9d] -./build/X86/gem5.opt(_ZN12CoherentXBar10recvAtomicEP6Packets+0x3e7)[0x1415b77] -./build/X86/gem5.opt(_ZN15AtomicSimpleCPU8writeMemEPhjm5FlagsIjEPm+0x327)[0xa790a7] -./build/X86/gem5.opt(_ZN17SimpleExecContext8writeMemEPhjm5FlagsIjEPm+0x19)[0xa856b9] -./build/X86/gem5.opt(_ZNK10X86ISAInst2St7executeEP11ExecContextPN5Trace10InstRecordE+0x235)[0xfb9e65] -./build/X86/gem5.opt(_ZN15AtomicSimpleCPU4tickEv+0x23c)[0xa784fc] -./build/X86/gem5.opt(_ZN10EventQueue10serviceOneEv+0xc5)[0x12be0d5] -./build/X86/gem5.opt(_Z9doSimLoopP10EventQueue+0x38)[0x12cd558] -./build/X86/gem5.opt(_Z8simulatem+0x2eb)[0x12cdbdb] -./build/X86/gem5.opt(_ZZN8pybind1112cpp_function10initializeIRPFP22GlobalSimLoopExitEventmES3_ImEINS_4nameENS_5scopeENS_7siblingENS_5arg_vEEEEvOT_PFT0_DpT1_EDpRKT2_ENUlRNS_6detail13function_callEE1_4_FUNESO_+0x41)[0x13fca11] -./build/X86/gem5.opt(_ZN8pybind1112cpp_function10dispatcherEP7_objectS2_S2_+0x8d8)[0xfc7398] -/usr/lib/x86_64-linux-gnu/libpython2.7.so.1.0(PyEval_EvalFrameEx+0x7852)[0x7fe047d3b552] -/usr/lib/x86_64-linux-gnu/libpython2.7.so.1.0(PyEval_EvalCodeEx+0x85c)[0x7fe047e6501c] -/usr/lib/x86_64-linux-gnu/libpython2.7.so.1.0(PyEval_EvalFrameEx+0x6ffd)[0x7fe047d3acfd] -/usr/lib/x86_64-linux-gnu/libpython2.7.so.1.0(PyEval_EvalFrameEx+0x7124)[0x7fe047d3ae24] -/usr/lib/x86_64-linux-gnu/libpython2.7.so.1.0(PyEval_EvalFrameEx+0x7124)[0x7fe047d3ae24] -/usr/lib/x86_64-linux-gnu/libpython2.7.so.1.0(PyEval_EvalCodeEx+0x85c)[0x7fe047e6501c] -/usr/lib/x86_64-linux-gnu/libpython2.7.so.1.0(PyEval_EvalCode+0x19)[0x7fe047d33b89] -/usr/lib/x86_64-linux-gnu/libpython2.7.so.1.0(PyEval_EvalFrameEx+0x613b)[0x7fe047d39e3b] -/usr/lib/x86_64-linux-gnu/libpython2.7.so.1.0(PyEval_EvalCodeEx+0x85c)[0x7fe047e6501c] -/usr/lib/x86_64-linux-gnu/libpython2.7.so.1.0(PyEval_EvalFrameEx+0x6ffd)[0x7fe047d3acfd] -/usr/lib/x86_64-linux-gnu/libpython2.7.so.1.0(PyEval_EvalCodeEx+0x85c)[0x7fe047e6501c] -/usr/lib/x86_64-linux-gnu/libpython2.7.so.1.0(PyEval_EvalCode+0x19)[0x7fe047d33b89] ---- END LIBC BACKTRACE --- -Aborted (core dumped) -.... - -Boot goes quite far, on telnet: - -.... -ALSA device list: - No soundcards found. -.... - -So just looks like we have to disable some Linux configs which GEM5 does not support... so fragile. - -===== GEM5 x86 copy upstream 2.6 configs on 4.9 kernel - -The magic image provides its kernel configurations, so let's try that. - -The configs are present at: - -.... -wget http://www.gem5.org/dist/current/x86/config-x86.tar.bz2 -.... - -backed up at: https://github.com/cirosantilli/media/releases/tag/gem5 - -Copy `linux-2.6.22.9` into the kernel tree as `.config`, `git checkout v4.9.6`, `make olddefconfig`, `make`, then use the Buildroot filesystem as above, failure: - -.... -panic: Invalid IDE control register offset: 0 -Memory Usage: 931272 KBytes -Program aborted at tick 382834812000 ---- BEGIN LIBC BACKTRACE --- -./build/X86/gem5.opt(_Z15print_backtracev+0x15)[0x12b8165] -./build/X86/gem5.opt(_Z12abortHandleri+0x39)[0x12c32f9] -/lib/x86_64-linux-gnu/libpthread.so.0(+0x11390)[0x7fc2081c6390] -/lib/x86_64-linux-gnu/libc.so.6(gsignal+0x38)[0x7fc206d56428] -/lib/x86_64-linux-gnu/libc.so.6(abort+0x16a)[0x7fc206d5802a] -./build/X86/gem5.opt(_ZN7IdeDisk11readControlEmiPh+0xd9)[0xa96989] -./build/X86/gem5.opt(_ZN13IdeController14dispatchAccessEP6Packetb+0x53e)[0xa947ae] -./build/X86/gem5.opt(_ZN13IdeController4readEP6Packet+0xe)[0xa94a5e] -./build/X86/gem5.opt(_ZN7PioPort10recvAtomicEP6Packet+0x3f)[0x139f78f] -./build/X86/gem5.opt(_ZN15NoncoherentXBar10recvAtomicEP6Packets+0x200)[0x1434af0] -./build/X86/gem5.opt(_ZN6Bridge15BridgeSlavePort10recvAtomicEP6Packet+0x5d)[0x140ee9d] -./build/X86/gem5.opt(_ZN12CoherentXBar10recvAtomicEP6Packets+0x3e7)[0x1415b77] -./build/X86/gem5.opt(_ZN15AtomicSimpleCPU7readMemEmPhj5FlagsIjE+0x3ef)[0xa780ef] -./build/X86/gem5.opt(_ZN17SimpleExecContext7readMemEmPhj5FlagsIjE+0x11)[0xa85671] -./build/X86/gem5.opt(_ZNK10X86ISAInst2Ld7executeEP11ExecContextPN5Trace10InstRecordE+0x130)[0xfb6c00] -./build/X86/gem5.opt(_ZN15AtomicSimpleCPU4tickEv+0x23c)[0xa784fc] -./build/X86/gem5.opt(_ZN10EventQueue10serviceOneEv+0xc5)[0x12be0d5] -./build/X86/gem5.opt(_Z9doSimLoopP10EventQueue+0x38)[0x12cd558] -./build/X86/gem5.opt(_Z8simulatem+0x2eb)[0x12cdbdb] -./build/X86/gem5.opt(_ZZN8pybind1112cpp_function10initializeIRPFP22GlobalSimLoopExitEventmES3_ImEINS_4nameENS_5scopeENS_7siblingENS_5arg_vEEEEvOT_PFT0_DpT1_EDpRKT2_ENUlRNS_6detail13function_callEE1_4_FUNESO_+0x41)[0x13fca11] -./build/X86/gem5.opt(_ZN8pybind1112cpp_function10dispatcherEP7_objectS2_S2_+0x8d8)[0xfc7398] -/usr/lib/x86_64-linux-gnu/libpython2.7.so.1.0(PyEval_EvalFrameEx+0x7852)[0x7fc208490552] -/usr/lib/x86_64-linux-gnu/libpython2.7.so.1.0(PyEval_EvalCodeEx+0x85c)[0x7fc2085ba01c] -/usr/lib/x86_64-linux-gnu/libpython2.7.so.1.0(PyEval_EvalFrameEx+0x6ffd)[0x7fc20848fcfd] -/usr/lib/x86_64-linux-gnu/libpython2.7.so.1.0(PyEval_EvalFrameEx+0x7124)[0x7fc20848fe24] -/usr/lib/x86_64-linux-gnu/libpython2.7.so.1.0(PyEval_EvalFrameEx+0x7124)[0x7fc20848fe24] -/usr/lib/x86_64-linux-gnu/libpython2.7.so.1.0(PyEval_EvalCodeEx+0x85c)[0x7fc2085ba01c] -/usr/lib/x86_64-linux-gnu/libpython2.7.so.1.0(PyEval_EvalCode+0x19)[0x7fc208488b89] -/usr/lib/x86_64-linux-gnu/libpython2.7.so.1.0(PyEval_EvalFrameEx+0x613b)[0x7fc20848ee3b] -/usr/lib/x86_64-linux-gnu/libpython2.7.so.1.0(PyEval_EvalCodeEx+0x85c)[0x7fc2085ba01c] -/usr/lib/x86_64-linux-gnu/libpython2.7.so.1.0(PyEval_EvalFrameEx+0x6ffd)[0x7fc20848fcfd] -/usr/lib/x86_64-linux-gnu/libpython2.7.so.1.0(PyEval_EvalCodeEx+0x85c)[0x7fc2085ba01c] ---- END LIBC BACKTRACE --- -Aborted (core dumped) -.... - -===== GEM5 x86 use upstream 2.6 configs and 2.6 kernel - -If we checkout to the ancient kernel `v2.6.22.9`, it fails to compile with modern GNU make 4.1: https://stackoverflow.com/questions/35002691/makefile-make-clean-why-getting-mixed-implicit-and-normal-rules-deprecated-s lol - == Failed action === Record and replay diff --git a/build b/build index 1c35ea7..13e9d0f 100755 --- a/build +++ b/build @@ -48,16 +48,7 @@ case "$arch" in defconfig=qemu_x86_64_defconfig ;; arm) - if "$gem5"; then - # Ideally we should use a custom clean config here. - # But let's just use this one as a starting point for now. - defconfig=qemu_arm_versatile_defconfig - post_script_args="$post_script_args -n" - else - # qemu_arm_vexpress_defconfig required a newer QEMU than 2.0.0 on a Ubuntu host. - # so let's stick to versatile for now. - defconfig=qemu_arm_versatile_defconfig - fi + defconfig=qemu_arm_versatile_defconfig ;; aarch64) defconfig=qemu_aarch64_virt_defconfig @@ -69,6 +60,8 @@ esac arch_dir="$arch" if "$gem5"; then arch_dir="${arch}-gem5" + # Networking was not working, so disable it to speed things up. + post_script_args="$post_script_args -n" fi cd kernel_module diff --git a/build-gem5 b/build-gem5 index e06c46e..eb23f8e 100755 --- a/build-gem5 +++ b/build-gem5 @@ -1,6 +1,6 @@ #/usr/bin/env bash set -eu -arch=arm +arch=x86_64 while getopts 'a:' OPT; do case "$OPT" in a) diff --git a/gem5-shell b/gem5-shell index c67758c..7150b8b 100755 --- a/gem5-shell +++ b/gem5-shell @@ -1,2 +1,2 @@ #!/usr/bin/env bash -telnet localhost 3456 +telnet localhost "${1:-3456}" diff --git a/gem5/gem5 b/gem5/gem5 index da79d6c..fbe6307 160000 --- a/gem5/gem5 +++ b/gem5/gem5 @@ -1 +1 @@ -Subproject commit da79d6c6cde0fbe5473ce868c9be4771160a003b +Subproject commit fbe63074e3a8128bdbe1a5e8f6509c565a3abbd4 diff --git a/run b/run index b10ac76..b6fcc49 100755 --- a/run +++ b/run @@ -4,6 +4,7 @@ set -e # CLI handling. arch=x86_64 +cpus=1 debug_vm='' kgdb=false nographic=false @@ -17,11 +18,14 @@ gem5=false gem5opts='' initrd=false root='' -while getopts a:Dde:G:giknt:x OPT; do +while getopts a:c:Dde:G:giknt:x OPT; do case "$OPT" in a) arch="$OPTARG" ;; + c) + cpus="$OPTARG" + ;; d) extra_flags="$extra_flags -S -s" ;; @@ -66,9 +70,10 @@ M5_PATH='$(pwd)/gem5/gem5-system' \ '${gem5_dir}/build/X86/gem5.opt' \ ${gem5opts} \ '${gem5_dir}/configs/example/fs.py' \ +--command-line='earlyprintk=ttyS0 console=ttyS0 lpj=7999923 root=/dev/hda $extra_append' \ --disk-image='${outdir}/images/rootfs.ext2' \ --kernel='${outdir}/build/linux-custom/vmlinux' \ ---root-device=/dev/sda \ +--num-cpus=${cpus} \ $extra_flags \ " elif [ "$arch" = arm ] || [ "$arch" = aarch64 ]; then @@ -80,9 +85,10 @@ ${gem5opts} \ '${gem5_dir}/configs/example/fs.py' \ --command-line='earlyprintk=pl011,0x1c090000 console=ttyAMA0 lpj=19988480 rw loglevel=8 mem=512MB root=/dev/sda $extra_append' \ --disk-image='${outdir}/images/rootfs.ext2' \ ---dtb-file='${gem5_dir}/system/arm/dt/$([ "$arch" = arm ] && echo armv7_gem5_v1_1cpu || echo armv8_gem5_v1_1cpu).dtb' \ +--dtb-file='${gem5_dir}/system/arm/dt/$([ "$arch" = arm ] && echo "armv7_gem5_v1_${cpus}cpu" || echo "armv8_gem5_v1_${cpus}cpu").dtb' \ --kernel='${outdir}/build/linux-custom/vmlinux' \ --machine-type=VExpress_GEM5_V1 \ +--num-cpus=${cpus} \ $extra_flags \ " fi @@ -95,7 +101,7 @@ $buildroot_out_dir/host/usr/bin/qemu-system-${arch} \ -m 128M \ -monitor telnet::45454,server,nowait \ -netdev user,hostfwd=tcp::45455-:45455,id=net0 \ --smp 1 \ +-smp $cpus \ " if $initrd; then extra_flags="$extra_flags -initrd '${images_dir}/rootfs.cpio'" diff --git a/rungdb b/rungdb index 4ed7cfe..6202378 100755 --- a/rungdb +++ b/rungdb @@ -80,5 +80,5 @@ $brk \ esac fi cmd="$cmd $after" -echo "$cmd" +echo "$cmd" | tee rungdb.log eval "$cmd"