From e267435f6a4daaa38afa8589409fc569db95bfb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ciro=20Santilli=20=E5=85=AD=E5=9B=9B=E4=BA=8B=E4=BB=B6=20?= =?UTF-8?q?=E6=B3=95=E8=BD=AE=E5=8A=9F?= Date: Mon, 29 Oct 2018 00:00:00 +0000 Subject: [PATCH] fix most of GDB and remove most kernel_modules- references --- README.adoc | 89 +++++++++++++++++++++++++++++++++------------------ build-modules | 3 +- common.py | 1 + run-gdb | 2 +- run-toolchain | 2 +- 5 files changed, 61 insertions(+), 36 deletions(-) diff --git a/README.adoc b/README.adoc index fa80fc2..ce49725 100644 --- a/README.adoc +++ b/README.adoc @@ -153,6 +153,14 @@ hello /root/.profile Besides a seamless <>, this project also aims to make it effortless to modify and rebuild several major components of the system, to serve as an awesome development setup. +While developing individual components, you will most often want to use specific build commands such as `./build-linux` instead of the more generic `./build` helper. + +You can see what `./build` does with: + +.... +./build --dry-run +.... + ===== Your first Linux kernel hack Let's hack up the <>, which is an easy place to start. @@ -182,12 +190,6 @@ We could have used just `./build` as in the initial build, but doing just `./bui The link:build[`./build`] script is just a lightweight wrapper, but when you start modifying components such as the Linux kernel, it is better to run individual steps directly. -You can see that `./build` does `./build-linux` by running: - -.... -./build --dry-run -.... - So you are now officially a Linux kernel hacker, way to go! ===== Your first kernel module hack @@ -228,7 +230,13 @@ The safe way, is to fist quit QEMU, then rebuild the modules, root filesystem, a ./run --eval-busybox 'insmod /hello.ko' .... -`./build-buildroot` generates the root filesystem with the modules that we compiled at `./build-modules`. +`./build-buildroot` is required after `./build-modules` because it generates the root filesystem with the modules that we compiled at `./build-modules`. + +You can see that `./build` does that as well, by running: + +.... +./build --dry-run +.... `--eval-busybox` is optional: you could just type `insmod /hello.ko` in the terminal, but this makes it run automatically at the end of boot, and then drops you into a shell. @@ -1059,8 +1067,8 @@ Shell 2: In GDB, hit `Ctrl-C`, and note how it says: .... -scanning for modules in /linux-kernel-module-cheat//out/x86_64/buildroot/build/linux-custom -loading @0xffffffffc0000000: ../kernel_modules-1.0//timer.ko +scanning for modules in /full/path/to/linux-kernel-module-cheat/out/kernel_modules/x86_64/kernel_modules +loading @0xffffffffc0000000: /full/path/to/linux-kernel-module-cheat/out/kernel_modules/x86_64/kernel_modules/timer.ko .... That's `lx-symbols` working! Now simply: @@ -1076,9 +1084,9 @@ and we now control the callback from GDB! Just don't forget to remove your breakpoints after `rmmod`, or they will point to stale memory locations. -TODO: why does `break work_func` for `insmod kthread.ko` not break the first time I `insmod`, but breaks the second time? +TODO: why does `break work_func` for `insmod kthread.ko` not very well? Sometimes it breaks but not others. -==== GDB step debug kernel module ARM +==== GDB step debug kernel module insmodded by init on ARM TODO on `arm` 51e31cdc2933a774c2a0dc62664ad8acec1d2dbe it does not always work, and `lx-symbols` fails with the message: @@ -1130,7 +1138,7 @@ so it is close to the failing `0xbf0000cc`. `readelf`: .... -./run-toolchain readelf -- -s "$(./getvar build_dir)/kernel_modules-1.0/hello.ko" +./run-toolchain readelf -- -s "$(./getvar kernel_modules_build_subdir)/hello.ko" .... does not give any interesting hits at `cc`, no symbol was placed that far. @@ -1148,6 +1156,8 @@ Possibly asked at: ===== GDB module_init step into it +This is the best method we've found so far. + The kernel calls `module_init` synchronously, therefore it is not hard to step into that call. As of 4.16, the call happens in `do_one_initcall`, so we can do in shell 1: @@ -1204,7 +1214,7 @@ Now let's find the offset of `myinit`: .... ./run-toolchain readelf -- \ - -s "$(./getvar build_dir)/kernel_modules-1.0/fops.ko" | \ + -s "$(./getvar kernel_modules_build_subdir)/fops.ko" | \ grep myinit .... @@ -1341,6 +1351,8 @@ And then search for a line of type: === GDB step debug early boot +TODO sucessfully debu the very first instruction that the Linux kernel runs, before `start_kernel`! + Break at the very first instruction executed by QEMU: .... @@ -1360,6 +1372,8 @@ See also: https://stackoverflow.com/questions/2589845/what-are-the-first-operati <> with `--debug-flags=Exec` does show the right symbols however! So in the worst case, we can just read their source. Amazing. +TODO: try out `CONFIG_HAVE_KERNEL_UNCOMPRESSED=y` from Linux v4.19 and see if it gives us any extra visibility. + ==== GDB step debug early boot by address One possibility is to run: @@ -1434,7 +1448,7 @@ since GDB does not know that libc is loaded. * Shell 2: + .... -./run-gdb-user kernel_modules-1.0/user/sleep_forever.out main +./run-gdb-user "$(./getvar userland_build_dir)/sleep_forever.out" main .... TODO not working as of f8c0502bb2680f2dbe7c1f3d7958f60265347005, does not break. Bisect on recent QEMU and kernel. Debug by creating an executable that prints the address of `main`. @@ -1487,7 +1501,7 @@ Non-init process: * Shell 2: + .... -./run-gdb-user kernel_modules-1.0/user/myinsmod.out main +./run-gdb-user "$(./getvar userland_build_dir)/myinsmod.out" main .... * Shell 1 after the boot finishes: + @@ -1509,7 +1523,7 @@ TODO: on QEMU bfba11afddae2f7b2c1335b4e23133e9cd3c9126, it works on `x86_64` and * Shell 2: wait for boot to finish, and run: + .... -./run-gdb-user --arch arm kernel_modules-1.0/user/hello.out main +./run-gdb-user --arch arm "$(./getvar userland_build_dir)/hello.out" main .... * Shell 1: + @@ -1527,7 +1541,7 @@ We have also double checked the address with: .... ./run-toolchain --arch arm readelf -- \ - -s "$(./getvar --arch arm build_dir)/kernel_modules-1.0/fops.ko" | \ + -s "$(./getvar --arch arm kernel_modules_build_subdir)/fops.ko" | \ grep main .... @@ -1607,7 +1621,9 @@ See also: https://github.com/cirosantilli/linux-kernel-module-cheat/issues/19 === GDB view ARM system registers -Not possible as of QEMU 3.0.0 it seems: https://stackoverflow.com/questions/46415059/how-to-observe-aarch64-system-registers-in-qemu +`info all-registers` shows some of them. + +The implementation is described at: https://stackoverflow.com/questions/46415059/how-to-observe-aarch64-system-registers-in-qemu/53043044#53043044 === GDB step debug multicore @@ -1673,7 +1689,7 @@ We will run our `/sched_getaffinity.out` infinitely many time, on core 0 and cor on another shell: .... -./run-gdb-user kernel_modules-1.0/user/sched_getaffinity.out main +./run-gdb-user "$(./getvar userland_build_dir)/sched_getaffinity.out" main .... Then, inside GDB: @@ -1704,7 +1720,7 @@ TODO we then tried: and: .... -./run-gdb-user kernel_modules-1.0/user/sched_getaffinity_threads.out +./run-gdb-user "$(./getvar userland_build_dir)/sched_getaffinity_threads.out" .... to switch between two simultaneous live threads with different affinities, it just didn't break on our threads: @@ -1966,7 +1982,7 @@ Source: link:rootfs_overlay/gdbserver.sh[]. Host: .... -./run-gdbserver kernel_modules-1.0/user/myinsmod.out +./run-gdbserver "$(./getvar userland_build_dir)/myinsmod.out" .... You can find the executable with: @@ -2001,7 +2017,7 @@ An implementation overview can be found at: https://reverseengineering.stackexch As usual, different archs work with: .... -./run-gdbserver --arch arm kernel_modules-1.0/user/myinsmod.out +./run-gdbserver --arch arm "$(./getvar userland_build_dir)/myinsmod.out" .... === gdbserver BusyBox @@ -3155,7 +3171,7 @@ More concretely: git -C "$(./getvar linux_src_dir)" checkout gem5/v4.15 ./build-linux \ --arch arm \ - --custom-config-file submodules/linux/arch/arm/configs/gem5_defconfig \ + --custom-config-file "$(./getvar linux_src_dir)/arch/arm/configs/gem5_defconfig" \ --linux-build-id gem5-v4.15 \ ; git -C "$(./getvar linux_src_dir)" checkout - @@ -3224,7 +3240,7 @@ git -C "$(./getvar linux_src_dir)" checkout gem5/v4.15 ./build-linux \ --arch aarch64 \ --config-fragment linux_config/display \ - --custom-config-file submodules/linux/arch/arm64/configs/gem5_defconfig \ + --custom-config-file "$(./getvar linux_src_dir)/arch/arm64/configs/gem5_defconfig" \ --linux-build-id gem5-v4.15 \ ; git -C "$(./getvar linux_src_dir)" checkout - @@ -4558,7 +4574,7 @@ The exact same thing can be done post mortem with: ./run-toolchain gdb -- \ -batch \ -ex 'info line *(myinit+0x1d)' \ - "$(./getvar build_dir)/kernel_modules-1.0/panic.ko" \ + "$(./getvar kernel_modules_build_subdir)/panic.ko" \ ; .... @@ -5859,7 +5875,7 @@ Meaning of the flags: * `vaddr`: first virtual address of a page the belongs to the process. Notably: + .... -./run-toolchain readelf -- -l "$(./getvar build_dir)/kernel_modules-1.0/user/virt_to_phys_test.out" +./run-toolchain readelf -- -l "$(./getvar userland_build_dir)/virt_to_phys_test.out" .... + contains: @@ -6267,7 +6283,7 @@ TODO `--arch arm` and `--arch aarch64` does not count firmware instructions prop * We can also discount the instructions after `init` runs by using `readelf` to get the initial address of `init`. One easy way to do that now is to just run: + .... -./run-gdb-user kernel_modules-1.0/user/poweroff.out main +./run-gdb-user "$(./getvar userland_build_dir)/poweroff.out" main .... + And get that from the traces, e.g. if the address is `4003a0`, then we search: @@ -7649,12 +7665,12 @@ GDB step debugging is also possible with: .... cd "$(./getvar --arch arm target_dir)" -qemu-arm -g 1234 -L . ../build/kernel_modules-1.0/user/myinsmod.out +qemu-arm -g 1234 -L . "$(./getvar userland_build_dir)/myinsmod.out" ../host/usr/bin/arm-buildroot-linux-uclibcgnueabihf-gdb \ --nh \ -ex 'set architecture arm' \ -ex 'set sysroot .' \ - -ex 'file ../build/kernel_modules-1.0/user/myinsmod.out' \ + -ex "file $(./getvar userland_build_dir)/myinsmod.out" \ -ex 'target remote localhost:1234' \ -ex 'break main' \ -ex 'continue' \ @@ -8244,6 +8260,8 @@ Using text mode is another workaround if you don't need GUI features. == gem5 +Getting started at: <>. + === gem5 vs QEMU * advantages of gem5: @@ -9623,7 +9641,13 @@ We provide the following mechanisms: For example, if you decide to <> after an initial build is finished, you must <> and rebuild: .... -./build-buildroot --config 'BR2_OPTIMIZE_3=y' kernel_modules-dirclean kernel_modules-reconfigure +./build-buildroot \ + --config 'BR2_OPTIMIZE_3=y' \ + --config 'BR2_SAMPLE_PACKAGE=y' \ + -- + sample_package-dirclean \ + sample_package-reconfigure \ +; .... as explained at: https://buildroot.org/downloads/manual/manual.html#rebuild-pkg @@ -9659,9 +9683,10 @@ Then, you have two choices: .... ./build-buildroot \ --config 'BR2_OPTIMIZE_3=y' \ + --config 'BR2_SAMPLE_PACKAGE=y' \ -- \ - kernel_modules-dirclean \ - kernel_modules-reconfigure \ + sample_package-dirclean \ + sample_package-reconfigure \ ; .... + diff --git a/build-modules b/build-modules index 75de699..b924666 100755 --- a/build-modules +++ b/build-modules @@ -82,7 +82,6 @@ class ModulesComponent(common.Component): linux_dir = os.path.join('/lib', 'modules', platform.uname().release, 'build') else: linux_dir = common.linux_build_dir - build_subdir = os.path.join(build_dir, common.kernel_modules_subdir) common.run_cmd( ( [ @@ -92,7 +91,7 @@ class ModulesComponent(common.Component): 'CC={}'.format(cc), 'CROSS_COMPILE={}'.format(prefix), 'LINUX_DIR={}'.format(linux_dir), - 'M={}'.format(build_subdir), + 'M={}'.format(common.kernel_modules_build_subdir), 'OBJECT_FILES={}'.format(' '.join(object_files)), ] + verbose diff --git a/common.py b/common.py index ebd7ca4..3915f40 100644 --- a/common.py +++ b/common.py @@ -792,6 +792,7 @@ def setup(parser): # Kernel modules. this_module.kernel_modules_build_base_dir = os.path.join(this_module.out_dir, 'kernel_modules') this_module.kernel_modules_build_dir = os.path.join(this_module.kernel_modules_build_base_dir, args.arch) + this_module.kernel_modules_build_subdir = os.path.join(this_module.kernel_modules_build_dir, kernel_modules_subdir) this_module.kernel_modules_build_host_dir = os.path.join(this_module.kernel_modules_build_base_dir, 'host') this_module.userland_build_dir = os.path.join(this_module.out_dir, 'userland', args.arch) this_module.out_rootfs_overlay_dir = os.path.join(this_module.out_dir, 'rootfs_overlay', args.arch) diff --git a/run-gdb b/run-gdb index 45ac5e0..c9e7f60 100755 --- a/run-gdb +++ b/run-gdb @@ -37,7 +37,7 @@ def main(args, extra_args=None): if args.no_lxsymbols or args.baremetal is not None: lx_symbols = [] else: - lx_symbols = ['-ex', 'lx-symbols ../kernel_modules-1.0/'] + lx_symbols = ['-ex', 'lx-symbols {}'.format(common.kernel_modules_build_subdir)] if args.break_at is not None: break_at = ['-ex', 'break {}'.format(args.break_at)] else: diff --git a/run-toolchain b/run-toolchain index 1afcc4a..3ce0c95 100755 --- a/run-toolchain +++ b/run-toolchain @@ -40,7 +40,7 @@ if args.baremetal is None: image = common.vmlinux else: image = common.image -tool= common.get_toolchain_tool(args.tool, allowed_toolchains=allowed_toolchains) +tool= common.get_toolchain_tool(args.tool) if args.dry: print(tool) else: