diff --git a/README.adoc b/README.adoc index 347b047..65663e3 100644 --- a/README.adoc +++ b/README.adoc @@ -225,7 +225,7 @@ The fast is slightly risky because your kernel module might have corrupted the k Such failures are however unlikely, and you should be fine if you don't see anything weird happening. -The safe way, is to fist quit QEMU, then rebuild the modules, root filesystem, and then reboot: +The safe way, is to fist quit QEMU, then rebuild the modules, put them in the root filesystem, and then reboot: .... ./build-modules @@ -3109,28 +3109,6 @@ link:https://git.busybox.net/busybox/tree/modutils/insmod.c?h=1_29_3[Provided by ./run --eval-busybox 'insmod /hello.ko' .... -=== modprobe - -If you are feeling fancy, you can also insert modules with: - -.... -modprobe hello -.... - -which insmods link:kernel_modules/hello.c[]. - -`modprobe` searches for modules under: - -.... -ls /lib/modules/*/extra/ -.... - -Kernel modules built from the Linux mainline tree with `CONFIG_SOME_MOD=m`, are automatically available with `modprobe`, e.g.: - -.... -modprobe dummy-irq irq=1 -.... - === myinsmod If you are feeling raw, you can insert and remove modules with our own minimal module inserter and remover! @@ -3171,11 +3149,39 @@ ____ Bibliography: https://stackoverflow.com/questions/5947286/how-to-load-linux-kernel-modules-from-c-code +=== modprobe + +Implemented as a BusyBox applet by default: https://git.busybox.net/busybox/tree/modutils/modprobe.c?h=1_29_stable + +`modprobe` searches for modules installed under: + +.... +ls /lib/modules/ +.... + +and specified in the `modules.order` file. + +This is the default install path for `CONFIG_SOME_MOD=m` modules built with `make modules_install` in the Linux kernel tree, with root path given by `INSTALL_MOD_PATH`, and therefore canonical in that sense. + +Currently, there are only two kinds of kernel modules that you can try out with `modprobe`: + +* modules built with Buildroot, see: <> +* modules built from the kernel tree itself, see: <> + +We are not installing out custom `./build-modules` modules there, because: + +* we don't know the right way. Why is there no `install` or `install_modules` target for kernel modules? ++ +This can of course be solved by running Buildroot in verbose mode, and copying whatever it is doing. ++ +See also: https://askubuntu.com/questions/299676/how-to-install-3rd-party-module-so-that-it-is-loaded-on-boot +* we would have to think how to not have to include the kernel modules twice in the root filesystem, but still have <<9p>> working for fast development as described at: <> + === kmod -https://git.kernel.org/pub/scm/utils/kernel/kmod/kmod.git +The more "reference" kernel.org implementation of `lsmod`, `insmod`, `rmmod`, etc.: https://git.kernel.org/pub/scm/utils/kernel/kmod/kmod.git -Multi-call executable that implements: `lsmod`, `insmod`, `rmmod`, and other tools on desktop distros such as Ubuntu 16.04, where e.g.: +Default implementation on desktop distros such as Ubuntu 16.04, where e.g.: .... ls -l /bin/lsmod @@ -3199,11 +3205,7 @@ contains: ii kmod 22-1ubuntu5 amd64 tools for managing Linux kernel modules .... -BusyBox also implements its own version of those executables. There are some differences. - -Buildroot also has a kmod package, but we are not using it since BusyBox' version is good enough so far. - -This page will only describe features that differ from kmod to the BusyBox implementation. +BusyBox also implements its own version of those executables, see e.g. <>. Here we will only describe features that differ from kmod to the BusyBox implementation. ==== module-init-tools @@ -4550,7 +4552,7 @@ ffffffffc0002300 B lkmc_dep [dep] This requires `CONFIG_KALLSYMS_ALL=y`. -Dependency information is stored by the kernel module build system in the `.ko` files' <>, e.g.: +Dependency information is stored by the kernel module build system in the `.ko` files' <>, e.g.: .... modinfo /dep2.ko @@ -4592,16 +4594,51 @@ TODO: what for, and at which point point does Buildroot / BusyBox generate that ===== Kernel module dependencies with modprobe -Unlike `insmod`, `modprobe` deals with kernel module dependencies for us: +Unlike `insmod`, <> deals with kernel module dependencies for us. + +First get <> working. + +Then, for example: .... -modprobe dep2 +modprobe buildroot_dep2 .... +outputs to dmesg: + +.... +42 +.... + +and then: + +.... +lsmod +.... + +outputs: + +.... +Module Size Used by Tainted: G +buildroot_dep2 16384 0 +buildroot_dep 16384 1 buildroot_dep2 +.... + +Sources: + +* link:buildroot_packages/kernel_modules/buildroot_dep.c[] +* link:buildroot_packages/kernel_modules/buildroot_dep2.c[] + Removal also removes required modules that have zero usage count: .... -modprobe -r dep2 +modprobe -r buildroot_dep2 +.... + +`modprobe` uses information from the `modules.dep` file to decide the required dependencies. That file contains: + +.... +extra/buildroot_dep2.ko: extra/buildroot_dep.ko .... Bibliography: @@ -4609,9 +4646,7 @@ Bibliography: * https://askubuntu.com/questions/20070/whats-the-difference-between-insmod-and-modprobe * https://stackoverflow.com/questions/22891705/whats-the-difference-between-insmod-and-modprobe -`modprobe` seems to use information contained in the kernel module itself for the dependencies since `modprobe dep2` still works even if we modify `modules.dep` to remove the dependency. - -==== modinfo +==== MODULE_INFO Module metadata is stored on module files at compile time. Some of the fields can be retrieved through the `THIS_MODULE` `struct module`: @@ -4700,7 +4735,7 @@ Bibliography: ==== vermagic -Vermagic is a magic string present in the kernel and on <> of kernel modules. It is used to verify that the kernel module was compiled against a compatible kernel version and relevant configuration: +Vermagic is a magic string present in the kernel and on <> of kernel modules. It is used to verify that the kernel module was compiled against a compatible kernel version and relevant configuration: .... insmod /vermagic.ko @@ -11249,6 +11284,8 @@ You won't be able to run those executables directly, but this is interesting if ==== buildroot_packages directory +Source: link:buildroot_packages/[] + Every directory inside it is a Buildroot package. Those packages get automatically added to Buildroot's `BR2_EXTERNAL`, so all you need to do is to turn them on during build, e.g.: @@ -11281,6 +11318,38 @@ Buildroot packages are convenient, but in general, if a package if very importan A custom build script can give you more flexibility: e.g. the package can be made work with other root filesystems more easily, have better <<9p>> support, and rebuild faster as it evades some Buildroot boilerplate. +===== kernel_modules package + +Source: link:buildroot_packages/kernel_modules/[] + +An example of how to kernel modules in Buildroot. + +Procedure described in detail at: https://stackoverflow.com/questions/40307328/how-to-add-a-linux-kernel-driver-module-as-a-buildroot-package/43874273#43874273 + +Usage: + +.... +rm -rf "$(./getvar out_rootfs_overlay_dir)/lib/modules" +./build-buildroot \ + --build-linux \ + --config 'BR2_PACKAGE_KERNEL_MODULES=y' \ + -- \ + kernel_modules-reconfigure \ +; +.... + +Then test one of the modules with: + +.... +./run --buildroot-linux --eval-busybox 'modprobe buildroot_hello' +.... + +Source: link:buildroot_packages/kernel_modules/buildroot_hello.c[] + +The `rm -rf` is required otherwise our `modules.order` generated by `./build-linux` and installed with `BR2_ROOTFS_OVERLAY` overwrites the Buildroot generated one. + +`./build-buildroot --build-linux` and `./run --buildroot-linux` are needed because the Buildroot kernel modules must use the Buildroot Linux kernel at build and run time. + ==== patches directory ===== patches/global diff --git a/build-linux b/build-linux index 9b2ca53..15ac24d 100755 --- a/build-linux +++ b/build-linux @@ -121,6 +121,19 @@ Configure the kernel, but don't build it. ), **common_args ) + common.run_cmd( + ( + common_make_args + + [ + 'INSTALL_MOD_PATH={}'.format(common.out_rootfs_overlay_dir), common.Newline, + 'modules_install', common.Newline, + ] + ), + **common_args + ) + # TODO: remove build and source https://stackoverflow.com/questions/13578618/what-does-build-and-source-link-do-in-lib-modules-kernel-version + # TODO Basically all kernel modules also basically leak full host paths. Just terrible. Buildroot deals with that stuff nicely for us. + # common.rmrf() def get_argparse_args(self): return { diff --git a/build-modules b/build-modules index dcfc62d..73432e5 100755 --- a/build-modules +++ b/build-modules @@ -12,7 +12,6 @@ class ModulesComponent(common.Component): parser.add_argument( '--make-args', default='', - nargs='*' ) parser.add_argument( '--host', diff --git a/buildroot_packages/kernel_modules/Config.in b/buildroot_packages/kernel_modules/Config.in new file mode 100644 index 0000000..f897377 --- /dev/null +++ b/buildroot_packages/kernel_modules/Config.in @@ -0,0 +1,5 @@ +config BR2_PACKAGE_KERNEL_MODULES + bool "kernel_modules" + depends on BR2_LINUX_KERNEL + help + https://github.com/cirosantilli/linux-kernel-module-cheat#kernel_modules-package diff --git a/buildroot_packages/kernel_modules/Makefile b/buildroot_packages/kernel_modules/Makefile new file mode 100644 index 0000000..9630974 --- /dev/null +++ b/buildroot_packages/kernel_modules/Makefile @@ -0,0 +1,10 @@ +obj-m += $(addsuffix .o, $(notdir $(basename $(filter-out %.mod.c, $(wildcard $(BR2_EXTERNAL_KERNEL_MODULES_PATH)/*.c))))) +ccflags-y := -DDEBUG -g -std=gnu99 -Werror -Wno-declaration-after-statement -Wframe-larger-than=1000000000 + +.PHONY: all clean + +all: + $(MAKE) -C '/lib/modules/$(shell uname -r)/build' M='$(PWD)' modules + +clean: + $(MAKE) -C '$(LINUX_DIR)' M='$(PWD)' clean diff --git a/buildroot_packages/kernel_modules/buildroot_dep.c b/buildroot_packages/kernel_modules/buildroot_dep.c new file mode 100644 index 0000000..79e03cc --- /dev/null +++ b/buildroot_packages/kernel_modules/buildroot_dep.c @@ -0,0 +1,18 @@ +/* https://github.com/cirosantilli/linux-kernel-module-cheat#kernel-module-dependencies */ + +#include +#include + +u32 lkmc_dep = 42; +EXPORT_SYMBOL(lkmc_dep); + +static int myinit(void) +{ + return 0; +} + +static void myexit(void) {} + +module_init(myinit) +module_exit(myexit) +MODULE_LICENSE("GPL"); diff --git a/buildroot_packages/kernel_modules/buildroot_dep2.c b/buildroot_packages/kernel_modules/buildroot_dep2.c new file mode 100644 index 0000000..dcd4103 --- /dev/null +++ b/buildroot_packages/kernel_modules/buildroot_dep2.c @@ -0,0 +1,18 @@ +/* https://github.com/cirosantilli/linux-kernel-module-cheat#kernel-module-dependencies */ + +#include +#include + +extern u32 lkmc_dep; + +static int myinit(void) +{ + pr_info("%llu\n", (long long unsigned)lkmc_dep); + return 0; +} + +static void myexit(void) {} + +module_init(myinit) +module_exit(myexit) +MODULE_LICENSE("GPL"); diff --git a/buildroot_packages/kernel_modules/buildroot_hello.c b/buildroot_packages/kernel_modules/buildroot_hello.c new file mode 100644 index 0000000..0574b5a --- /dev/null +++ b/buildroot_packages/kernel_modules/buildroot_hello.c @@ -0,0 +1,19 @@ +/* https://github.com/cirosantilli/linux-kernel-module-cheat#getting-started-natively */ + +#include +#include + +static int myinit(void) +{ + pr_info("init buildroot\n"); + return 0; +} + +static void myexit(void) +{ + pr_info("exit buildroot\n"); +} + +module_init(myinit) +module_exit(myexit) +MODULE_LICENSE("GPL"); diff --git a/buildroot_packages/kernel_modules/external.desc b/buildroot_packages/kernel_modules/external.desc new file mode 100644 index 0000000..d480845 --- /dev/null +++ b/buildroot_packages/kernel_modules/external.desc @@ -0,0 +1 @@ +name: KERNEL_MODULES diff --git a/buildroot_packages/kernel_modules/external.mk b/buildroot_packages/kernel_modules/external.mk new file mode 100644 index 0000000..a7ecb38 --- /dev/null +++ b/buildroot_packages/kernel_modules/external.mk @@ -0,0 +1,12 @@ +################################################################################ +# +# kernel_modules +# +################################################################################ + +KERNEL_MODULES_VERSION = 1.0 +KERNEL_MODULES_SITE = $(BR2_EXTERNAL_KERNEL_MODULES_PATH) +KERNEL_MODULES_SITE_METHOD = local + +$(eval $(kernel-module)) +$(eval $(generic-package)) diff --git a/common.py b/common.py index 66456a5..1ea6da0 100644 --- a/common.py +++ b/common.py @@ -243,6 +243,10 @@ inside baremetal/ and then try to use corresponding executable. default=default_build_id, help='Buildroot build ID. Allows you to keep multiple separate gem5 builds. Default: %(default)s' ) + parser.add_argument( + '--buildroot-linux', default=False, action='store_true', + help='Boot with the Buildroot Linux kernel instead of our custom built one. Mostly for sanity checks.' + ) parser.add_argument( '--crosstool-ng-build-id', default=default_build_id, help='Crosstool-NG build ID. Allows you to keep multiple separate crosstool-NG builds. Default: %(default)s' @@ -749,6 +753,8 @@ def setup(parser): this_module.buildroot_download_dir = os.path.join(this_module.buildroot_out_dir, 'download') this_module.buildroot_config_file = os.path.join(this_module.buildroot_build_dir, '.config') this_module.buildroot_build_build_dir = os.path.join(this_module.buildroot_build_dir, 'build') + this_module.buildroot_linux_build_dir = os.path.join(this_module.buildroot_build_build_dir, 'linux-custom') + this_module.buildroot_vmlinux = os.path.join(this_module.buildroot_linux_build_dir, "vmlinux") this_module.qemu_build_dir = os.path.join(this_module.out_dir, 'qemu', args.qemu_build_id) this_module.qemu_executable_basename = 'qemu-system-{}'.format(args.arch) this_module.qemu_executable = os.path.join(this_module.qemu_build_dir, '{}-softmmu'.format(args.arch), this_module.qemu_executable_basename) @@ -823,17 +829,24 @@ def setup(parser): # Linux this_module.linux_buildroot_build_dir = os.path.join(this_module.buildroot_build_build_dir, 'linux-custom') this_module.linux_build_dir = os.path.join(this_module.out_dir, 'linux', args.linux_build_id, args.arch) - this_module.vmlinux = os.path.join(this_module.linux_build_dir, "vmlinux") + this_module.lkmc_vmlinux = os.path.join(this_module.linux_build_dir, "vmlinux") if args.arch == 'arm': this_module.linux_arch = 'arm' - this_module.linux_image = os.path.join('arch', this_module.linux_arch, 'boot', 'zImage') + this_module.linux_image_prefix = os.path.join('arch', this_module.linux_arch, 'boot', 'zImage') elif args.arch == 'aarch64': this_module.linux_arch = 'arm64' - this_module.linux_image = os.path.join('arch', this_module.linux_arch, 'boot', 'Image') + this_module.linux_image_prefix = os.path.join('arch', this_module.linux_arch, 'boot', 'Image') elif args.arch == 'x86_64': this_module.linux_arch = 'x86' - this_module.linux_image = os.path.join('arch', this_module.linux_arch, 'boot', 'bzImage') - this_module.linux_image = os.path.join(this_module.linux_build_dir, linux_image) + this_module.linux_image_prefix = os.path.join('arch', this_module.linux_arch, 'boot', 'bzImage') + this_module.lkmc_linux_image = os.path.join(this_module.linux_build_dir, this_module.linux_image_prefix) + this_module.buildroot_linux_image = os.path.join(this_module.buildroot_linux_build_dir, linux_image_prefix) + if args.buildroot_linux: + this_module.vmlinux = this_module.buildroot_vmlinux + this_module.linux_image = this_module.buildroot_linux_image + else: + this_module.vmlinux = this_module.lkmc_vmlinux + this_module.linux_image = this_module.lkmc_linux_image # Kernel modules. this_module.kernel_modules_build_base_dir = os.path.join(this_module.out_dir, 'kernel_modules')