From 57dbe81f3369e8259ef2c0e7d9a6f51b0ec31deb Mon Sep 17 00:00:00 2001 From: Ciro Santilli Date: Sun, 10 Jun 2018 13:55:45 +0100 Subject: [PATCH] lkmc_many_files: create package to test a rootfs with many files Automatically pick up packages from under packages/ into BR2_EXTERNAL. Move many subdirectory READMEs into the toplevel and link to toplevel from those subreadmes instead. --- README.adoc | 55 +++++++++++++++++-- build | 7 ++- gem5/README.adoc | 11 +--- global_patch_dir/README.adoc | 8 +-- packages/README.adoc | 1 + packages/lkmc_many_files/Config.in | 4 ++ .../lkmc_many_files}/Makefile | 0 packages/lkmc_many_files/external.desc | 1 + packages/lkmc_many_files/external.mk | 30 ++++++++++ packages/lkmc_many_files/lkmc_many_files.c | 7 +++ .../sample_package}/Config.in | 0 packages/sample_package/Makefile | 16 ++++++ .../sample_package}/external.desc | 0 .../sample_package}/external.mk | 0 .../sample_package}/sample_package.c | 0 rootfs_overlay/README.adoc | 9 +-- 16 files changed, 118 insertions(+), 31 deletions(-) create mode 100644 packages/README.adoc create mode 100644 packages/lkmc_many_files/Config.in rename {sample_package => packages/lkmc_many_files}/Makefile (100%) create mode 100644 packages/lkmc_many_files/external.desc create mode 100644 packages/lkmc_many_files/external.mk create mode 100644 packages/lkmc_many_files/lkmc_many_files.c rename {sample_package => packages/sample_package}/Config.in (100%) create mode 100644 packages/sample_package/Makefile rename {sample_package => packages/sample_package}/external.desc (100%) rename {sample_package => packages/sample_package}/external.mk (100%) rename {sample_package => packages/sample_package}/sample_package.c (100%) diff --git a/README.adoc b/README.adoc index c97dba9..404f7c1 100644 --- a/README.adoc +++ b/README.adoc @@ -6578,7 +6578,7 @@ First, see if you can't get away without actually adding a new package, for exam If none of those methods are flexible enough for you, create a new package as follows: -* use link:sample_package[] as a starting point +* use link:packages/sample_package[] as a starting point * fork this repository, and modify that package to do what you want * read the comments on that package to get an idea of how to start * check the main manual for more complicated things: https://buildroot.org/downloads/manual/manual.html @@ -6586,6 +6586,7 @@ If none of those methods are flexible enough for you, create a new package as fo + .... ./build -- sample_package-reconfigure +./run -F '/sample_package.out' .... + if you make any changes to that package after the initial build: <> @@ -6744,6 +6745,15 @@ We have applied link:https://github.com/cirosantilli/buildroot/commit/983fe7910a which contributed to a large part of the slowness. +Test how Buildroot deals with many files with: + +.... +./build -B BR2_PACKAGE_LKMC_MANY_FILES=y -- lkmc_many_files-reconfigure |& ts -i '%.s' +./build |& ts -i '%.s' +.... + +and notice how the second build, which does not rebuilt the package at all, still gets stuck in the `RPATH` check forever without our Buildroot patch. + === Report upstream bugs When asking for help on upstream repositories outside of this repository, you will need to provide the commands that you are running in detail without referencing our scripts. @@ -7206,20 +7216,55 @@ Every `.patch` file in this directory gets applied to Buildroot before anything This directory has been made kind of useless when we decided to use our own Buildroot fork, but we've kept the functionality just in case we someday go back to upstream Buildroot. -:leveloffset: +3 +==== gem5 directory -include::global_patch_dir/README.adoc[] +We Build the gem5 emulator through Buildroot basically just to reuse its timestamping system to avoid rebuilds. + +There is also the `m5` tool that we must build through Buildroot ans install on the root filesystem, but we could just make two separate builds. + +This directory has the following structure: + +==== global_patch_dir + +Has the following structure: + +.... +package-name/00001-do-something.patch +.... + +The patches are then applied to the corresponding packages before build. + +Uses `BR2_GLOBAL_PATCH_DIR`. + +==== packages directory + +Any directory in that subdirectory is added to `BR2_EXTERNAL` and become available to the build. + +==== rootfs_overlay + +Copied into the target filesystem. + +We use it for: + +* customized configuration files +* userland module test scripts that don't need to be compiled. ++ +C files for example need compilation, and must go through the regular package system, e.g. through link:kernel_module/user[]. + +:leveloffset: +3 include::kernel_module/README.adoc[] include::kernel_module/user/README.adoc[] -include::rootfs_overlay/README.adoc[] - :leveloffset: -3 === Script man pages +These appear when you do `./some-script -h`. + +We have to keep them as separate files from the README for that to be possible. + :leveloffset: +3 include::build-usage.adoc[] diff --git a/build b/build index 2576bd3..fc3914e 100755 --- a/build +++ b/build @@ -137,7 +137,12 @@ if "$configure"; then for p in $(find "${root_dir}/buildroot_patches/" -maxdepth 1 -name '*.patch' -print); do patch -N -r - -p 1 < "$p" || : done - make O="$buildroot_out_dir" BR2_EXTERNAL="../kernel_module:../gem5:../parsec-benchmark:../sample_package" "$defconfig" + br2_external='../kernel_module:../gem5:../parsec-benchmark' + packages_dir="${root_dir}/packages" + for package_dir in "${packages_dir}"/*/; do + br2_external="${br2_external}:../packages/$(basename "${package_dir}")" + done + make O="$buildroot_out_dir" BR2_EXTERNAL="$br2_external" "$defconfig" # TODO Can't get rid of these for now. # http://stackoverflow.com/questions/44078245/is-it-possible-to-use-config-fragments-with-buildroots-config for config_fragment in $config_fragments; do diff --git a/gem5/README.adoc b/gem5/README.adoc index 82e0c86..16bc6dc 100644 --- a/gem5/README.adoc +++ b/gem5/README.adoc @@ -1,10 +1 @@ -= gem5 - -We Build gem5 as a Buildroot package for the following reasons: - -* if we build in tree without Buildroot, we still want to build the `m5` tool through Buildroot. But then Buildroot rsyncs the build for all archs into `output/build`, which takes quite some disk space. There seems to be no way to exclude some packages from the sync. -* it allows us to reuse Buildroot's: -** configuration system, which reduces duplication: set the gem5 package option, everything just gets built -** timestamping system, which could save a few seconds on rebuilds, since we then don't ever run `scons` when gem5 is not modified - -What is preventing upstreaming to Buildroot: a minimal kernel config like the ones QEMU has. +https://github.com/cirosantilli/linux-kernel-module-cheat#gem5-directory diff --git a/global_patch_dir/README.adoc b/global_patch_dir/README.adoc index 7da6d73..a680752 100644 --- a/global_patch_dir/README.adoc +++ b/global_patch_dir/README.adoc @@ -1,7 +1 @@ -= global_patch_dir - -This directory has the following structure: - -.... -package-name/00001-do-something.patch -.... +https://github.com/cirosantilli/linux-kernel-module-cheat#global_patch_dir diff --git a/packages/README.adoc b/packages/README.adoc new file mode 100644 index 0000000..f64a50e --- /dev/null +++ b/packages/README.adoc @@ -0,0 +1 @@ +https://github.com/cirosantilli/linux-kernel-module-cheat#packages_directory diff --git a/packages/lkmc_many_files/Config.in b/packages/lkmc_many_files/Config.in new file mode 100644 index 0000000..8c972ec --- /dev/null +++ b/packages/lkmc_many_files/Config.in @@ -0,0 +1,4 @@ +config BR2_PACKAGE_LKMC_MANY_FILES + bool "lkmc_many_files" + help + Create a bunch of files in target/ to benchmark Buildroot. diff --git a/sample_package/Makefile b/packages/lkmc_many_files/Makefile similarity index 100% rename from sample_package/Makefile rename to packages/lkmc_many_files/Makefile diff --git a/packages/lkmc_many_files/external.desc b/packages/lkmc_many_files/external.desc new file mode 100644 index 0000000..33f2752 --- /dev/null +++ b/packages/lkmc_many_files/external.desc @@ -0,0 +1 @@ +name: LKMC_MANY_FILES diff --git a/packages/lkmc_many_files/external.mk b/packages/lkmc_many_files/external.mk new file mode 100644 index 0000000..b79ff00 --- /dev/null +++ b/packages/lkmc_many_files/external.mk @@ -0,0 +1,30 @@ +################################################################################ +# +# lkmc_many_files +# +################################################################################ + +LKMC_MANY_FILES_VERSION = 1.0 +LKMC_MANY_FILES_SITE = $(BR2_EXTERNAL_LKMC_MANY_FILES_PATH) +LKMC_MANY_FILES_SITE_METHOD = local + +define LKMC_MANY_FILES_BUILD_CMDS + # D contains the source code of this package. + # TARGET_CONFIGURE_OPTS contains several common options such as CFLAGS and LDFLAGS. + $(MAKE) -C '$(@D)' $(TARGET_CONFIGURE_OPTS) +endef + +define LKMC_MANY_FILES_INSTALL_TARGET_CMDS + # Anything put inside TARGET_DIR will end up on the guest relative to the root directory. + i=0; \ + while [ $$i -le 128 ]; do \ + j=0; \ + while [ $$j -le 128 ]; do \ + $(INSTALL) -D -m 0755 $(@D)/lkmc_many_files.out $(TARGET_DIR)/lkmc_many_files/$${i}_$${j}.out; \ + j=$$(($$j+1)); \ + done; \ + i=$$(($$i+1)); \ + done +endef + +$(eval $(generic-package)) diff --git a/packages/lkmc_many_files/lkmc_many_files.c b/packages/lkmc_many_files/lkmc_many_files.c new file mode 100644 index 0000000..883ec6f --- /dev/null +++ b/packages/lkmc_many_files/lkmc_many_files.c @@ -0,0 +1,7 @@ +#include +#include + +int main(void) { + puts("hello lkmc_many_files"); + return EXIT_SUCCESS; +} diff --git a/sample_package/Config.in b/packages/sample_package/Config.in similarity index 100% rename from sample_package/Config.in rename to packages/sample_package/Config.in diff --git a/packages/sample_package/Makefile b/packages/sample_package/Makefile new file mode 100644 index 0000000..3a0b40c --- /dev/null +++ b/packages/sample_package/Makefile @@ -0,0 +1,16 @@ +.PHONY: all clean + +CFLAGS_EXTRA ?= -fopenmp -std=c99 -Wall -Werror -Wextra +IN_EXT ?= .c +LIBS := +OUT_EXT ?= .out + +OUTS := $(addsuffix $(OUT_EXT), $(basename $(wildcard *$(IN_EXT)))) + +all: $(OUTS) + +%$(OUT_EXT): %$(IN_EXT) + $(CC) $(CFLAGS) $(CFLAGS_EXTRA) -o '$@' '$<' $(LIBS) + +clean: + rm -f *'$(OUT_EXT)' diff --git a/sample_package/external.desc b/packages/sample_package/external.desc similarity index 100% rename from sample_package/external.desc rename to packages/sample_package/external.desc diff --git a/sample_package/external.mk b/packages/sample_package/external.mk similarity index 100% rename from sample_package/external.mk rename to packages/sample_package/external.mk diff --git a/sample_package/sample_package.c b/packages/sample_package/sample_package.c similarity index 100% rename from sample_package/sample_package.c rename to packages/sample_package/sample_package.c diff --git a/rootfs_overlay/README.adoc b/rootfs_overlay/README.adoc index 4d99b83..97100ed 100644 --- a/rootfs_overlay/README.adoc +++ b/rootfs_overlay/README.adoc @@ -1,8 +1 @@ -= rootfs_overlay - -This directory copied into the target filesystem. - -We use it to for things like: - -* customized configuration files -* userland module test scripts +https://github.com/cirosantilli/linux-kernel-module-cheat#rootfs_overlay