From 58de3f7243016c052ad080f82dd757d61878219b Mon Sep 17 00:00:00 2001 From: Ciro Santilli Date: Sat, 8 Sep 2018 23:20:08 +0100 Subject: [PATCH] port bisect-linux-boot-gem5 and zip-img fix m5 build and a bunch other things --- README.adoc | 20 +++++++++---- bisect-linux-boot-gem5 | 56 +++++++++++++++++++---------------- build | 29 +++++++++--------- buildroot_config/default | 39 ++++++++++++------------ common.py | 62 +++++++++++++++++++++------------------ packages/gem5/external.mk | 6 ++-- release | 20 ++++++------- run | 6 ++-- rungdb-user | 3 +- zip-img | 27 ++++++++++------- 10 files changed, 147 insertions(+), 121 deletions(-) diff --git a/README.adoc b/README.adoc index 615da71..6c99633 100644 --- a/README.adoc +++ b/README.adoc @@ -3335,7 +3335,7 @@ cat /proc/version or in the source: .... -cd linux +cd "$(./getvar linux_src_dir)" git log | grep -E ' Linux [0-9]+\.' | head .... @@ -7192,6 +7192,14 @@ Getting everything to work required careful choice of QEMU command line options: Peter Maydell said potentially not possible nicely as of August 2018: https://stackoverflow.com/questions/51747744/how-to-run-a-qemu-monitor-command-from-inside-the-guest/51764110#51764110 +It is also worth looking into the QEMU Guest Agent tool `qemu-gq` that can be enabled with: + +.... +./build -B 'BR2_PACKAGE_QEMU=y' +.... + +See also: https://superuser.com/questions/930588/how-to-pass-commands-noninteractively-to-running-qemu-from-the-guest-qmp-via-te + ==== QEMU monitor from GDB When doing <> it is possible to send QEMU monitor commands through the GDB `monitor` command, which saves you the trouble of opening yet another shell. @@ -8151,7 +8159,7 @@ If you want to remove PARSEC later, Buildroot doesn't provide an automated packa .... rm -rf \ - ./out/common/dl/parsec-* \ + "$(./getvar dl_dir)"/parsec-* \ "$(./getvar buildroot_out_dir)"/build/parsec-* \ "$(./getvar buildroot_out_dir)"/build/packages-file-list.txt \ "$(./getvar buildroot_out_dir)"/images/rootfs.* \ @@ -8173,7 +8181,7 @@ A few workarounds are: If you do this, don't forget to do a: + .... -cd submodules/parsec-benchmark +cd "$(./getvar parsec_src_dir)" git clean -xdf . .... before going for the cross compile build. @@ -9788,11 +9796,11 @@ For example, when updating from QEMU `v2.12.0` to `v3.0.0-rc3`, the Linux kernel We then bisected it as explained at: https://stackoverflow.com/questions/4713088/how-to-use-git-bisect/22592593#22592593 with the link:qemu-bisect-boot[] script: .... -cd qemu +cd "$(./getvar qemu_src_dir)" git bisect start # Check that our test script fails on v3.0.0-rc3 as expected, and mark it as bad. -../qemu-bisect-boot +../../qemu-bisect-boot # Should output 1. echo #? git bisect bad @@ -9826,7 +9834,7 @@ This example is based on the Linux kernel, for which we used to have patches, bu # Last point before out patches. last_mainline_revision=v4.15 next_mainline_revision=v4.16 -cd linux +cd "$(./getvar linux_src_dir)" # Create a branch before the rebase in case things go wrong. git checkout -b "lkmc-${last_mainline_revision}" diff --git a/bisect-linux-boot-gem5 b/bisect-linux-boot-gem5 index 6e28652..534b028 100755 --- a/bisect-linux-boot-gem5 +++ b/bisect-linux-boot-gem5 @@ -1,28 +1,34 @@ -#!/usr/bin/env bash -. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/common" -while getopts "h${common_getopts_flags}" OPT; do - case "$OPT" in - h) - echo "https://github.com/cirosantilli/linux-kernel-module-cheat#bisection" 2>&1 - exit - ;; - ?) - common_getopts_case "$OPT" - ;; - esac -done -shift "$(($OPTIND - 1))" -common_linux_variant=bisect -common_setup -cd "$common_root_dir" +#!/usr/bin/env python3 + +import imp +import os +import shutil +import sys + +import common +build = imp.load_source('build', os.path.join(common.root_dir, 'build')) +run = imp.load_source('run', os.path.join(common.root_dir, 'run')) + +parser = common.get_argparse( + argparse_args={ + 'description': '''Bisect the Linux kernel on gem5 boots. + +More information at: https://github.com/cirosantilli/linux-kernel-module-cheat#bisection +'''}, + default_args={ + 'gem5': True, + 'linux_build_id': 'bisect', + }, +) +args = common.setup(parser) # We need a clean rebuild becuase rebuilds at different revisions: # - may fail # - may not actually rebuild all files, e.g. on header changes -rm -rf "$(./getvar -a "$common_arch" -L "$common_linux_variant" linux_variant_dir)" -./build -a "$common_arch" -L "$common_linux_variant" -status=0 -./run -a "$common_arch" -E 'm5 exit' -L "$common_linux_variant" -g || status=$? -if [ "$status" -eq 125 ] || [ "$status" -gt 127 ]; then - status=1 -fi -exit "$status" +shutil.rmtree(common.linux_variant_dir) +assert build.main(args) == 0 +status = run.main(args, { + 'eval': 'm5 exit', +}) +if status == 125 or status == 127: + status = 1 +sys.exit(status) diff --git a/build b/build index 0251577..71a88e6 100755 --- a/build +++ b/build @@ -3,7 +3,6 @@ import multiprocessing import os import pathlib -import shlex import shutil import subprocess import sys @@ -25,7 +24,7 @@ defaults = { 'nproc': None, 'skip_configure': False, 'verbose': False, - 'extra_make_args': '', + 'extra_make_args': [], } def path_relative_to_buildroot(abspath): @@ -35,7 +34,7 @@ def main(args, extra_args=None): global defaults args = common.resolve_args(defaults, args, extra_args) os.makedirs(common.out_dir, exist_ok=True) - extra_make_args = shlex.split(args.extra_make_args) + extra_make_args = args.extra_make_args.copy() if args.kernel_modules_reconfigure: extra_make_args.append('kernel_modules-reconfigure') if args.linux_reconfigure: @@ -81,7 +80,7 @@ def main(args, extra_args=None): buildroot_configs = args.buildroot_config buildroot_configs.extend([ 'BR2_JLEVEL={}'.format(nproc), - 'BR2_DL_DIR="{}"'.format(os.path.join(common.common_dir, 'dl')), + 'BR2_DL_DIR="{}"'.format(common.dl_dir), 'BR2_GLOBAL_PATCH_DIR="{}"'.format( path_relative_to_buildroot(os.path.join(common.root_dir, 'patches', 'global'))), 'BR2_PACKAGE_BUSYBOX_CONFIG_FRAGMENT_FILES="{}"'.format( @@ -195,6 +194,7 @@ def main(args, extra_args=None): assert common.run_cmd( [ 'make', + 'GEM5_LKMC_SRCDIR="{}"'.format(common.gem5_src_dir), 'O={}'.format(common.buildroot_out_dir), 'V={}'.format(int(args.verbose)), ] + @@ -207,15 +207,16 @@ def main(args, extra_args=None): ) == 0 # Create the qcow2 from ext2. - assert common.run_cmd([ - common.qemu_img_executable, - '-T', 'pr_manager_run,file=/dev/null', - 'convert', - '-f', 'raw', - '-O', 'qcow2', - common.ext2_file, - common.qcow2_file, - ]) == 0 + if os.path.exists(common.qemu_img_executable): + assert common.run_cmd([ + common.qemu_img_executable, + '-T', 'pr_manager_run,file=/dev/null', + 'convert', + '-f', 'raw', + '-O', 'qcow2', + common.ext2_file, + common.qcow2_file, + ]) == 0 return 0 @@ -282,7 +283,7 @@ https://stackoverflow.com/questions/49260466/why-when-i-change-br2-linux-kernel- help='Do a verbose build' ) parser.add_argument( - 'extra-make-args', default=defaults['extra_make_args'], nargs='?' + 'extra-make-args', default=defaults['extra_make_args'], nargs='*' ) return parser diff --git a/buildroot_config/default b/buildroot_config/default index fe1872a..103567f 100644 --- a/buildroot_config/default +++ b/buildroot_config/default @@ -42,27 +42,24 @@ BR2_SAMPLE_PACKAGE=y #BR2_PACKAGE_IFUPDOWN_SCRIPTS=n # misc packages -# BR2_PACKAGE_DHRYSTONE=y -# BR2_PACKAGE_FILE=y -# BR2_PACKAGE_PCIUTILS=y -# # For qemu-ga on guest. TODO: do something with it, and document it. -# # Maybe: https://superuser.com/questions/930588/how-to-pass-commands-noninteractively-to-running-qemu-from-the-guest-qmp-via-te -# BR2_PACKAGE_QEMU=y -# BR2_PACKAGE_STRACE=y +BR2_PACKAGE_DHRYSTONE=y +BR2_PACKAGE_FILE=y +BR2_PACKAGE_PCIUTILS=y +BR2_PACKAGE_STRACE=y -# # lscpu: TODO not installing? -# BR2_PACKAGE_UTIL_LINUX=y -# BR2_PACKAGE_UTIL_LINUX_BINARIES=y -# # taskset -# BR2_PACKAGE_UTIL_LINUX_SCHEDUTILS=y +# lscpu: TODO not installing? +BR2_PACKAGE_UTIL_LINUX=y +BR2_PACKAGE_UTIL_LINUX_BINARIES=y +# taskset +BR2_PACKAGE_UTIL_LINUX_SCHEDUTILS=y -# # gdbserver -# BR2_PACKAGE_GDB=y +# gdbserver +BR2_PACKAGE_GDB=y -# # ftrace -# BR2_PACKAGE_TRACE_CMD=y -# -# # DTC -# BR2_PACKAGE_DTC=y -# BR2_PACKAGE_DTC_PROGRAMS=y -# BR2_PACKAGE_HOST_DTC=y +# ftrace +BR2_PACKAGE_TRACE_CMD=y + +# DTC +BR2_PACKAGE_DTC=y +BR2_PACKAGE_DTC_PROGRAMS=y +BR2_PACKAGE_HOST_DTC=y diff --git a/common.py b/common.py index d403756..75d0c61 100644 --- a/common.py +++ b/common.py @@ -15,6 +15,37 @@ import sys this = sys.modules[__name__] +# Default paths. +root_dir = os.path.dirname(os.path.abspath(__file__)) +data_dir = os.path.join(root_dir, 'data') +p9_dir = os.path.join(data_dir, '9p') +gem5_non_default_src_root_dir = os.path.join(data_dir, 'gem5') +gem5_readfile_file = os.path.join(data_dir, 'readfile') +out_dir = os.path.join(root_dir, 'out') +bench_boot = os.path.join(out_dir, 'bench-boot.txt') +common_dir = os.path.join(out_dir, 'common') +dl_dir = os.path.join(common_dir, 'dl') +submodules_dir = os.path.join(root_dir, 'submodules') +buildroot_src_dir = os.path.join(submodules_dir, 'buildroot') +gem5_default_src_dir = os.path.join(submodules_dir, 'gem5') +linux_src_dir = os.path.join(submodules_dir, 'linux') +qemu_src_dir = os.path.join(submodules_dir, 'qemu') +parsec_src_dir = os.path.join(submodules_dir, 'parsec-benchmark') + +# Other default variables. +arch_map = { + 'a': 'arm', + 'A': 'aarch64', + 'x': 'x86_64', +} +arches = [arch_map[k] for k in arch_map] +gem5_cpt_prefix = '^cpt\.' +sha = subprocess.check_output(['git', '-C', root_dir, 'log', '-1', '--format=%H']).decode().rstrip() +config_file = os.path.join(data_dir, 'config') +if os.path.exists(config_file): + config = imp.load_source('config', config_file) + configs = {x:getattr(config, x) for x in dir(config) if not x.startswith('__')} + # TODO ## Benchmark a command. ## @@ -50,6 +81,9 @@ def gem_list_checkpoint_dirs(): def get_argparse(default_args=None, argparse_args=None): ''' Return an argument parser with common arguments set. + + :type default_args: Dict[str,str] + :type argparse_args: Dict ''' global this if default_args is None: @@ -370,31 +404,3 @@ def mkdir(): os.makedirs(this.gem5_run_dir, exist_ok=True) os.makedirs(this.qemu_run_dir, exist_ok=True) os.makedirs(this.p9_dir, exist_ok=True) - -# Default paths. -root_dir = os.path.dirname(os.path.abspath(__file__)) -data_dir = os.path.join(root_dir, 'data') -p9_dir = os.path.join(data_dir, '9p') -gem5_non_default_src_root_dir = os.path.join(data_dir, 'gem5') -gem5_readfile_file = os.path.join(data_dir, 'readfile') -out_dir = os.path.join(root_dir, 'out') -bench_boot = os.path.join(out_dir, 'bench-boot.txt') -common_dir = os.path.join(out_dir, 'common') -submodules_dir = os.path.join(root_dir, 'submodules') -buildroot_src_dir = os.path.join(submodules_dir, 'buildroot') -gem5_default_src_dir = os.path.join(submodules_dir, 'gem5') -linux_src_dir = os.path.join(submodules_dir, 'linux') -qemu_src_dir = os.path.join(submodules_dir, 'qemu') - -# Other default variables. -arch_map = { - 'a': 'arm', - 'A': 'aarch64', - 'x': 'x86_64', -} -gem5_cpt_prefix = '^cpt\.' -sha = subprocess.check_output(['git', '-C', root_dir, 'log', '-1', '--format=%H']).decode().rstrip() -config_file = os.path.join(data_dir, 'config') -if os.path.exists(config_file): - config = imp.load_source('config', config_file) - configs = {x:getattr(config, x) for x in dir(config) if not x.startswith('__')} diff --git a/packages/gem5/external.mk b/packages/gem5/external.mk index 6ef37ed..90cef85 100644 --- a/packages/gem5/external.mk +++ b/packages/gem5/external.mk @@ -5,7 +5,7 @@ ################################################################################ GEM5_VERSION = 1.0 -GEM5_SITE = $(BR2_EXTERNAL_GEM5_PATH) +GEM5_SITE = $(GEM5_LKMC_SRCDIR) GEM5_SITE_METHOD = local ifeq ($(ARCH),x86_64) @@ -17,11 +17,11 @@ endif define GEM5_BUILD_CMDS # TODO cannot use TARGET_CONFIGURE_OPTS here because it overrides the CFLAGS on m5, # which have an include. We should patch gem5 to add a += instead of = there. - cd '$(@D)/gem5/util/m5' && $(MAKE) -f 'Makefile.$(ARCH_MAKE)' CC='$(TARGET_CC)' LD='$(TARGET_LD)' + cd '$(@D)/util/m5' && $(MAKE) -f 'Makefile.$(ARCH_MAKE)' CC='$(TARGET_CC)' LD='$(TARGET_LD)' endef define GEM5_INSTALL_TARGET_CMDS - $(INSTALL) -D -m 0755 '$(@D)/gem5/util/m5/m5' '$(TARGET_DIR)/usr/bin' + $(INSTALL) -D -m 0755 '$(@D)/util/m5/m5' '$(TARGET_DIR)/usr/bin' endef $(eval $(generic-package)) diff --git a/release b/release index cd67b69..c1c8597 100644 --- a/release +++ b/release @@ -1,19 +1,19 @@ -#!/usr/bin/env bash -. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/common" -common_setup -./build-all -./zip-img -tag="sha-${common_sha}" -upload_basename="images-${common_sha}.zip" -git tag "$tag" -git push --tags +#!/usr/bin/env python3 +import subprocess +import common +subprocess.check_call(['./build-all']) +subprocess.check_call(['./zip-img']) +tag = 'sha-{}'.format(common.sha) +subprocess.check_call(['git', 'tag' tag]) +subprocess.check_call(['git', 'push' '--tags']) # TODO # - https://stackoverflow.com/questions/41022470/curl-request-to-add-file-to-github-release # - https://stackoverflow.com/questions/38627115/upload-files-to-github-directory-using-github-api +# upload_basename = 'images-{}.zip'.format(common.sha) #curl "https://api.github.com/repos/cirosantilli/linux-kernel-module-cheat/releases/tags/${tag}/assets?access_token=$(cat data/access_token)&tag_name=${upload_basename}" \ # --header 'Content-Type: application/zip' \ # --upload-file "${common_out_dir}/${upload_basename}" \ # -H 'Accept: application/vnd.github.v3+json' \ # -X POST \ #; -./bench-all -Au +subprocess.check_call(['./bench-all', '-A', '-u']) diff --git a/run b/run index a5393e6..86220a0 100755 --- a/run +++ b/run @@ -13,7 +13,7 @@ defaults = { 'debug_guest': False, 'debug_vm': False, 'eval': None, - 'extra_emulator_args': None, + 'extra_emulator_args': [], 'gem5_biglittle': False, 'gem5_exe_args':'', 'gem5_restore_last_checkpoint': None, @@ -164,7 +164,7 @@ def main(args, extra_args=None): qemu_executable = "qemu-system-{}".format(args.arch) else: qemu_executable = common.qemu_executable - extra_emulator_args = extra_qemu_args + extra_emulator_args + extra_emulator_args = extra_qemu_args + args.extra_emulator_args cmd = ( debug_vm + [ @@ -435,7 +435,7 @@ Run QEMU with VNC instead of the default SDL. Connect to it with: ''' ) parser.add_argument( - 'extra_emulator_args', nargs='*', + 'extra_emulator_args', nargs='*', default=defaults['extra_emulator_args'], help='Extra options to append at the end of the emulator command line' ) return parser diff --git a/rungdb-user b/rungdb-user index 49e477b..a97c751 100755 --- a/rungdb-user +++ b/rungdb-user @@ -2,6 +2,7 @@ import imp import os +import sys import common rungdb = imp.load_source('rungdb', os.path.join(common.root_dir, 'rungdb')) @@ -31,4 +32,4 @@ extra_args['before'] = '-ex \"add-symbol-file {} {}\"'.format(args.executable, h # TODO understand better. # Also, lx-symbols overrides the add-symbol-file commands. extra_args['no_lxsymbols'] = True -rungdb.main(args, extra_args) +sys.exit(rungdb.main(args, extra_args)) diff --git a/zip-img b/zip-img index c5ea936..548d044 100755 --- a/zip-img +++ b/zip-img @@ -1,10 +1,17 @@ -#!/usr/bin/env bash -. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/common" -common_setup -cd "$common_root_dir" -outfile="${common_out_dir}/lkmc-${common_sha}.zip" -rm -f "$outfile" -for common_arch in x86_64 arm aarch64; do - common_setup - zip -r "$outfile" "${common_qcow2_file#${common_root_dir}/}" "${common_linux_image#${common_root_dir}/}" -done +#!/usr/bin/env python3 + +import os +import subprocess +import zipfile + +import common + +outfile = os.path.join(common.out_dir, 'lkmc-{}.zip'.format(common.sha)) +if os.path.exists(outfile): + os.unlink(outfile) +zipf = zipfile.ZipFile(outfile, 'w', zipfile.ZIP_DEFLATED) +for arch in common.arches: + common.setup(common.get_argparse(default_args={'arch': arch})) + zipf.write(common.qcow2_file, arcname=os.path.relpath(common.qcow2_file, common.root_dir)) + zipf.write(common.linux_image, arcname=os.path.relpath(common.linux_image, common.root_dir)) +zipf.close()