port bisect-linux-boot-gem5 and zip-img

fix m5 build and a bunch other things

README.adoc

@@ -3335,7 +3335,7 @@ cat /proc/version
 or in the source:
 
 ....
-cd linux
+cd "$(./getvar linux_src_dir)"
 git log | grep -E '    Linux [0-9]+\.' | head
 ....
 
@@ -7192,6 +7192,14 @@ Getting everything to work required careful choice of QEMU command line options:
 
 Peter Maydell said potentially not possible nicely as of August 2018: https://stackoverflow.com/questions/51747744/how-to-run-a-qemu-monitor-command-from-inside-the-guest/51764110#51764110
 
+It is also worth looking into the QEMU Guest Agent tool `qemu-gq` that can be enabled with:
+
+....
+./build -B 'BR2_PACKAGE_QEMU=y'
+....
+
+See also: https://superuser.com/questions/930588/how-to-pass-commands-noninteractively-to-running-qemu-from-the-guest-qmp-via-te
+
 ==== QEMU monitor from GDB
 
 When doing <<gdb>> it is possible to send QEMU monitor commands through the GDB `monitor` command, which saves you the trouble of opening yet another shell.
@@ -8151,7 +8159,7 @@ If you want to remove PARSEC later, Buildroot doesn't provide an automated packa
 
 ....
 rm -rf \
-  ./out/common/dl/parsec-* \
+  "$(./getvar dl_dir)"/parsec-* \
   "$(./getvar buildroot_out_dir)"/build/parsec-* \
   "$(./getvar buildroot_out_dir)"/build/packages-file-list.txt \
   "$(./getvar buildroot_out_dir)"/images/rootfs.* \
@@ -8173,7 +8181,7 @@ A few workarounds are:
 If you do this, don't forget to do a:
 +
 ....
-cd submodules/parsec-benchmark
+cd "$(./getvar parsec_src_dir)"
 git clean -xdf .
 ....
 before going for the cross compile build.
@@ -9788,11 +9796,11 @@ For example, when updating from QEMU `v2.12.0` to `v3.0.0-rc3`, the Linux kernel
 We then bisected it as explained at: https://stackoverflow.com/questions/4713088/how-to-use-git-bisect/22592593#22592593 with the link:qemu-bisect-boot[] script:
 
 ....
-cd qemu
+cd "$(./getvar qemu_src_dir)"
 git bisect start
 
 # Check that our test script fails on v3.0.0-rc3 as expected, and mark it as bad.
-../qemu-bisect-boot
+../../qemu-bisect-boot
 # Should output 1.
 echo #?
 git bisect bad
@@ -9826,7 +9834,7 @@ This example is based on the Linux kernel, for which we used to have patches, bu
 # Last point before out patches.
 last_mainline_revision=v4.15
 next_mainline_revision=v4.16
-cd linux
+cd "$(./getvar linux_src_dir)"
 
 # Create a branch before the rebase in case things go wrong.
 git checkout -b "lkmc-${last_mainline_revision}"

bisect-linux-boot-gem5

@@ -1,28 +1,34 @@
-#!/usr/bin/env bash
+#!/usr/bin/env python3
-. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/common"
+
-while getopts "h${common_getopts_flags}" OPT; do
+import imp
-  case "$OPT" in
+import os
-    h)
+import shutil
-      echo "https://github.com/cirosantilli/linux-kernel-module-cheat#bisection" 2>&1
+import sys
-      exit
+
-      ;;
+import common
-    ?)
+build = imp.load_source('build', os.path.join(common.root_dir, 'build'))
-      common_getopts_case "$OPT"
+run = imp.load_source('run', os.path.join(common.root_dir, 'run'))
-      ;;
+
-  esac
+parser = common.get_argparse(
-done
+    argparse_args={
-shift "$(($OPTIND - 1))"
+        'description': '''Bisect the Linux kernel on gem5 boots.
-common_linux_variant=bisect
+
-common_setup
+More information at: https://github.com/cirosantilli/linux-kernel-module-cheat#bisection
-cd "$common_root_dir"
+'''},
+    default_args={
+        'gem5': True,
+        'linux_build_id': 'bisect',
+    },
+)
+args = common.setup(parser)
 # We need a clean rebuild becuase rebuilds at different revisions:
 # - may fail
 # - may not actually rebuild all files, e.g. on header changes
-rm -rf "$(./getvar -a "$common_arch" -L "$common_linux_variant" linux_variant_dir)"
+shutil.rmtree(common.linux_variant_dir)
-./build -a "$common_arch" -L "$common_linux_variant"
+assert build.main(args) == 0
-status=0
+status = run.main(args, {
-./run -a "$common_arch" -E 'm5 exit' -L "$common_linux_variant" -g || status=$?
+    'eval': 'm5 exit',
-if [ "$status" -eq 125 ] || [ "$status" -gt 127 ]; then
+})
-  status=1
+if status == 125 or status == 127:
-fi
+    status = 1
-exit "$status"
+sys.exit(status)

build

@@ -3,7 +3,6 @@
 import multiprocessing
 import os
 import pathlib
-import shlex
 import shutil
 import subprocess
 import sys
@@ -25,7 +24,7 @@ defaults = {
     'nproc': None,
     'skip_configure': False,
     'verbose': False,
-    'extra_make_args': '',
+    'extra_make_args': [],
 }
 
 def path_relative_to_buildroot(abspath):
@@ -35,7 +34,7 @@ def main(args, extra_args=None):
     global defaults
     args = common.resolve_args(defaults, args, extra_args)
     os.makedirs(common.out_dir, exist_ok=True)
-    extra_make_args = shlex.split(args.extra_make_args)
+    extra_make_args = args.extra_make_args.copy()
     if args.kernel_modules_reconfigure:
         extra_make_args.append('kernel_modules-reconfigure')
     if args.linux_reconfigure:
@@ -81,7 +80,7 @@ def main(args, extra_args=None):
         buildroot_configs = args.buildroot_config
         buildroot_configs.extend([
             'BR2_JLEVEL={}'.format(nproc),
-            'BR2_DL_DIR="{}"'.format(os.path.join(common.common_dir, 'dl')),
+            'BR2_DL_DIR="{}"'.format(common.dl_dir),
             'BR2_GLOBAL_PATCH_DIR="{}"'.format(
                 path_relative_to_buildroot(os.path.join(common.root_dir, 'patches', 'global'))),
             'BR2_PACKAGE_BUSYBOX_CONFIG_FRAGMENT_FILES="{}"'.format(
@@ -195,6 +194,7 @@ def main(args, extra_args=None):
     assert common.run_cmd(
         [
             'make',
+            'GEM5_LKMC_SRCDIR="{}"'.format(common.gem5_src_dir),
             'O={}'.format(common.buildroot_out_dir),
             'V={}'.format(int(args.verbose)),
         ] +
@@ -207,15 +207,16 @@ def main(args, extra_args=None):
     ) == 0
 
     # Create the qcow2 from ext2.
-    assert common.run_cmd([
+    if os.path.exists(common.qemu_img_executable):
-        common.qemu_img_executable,
+        assert common.run_cmd([
-        '-T', 'pr_manager_run,file=/dev/null',
+            common.qemu_img_executable,
-        'convert',
+            '-T', 'pr_manager_run,file=/dev/null',
-        '-f', 'raw',
+            'convert',
-        '-O', 'qcow2',
+            '-f', 'raw',
-        common.ext2_file,
+            '-O', 'qcow2',
-        common.qcow2_file,
+            common.ext2_file,
-    ]) == 0
+            common.qcow2_file,
+        ]) == 0
 
     return 0
 
@@ -282,7 +283,7 @@ https://stackoverflow.com/questions/49260466/why-when-i-change-br2-linux-kernel-
         help='Do a verbose build'
     )
     parser.add_argument(
-        'extra-make-args', default=defaults['extra_make_args'], nargs='?'
+        'extra-make-args', default=defaults['extra_make_args'], nargs='*'
     )
     return parser
 

buildroot_config/default

@@ -42,27 +42,24 @@ BR2_SAMPLE_PACKAGE=y
 #BR2_PACKAGE_IFUPDOWN_SCRIPTS=n
 
 # misc packages
-# BR2_PACKAGE_DHRYSTONE=y
+BR2_PACKAGE_DHRYSTONE=y
-# BR2_PACKAGE_FILE=y
+BR2_PACKAGE_FILE=y
-# BR2_PACKAGE_PCIUTILS=y
+BR2_PACKAGE_PCIUTILS=y
-# # For qemu-ga on guest. TODO: do something with it, and document it.
+BR2_PACKAGE_STRACE=y
-# # Maybe: https://superuser.com/questions/930588/how-to-pass-commands-noninteractively-to-running-qemu-from-the-guest-qmp-via-te
-# BR2_PACKAGE_QEMU=y
-# BR2_PACKAGE_STRACE=y
 
-# # lscpu: TODO not installing?
+# lscpu: TODO not installing?
-# BR2_PACKAGE_UTIL_LINUX=y
+BR2_PACKAGE_UTIL_LINUX=y
-# BR2_PACKAGE_UTIL_LINUX_BINARIES=y
+BR2_PACKAGE_UTIL_LINUX_BINARIES=y
-# # taskset
+# taskset
-# BR2_PACKAGE_UTIL_LINUX_SCHEDUTILS=y
+BR2_PACKAGE_UTIL_LINUX_SCHEDUTILS=y
 
-# # gdbserver
+# gdbserver
-# BR2_PACKAGE_GDB=y
+BR2_PACKAGE_GDB=y
 
-# # ftrace
+# ftrace
-# BR2_PACKAGE_TRACE_CMD=y
+BR2_PACKAGE_TRACE_CMD=y
-#
+
-# # DTC
+# DTC
-# BR2_PACKAGE_DTC=y
+BR2_PACKAGE_DTC=y
-# BR2_PACKAGE_DTC_PROGRAMS=y
+BR2_PACKAGE_DTC_PROGRAMS=y
-# BR2_PACKAGE_HOST_DTC=y
+BR2_PACKAGE_HOST_DTC=y

common.py

@@ -15,6 +15,37 @@ import sys
 
 this = sys.modules[__name__]
 
+# Default paths.
+root_dir = os.path.dirname(os.path.abspath(__file__))
+data_dir = os.path.join(root_dir, 'data')
+p9_dir = os.path.join(data_dir, '9p')
+gem5_non_default_src_root_dir = os.path.join(data_dir, 'gem5')
+gem5_readfile_file = os.path.join(data_dir, 'readfile')
+out_dir = os.path.join(root_dir, 'out')
+bench_boot = os.path.join(out_dir, 'bench-boot.txt')
+common_dir = os.path.join(out_dir, 'common')
+dl_dir = os.path.join(common_dir, 'dl')
+submodules_dir = os.path.join(root_dir, 'submodules')
+buildroot_src_dir = os.path.join(submodules_dir, 'buildroot')
+gem5_default_src_dir = os.path.join(submodules_dir, 'gem5')
+linux_src_dir = os.path.join(submodules_dir, 'linux')
+qemu_src_dir = os.path.join(submodules_dir, 'qemu')
+parsec_src_dir = os.path.join(submodules_dir, 'parsec-benchmark')
+
+# Other default variables.
+arch_map = {
+    'a': 'arm',
+    'A': 'aarch64',
+    'x': 'x86_64',
+}
+arches = [arch_map[k] for k in arch_map]
+gem5_cpt_prefix = '^cpt\.'
+sha = subprocess.check_output(['git', '-C', root_dir, 'log', '-1', '--format=%H']).decode().rstrip()
+config_file = os.path.join(data_dir, 'config')
+if os.path.exists(config_file):
+    config = imp.load_source('config', config_file)
+    configs = {x:getattr(config, x) for x in dir(config) if not x.startswith('__')}
+
 # TODO
 ## Benchmark a command.
 ##
@@ -50,6 +81,9 @@ def gem_list_checkpoint_dirs():
 def get_argparse(default_args=None, argparse_args=None):
     '''
     Return an argument parser with common arguments set.
+
+    :type default_args: Dict[str,str]
+    :type argparse_args: Dict
     '''
     global this
     if default_args is None:
@@ -370,31 +404,3 @@ def mkdir():
     os.makedirs(this.gem5_run_dir, exist_ok=True)
     os.makedirs(this.qemu_run_dir, exist_ok=True)
     os.makedirs(this.p9_dir, exist_ok=True)
-
-# Default paths.
-root_dir = os.path.dirname(os.path.abspath(__file__))
-data_dir = os.path.join(root_dir, 'data')
-p9_dir = os.path.join(data_dir, '9p')
-gem5_non_default_src_root_dir = os.path.join(data_dir, 'gem5')
-gem5_readfile_file = os.path.join(data_dir, 'readfile')
-out_dir = os.path.join(root_dir, 'out')
-bench_boot = os.path.join(out_dir, 'bench-boot.txt')
-common_dir = os.path.join(out_dir, 'common')
-submodules_dir = os.path.join(root_dir, 'submodules')
-buildroot_src_dir = os.path.join(submodules_dir, 'buildroot')
-gem5_default_src_dir = os.path.join(submodules_dir, 'gem5')
-linux_src_dir = os.path.join(submodules_dir, 'linux')
-qemu_src_dir = os.path.join(submodules_dir, 'qemu')
-
-# Other default variables.
-arch_map = {
-    'a': 'arm',
-    'A': 'aarch64',
-    'x': 'x86_64',
-}
-gem5_cpt_prefix = '^cpt\.'
-sha = subprocess.check_output(['git', '-C', root_dir, 'log', '-1', '--format=%H']).decode().rstrip()
-config_file = os.path.join(data_dir, 'config')
-if os.path.exists(config_file):
-    config = imp.load_source('config', config_file)
-    configs = {x:getattr(config, x) for x in dir(config) if not x.startswith('__')}

packages/gem5/external.mk

@@ -5,7 +5,7 @@
 ################################################################################
 
 GEM5_VERSION = 1.0
-GEM5_SITE = $(BR2_EXTERNAL_GEM5_PATH)
+GEM5_SITE = $(GEM5_LKMC_SRCDIR)
 GEM5_SITE_METHOD = local
 
 ifeq ($(ARCH),x86_64)
@@ -17,11 +17,11 @@ endif
 define GEM5_BUILD_CMDS
 	# TODO cannot use TARGET_CONFIGURE_OPTS here because it overrides the CFLAGS on m5,
 	# which have an include. We should patch gem5 to add a += instead of = there.
-	cd '$(@D)/gem5/util/m5' && $(MAKE) -f 'Makefile.$(ARCH_MAKE)' CC='$(TARGET_CC)' LD='$(TARGET_LD)'
+	cd '$(@D)/util/m5' && $(MAKE) -f 'Makefile.$(ARCH_MAKE)' CC='$(TARGET_CC)' LD='$(TARGET_LD)'
 endef
 
 define GEM5_INSTALL_TARGET_CMDS
-	$(INSTALL) -D -m 0755 '$(@D)/gem5/util/m5/m5' '$(TARGET_DIR)/usr/bin'
+	$(INSTALL) -D -m 0755 '$(@D)/util/m5/m5' '$(TARGET_DIR)/usr/bin'
 endef
 
 $(eval $(generic-package))

release

@@ -1,19 +1,19 @@
-#!/usr/bin/env bash
+#!/usr/bin/env python3
-. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/common"
+import subprocess
-common_setup
+import common
-./build-all
+subprocess.check_call(['./build-all'])
-./zip-img
+subprocess.check_call(['./zip-img'])
-tag="sha-${common_sha}"
+tag = 'sha-{}'.format(common.sha)
-upload_basename="images-${common_sha}.zip"
+subprocess.check_call(['git', 'tag' tag])
-git tag "$tag"
+subprocess.check_call(['git', 'push' '--tags'])
-git push --tags
 # TODO
 # - https://stackoverflow.com/questions/41022470/curl-request-to-add-file-to-github-release
 # - https://stackoverflow.com/questions/38627115/upload-files-to-github-directory-using-github-api
+# upload_basename = 'images-{}.zip'.format(common.sha)
 #curl "https://api.github.com/repos/cirosantilli/linux-kernel-module-cheat/releases/tags/${tag}/assets?access_token=$(cat data/access_token)&tag_name=${upload_basename}" \
 #  --header 'Content-Type: application/zip' \
 #  --upload-file "${common_out_dir}/${upload_basename}" \
 #  -H 'Accept: application/vnd.github.v3+json' \
 #  -X POST \
 #;
-./bench-all -Au
+subprocess.check_call(['./bench-all', '-A', '-u'])

run

@@ -13,7 +13,7 @@ defaults = {
     'debug_guest': False,
     'debug_vm': False,
     'eval': None,
-    'extra_emulator_args': None,
+    'extra_emulator_args': [],
     'gem5_biglittle': False,
     'gem5_exe_args':'',
     'gem5_restore_last_checkpoint': None,
@@ -164,7 +164,7 @@ def main(args, extra_args=None):
             qemu_executable = "qemu-system-{}".format(args.arch)
         else:
             qemu_executable = common.qemu_executable
-        extra_emulator_args = extra_qemu_args + extra_emulator_args
+        extra_emulator_args = extra_qemu_args + args.extra_emulator_args
         cmd = (
             debug_vm +
             [
@@ -435,7 +435,7 @@ Run QEMU with VNC instead of the default SDL. Connect to it with:
 '''
     )
     parser.add_argument(
-        'extra_emulator_args', nargs='*',
+        'extra_emulator_args', nargs='*', default=defaults['extra_emulator_args'],
         help='Extra options to append at the end of the emulator command line'
     )
     return parser

rungdb-user

@@ -2,6 +2,7 @@
 
 import imp
 import os
+import sys
 
 import common
 rungdb = imp.load_source('rungdb', os.path.join(common.root_dir, 'rungdb'))
@@ -31,4 +32,4 @@ extra_args['before'] = '-ex \"add-symbol-file {} {}\"'.format(args.executable, h
 # TODO understand better.
 # Also, lx-symbols overrides the add-symbol-file commands.
 extra_args['no_lxsymbols'] = True
-rungdb.main(args, extra_args)
+sys.exit(rungdb.main(args, extra_args))

zip-img

@@ -1,10 +1,17 @@
-#!/usr/bin/env bash
+#!/usr/bin/env python3
-. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/common"
+
-common_setup
+import os
-cd "$common_root_dir"
+import subprocess
-outfile="${common_out_dir}/lkmc-${common_sha}.zip"
+import zipfile
-rm -f "$outfile"
+
-for common_arch in x86_64 arm aarch64; do
+import common
-  common_setup
+
-  zip -r "$outfile" "${common_qcow2_file#${common_root_dir}/}" "${common_linux_image#${common_root_dir}/}"
+outfile = os.path.join(common.out_dir, 'lkmc-{}.zip'.format(common.sha))
-done
+if os.path.exists(outfile):
+    os.unlink(outfile)
+zipf = zipfile.ZipFile(outfile, 'w', zipfile.ZIP_DEFLATED)
+for arch in common.arches:
+    common.setup(common.get_argparse(default_args={'arch': arch}))
+    zipf.write(common.qcow2_file, arcname=os.path.relpath(common.qcow2_file, common.root_dir))
+    zipf.write(common.linux_image, arcname=os.path.relpath(common.linux_image, common.root_dir))
+zipf.close()