port bisect-linux-boot-gem5 and zip-img

fix m5 build and a bunch other things
2026-01-26 03:31:36 +01:00 · 2018-09-08 23:20:08 +01:00
parent dd7614cbce
commit 58de3f7243
10 changed files with 147 additions and 121 deletions
--- a/README.adoc
+++ b/README.adoc
@@ -3335,7 +3335,7 @@ cat /proc/version
 or in the source:

 ....
-cd linux
+cd "$(./getvar linux_src_dir)"
 git log | grep -E '    Linux [0-9]+\.' | head
 ....

@@ -7192,6 +7192,14 @@ Getting everything to work required careful choice of QEMU command line options:

 Peter Maydell said potentially not possible nicely as of August 2018: https://stackoverflow.com/questions/51747744/how-to-run-a-qemu-monitor-command-from-inside-the-guest/51764110#51764110

+It is also worth looking into the QEMU Guest Agent tool `qemu-gq` that can be enabled with:
+
+....
+./build -B 'BR2_PACKAGE_QEMU=y'
+....
+
+See also: https://superuser.com/questions/930588/how-to-pass-commands-noninteractively-to-running-qemu-from-the-guest-qmp-via-te
+
 ==== QEMU monitor from GDB

 When doing <<gdb>> it is possible to send QEMU monitor commands through the GDB `monitor` command, which saves you the trouble of opening yet another shell.
@@ -8151,7 +8159,7 @@ If you want to remove PARSEC later, Buildroot doesn't provide an automated packa

 ....
 rm -rf \
-  ./out/common/dl/parsec-* \
+  "$(./getvar dl_dir)"/parsec-* \
  "$(./getvar buildroot_out_dir)"/build/parsec-* \
  "$(./getvar buildroot_out_dir)"/build/packages-file-list.txt \
  "$(./getvar buildroot_out_dir)"/images/rootfs.* \
@@ -8173,7 +8181,7 @@ A few workarounds are:
 If you do this, don't forget to do a:
 +
 ....
-cd submodules/parsec-benchmark
+cd "$(./getvar parsec_src_dir)"
 git clean -xdf .
 ....
 before going for the cross compile build.
@@ -9788,11 +9796,11 @@ For example, when updating from QEMU `v2.12.0` to `v3.0.0-rc3`, the Linux kernel
 We then bisected it as explained at: https://stackoverflow.com/questions/4713088/how-to-use-git-bisect/22592593#22592593 with the link:qemu-bisect-boot[] script:

 ....
-cd qemu
+cd "$(./getvar qemu_src_dir)"
 git bisect start

 # Check that our test script fails on v3.0.0-rc3 as expected, and mark it as bad.
-../qemu-bisect-boot
+../../qemu-bisect-boot
 # Should output 1.
 echo #?
 git bisect bad
@@ -9826,7 +9834,7 @@ This example is based on the Linux kernel, for which we used to have patches, bu
 # Last point before out patches.
 last_mainline_revision=v4.15
 next_mainline_revision=v4.16
-cd linux
+cd "$(./getvar linux_src_dir)"

 # Create a branch before the rebase in case things go wrong.
 git checkout -b "lkmc-${last_mainline_revision}"
--- a/56
+++ b/56
@@ -1,28 +1,34 @@
-#!/usr/bin/env bash
-. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/common"
-while getopts "h${common_getopts_flags}" OPT; do
-  case "$OPT" in
-    h)
-      echo "https://github.com/cirosantilli/linux-kernel-module-cheat#bisection" 2>&1
-      exit
-      ;;
-    ?)
-      common_getopts_case "$OPT"
-      ;;
-  esac
-done
-shift "$(($OPTIND - 1))"
-common_linux_variant=bisect
-common_setup
-cd "$common_root_dir"
+#!/usr/bin/env python3
+
+import imp
+import os
+import shutil
+import sys
+
+import common
+build = imp.load_source('build', os.path.join(common.root_dir, 'build'))
+run = imp.load_source('run', os.path.join(common.root_dir, 'run'))
+
+parser = common.get_argparse(
+    argparse_args={
+        'description': '''Bisect the Linux kernel on gem5 boots.
+
+More information at: https://github.com/cirosantilli/linux-kernel-module-cheat#bisection
+'''},
+    default_args={
+        'gem5': True,
+        'linux_build_id': 'bisect',
+    },
+)
+args = common.setup(parser)
 # We need a clean rebuild becuase rebuilds at different revisions:
 # - may fail
 # - may not actually rebuild all files, e.g. on header changes
-rm -rf "$(./getvar -a "$common_arch" -L "$common_linux_variant" linux_variant_dir)"
-./build -a "$common_arch" -L "$common_linux_variant"
-status=0
-./run -a "$common_arch" -E 'm5 exit' -L "$common_linux_variant" -g || status=$?
-if [ "$status" -eq 125 ] || [ "$status" -gt 127 ]; then
-  status=1
-fi
-exit "$status"
+shutil.rmtree(common.linux_variant_dir)
+assert build.main(args) == 0
+status = run.main(args, {
+    'eval': 'm5 exit',
+})
+if status == 125 or status == 127:
+    status = 1
+sys.exit(status)
--- a/29
+++ b/29
@@ -3,7 +3,6 @@
 import multiprocessing
 import os
 import pathlib
-import shlex
 import shutil
 import subprocess
 import sys
@@ -25,7 +24,7 @@ defaults = {
    'nproc': None,
    'skip_configure': False,
    'verbose': False,
-    'extra_make_args': '',
+    'extra_make_args': [],
 }

 def path_relative_to_buildroot(abspath):
@@ -35,7 +34,7 @@ def main(args, extra_args=None):
    global defaults
    args = common.resolve_args(defaults, args, extra_args)
    os.makedirs(common.out_dir, exist_ok=True)
-    extra_make_args = shlex.split(args.extra_make_args)
+    extra_make_args = args.extra_make_args.copy()
    if args.kernel_modules_reconfigure:
        extra_make_args.append('kernel_modules-reconfigure')
    if args.linux_reconfigure:
@@ -81,7 +80,7 @@ def main(args, extra_args=None):
        buildroot_configs = args.buildroot_config
        buildroot_configs.extend([
            'BR2_JLEVEL={}'.format(nproc),
-            'BR2_DL_DIR="{}"'.format(os.path.join(common.common_dir, 'dl')),
+            'BR2_DL_DIR="{}"'.format(common.dl_dir),
            'BR2_GLOBAL_PATCH_DIR="{}"'.format(
                path_relative_to_buildroot(os.path.join(common.root_dir, 'patches', 'global'))),
            'BR2_PACKAGE_BUSYBOX_CONFIG_FRAGMENT_FILES="{}"'.format(
@@ -195,6 +194,7 @@ def main(args, extra_args=None):
    assert common.run_cmd(
        [
            'make',
+            'GEM5_LKMC_SRCDIR="{}"'.format(common.gem5_src_dir),
            'O={}'.format(common.buildroot_out_dir),
            'V={}'.format(int(args.verbose)),
        ] +
@@ -207,15 +207,16 @@ def main(args, extra_args=None):
    ) == 0

    # Create the qcow2 from ext2.
-    assert common.run_cmd([
-        common.qemu_img_executable,
-        '-T', 'pr_manager_run,file=/dev/null',
-        'convert',
-        '-f', 'raw',
-        '-O', 'qcow2',
-        common.ext2_file,
-        common.qcow2_file,
-    ]) == 0
+    if os.path.exists(common.qemu_img_executable):
+        assert common.run_cmd([
+            common.qemu_img_executable,
+            '-T', 'pr_manager_run,file=/dev/null',
+            'convert',
+            '-f', 'raw',
+            '-O', 'qcow2',
+            common.ext2_file,
+            common.qcow2_file,
+        ]) == 0

    return 0

@@ -282,7 +283,7 @@ https://stackoverflow.com/questions/49260466/why-when-i-change-br2-linux-kernel-
        help='Do a verbose build'
    )
    parser.add_argument(
-        'extra-make-args', default=defaults['extra_make_args'], nargs='?'
+        'extra-make-args', default=defaults['extra_make_args'], nargs='*'
    )
    return parser

--- a/buildroot_config/default
+++ b/buildroot_config/default
@@ -42,27 +42,24 @@ BR2_SAMPLE_PACKAGE=y
 #BR2_PACKAGE_IFUPDOWN_SCRIPTS=n

 # misc packages
-# BR2_PACKAGE_DHRYSTONE=y
-# BR2_PACKAGE_FILE=y
-# BR2_PACKAGE_PCIUTILS=y
-# # For qemu-ga on guest. TODO: do something with it, and document it.
-# # Maybe: https://superuser.com/questions/930588/how-to-pass-commands-noninteractively-to-running-qemu-from-the-guest-qmp-via-te
-# BR2_PACKAGE_QEMU=y
-# BR2_PACKAGE_STRACE=y
+BR2_PACKAGE_DHRYSTONE=y
+BR2_PACKAGE_FILE=y
+BR2_PACKAGE_PCIUTILS=y
+BR2_PACKAGE_STRACE=y

-# # lscpu: TODO not installing?
-# BR2_PACKAGE_UTIL_LINUX=y
-# BR2_PACKAGE_UTIL_LINUX_BINARIES=y
-# # taskset
-# BR2_PACKAGE_UTIL_LINUX_SCHEDUTILS=y
+# lscpu: TODO not installing?
+BR2_PACKAGE_UTIL_LINUX=y
+BR2_PACKAGE_UTIL_LINUX_BINARIES=y
+# taskset
+BR2_PACKAGE_UTIL_LINUX_SCHEDUTILS=y

-# # gdbserver
-# BR2_PACKAGE_GDB=y
+# gdbserver
+BR2_PACKAGE_GDB=y

-# # ftrace
-# BR2_PACKAGE_TRACE_CMD=y
-#
-# # DTC
-# BR2_PACKAGE_DTC=y
-# BR2_PACKAGE_DTC_PROGRAMS=y
-# BR2_PACKAGE_HOST_DTC=y
+# ftrace
+BR2_PACKAGE_TRACE_CMD=y
+
+# DTC
+BR2_PACKAGE_DTC=y
+BR2_PACKAGE_DTC_PROGRAMS=y
+BR2_PACKAGE_HOST_DTC=y
--- a/common.py
+++ b/common.py
@@ -15,6 +15,37 @@ import sys

 this = sys.modules[__name__]

+# Default paths.
+root_dir = os.path.dirname(os.path.abspath(__file__))
+data_dir = os.path.join(root_dir, 'data')
+p9_dir = os.path.join(data_dir, '9p')
+gem5_non_default_src_root_dir = os.path.join(data_dir, 'gem5')
+gem5_readfile_file = os.path.join(data_dir, 'readfile')
+out_dir = os.path.join(root_dir, 'out')
+bench_boot = os.path.join(out_dir, 'bench-boot.txt')
+common_dir = os.path.join(out_dir, 'common')
+dl_dir = os.path.join(common_dir, 'dl')
+submodules_dir = os.path.join(root_dir, 'submodules')
+buildroot_src_dir = os.path.join(submodules_dir, 'buildroot')
+gem5_default_src_dir = os.path.join(submodules_dir, 'gem5')
+linux_src_dir = os.path.join(submodules_dir, 'linux')
+qemu_src_dir = os.path.join(submodules_dir, 'qemu')
+parsec_src_dir = os.path.join(submodules_dir, 'parsec-benchmark')
+
+# Other default variables.
+arch_map = {
+    'a': 'arm',
+    'A': 'aarch64',
+    'x': 'x86_64',
+}
+arches = [arch_map[k] for k in arch_map]
+gem5_cpt_prefix = '^cpt\.'
+sha = subprocess.check_output(['git', '-C', root_dir, 'log', '-1', '--format=%H']).decode().rstrip()
+config_file = os.path.join(data_dir, 'config')
+if os.path.exists(config_file):
+    config = imp.load_source('config', config_file)
+    configs = {x:getattr(config, x) for x in dir(config) if not x.startswith('__')}
+
 # TODO
 ## Benchmark a command.
 ##
@@ -50,6 +81,9 @@ def gem_list_checkpoint_dirs():
 def get_argparse(default_args=None, argparse_args=None):
    '''
    Return an argument parser with common arguments set.
+
+    :type default_args: Dict[str,str]
+    :type argparse_args: Dict
    '''
    global this
    if default_args is None:
@@ -370,31 +404,3 @@ def mkdir():
    os.makedirs(this.gem5_run_dir, exist_ok=True)
    os.makedirs(this.qemu_run_dir, exist_ok=True)
    os.makedirs(this.p9_dir, exist_ok=True)
-
-# Default paths.
-root_dir = os.path.dirname(os.path.abspath(__file__))
-data_dir = os.path.join(root_dir, 'data')
-p9_dir = os.path.join(data_dir, '9p')
-gem5_non_default_src_root_dir = os.path.join(data_dir, 'gem5')
-gem5_readfile_file = os.path.join(data_dir, 'readfile')
-out_dir = os.path.join(root_dir, 'out')
-bench_boot = os.path.join(out_dir, 'bench-boot.txt')
-common_dir = os.path.join(out_dir, 'common')
-submodules_dir = os.path.join(root_dir, 'submodules')
-buildroot_src_dir = os.path.join(submodules_dir, 'buildroot')
-gem5_default_src_dir = os.path.join(submodules_dir, 'gem5')
-linux_src_dir = os.path.join(submodules_dir, 'linux')
-qemu_src_dir = os.path.join(submodules_dir, 'qemu')
-
-# Other default variables.
-arch_map = {
-    'a': 'arm',
-    'A': 'aarch64',
-    'x': 'x86_64',
-}
-gem5_cpt_prefix = '^cpt\.'
-sha = subprocess.check_output(['git', '-C', root_dir, 'log', '-1', '--format=%H']).decode().rstrip()
-config_file = os.path.join(data_dir, 'config')
-if os.path.exists(config_file):
-    config = imp.load_source('config', config_file)
-    configs = {x:getattr(config, x) for x in dir(config) if not x.startswith('__')}
--- a/packages/gem5/external.mk
+++ b/packages/gem5/external.mk
@@ -5,7 +5,7 @@
 ################################################################################

 GEM5_VERSION = 1.0
-GEM5_SITE = $(BR2_EXTERNAL_GEM5_PATH)
+GEM5_SITE = $(GEM5_LKMC_SRCDIR)
 GEM5_SITE_METHOD = local

 ifeq ($(ARCH),x86_64)
@@ -17,11 +17,11 @@ endif
 define GEM5_BUILD_CMDS
 	# TODO cannot use TARGET_CONFIGURE_OPTS here because it overrides the CFLAGS on m5,
 	# which have an include. We should patch gem5 to add a += instead of = there.
-	cd '$(@D)/gem5/util/m5' && $(MAKE) -f 'Makefile.$(ARCH_MAKE)' CC='$(TARGET_CC)' LD='$(TARGET_LD)'
+	cd '$(@D)/util/m5' && $(MAKE) -f 'Makefile.$(ARCH_MAKE)' CC='$(TARGET_CC)' LD='$(TARGET_LD)'
 endef

 define GEM5_INSTALL_TARGET_CMDS
-	$(INSTALL) -D -m 0755 '$(@D)/gem5/util/m5/m5' '$(TARGET_DIR)/usr/bin'
+	$(INSTALL) -D -m 0755 '$(@D)/util/m5/m5' '$(TARGET_DIR)/usr/bin'
 endef

 $(eval $(generic-package))
--- a/20
+++ b/20
@@ -1,19 +1,19 @@
-#!/usr/bin/env bash
-. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/common"
-common_setup
-./build-all
-./zip-img
-tag="sha-${common_sha}"
-upload_basename="images-${common_sha}.zip"
-git tag "$tag"
-git push --tags
+#!/usr/bin/env python3
+import subprocess
+import common
+subprocess.check_call(['./build-all'])
+subprocess.check_call(['./zip-img'])
+tag = 'sha-{}'.format(common.sha)
+subprocess.check_call(['git', 'tag' tag])
+subprocess.check_call(['git', 'push' '--tags'])
 # TODO
 # - https://stackoverflow.com/questions/41022470/curl-request-to-add-file-to-github-release
 # - https://stackoverflow.com/questions/38627115/upload-files-to-github-directory-using-github-api
+# upload_basename = 'images-{}.zip'.format(common.sha)
 #curl "https://api.github.com/repos/cirosantilli/linux-kernel-module-cheat/releases/tags/${tag}/assets?access_token=$(cat data/access_token)&tag_name=${upload_basename}" \
 #  --header 'Content-Type: application/zip' \
 #  --upload-file "${common_out_dir}/${upload_basename}" \
 #  -H 'Accept: application/vnd.github.v3+json' \
 #  -X POST \
 #;
-./bench-all -Au
+subprocess.check_call(['./bench-all', '-A', '-u'])
--- a/6
+++ b/6
@@ -13,7 +13,7 @@ defaults = {
    'debug_guest': False,
    'debug_vm': False,
    'eval': None,
-    'extra_emulator_args': None,
+    'extra_emulator_args': [],
    'gem5_biglittle': False,
    'gem5_exe_args':'',
    'gem5_restore_last_checkpoint': None,
@@ -164,7 +164,7 @@ def main(args, extra_args=None):
            qemu_executable = "qemu-system-{}".format(args.arch)
        else:
            qemu_executable = common.qemu_executable
-        extra_emulator_args = extra_qemu_args + extra_emulator_args
+        extra_emulator_args = extra_qemu_args + args.extra_emulator_args
        cmd = (
            debug_vm +
            [
@@ -435,7 +435,7 @@ Run QEMU with VNC instead of the default SDL. Connect to it with:
 '''
    )
    parser.add_argument(
-        'extra_emulator_args', nargs='*',
+        'extra_emulator_args', nargs='*', default=defaults['extra_emulator_args'],
        help='Extra options to append at the end of the emulator command line'
    )
    return parser
--- a/3
+++ b/3
@@ -2,6 +2,7 @@

 import imp
 import os
+import sys

 import common
 rungdb = imp.load_source('rungdb', os.path.join(common.root_dir, 'rungdb'))
@@ -31,4 +32,4 @@ extra_args['before'] = '-ex \"add-symbol-file {} {}\"'.format(args.executable, h
 # TODO understand better.
 # Also, lx-symbols overrides the add-symbol-file commands.
 extra_args['no_lxsymbols'] = True
-rungdb.main(args, extra_args)
+sys.exit(rungdb.main(args, extra_args))
--- a/27
+++ b/27
@@ -1,10 +1,17 @@
-#!/usr/bin/env bash
-. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/common"
-common_setup
-cd "$common_root_dir"
-outfile="${common_out_dir}/lkmc-${common_sha}.zip"
-rm -f "$outfile"
-for common_arch in x86_64 arm aarch64; do
-  common_setup
-  zip -r "$outfile" "${common_qcow2_file#${common_root_dir}/}" "${common_linux_image#${common_root_dir}/}"
-done
+#!/usr/bin/env python3
+
+import os
+import subprocess
+import zipfile
+
+import common
+
+outfile = os.path.join(common.out_dir, 'lkmc-{}.zip'.format(common.sha))
+if os.path.exists(outfile):
+    os.unlink(outfile)
+zipf = zipfile.ZipFile(outfile, 'w', zipfile.ZIP_DEFLATED)
+for arch in common.arches:
+    common.setup(common.get_argparse(default_args={'arch': arch}))
+    zipf.write(common.qcow2_file, arcname=os.path.relpath(common.qcow2_file, common.root_dir))
+    zipf.write(common.linux_image, arcname=os.path.relpath(common.linux_image, common.root_dir))
+zipf.close()