rungdb, gem5-shell and ./run -u ported

README.adoc

@@ -1112,7 +1112,7 @@ Bibliography: https://stackoverflow.com/questions/5947286/how-to-load-linux-kern
 
 When doing long simulations sweeping across multiple system parameters, it becomes fundamental to do multiple simulations in parallel.
 
-This is specially true for gem5, which runs much slower than QEMU, and cannot use multiple host cores to speed up the simulation: https://github.com/cirosantilli-work/gem5-issues/issues/15
+This is specially true for gem5, which runs much slower than QEMU, and cannot use multiple host cores to speed up the simulation: link:https://github.com/cirosantilli-work/gem5-issues/issues/15[], so the only way to parallelize is to run multiple instances in parallel.
 
 This also has a good synergy with <<build-variants>>.
 
@@ -1128,9 +1128,16 @@ Another shell:
 ./run -n 1
 ....
 
+and now you have two QEMU instances running in parallel.
+
 The default run id is `0`.
 
-This method also allows us to keep run outputs in separate directories for later inspection, e.g.:
+Our scripts solve two difficulties with simultaneous runs:
+
+* port conflicts, e.g. GDB and link:gem5-shell[]
+* output directory conflicts, e.g. traces and gem5 stats overwriting one another
+
+Each run gets a separate output directory. For example:
 
 ....
 ./run -a A -g -n 0 &>/dev/null &
@@ -1140,8 +1147,8 @@ This method also allows us to keep run outputs in separate directories for later
 produces two separate `m5out` directories:
 
 ....
-ls "$(./getvar -a A -g -n 0 m5out_dir)"
-ls "$(./getvar -a A -g -n 1 m5out_dir)"
+echo "$(./getvar -a A -g -n 0 m5out_dir)"
+echo "$(./getvar -a A -g -n 1 m5out_dir)"
 ....
 
 and the gem5 host executable stdout and stderr can be found at:
@@ -1153,21 +1160,13 @@ less "$(./getvar -a A -g -n 1 termout_file)"
 
 Each line is prepended with the timestamp in seconds since the start of the program when it appeared.
 
-You can also add a prefix to the build ID before a period:
+To have more semantic output directories names for later inspection, you can use a non numeric string for the run ID, and indicate the port offset explicitly:
 
 ....
-./run -a A -g -n some-experiment.1
+./run -a A -g -n some-experiment --port-offset 1
 ....
 
-and makes it easier to remember afterwards which directory contains what.
-
-However this still takes up the same ports as:
-
-....
-./run -a A -g -n 1
-....
-
-so you cannot run both at the same time.
+`--port-offset` defaults to the run ID when that is a number.
 
 Like <<cpu-architecture>>, you will need to pass the `-n` option to anything that needs to know runtime information, e.g. <<gdb>>:
 

common.py

@@ -32,19 +32,23 @@ this = sys.modules[__name__]
 def base64_encode(string):
     return base64.b64encode(string.encode()).decode()
 
-def get_argparse(**kwargs):
+def get_argparse(default_args=None, argparse_args=None):
     """
     Return an argument parser with common arguments set.
     """
     global this
+    if default_args is None:
+        default_args = {}
+    if argparse_args is None:
+        argparse_args = {}
     arch_choices = []
     for key in this.arch_map:
         arch_choices.append(key)
         arch_choices.append(this.arch_map[key])
-    default_build_id='default'
+    default_build_id = 'default'
     parser = argparse.ArgumentParser(
         formatter_class=argparse.RawTextHelpFormatter,
-        **kwargs
+        **argparse_args
     )
     parser.add_argument(
         '-a', '--arch', choices=arch_choices, default='x86_64',
@@ -102,11 +106,13 @@ around when you checkout between branches.
         '-t', '--gem5-build-type', default='opt',
         help='gem5 build type, most often used for "debug" builds. Default: %(default)s'
     )
+    defaults = this.configs.copy()
+    defaults.update(default_args)
     # A bit ugly as it actually changes the defaults shown on --help, but we can't do any better
     # because it is impossible to check if arguments were given or not...
     # - https://stackoverflow.com/questions/30487767/check-if-argparse-optional-argument-is-set-or-not
     # - https://stackoverflow.com/questions/3609852/which-is-the-best-way-to-allow-configuration-options-be-overridden-at-the-comman
-    parser.set_defaults(**this.configs)
+    parser.set_defaults(**defaults)
     return parser
 
 def print_cmd(cmd):
@@ -115,7 +121,7 @@ def print_cmd(cmd):
         out.extend([shlex.quote(arg), ' \\\n'])
     print(''.join(out))
 
-def setup(parser):
+def setup(parser, **extra_args):
     """
     Parse the command line arguments, and setup several variables based on them.
     Typically done after getting inputs from the command line arguments.

gem5-shell

@@ -1,13 +1,13 @@
-#!/usr/bin/env bash
-. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/common"
-common_gem5=true
-while getopts "${common_getopts_flags}" OPT; do
-  case "$OPT" in
-    ?)
-      common_getopts_case "$OPT"
-      ;;
-  esac
-done
-shift "$(($OPTIND - 1))"
-common_setup
-"${common_gem5_m5term}" localhost "$common_gem5_telnet_port"
+#!/usr/bin/env python3
+
+import subprocess
+import sys
+
+import common
+
+parser = common.get_argparse(
+    default_args={'gem5':True},
+    argparse_args={'description':'Connect a terminal to a running gem5 instance'}
+)
+args = common.setup(parser)
+sys.exit(subprocess.Popen([str(common.gem5_m5term), 'localhost', str(common.gem5_telnet_port)]).wait())

getvar

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 import common
 parser = common.get_argparse(
-    description='https://github.com/cirosantilli/linux-kernel-module-cheat#getvar'
+    argparse_args={'description':'https://github.com/cirosantilli/linux-kernel-module-cheat#getvar'}
 )
 parser.add_argument('variable')
 args = common.setup(parser)

run

@@ -2,13 +2,14 @@
 
 import os
 import shlex
+import signal
 import subprocess
 import sys
 
 import common
 
 # Argparse.
-parser = common.get_argparse(description='Run Linux on an emulator')
+parser = common.get_argparse(argparse_args={'description':'Run Linux on an emulator'})
 init_group = parser.add_mutually_exclusive_group()
 kvm_group = parser.add_mutually_exclusive_group()
 parser.add_argument(
@@ -123,7 +124,7 @@ rare and don't affect performance, because `./configure
 """
 )
 parser.add_argument(
-    '-U', '--tmux-args',
+    '-U', '--tmux-args', default='',
     help='Pass extra parameters to the program running on the `-u` tmux split'
 )
 parser.add_argument(
@@ -358,28 +359,37 @@ else:
             virtio_gpu_pci
         )
 
-#if args.tmux:
-#    if args.gem5:
-#        eval "./tmu 'sleep 2;./gem5-shell -n ${common_run_id} ${tmux_args};'"
-#    elif args.debug:
-#        eval "./tmu ./rungdb -a '${args.arch} -L ${common_linux_variant}' -n ${common_run_id} ${tmux_args}"
-#if [ -n "${1:-}" ]; then
-#  extra_emulator_args="${extra_emulator_args}${@} \\
-#"
-#fi
+if args.tmux:
+    if args.gem5:
+        subprocess.Popen([os.path.join(common.root_dir, 'tmu'),
+            'sleep 2;./gem5-shell -n {} {}' \
+            .format(args.run_id, args.tmux_args)
+        ])
+    elif args.debug:
+        # TODO find a nicer way to forward all those args automatically.
+        # Part of me wants to: https://github.com/jonathanslenders/pymux
+        # but it cannot be used as a library properly it seems, and it is
+        # slower than tmux.
+        subprocess.Popen([os.path.join(common.root_dir, 'tmu'),
+            "sleep 2;./rungdb -a '{}' -L '{}' -n '{}' {}" \
+            .format(args.arch, args.linux_build_id, args.run_id, args.tmux_args)
+        ])
 
 cmd += extra_emulator_args
 common.print_cmd(cmd)
+# Otherwise Ctrl + C gives an ugly Python stack trace for gem5 (QEMU takes over terminal and is fine).
+signal.signal(signal.SIGINT, signal.SIG_IGN)
 subprocess.Popen(cmd, env=env).wait()
+signal.signal(signal.SIGINT, signal.SIG_DFL)
 
 #cmd="time \\
 #${cmd}${extra_emulator_args}"
 #if [ -z "$debug_vm" ]; then
 #  cmd="${cmd}\
-#|& tee >(ts -s %.s > ${common_termout_file})\
+#|& tee >(ts -s %.s > ${common.termout_file})\
 #"
 #fi
-#"${common_root_dir}/eeval" "$cmd" "${common_run_dir}/run.sh"
+#"${common.root_dir}/eeval" "$cmd" "${common.run_dir}/run.sh"
 #cmd_out=$?
 #if [ "$cmd_out" -ne 0 ]; then
 #  exit "$cmd_out"

rungdb

@@ -2,12 +2,13 @@
 
 import os
 import shlex
+import sys
 import signal
 import subprocess
 
 import common
 
-parser = common.get_argparse(description='Connect with GDB to an emulator to debug Linux itself')
+parser = common.get_argparse(argparse_args={'description':'Connect with GDB to an emulator to debug Linux itself'})
 parser.add_argument(
     '-A', '--after', default='',
     help='Pass extra arguments to GDB, to be appended after all other arguments'
@@ -82,6 +83,9 @@ common.print_cmd(cmd)
 
 # TODO eeval
 # "${common.root_dir}/eeval" "$cmd $after" "${common.run_dir}/rungdb.sh"
-def signal_handler(sig, frame): pass
-signal.signal(signal.SIGINT, signal_handler)
-subprocess.Popen(cmd, cwd=common.linux_variant_dir).wait()
+
+# Required, otherwise Ctrl + C kills Python and that kills GDB.
+# https://stackoverflow.com/questions/19807134/does-python-always-raise-an-exception-if-you-do-ctrlc-when-a-subprocess-is-exec
+signal.signal(signal.SIGINT, lambda *args: None)
+
+sys.exit(subprocess.Popen(cmd, cwd=common.linux_variant_dir).wait())