common: convert scripts to python

This commit is contained in:
Ciro Santilli
2018-08-23 09:25:21 +01:00
parent f8c0502bb2
commit 9b30ac217f
6 changed files with 226 additions and 235 deletions

4
.gitignore vendored
View File

@@ -11,3 +11,7 @@ gitignore*
/images-*.zip
/out
/out.*
# Python trash.
*.pyc
__pycache__

View File

@@ -9553,22 +9553,15 @@ Otherwise, it becomes very difficult to keep everything working across path refa
Mnemonic: `fast`.
|`-g` | |Enable gem5 build or force its rebuild.
|`-h` | |Show this help message.
|`-L` |`VARIANT` |Linux kernel build variant.
|`-I` | |Enable initramfs for the current build.
|`-i` | |Enable initrd for the current build.
|`-K` |`KERNEL_CONFIG_FILE` |Use `KERNEL_CONFIG_FILE` as the exact Linux kernel
configuration. Ignore the default kernel config fragments,
but still add options explicitly passed with `-C` and `-c`.
on top of it.
|`-M` |`VARIANT` |gem5 build variant.
|`-p` | |Pass extra arguments to the `rootfs_post_build_script`.
|`-Q` |`VARIANT`` |QEMU build variant.
|`-S` | |Don't build QEMU with SDL support.
Graphics such as X11 won't work, only the terminal.
|`-s` | |Add a custom suffix to the build.
E.g., doing `./build -s mysuf` puts all the build output
into `out/x86_64-mysuf`. This allows keep multiple builds around
when you checkout between branches.
|`-v` | |Do a verbose build.
|===
@@ -9595,7 +9588,6 @@ Otherwise, it becomes very difficult to keep everything working across path refa
[options="header"]
|===
|Name |Argument name |Description
|`-a` |`ARCH` |Run architecture `ARCH`.
|`-c` |`NCPUS` |Emulate `NCPUS` guest CPUs.
|`-D` | |Run GDB on the emulator itself.
|`-d` | |Wait for GDB to connect before starting execution.
@@ -9618,7 +9610,6 @@ Otherwise, it becomes very difficult to keep everything working across path refa
|`-G` | |Pass extra options to the gem5 executable.
Do not confuse with the arguments passed to config scripts,
like `fs.py`. Example: `./run -G '--debug-flags=Exec --debug' -g`
|`-g` | |Use gem5 instead of QEMU.
|`-h` | |Show this help message.
|`-I` | |Run with initramfs.
|`-i` | |Run with initrd.
@@ -9632,8 +9623,6 @@ Otherwise, it becomes very difficult to keep everything working across path refa
The default is the minimum amount that boots all archs without extra
options added. Anything lower will lead some arch to fail to boot.
Any
|`-N` |`VARIANT` |gem5 source input variant.
|`-n` | |Run ID.
|`-R` | |Replay a QEMU run record deterministically.
|`-r` | |Record a QEMU run record for later replay with `-R`.
|`-P` | |Run the downloaded prebuilt images.

204
common
View File

@@ -1,204 +0,0 @@
#!/usr/bin/env bash
set -eu
common_abspath() (
echo "$(cd "$(dirname "$1")"; pwd)/$(basename "$1")"
)
# Benchmark a command.
#
# $1: command to benchmark
# $2: where to append write results to. Default: /dev/null.
#
# Result format:
#
# cmd <command run>
# time <time in seconds to finish>
# exit_status <exit status>
common_bench_cmd() (
cmd="$1"
results_file="${2:-/dev/null}"
printf 'cmd ' >> "$results_file"
env time --append -f 'time %e' --output="$results_file" "${common_root_dir}/eeval" -a "$cmd" "$results_file"
printf "exit_status $?\n" >> "$results_file"
)
# Handle options common across multiple scripts.
common_getopts_case() {
case "$1" in
a)
common_arch="$OPTARG"
;;
g)
common_gem5=true
;;
L)
common_linux_variant="$OPTARG"
;;
M)
common_gem5_variant="$OPTARG"
;;
N)
common_gem5_worktree="$OPTARG"
;;
n)
common_run_id="$OPTARG"
;;
Q)
common_qemu_variant="$OPTARG"
;;
s)
common_suffix="$OPTARG"
;;
t)
common_gem5_build_type="$OPTARG"
;;
?)
exit 2
;;
esac
}
common_getopts_flags='a:gL:M:N:n:Q:s:t:'
# Setup several variables and do other initialization common to most scripts.
# Typically done after getting inputs from the command line arguments.
common_setup() {
case "$common_arch" in
a|arm)
common_arch=arm
common_gem5_arch=ARM
;;
A|aarch64)
common_arch=aarch64
common_gem5_arch=ARM
;;
m|mips64)
common_arch=mips64
;;
x|x86_64)
common_arch=x86_64
common_gem5_arch=X86
;;
*)
printf "unknown arch: ${common_arch}\n" 1>&2
exit 2
;;
esac
common_buildroot_dir="${common_root_dir}/buildroot"
common_arch_dir="$common_arch"
if [ -n "$common_suffix" ]; then
common_arch_dir="${common_arch_dir}-${common_suffix}"
fi
common_out_arch_dir="${common_out_dir}/${common_arch_dir}"
common_buildroot_out_dir="${common_out_arch_dir}/buildroot"
common_build_dir="${common_buildroot_out_dir}/build"
common_linux_custom_dir="${common_build_dir}/linux-custom"
common_linux_variant_dir="${common_linux_custom_dir}.${common_linux_variant}"
common_vmlinux="${common_linux_variant_dir}/vmlinux"
common_qemu_custom_dir="${common_build_dir}/host-qemu-custom"
common_qemu_guest_variant_dir="${common_qemu_custom_dir}.${common_qemu_variant}"
common_qemu_variant_dir="${common_qemu_custom_dir}.${common_qemu_variant}"
common_qemu_exec="${common_qemu_variant_dir}/${common_arch}-softmmu/qemu-system-${common_arch}"
common_qemu_guest_custom_dir="${common_build_dir}/qemu-custom"
common_host_dir="${common_buildroot_out_dir}/host"
common_images_dir="${common_buildroot_out_dir}/images"
common_qcow2_file="${common_images_dir}/rootfs.ext2.qcow2"
common_staging_dir="${common_buildroot_out_dir}/staging"
common_target_dir="${common_buildroot_out_dir}/target"
common_gem5_run_dir="${common_out_arch_dir}/gem5/${common_run_id}"
common_m5out_dir="${common_gem5_run_dir}/m5out"
common_trace_txt_file="${common_m5out_dir}/trace.txt"
common_gem5_termout_file="${common_gem5_run_dir}/termout.txt"
common_qemu_run_dir="${common_out_arch_dir}/qemu/${common_run_id}"
common_qemu_termout_file="${common_qemu_run_dir}/termout.txt"
common_qemu_rrfile="${common_qemu_run_dir}/rrfile"
common_gem5_out_dir="${common_dir}/gem5/${common_gem5_variant}"
common_gem5_m5term="${common_gem5_out_dir}/m5term"
common_gem5_build_dir="${common_gem5_out_dir}/build"
common_gem5_exec="${common_gem5_build_dir}/${common_gem5_arch}/gem5.${common_gem5_build_type}"
common_gem5_system_dir="${common_gem5_out_dir}/system"
if [ -n "$common_gem5_worktree" ]; then
common_gem5_src_dir="${common_gem5_non_default_src_root_dir}/${common_gem5_worktree}"
else
common_gem5_src_dir="$common_gem5_default_src_dir"
fi
if "$common_gem5"; then
common_exec="$common_gem5_exec"
common_run_dir="$common_gem5_run_dir"
common_termout_file="$common_gem5_termout_file"
else
common_exec="$common_qemu_exec"
common_run_dir="$common_qemu_run_dir"
common_termout_file="$common_qemu_termout_file"
fi
case "$common_arch" in
arm)
common_linux_image=arch/arm/boot/zImage
;;
aarch64)
common_linux_image=arch/arm64/boot/Image
;;
mips64)
common_linux_image=vmlinux
;;
x86_64)
common_linux_image=arch/x86/boot/bzImage
;;
esac
common_linux_image="${common_linux_variant_dir}/${common_linux_image}"
# Ports.
common_run_id_number="$(echo "$common_run_id" | cut -d . -f 2)"
if "$common_gem5"; then
common_gem5_telnet_port="$((3456 + $common_run_id_number))"
common_gdb_port="$((7000 + $common_run_id_number))"
else
common_qemu_base_port="$((45454 + 10 * $common_run_id_number))"
common_qemu_monitor_port="$(($common_qemu_base_port + 0))"
common_qemu_hostfwd_generic_port="$(($common_qemu_base_port + 1))"
common_qemu_hostfwd_ssh_port="$(($common_qemu_base_port + 2))"
common_qemu_gdb_port="$(($common_qemu_base_port + 3))"
common_gdb_port="$common_qemu_gdb_port"
fi
}
common_mkdir() (
mkdir -p \
"$common_build_dir" \
"$common_gem5_out_dir" \
"$common_gem5_run_dir" \
"$common_qemu_run_dir" \
"$common_9p_dir" \
;
)
# Default paths.
common_root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
common_data_dir="${common_root_dir}/data"
common_9p_dir="${common_data_dir}/9p"
common_gem5_non_default_src_root_dir="${common_data_dir}/gem5"
common_gem5_readfile_file="${common_data_dir}/readfile"
common_gem5_default_src_dir="${common_root_dir}/gem5/gem5"
common_out_dir="${common_root_dir}/out"
common_bench_boot="${common_out_dir}/bench-boot.txt"
common_dir="${common_out_dir}/common"
# Other default variables.
common_arch=x86_64
common_gem5=false
common_gem5_build_type=opt
common_gem5_cpt_pref='^cpt\.'
common_gem5_variant=default
common_gem5_worktree=
common_linux_variant=default
common_qemu_variant=default
common_run_id=0
common_sha="$(git log -1 --format="%H")"
common_suffix=
f="${common_data_dir}/config"
if [ -f "$f" ]; then
. "$f"
fi

210
common.py Normal file
View File

@@ -0,0 +1,210 @@
#!/usr/bin/env python3
import argparse
import subprocess
import os
import sys
this = sys.modules[__name__]
# TODO
## Benchmark a command.
##
## $1: command to benchmark
## $2: where to append write results to. Default: /dev/null.
##
## Result format:
##
## cmd <command run>
## time <time in seconds to finish>
## exit_status <exit status>
#bench_cmd() (
# cmd = "$1"
# results_file = "${2:-/dev/null}"
# printf 'cmd ' >> "$results_file"
# env time --append -f 'time %e' --output = "$results_file" "${root_dir}/eeval" -a "$cmd" "$results_file"
# printf "exit_status $?\n" >> "$results_file"
#)
def get_argparse(**kwargs):
"""
Return an argument parser with common arguments set.
"""
default_build_id='default'
parser = argparse.ArgumentParser(
formatter_class=argparse.RawTextHelpFormatter,
**kwargs
)
parser.add_argument(
'-a', '--arch', default='x86_64',
help='CPU architecture. Default: %(default)s'
)
parser.add_argument(
'-g', '--gem5', default=False, action='store_true',
help='Use gem5 instead of QEMU'
)
parser.add_argument(
'-L', '--linux-build-id', default=default_build_id,
help='Linux build ID. Allows you to keep multiple separate Linux builds. Default: %(default)s'
)
parser.add_argument(
'-M', '--gem5-build-id', default=default_build_id,
help='gem5 build ID. Allows you to keep multiple separate gem5 builds. Default: %(default)s'
)
parser.add_argument(
'-N', '--gem5-worktree',
help="""\
gem5 worktree to use for build and Python scripts at runtime.
If not given, just use the submodule source.
"""
)
parser.add_argument(
'-n', '--run-id', default='0',
help="""\
ID for run outputs such as gem5's m5out. Allows you to do multiple runs,
and then inspect separate outputs later in different output directories.
Default: %(default)s
"""
)
parser.add_argument(
'--port-offset', type=int,
help="""\
Increase the ports to be used such as for GDB by an offset to run multiple
instances in parallel.
Default: %(default)s
"""
)
parser.add_argument(
'-Q', '--qemu-build-id', default=default_build_id,
help='QEMU build ID. Allows you to keep multiple separate QEMU builds. Default: %(default)s'
)
parser.add_argument(
'-s', '--suffix',
help="""\
Add a custom suffix to the build. E.g., doing `./build -s mysuf` puts all
the build output into `out/x86_64-mysuf`. This allows keep multiple builds
around when you checkout between branches.
"""
)
parser.add_argument(
'-t', '--gem5-build-type', default='opt',
help='gem5 build type, most often used for "debug" builds. Default: %(default)s'
)
return parser
def setup(parser):
"""
Parse the command line arguments, and setup several variables based on them.
Typically done after getting inputs from the command line arguments.
"""
global this
args = parser.parse_args()
if args.arch == 'a' or args.arch == 'arm':
args.arch = 'arm'
gem5_arch = 'ARM'
elif args.arch == 'A' or args.arch == 'aarch64':
args.arch = 'aarch64'
gem5_arch = 'ARM'
elif args.arch == 'x' or args.arch == 'x86_64':
args.arch = 'x86_64'
gem5_arch = 'X86'
else:
print('unknown arch: {}'.format(args.arch), file=sys.stderr)
sys.exit(2)
this.buildroot_dir = os.path.join(root_dir, 'buildroot')
this.arch_dir = args.arch
if args.suffix is not None:
this.arch_dir = '{}-{}'.format(arch_dir, args.suffix)
global out_arch_dir
this.out_arch_dir = os.path.join(this.out_dir, this.arch_dir)
this.buildroot_out_dir = os.path.join(this.out_arch_dir, 'buildroot')
this.build_dir = os.path.join(this.buildroot_out_dir, 'build')
this.linux_custom_dir = os.path.join(this.build_dir, 'linux-custom')
this.linux_variant_dir = '{}.{}'.format(this.linux_custom_dir, args.linux_build_id)
this.vmlinux = os.path.join(this.linux_variant_dir, "vmlinux")
this.qemu_custom_dir = os.path.join(this.build_dir, 'host-qemu-custom')
this.qemu_guest_variant_dir = os.path.join(this.qemu_custom_dir, args.qemu_build_id)
this.qemu_variant_dir = '{}.{}'.format(this.qemu_custom_dir, args.qemu_build_id)
this.qemu_executable = os.path.join(this.qemu_variant_dir, '{}-softmmu'.format(args.arch), 'qemu-system-{}'.format(args.arch))
this.qemu_guest_custom_dir = os.path.join(this.build_dir, 'qemu-custom')
this.host_dir = os.path.join(this.buildroot_out_dir, 'host')
this.images_dir = os.path.join(this.buildroot_out_dir, 'images')
this.qcow2_file = os.path.join(this.images_dir, 'rootfs.ext2.qcow2')
this.staging_dir = os.path.join(this.buildroot_out_dir, 'staging')
this.target_dir = os.path.join(this.buildroot_out_dir, 'target')
this.gem5_run_dir = os.path.join(this.out_arch_dir, 'gem5', str(args.run_id))
this.m5out_dir = os.path.join(this.gem5_run_dir, 'm5out')
this.trace_txt_file = os.path.join(this.m5out_dir, 'trace.txt')
this.gem5_termout_file = os.path.join(this.gem5_run_dir, 'termout.txt')
this.qemu_run_dir = os.path.join(this.out_arch_dir, 'qemu', str(args.run_id))
this.qemu_termout_file = os.path.join(this.qemu_run_dir, 'termout.txt')
this.qemu_rrfile = os.path.join(this.qemu_run_dir, 'rrfile')
this.gem5_out_dir = os.path.join(this.common_dir, 'gem5', args.gem5_build_id)
this.gem5_m5term = os.path.join(this.gem5_out_dir, 'm5term')
this.gem5_build_dir = os.path.join(this.gem5_out_dir, 'build')
this.gem5_executable = os.path.join(this.gem5_build_dir, gem5_arch, 'gem5.{}'.format(args.gem5_build_type))
this.gem5_system_dir = os.path.join(this.gem5_out_dir, 'system')
if args.gem5_worktree is not None:
this.gem5_src_dir = os.path.join(this.gem5_non_default_src_root_dir, args.gem5_worktree)
else:
this.gem5_src_dir = this.gem5_default_src_dir
if args.gem5:
this.executable = this.gem5_executable
this.run_dir = this.gem5_run_dir
this.termout_file = this.gem5_termout_file
else:
this.executable = this.qemu_executable
this.run_dir = this.qemu_run_dir
this.termout_file = this.qemu_termout_file
if args.arch == 'arm':
this.linux_image = os.path.join('arch', 'arm', 'boot', 'zImage')
elif args.arch == 'aarch64':
this.linux_image = os.path.join('arch', 'arm64', 'boot', 'Image')
elif args.arch == 'x86_64':
this.linux_image = os.path.join('arch', 'x86', 'boot', 'bzImage')
this.linux_image = os.path.join(this.linux_variant_dir, linux_image)
# Ports.
if args.port_offset is None:
args.port_offset = int(args.run_id)
if args.gem5:
this.gem5_telnet_port = 3456 + args.port_offset
this.gdb_port = 7000 + args.port_offset
else:
this.qemu_base_port = 45454 + 10 * args.port_offset
this.qemu_monitor_port = this.qemu_base_port + 0
this.qemu_hostfwd_generic_port = this.qemu_base_port + 1
this.qemu_hostfwd_ssh_port = this.qemu_base_port + 2
this.qemu_gdb_port = this.qemu_base_port + 3
this.gdb_port = this.qemu_gdb_port
return args
def mkdir():
global this
os.makedirs(this.build_dir, exist_ok=True)
os.mkdirs(this.gem5_out_dir, exit_ok=True)
os.mkdirs(this.gem5_run_dir, exit_ok=True)
os.mkdirs(this.qemu_run_dir, exit_ok=True)
os.mkdirs(this.p9_dir, exit_ok=True)
# Default paths.
root_dir = os.path.dirname(os.path.abspath(__file__))
data_dir = os.path.join(root_dir, 'data')
p9_dir = os.path.join(data_dir, '9p')
gem5_non_default_src_root_dir = os.path.join(data_dir, 'gem5')
gem5_readfile_file = os.path.join(data_dir, 'readfile')
gem5_default_src_dir = os.path.join(root_dir, 'gem5', 'gem5')
out_dir = os.path.join(root_dir, 'out')
bench_boot = os.path.join(out_dir, 'bench-boot.txt')
common_dir = os.path.join(out_dir, 'common')
# Other default variables.
gem5_cpt_pref = '^cpt\.'
sha = subprocess.check_output(['git', '-C', root_dir, 'log', '-1', '--format=%H']).decode().rstrip()
# Config file. TODO move to decent python setup.
config_file = os.path.join(data_dir, 'config')
if os.path.exists(config_file):
exec(open(config_file).read())
with open(config_file) as f:
exec(f.read())

24
getvar
View File

@@ -1,16 +1,8 @@
#!/usr/bin/env bash
. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/common"
while getopts "h${common_getopts_flags}" OPT; do
case "$OPT" in
h)
echo "https://github.com/cirosantilli/linux-kernel-module-cheat#getvar" 2>&1
exit
;;
?)
common_getopts_case "$OPT"
;;
esac
done
shift "$(($OPTIND - 1))"
common_setup
eval "echo \$common_${1}"
#!/usr/bin/env python3
import common
parser = common.get_argparse(
description='https://github.com/cirosantilli/linux-kernel-module-cheat#getvar'
)
parser.add_argument('variable')
args = common.setup(parser)
print(getattr(common, args.variable))

8
run
View File

@@ -176,7 +176,7 @@ if "$common_gem5"; then
gem5_common="\
M5_PATH='${common_gem5_system_dir}' \\
${debug_vm}\
'${common_exec}' \\
'${common_executable}' \\
--debug-file=trace.txt \\
${gem5opts}\
-d '${common_m5out_dir}' \\
@@ -248,14 +248,14 @@ else
fi
if "$prebuilt"; then
common_mkdir
qemu_exec="qemu-system-${common_arch}"
qemu_executable="qemu-system-${common_arch}"
else
qemu_exec="${common_qemu_exec}"
qemu_executable="${common_qemu_exec}"
fi
extra_flags="${extra_flags_qemu}${extra_flags}"
qemu_common="\
${debug_vm}\
${qemu_exec} \\
${qemu_executable} \\
-device rtl8139,netdev=net0 \\
-gdb 'tcp::${common_gdb_port}' \\
-kernel '${common_linux_image}' \\