test-gdb: move to pure python calls

This commit is contained in:
Ciro Santilli 六四事件 法轮功
2019-01-22 00:00:00 +00:00
parent d923c606f3
commit 3ce152f61c
10 changed files with 182 additions and 100 deletions

2
build
View File

@@ -86,7 +86,7 @@ buildroot_component = Component(
name_to_component_map = { name_to_component_map = {
# Leaves without dependencies. # Leaves without dependencies.
'baremetal-qemu': Component( 'baremetal-qemu': Component(
lambda arch: run_cmd(['build-baremetal', '--qemu'], arch), lambda arch: run_cmd(['build-baremetal', '--emulator', 'qemu'], arch),
supported_archs=kwargs['crosstool_ng_supported_archs'], supported_archs=kwargs['crosstool_ng_supported_archs'],
), ),
'baremetal-gem5': Component( 'baremetal-gem5': Component(

View File

@@ -92,10 +92,9 @@ Build the baremetal examples with crosstool-NG.
bootloader_obj=bootloader_obj, bootloader_obj=bootloader_obj,
common_objs=common_objs, common_objs=common_objs,
) )
arch_dir = os.path.join('arch', self.env['arch']) if os.path.isdir(os.path.join(self.env['baremetal_src_arch_dir'])):
if os.path.isdir(os.path.join(self.env['baremetal_src_dir'], arch_dir)):
self._build_dir( self._build_dir(
arch_dir, self.env['baremetal_src_arch_subpath'],
gcc=gcc, gcc=gcc,
cflags=cflags, cflags=cflags,
entry_address=entry_address, entry_address=entry_address,
@@ -127,12 +126,12 @@ Build the baremetal examples with crosstool-NG.
common_objs, common_objs,
bootloader=True bootloader=True
): ):
""" '''
Build all .c and .S files in a given subpath of the baremetal source Build all .c and .S files in a given subpath of the baremetal source
directory non recursively. directory non recursively.
Place outputs on the same subpath or the output directory. Place outputs on the same subpath or the output directory.
""" '''
in_dir = os.path.join(self.env['baremetal_src_dir'], subpath) in_dir = os.path.join(self.env['baremetal_src_dir'], subpath)
out_dir = os.path.join(self.env['baremetal_build_dir'], subpath) out_dir = os.path.join(self.env['baremetal_build_dir'], subpath)
os.makedirs(out_dir, exist_ok=True) os.makedirs(out_dir, exist_ok=True)

View File

@@ -4,7 +4,7 @@ import argparse
import imp import imp
import os import os
class Argument: class _Argument:
def __init__( def __init__(
self, self,
long_or_short_1, long_or_short_1,
@@ -14,28 +14,22 @@ class Argument:
nargs=None, nargs=None,
**kwargs **kwargs
): ):
if long_or_short_2 is None:
shortname = None
longname = long_or_short_1
else:
shortname = long_or_short_1
longname = long_or_short_2
self.args = [] self.args = []
# argparse is crappy and cannot tell us if arguments were given or not. # argparse is crappy and cannot tell us if arguments were given or not.
# We need that information to decide if the config file should override argparse or not. # We need that information to decide if the config file should override argparse or not.
# So we just use None as a sentinel. # So we just use None as a sentinel.
self.kwargs = {'default': None} self.kwargs = {'default': None}
shortname, longname, key, is_option = self.get_key(
long_or_short_1,
long_or_short_2
)
if shortname is not None: if shortname is not None:
self.args.append(shortname) self.args.append(shortname)
if longname[0] == '-': if is_option:
self.args.append(longname) self.args.append(longname)
self.key = longname.lstrip('-').replace('-', '_')
self.is_option = True
else: else:
self.key = longname.replace('-', '_') self.args.append(key)
self.args.append(self.key)
self.kwargs['metavar'] = longname self.kwargs['metavar'] = longname
self.is_option = False
if default is not None and nargs is None: if default is not None and nargs is None:
self.kwargs['nargs'] = '?' self.kwargs['nargs'] = '?'
if nargs is not None: if nargs is not None:
@@ -53,22 +47,51 @@ class Argument:
if self.is_bool and not 'action' in kwargs: if self.is_bool and not 'action' in kwargs:
self.kwargs['action'] = bool_action self.kwargs['action'] = bool_action
if help is not None: if help is not None:
if not self.is_bool and default: if default is not None:
help += ' Default: {}'.format(default) if help[-1] == '\n':
if '\n\n' in help[:-1]:
help += '\n'
elif help[-1] == ' ':
pass
else:
help += ' '
help += 'Default: {}'.format(default)
self.kwargs['help'] = help self.kwargs['help'] = help
self.optional = ( self.optional = (
default is not None or default is not None or
self.is_bool or self.is_bool or
self.is_option or is_option or
nargs in ('?', '*', '+') nargs in ('?', '*', '+')
) )
self.kwargs.update(kwargs) self.kwargs.update(kwargs)
self.default = default self.default = default
self.longname = longname self.longname = longname
self.key = key
self.is_option = is_option
def __str__(self): def __str__(self):
return str(self.args) + ' ' + str(self.kwargs) return str(self.args) + ' ' + str(self.kwargs)
@staticmethod
def get_key(
long_or_short_1,
long_or_short_2=None,
**kwargs
):
if long_or_short_2 is None:
shortname = None
longname = long_or_short_1
else:
shortname = long_or_short_1
longname = long_or_short_2
if longname[0] == '-':
key = longname.lstrip('-').replace('-', '_')
is_option = True
else:
key = longname.replace('-', '_')
is_option = False
return shortname, longname, key, is_option
class CliFunction: class CliFunction:
''' '''
Represent a function that can be called either from Python code, or Represent a function that can be called either from Python code, or
@@ -141,7 +164,7 @@ class CliFunction:
*args, *args,
**kwargs **kwargs
): ):
argument = Argument(*args, **kwargs) argument = _Argument(*args, **kwargs)
self._arguments.append(argument) self._arguments.append(argument)
self._all_keys.add(argument.key) self._all_keys.add(argument.key)
@@ -170,6 +193,10 @@ class CliFunction:
args = parser.parse_args(args=cli_args) args = parser.parse_args(args=cli_args)
return self(**vars(args)) return self(**vars(args))
@staticmethod
def get_key(*args, **kwargs):
return _Argument.get_key(*args, **kwargs)
def main(self, **kwargs): def main(self, **kwargs):
''' '''
Do the main function call work. Do the main function call work.
@@ -217,7 +244,7 @@ amazing function!
'args_star': [] 'args_star': []
} }
# Default CLI call. # Default CLI call with programmatic CLI arguments.
out = one_cli_function.cli(['1']) out = one_cli_function.cli(['1'])
assert out == default assert out == default
@@ -258,5 +285,5 @@ amazing function!
# Force a boolean value set on the config to be False on CLI. # Force a boolean value set on the config to be False on CLI.
assert one_cli_function.cli(['--no-bool-cli', '1'])['bool_cli'] is False assert one_cli_function.cli(['--no-bool-cli', '1'])['bool_cli'] is False
# CLI call. # CLI call with argv command line arguments.
print(one_cli_function.cli()) print(one_cli_function.cli())

View File

@@ -92,6 +92,7 @@ consts['obj_ext'] = '.o'
consts['config_file'] = os.path.join(consts['data_dir'], 'config.py') consts['config_file'] = os.path.join(consts['data_dir'], 'config.py')
consts['magic_fail_string'] = b'lkmc_test_fail' consts['magic_fail_string'] = b'lkmc_test_fail'
consts['baremetal_lib_basename'] = 'lib' consts['baremetal_lib_basename'] = 'lib'
consts['emulators'] = ['qemu', 'gem5']
class LkmcCliFunction(cli_function.CliFunction): class LkmcCliFunction(cli_function.CliFunction):
''' '''
@@ -100,9 +101,15 @@ class LkmcCliFunction(cli_function.CliFunction):
* command timing * command timing
* some common flags, e.g.: --arch, --dry-run, --verbose * some common flags, e.g.: --arch, --dry-run, --verbose
''' '''
def __init__(self, *args, do_print_time=True, **kwargs): def __init__(self, *args, defaults=None, **kwargs):
'''
:ptype defaults: Dict[str,Any]
:param defaults: override the default value of an argument
'''
kwargs['config_file'] = consts['config_file'] kwargs['config_file'] = consts['config_file']
self._do_print_time = do_print_time if defaults is None:
defaults = {}
self._defaults = defaults
super().__init__(*args, **kwargs) super().__init__(*args, **kwargs)
# Args for all scripts. # Args for all scripts.
@@ -122,6 +129,10 @@ Bash equivalents even for actions taken directly in Python without shelling out.
mkdir are generally omitted since those are obvious mkdir are generally omitted since those are obvious
''' '''
) )
self.add_argument(
'--print-time', default=True,
help='Print how long it took to run the command at the end.'
)
self.add_argument( self.add_argument(
'-v', '--verbose', default=False, '-v', '--verbose', default=False,
help='Show full compilation commands when they are not shown by default.' help='Show full compilation commands when they are not shown by default.'
@@ -266,27 +277,29 @@ instances in parallel. Default: the run ID (-n) if that is an integer, otherwise
# Misc. # Misc.
self.add_argument( self.add_argument(
'-g', '--gem5', default=False, '--emulator', choices=consts['emulators'],
help='Use gem5 instead of QEMU.' help='''\
Set the emulator to use. Ignore --gem5.
'''
) )
self.add_argument( self.add_argument(
'--qemu', default=False, '-g', '--gem5', default=False,
help='''\ help='''\
Use QEMU as the emulator. This option exists in addition to --gem5 Use gem5 instead of QEMU. Shortcut for `--emulator gem5`.
to allow overriding configs from the CLI.
''' '''
) )
def _init_env(self, env): def _init_env(self, env):
''' '''
Update the kwargs from the command line with derived arguments. Update the kwargs from the command line with values derived from them.
''' '''
def join(*paths): def join(*paths):
return os.path.join(*paths) return os.path.join(*paths)
if env['qemu'] or not env['gem5']: if env['emulator'] is None:
env['emulator'] = 'qemu' if env['gem5']:
else: env['emulator'] = 'gem5'
env['emulator'] = 'gem5' else:
env['emulator'] = 'qemu'
if env['arch'] in env['arch_short_to_long_dict']: if env['arch'] in env['arch_short_to_long_dict']:
env['arch'] = env['arch_short_to_long_dict'][env['arch']] env['arch'] = env['arch_short_to_long_dict'][env['arch']]
if env['userland_build_id'] is None: if env['userland_build_id'] is None:
@@ -481,6 +494,8 @@ to allow overriding configs from the CLI.
# Baremetal. # Baremetal.
env['baremetal_src_dir'] = join(env['root_dir'], 'baremetal') env['baremetal_src_dir'] = join(env['root_dir'], 'baremetal')
env['baremetal_src_arch_subpath'] = join('arch', env['arch'])
env['baremetal_src_arch_dir'] = join(env['baremetal_src_dir'], env['baremetal_src_arch_subpath'])
env['baremetal_src_lib_dir'] = join(env['baremetal_src_dir'], env['baremetal_lib_basename']) env['baremetal_src_lib_dir'] = join(env['baremetal_src_dir'], env['baremetal_lib_basename'])
if env['emulator'] == 'gem5': if env['emulator'] == 'gem5':
env['simulator_name'] = 'gem5' env['simulator_name'] = 'gem5'
@@ -534,6 +549,12 @@ to allow overriding configs from the CLI.
env['image'] = path env['image'] = path
self.env = env self.env = env
def add_argument(self, *args, **kwargs):
shortname, longname, key, is_option = self.get_key(*args, **kwargs)
if key in self._defaults:
kwargs['default'] = self._defaults[key]
super().add_argument(*args, **kwargs)
def assert_crosstool_ng_supports_arch(self, arch): def assert_crosstool_ng_supports_arch(self, arch):
if arch not in self.env['crosstool_ng_supported_archs']: if arch not in self.env['crosstool_ng_supported_archs']:
raise Exception('arch not yet supported: ' + arch) raise Exception('arch not yet supported: ' + arch)
@@ -644,6 +665,14 @@ to allow overriding configs from the CLI.
_json = {} _json = {}
return _json return _json
@staticmethod
def import_path(path):
'''
https://stackoverflow.com/questions/2601047/import-a-python-module-without-the-py-extension
https://stackoverflow.com/questions/31773310/what-does-the-first-argument-of-the-imp-load-source-method-do
'''
return imp.load_source(os.path.split(path)[1].replace('-', '_'), path)
@staticmethod @staticmethod
def log_error(msg): def log_error(msg):
print('error: {}'.format(msg), file=sys.stderr) print('error: {}'.format(msg), file=sys.stderr)
@@ -687,7 +716,7 @@ to allow overriding configs from the CLI.
return False return False
def _print_time(self, ellapsed_seconds): def _print_time(self, ellapsed_seconds):
if self._do_print_time: if self.env['print_time']:
hours, rem = divmod(ellapsed_seconds, 3600) hours, rem = divmod(ellapsed_seconds, 3600)
minutes, seconds = divmod(rem, 60) minutes, seconds = divmod(rem, 60)
print("time {:02}:{:02}:{:02}".format(int(hours), int(minutes), int(seconds))) print("time {:02}:{:02}:{:02}".format(int(hours), int(minutes), int(seconds)))

7
getvar
View File

@@ -1,13 +1,13 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
import types
import common import common
from shell_helpers import LF
class Main(common.LkmcCliFunction): class Main(common.LkmcCliFunction):
def __init__(self): def __init__(self):
super().__init__( super().__init__(
defaults={
'print_time': False,
},
description='''\ description='''\
Print the value of a self.env['py'] variable. Print the value of a self.env['py'] variable.
@@ -28,7 +28,6 @@ List all available variables:
./%(prog)s ./%(prog)s
.... ....
''', ''',
do_print_time=False,
) )
self.add_argument('variable', nargs='?') self.add_argument('variable', nargs='?')

18
run
View File

@@ -40,6 +40,10 @@ https://superuser.com/questions/1373226/how-to-redirect-qemu-serial-output-to-bo
'--debug-vm-args', default='', '--debug-vm-args', default='',
help='Pass arguments to GDB.' help='Pass arguments to GDB.'
) )
self.add_argument(
'--dp650', default=False,
help='Use the ARM DP650 display processor instead of the default HDLCD on gem5.'
)
self.add_argument( self.add_argument(
'--dtb', '--dtb',
help='''\ help='''\
@@ -372,15 +376,15 @@ Run QEMU with VNC instead of the default SDL. Connect to it with:
dtb = None dtb = None
if self.env['dtb'] is not None: if self.env['dtb'] is not None:
dtb = self.env['dtb'] dtb = self.env['dtb']
elif args.dp650: elif self.env['dp650']:
dtb = os.path.join(common.gem5_system_dir, 'arm', 'dt', 'armv{}_gem5_v1_{}{}cpu.dtb'.format(common.armv, dp650_cmd, args.cpus)), common.Newline, dtb = os.path.join(common.gem5_system_dir, 'arm', 'dt', 'armv{}_gem5_v1_{}{}cpu.dtb'.format(common.armv, dp650_cmd, self.env['cpus'])), LF,
if dtb is None: if dtb is None:
cmd.extend(['--generate-dtb', common.Newline]) if not self.env['baremetal']:
cmd.extend(['--generate-dtb', LF])
else: else:
cmd.extend(['--dtb-filename', dtb, common.Newline]) cmd.extend(['--dtb-filename', dtb, LF])
if self.env['baremetal'] is None: if self.env['baremetal'] is None:
cmd.extend([ cmd.extend(['--param', 'system.panic_on_panic = True', LF ])
'--param', 'system.panic_on_panic = True', LF])
else: else:
cmd.extend([ cmd.extend([
'--bare-metal', LF, '--bare-metal', LF,
@@ -488,7 +492,7 @@ Run QEMU with VNC instead of the default SDL. Connect to it with:
vnc vnc
) )
if self.env['dtb'] is not None: if self.env['dtb'] is not None:
cmd.extend(['-dtb', self.env['dtb'], common.Newline]) cmd.extend(['-dtb', self.env['dtb'], LF])
if not qemu_executable_prebuilt: if not qemu_executable_prebuilt:
cmd.extend(qemu_user_and_system_options) cmd.extend(qemu_user_and_system_options)
if self.env['initrd']: if self.env['initrd']:

View File

@@ -23,8 +23,6 @@ class GdbTestcase:
''' '''
self.prompt = '\(gdb\) ' self.prompt = '\(gdb\) '
self.source_path = source_path self.source_path = source_path
self.print_cmd(cmd)
cmd = self.strip_newlines(cmd)
import pexpect import pexpect
self.child = pexpect.spawn( self.child = pexpect.spawn(
cmd[0], cmd[0],
@@ -198,10 +196,11 @@ See: https://github.com/cirosantilli/linux-kernel-module-cheat#gdb-builtin-cpu-s
cmd.extend(['-ex', 'lx-symbols {}'.format(self.env['kernel_modules_build_subdir']), LF]) cmd.extend(['-ex', 'lx-symbols {}'.format(self.env['kernel_modules_build_subdir']), LF])
cmd.extend(after) cmd.extend(after)
if self.env['test']: if self.env['test']:
self.sh.print_cmd(cmd)
GdbTestcase( GdbTestcase(
self.env['source_path'], self.env['source_path'],
test_script_path, test_script_path,
cmd, self.sh.strip_newlines(cmd),
verbose=self.env['verbose'], verbose=self.env['verbose'],
) )
else: else:

View File

@@ -5,7 +5,7 @@ import os
import sys import sys
import common import common
rungdb = imp.load_source('rungdb', os.path.join(kwargs['root_dir'], 'run-gdb')) rungdb = imp.load_source('run_gdb', os.path.join(kwargs['root_dir'], 'run-gdb'))
parser = self.get_argparse(argparse_args={ parser = self.get_argparse(argparse_args={
'description': '''GDB step debug guest userland processes without gdbserver. 'description': '''GDB step debug guest userland processes without gdbserver.

View File

@@ -9,6 +9,7 @@ import signal
import stat import stat
import subprocess import subprocess
import sys import sys
import threading
class LF: class LF:
''' '''
@@ -178,18 +179,21 @@ class ShellHelpers:
if show_cmd: if show_cmd:
self.print_cmd(cmd, cwd=cwd, cmd_file=cmd_file, extra_env=extra_env, extra_paths=extra_paths) self.print_cmd(cmd, cwd=cwd, cmd_file=cmd_file, extra_env=extra_env, extra_paths=extra_paths)
# Otherwise Ctrl + C gives: # Otherwise, if called from a non-main thread:
# - ugly Python stack trace for gem5 (QEMU takes over terminal and is fine). # ValueError: signal only works in main thread
# - kills Python, and that then kills GDB: https://stackoverflow.com/questions/19807134/does-python-always-raise-an-exception-if-you-do-ctrlc-when-a-subprocess-is-exec if threading.current_thread() == threading.main_thread():
sigint_old = signal.getsignal(signal.SIGINT) # Otherwise Ctrl + C gives:
signal.signal(signal.SIGINT, signal.SIG_IGN) # - ugly Python stack trace for gem5 (QEMU takes over terminal and is fine).
# - kills Python, and that then kills GDB: https://stackoverflow.com/questions/19807134/does-python-always-raise-an-exception-if-you-do-ctrlc-when-a-subprocess-is-exec
sigint_old = signal.getsignal(signal.SIGINT)
signal.signal(signal.SIGINT, signal.SIG_IGN)
# Otherwise BrokenPipeError when piping through | grep # Otherwise BrokenPipeError when piping through | grep
# But if I do this_module, my terminal gets broken at the end. Why, why, why. # But if I do this_module, my terminal gets broken at the end. Why, why, why.
# https://stackoverflow.com/questions/14207708/ioerror-errno-32-broken-pipe-python # https://stackoverflow.com/questions/14207708/ioerror-errno-32-broken-pipe-python
# Ignoring the exception is not enough as it prints a warning anyways. # Ignoring the exception is not enough as it prints a warning anyways.
#sigpipe_old = signal.getsignal(signal.SIGPIPE) #sigpipe_old = signal.getsignal(signal.SIGPIPE)
#signal.signal(signal.SIGPIPE, signal.SIG_DFL) #signal.signal(signal.SIGPIPE, signal.SIG_DFL)
cmd = self.strip_newlines(cmd) cmd = self.strip_newlines(cmd)
if not self.dry_run: if not self.dry_run:
@@ -211,8 +215,9 @@ class ShellHelpers:
logfile.write(byte) logfile.write(byte)
else: else:
break break
signal.signal(signal.SIGINT, sigint_old) if threading.current_thread() == threading.main_thread():
#signal.signal(signal.SIGPIPE, sigpipe_old) signal.signal(signal.SIGINT, sigint_old)
#signal.signal(signal.SIGPIPE, sigpipe_old)
returncode = proc.returncode returncode = proc.returncode
if returncode != 0 and raise_on_failure: if returncode != 0 and raise_on_failure:
raise Exception('Command exited with status: {}'.format(returncode)) raise Exception('Command exited with status: {}'.format(returncode))

View File

@@ -1,35 +1,55 @@
#!/usr/bin/env bash #!/usr/bin/env python3
set -eux
for emulator in --qemu --gem5; do
# Userland.
# TODO make work.
#./run --arch x86_64 --background --userland add "$emulator" --wait-gdb &
#./run-gdb --arch x86_64 --userland add "$emulator" --test "$@"
#wait
# Baremetal. import functools
./run --arch arm --background --baremetal add "$emulator" --wait-gdb & import threading
./run-gdb --arch arm --baremetal add "$emulator" --test "$@" import os
wait
./run --arch arm --background --baremetal arch/arm/add "$emulator" --wait-gdb & import common
./run-gdb --arch arm --baremetal arch/arm/add "$emulator" --test "$@"
wait def output_reader(proc, file):
./run --arch arm --background --baremetal arch/arm/regs "$emulator" --wait-gdb & while True:
./run-gdb --arch arm --baremetal arch/arm/regs "$emulator" --test "$@" byte = proc.stdout.read(1)
wait if byte:
./run --arch aarch64 --background --baremetal add "$emulator" --wait-gdb & sys.stdout.buffer.write(byte)
./run-gdb --arch aarch64 --baremetal add "$emulator" --test "$@" sys.stdout.flush()
wait file.buffer.write(byte)
./run --arch aarch64 --background --baremetal arch/aarch64/add "$emulator" --wait-gdb & else:
./run-gdb --arch aarch64 --baremetal arch/aarch64/add "$emulator" --test "$@" break
wait
./run --arch aarch64 --background --baremetal arch/aarch64/regs "$emulator" --wait-gdb & class Main(common.LkmcCliFunction):
./run-gdb --arch aarch64 --baremetal arch/aarch64/regs "$emulator" --test "$@" def timed_main(self):
wait run = self.import_path('run').Main()
./run --arch aarch64 --background --baremetal arch/aarch64/fadd "$emulator" --wait-gdb & run_gdb = self.import_path('run-gdb').Main()
./run-gdb --arch aarch64 --baremetal arch/aarch64/fadd "$emulator" --test "$@" for emulator in self.env['emulators']:
wait for arch in self.env['crosstool_ng_supported_archs']:
./run --arch aarch64 --background --baremetal arch/aarch64/regs "$emulator" --wait-gdb & test_scripts_noext = []
./run-gdb --arch aarch64 --baremetal arch/aarch64/regs "$emulator" --test "$@" for f in os.listdir(self.env['baremetal_src_dir']):
wait base, ext = os.path.splitext(f)
done if ext == '.py':
test_scripts_noext.append(base)
for root, dirs, files in os.walk(os.path.join(self.env['baremetal_src_dir'], 'arch', arch)):
for f in files:
base, ext = os.path.splitext(f)
if ext == '.py':
full_path = os.path.join(root, base)
relpath = os.path.relpath(full_path, self.env['baremetal_src_dir'])
test_scripts_noext.append(relpath)
for test_script_noext in test_scripts_noext:
run_thread = threading.Thread(target=lambda: run(
arch=arch,
background=True,
baremetal=test_script_noext,
print_time=False,
emulator=emulator,
wait_gdb=True
))
gdb_thread = threading.Thread(target=lambda: run_gdb(
arch=arch, baremetal=test_script_noext, print_time=False, emulator=emulator, test=True
))
run_thread.start()
gdb_thread.start()
run_thread.join()
gdb_thread.join()
if __name__ == '__main__':
Main().cli()