diff --git a/cli_function.py b/cli_function.py index eeca9b8..b635d77 100755 --- a/cli_function.py +++ b/cli_function.py @@ -1,6 +1,8 @@ #!/usr/bin/env python3 import argparse +import bisect +import collections import imp import os import sys @@ -108,46 +110,27 @@ class CliFunction: https://stackoverflow.com/questions/12834785/having-options-in-argparse-with-a-dash ** boolean defaults automatically use store_true or store_false, and add a --no-* CLI option to invert them if set from the config + * from a Python call, get the corresponding CLI. See get_cli. This somewhat duplicates: https://click.palletsprojects.com but: * that decorator API is insane * CLI + Python for single functions was wontfixed: https://github.com/pallets/click/issues/40 ''' - def __call__(self, **args): + def __call__(self, **kwargs): ''' - Python version of the function call. + Python version of the function call. Not called by cli() indirectly, + so can be overridden to distinguish between Python and CLI calls. :type arguments: Dict ''' - args_with_defaults = args.copy() - # Add missing args from config file. - if 'config_file' in args_with_defaults and args_with_defaults['config_file'] is not None: - config_file = args_with_defaults['config_file'] - else: - config_file = self._config_file - if os.path.exists(config_file): - config_configs = {} - config = imp.load_source('config', config_file) - config.set_args(config_configs) - for key in config_configs: - if key not in self._all_keys: - raise Exception('Unknown key in config file: ' + key) - if (not key in args_with_defaults) or args_with_defaults[key] is None: - args_with_defaults[key] = config_configs[key] - # Add missing args from hard-coded defaults. - for argument in self._arguments: - key = argument.key - if (not key in args_with_defaults) or args_with_defaults[key] is None: - if argument.optional: - args_with_defaults[key] = argument.default - else: - raise Exception('Value not given for mandatory argument: ' + key) - return self.main(**args_with_defaults) + return self._do_main(kwargs) + + def _do_main(self, kwargs): + return self.main(**self._get_args(kwargs)) def __init__(self, config_file=None, description=None): - self._all_keys = set() - self._arguments = [] + self._arguments = collections.OrderedDict() self._config_file = config_file self._description = description if self._config_file is not None: @@ -158,7 +141,35 @@ class CliFunction: ) def __str__(self): - return '\n'.join(str(arg) for arg in self._arguments) + return '\n'.join(str(arg[key]) for key in self._arguments) + + def _get_args(self, kwargs): + args_with_defaults = kwargs.copy() + # Add missing args from config file. + config_file = None + if 'config_file' in args_with_defaults and args_with_defaults['config_file'] is not None: + config_file = args_with_defaults['config_file'] + else: + config_file = self._config_file + if os.path.exists(config_file): + config_configs = {} + config = imp.load_source('config', config_file) + config.set_args(config_configs) + for key in config_configs: + if key not in self._arguments: + raise Exception('Unknown key in config file: ' + key) + if (not key in args_with_defaults) or args_with_defaults[key] is None: + args_with_defaults[key] = config_configs[key] + # Add missing args from hard-coded defaults. + for key in self._arguments: + argument = self._arguments[key] + if (not key in args_with_defaults) or args_with_defaults[key] is None: + if argument.optional: + args_with_defaults[key] = argument.default + else: + raise Exception('Value not given for mandatory argument: ' + key) + del args_with_defaults['config_file'] + return args_with_defaults def add_argument( self, @@ -166,8 +177,7 @@ class CliFunction: **kwargs ): argument = _Argument(*args, **kwargs) - self._arguments.append(argument) - self._all_keys.add(argument.key) + self._arguments[argument.key] = argument def cli(self, cli_args=None): ''' @@ -178,7 +188,8 @@ class CliFunction: description=self._description, formatter_class=argparse.RawTextHelpFormatter, ) - for argument in self._arguments: + for key in self._arguments: + argument = self._arguments[key] parser.add_argument(*argument.args, **argument.kwargs) if argument.is_bool: new_longname = '--no' + argument.longname[1:] @@ -192,13 +203,60 @@ class CliFunction: del kwargs['help'] parser.add_argument(new_longname, dest=argument.key, **kwargs) args = parser.parse_args(args=cli_args) - return self(**vars(args)) + return self._do_main(vars(args)) def cli_exit(self, *args, **kwargs): ''' - Same as cli, but also exit the program with int(cli(). + Same as cli, but also exit the program with status equal to the return value of main. + main must return an integer for this to be used. ''' - sys.exit(int(self.cli(*args, **kwargs))) + sys.exit(self.cli(*args, **kwargs)) + + def get_cli(self, **kwargs): + ''' + :rtype: List[Type(str)] + :return: the canonical command line arguments arguments that would + generate this Python function call. + + (--key, value) option pairs are grouped into tuples, and all + other values are grouped in their own tuple (positional_arg,) + or (--bool-arg,). + + Arguments with default values are not added, but arguments + that are set by the config are also given. + + The optional arguments are sorted alphabetically, followed by + positional arguments. + + The long option name is used if both long and short versions + are given. + ''' + options = [] + positional_dict = {} + kwargs = self._get_args(kwargs) + for key in kwargs: + argument = self._arguments[key] + default = argument.default + value = kwargs[key] + if value != default: + if argument.is_option: + if argument.is_bool: + val = (argument.longname,) + else: + val = (argument.longname, str(value)) + bisect.insort(options, val) + else: + if type(value) is list: + positional_dict[key] = [tuple(v,) for v in value] + else: + positional_dict[key] = [(str(value),)] + # Python built-in data structures suck. + # https://stackoverflow.com/questions/27726245/getting-the-key-index-in-a-python-ordereddict/27726534#27726534 + positional = [] + for key in self._arguments.keys(): + if key in positional_dict: + positional.extend(positional_dict[key]) + return options + positional @staticmethod def get_key(*args, **kwargs): @@ -232,7 +290,6 @@ amazing function! self.add_argument('pos-optional', default=0, help='Help for pos-optional', type=int), self.add_argument('args-star', help='Help for args-star', nargs='*'), def main(self, **kwargs): - del kwargs['config_file'] return kwargs one_cli_function = OneCliFunction() @@ -259,7 +316,7 @@ amazing function! out = one_cli_function(pos_mandatory=1, asdf='B') assert out['asdf'] == 'B' out['asdf'] = default['asdf'] - assert(out == default) + assert out == default # asdf and qwer out = one_cli_function(pos_mandatory=1, asdf='B', qwer='R') @@ -267,7 +324,7 @@ amazing function! assert out['qwer'] == 'R' out['asdf'] = default['asdf'] out['qwer'] = default['qwer'] - assert(out == default) + assert out == default if '--bool': out = one_cli_function(pos_mandatory=1, bool=False) @@ -275,22 +332,40 @@ amazing function! assert out == cli_out assert out['bool'] == False out['bool'] = default['bool'] - assert(out == default) + assert out == default if '--bool-nargs': - out = one_cli_function(pos_mandatory=1, bool_nargs=True) assert out['bool_nargs'] == True out['bool_nargs'] = default['bool_nargs'] - assert(out == default) + assert out == default out = one_cli_function(pos_mandatory=1, bool_nargs='asdf') assert out['bool_nargs'] == 'asdf' out['bool_nargs'] = default['bool_nargs'] - assert(out == default) + assert out == default + + # Positional + out = one_cli_function(pos_mandatory=1, pos_optional=2, args_star=['3', '4']) + assert out['pos_mandatory'] == 1 + assert out['pos_optional'] == 2 + assert out['args_star'] == ['3', '4'] + cli_out = one_cli_function.cli(['1', '2', '3', '4']) + assert out == cli_out + out['pos_mandatory'] = default['pos_mandatory'] + out['pos_optional'] = default['pos_optional'] + out['args_star'] = default['args_star'] + assert out == default # Force a boolean value set on the config to be False on CLI. assert one_cli_function.cli(['--no-bool-cli', '1'])['bool_cli'] is False - # CLI call with argv command line arguments. - print(one_cli_function.cli()) + # get_cli + assert one_cli_function.get_cli(pos_mandatory=1, asdf='B') == [('--asdf', 'B'), ('--bool-cli',), ('1',)] + assert one_cli_function.get_cli(pos_mandatory=1, asdf='B', qwer='R') == [('--asdf', 'B'), ('--bool-cli',), ('--qwer', 'R'), ('1',)] + assert one_cli_function.get_cli(pos_mandatory=1, bool=False) == [('--bool',), ('--bool-cli',), ('1',)] + assert one_cli_function.get_cli(pos_mandatory=1, pos_optional=2, args_star=['3', '4']) == [('--bool-cli',), ('1',), ('2',), ('3',), ('4',)] + + if len(sys.argv) > 1: + # CLI call with argv command line arguments. + print(one_cli_function.cli()) diff --git a/common.py b/common.py index ab54e35..2a1f7e9 100644 --- a/common.py +++ b/common.py @@ -7,6 +7,7 @@ import copy import datetime import glob import imp +import inspect import json import multiprocessing import os @@ -289,6 +290,17 @@ Use gem5 instead of QEMU. Shortcut for `--emulator gem5`. ''' ) + def __call__(self, **kwargs): + ''' + For Python code calls, print the CLI equivalent of the call. + ''' + print_cmd = ['./' + inspect.getfile(self.__class__), LF] + for line in self.get_cli(**kwargs): + print_cmd.extend(line) + print_cmd.append(LF) + shell_helpers.ShellHelpers.print_cmd(print_cmd) + return super().__call__(**kwargs) + def _init_env(self, env): ''' Update the kwargs from the command line with values derived from them. @@ -683,13 +695,14 @@ Use gem5 instead of QEMU. Shortcut for `--emulator gem5`. ''' Time the main of the derived class. ''' - if not kwargs['dry_run']: + myargs = kwargs.copy() + if not myargs['dry_run']: start_time = time.time() - kwargs.update(consts) - self._init_env(kwargs) + myargs.update(consts) + self._init_env(myargs) self.sh = shell_helpers.ShellHelpers(dry_run=self.env['dry_run']) ret = self.timed_main() - if not kwargs['dry_run']: + if not myargs['dry_run']: end_time = time.time() self._print_time(end_time - start_time) return ret diff --git a/run b/run index 5234b9f..544248d 100755 --- a/run +++ b/run @@ -618,7 +618,8 @@ Run QEMU with VNC instead of the default SDL. Connect to it with: if not self.env['userland']: if os.path.exists(self.env['guest_terminal_file']): with open(self.env['guest_terminal_file'], 'br') as logfile: - if logfile.readlines()[-1].rstrip() == self.env['magic_fail_string']: + lines = logfile.readlines() + if lines and lines[-1].rstrip() == self.env['magic_fail_string']: exit_status = 1 if exit_status != 0: self.log_error('simulation error detected by parsing logs') diff --git a/shell_helpers.py b/shell_helpers.py index 28d4f31..53eff69 100644 --- a/shell_helpers.py +++ b/shell_helpers.py @@ -28,6 +28,9 @@ class ShellHelpers: Attempt to print shell equivalents of all commands to make things easy to debug and understand what is going on. ''' + + _print_lock = threading.Lock() + def __init__(self, dry_run=False): ''' :param dry_run: don't run the commands, just potentially print them. Debug aid. @@ -35,6 +38,16 @@ class ShellHelpers: ''' self.dry_run = dry_run + @classmethod + def _print_thread_safe(cls, string): + # Python sucks: a naive print adds a bunch of random spaces to stdout, + # and then copy pasting the command fails. + # https://stackoverflow.com/questions/3029816/how-do-i-get-a-thread-safe-print-in-python-2-6 + cls._print_lock.acquire() + sys.stdout.write(string + '\n') + sys.stdout.flush() + cls._print_lock.release() + def add_newlines(self, cmd): out = [] for arg in cmd: @@ -46,7 +59,8 @@ class ShellHelpers: if not self.dry_run: shutil.copy2(src, dest) - def cmd_to_string(self, cmd, cwd=None, extra_env=None, extra_paths=None): + @staticmethod + def cmd_to_string(cmd, cwd=None, extra_env=None, extra_paths=None): ''' Format a command given as a list of strings so that it can be viewed nicely and executed by bash directly and print it to stdout. @@ -93,7 +107,8 @@ class ShellHelpers: update=1, ) - def print_cmd(self, cmd, cwd=None, cmd_file=None, extra_env=None, extra_paths=None): + @classmethod + def print_cmd(cls, cmd, cwd=None, cmd_file=None, extra_env=None, extra_paths=None): ''' Print cmd_to_string to stdout. @@ -106,8 +121,8 @@ class ShellHelpers: if type(cmd) is str: cmd_string = cmd else: - cmd_string = self.cmd_to_string(cmd, cwd=cwd, extra_env=extra_env, extra_paths=extra_paths) - print('+ ' + cmd_string) + cmd_string = cls.cmd_to_string(cmd, cwd=cwd, extra_env=extra_env, extra_paths=extra_paths) + cls._print_thread_safe('+ ' + cmd_string) if cmd_file is not None: with open(cmd_file, 'w') as f: f.write('#!/usr/bin/env bash\n') diff --git a/test-userland b/test-userland index b4f5d2e..35057d5 100755 --- a/test-userland +++ b/test-userland @@ -13,8 +13,7 @@ class Main(common.LkmcCliFunction): 'hello_cpp.cpp', 'print_argv.c', ] - # for emulator in self.env['emulators']: - for emulator in ['gem5']: + for emulator in self.env['emulators']: if emulator == 'gem5': extra_args = { 'userland_build_id': 'static',