print cli equivalent for commands called via python cli

cli_function.py

@@ -1,6 +1,8 @@
 #!/usr/bin/env python3
 
 import argparse
+import bisect
+import collections
 import imp
 import os
 import sys
@@ -108,46 +110,27 @@ class CliFunction:
        https://stackoverflow.com/questions/12834785/having-options-in-argparse-with-a-dash
     ** boolean defaults automatically use store_true or store_false, and add a --no-* CLI
        option to invert them if set from the config
+    * from a Python call, get the corresponding CLI. See get_cli.
 
     This somewhat duplicates: https://click.palletsprojects.com but:
 
     * that decorator API is insane
     * CLI + Python for single functions was wontfixed: https://github.com/pallets/click/issues/40
     '''
-    def __call__(self, **args):
+    def __call__(self, **kwargs):
         '''
-        Python version of the function call.
+        Python version of the function call. Not called by cli() indirectly,
+        so can be overridden to distinguish between Python and CLI calls.
 
         :type arguments: Dict
         '''
-        args_with_defaults = args.copy()
-        # Add missing args from config file.
-        if 'config_file' in args_with_defaults and args_with_defaults['config_file'] is not None:
-            config_file = args_with_defaults['config_file']
-        else:
-            config_file = self._config_file
-        if os.path.exists(config_file):
-            config_configs = {}
-            config = imp.load_source('config', config_file)
-            config.set_args(config_configs)
-            for key in config_configs:
-                if key not in self._all_keys:
-                    raise Exception('Unknown key in config file: ' + key)
-                if (not key in args_with_defaults) or args_with_defaults[key] is None:
-                    args_with_defaults[key] = config_configs[key]
-        # Add missing args from hard-coded defaults.
-        for argument in self._arguments:
-            key = argument.key
-            if (not key in args_with_defaults) or args_with_defaults[key] is None:
-                if argument.optional:
-                    args_with_defaults[key] = argument.default
-                else:
-                    raise Exception('Value not given for mandatory argument: ' + key)
-        return self.main(**args_with_defaults)
+        return self._do_main(kwargs)
+
+    def _do_main(self, kwargs):
+        return self.main(**self._get_args(kwargs))
 
     def __init__(self, config_file=None, description=None):
-        self._all_keys = set()
-        self._arguments = []
+        self._arguments = collections.OrderedDict()
         self._config_file = config_file
         self._description = description
         if self._config_file is not None:
@@ -158,7 +141,35 @@ class CliFunction:
             )
 
     def __str__(self):
-        return '\n'.join(str(arg) for arg in self._arguments)
+        return '\n'.join(str(arg[key]) for key in self._arguments)
+
+    def _get_args(self, kwargs):
+        args_with_defaults = kwargs.copy()
+        # Add missing args from config file.
+        config_file = None
+        if 'config_file' in args_with_defaults and args_with_defaults['config_file'] is not None:
+            config_file = args_with_defaults['config_file']
+        else:
+            config_file = self._config_file
+        if os.path.exists(config_file):
+            config_configs = {}
+            config = imp.load_source('config', config_file)
+            config.set_args(config_configs)
+            for key in config_configs:
+                if key not in self._arguments:
+                    raise Exception('Unknown key in config file: ' + key)
+                if (not key in args_with_defaults) or args_with_defaults[key] is None:
+                    args_with_defaults[key] = config_configs[key]
+        # Add missing args from hard-coded defaults.
+        for key in self._arguments:
+            argument = self._arguments[key]
+            if (not key in args_with_defaults) or args_with_defaults[key] is None:
+                if argument.optional:
+                    args_with_defaults[key] = argument.default
+                else:
+                    raise Exception('Value not given for mandatory argument: ' + key)
+        del args_with_defaults['config_file']
+        return args_with_defaults
 
     def add_argument(
             self,
@@ -166,8 +177,7 @@ class CliFunction:
             **kwargs
         ):
             argument = _Argument(*args, **kwargs)
-            self._arguments.append(argument)
-            self._all_keys.add(argument.key)
+            self._arguments[argument.key] = argument
 
     def cli(self, cli_args=None):
         '''
@@ -178,7 +188,8 @@ class CliFunction:
             description=self._description,
             formatter_class=argparse.RawTextHelpFormatter,
         )
-        for argument in self._arguments:
+        for key in self._arguments:
+            argument = self._arguments[key]
             parser.add_argument(*argument.args, **argument.kwargs)
             if argument.is_bool:
                 new_longname = '--no' + argument.longname[1:]
@@ -192,13 +203,60 @@ class CliFunction:
                     del kwargs['help']
                 parser.add_argument(new_longname, dest=argument.key, **kwargs)
         args = parser.parse_args(args=cli_args)
-        return self(**vars(args))
+        return self._do_main(vars(args))
 
     def cli_exit(self, *args, **kwargs):
         '''
-        Same as cli, but also exit the program with int(cli().
+        Same as cli, but also exit the program with status equal to the return value of main.
+        main must return an integer for this to be used.
         '''
-        sys.exit(int(self.cli(*args, **kwargs)))
+        sys.exit(self.cli(*args, **kwargs))
+
+    def get_cli(self, **kwargs):
+        '''
+        :rtype: List[Type(str)]
+        :return: the canonical command line arguments arguments that would
+                 generate this Python function call.
+
+                 (--key, value) option pairs are grouped into tuples, and all
+                 other values are grouped in their own tuple (positional_arg,)
+                 or (--bool-arg,).
+
+                 Arguments with default values are not added, but arguments
+                 that are set by the config are also given.
+
+                 The optional arguments are sorted alphabetically, followed by
+                 positional arguments.
+
+                 The long option name is used if both long and short versions
+                 are given.
+        '''
+        options = []
+        positional_dict = {}
+        kwargs = self._get_args(kwargs)
+        for key in kwargs:
+            argument = self._arguments[key]
+            default = argument.default
+            value = kwargs[key]
+            if value != default:
+                if argument.is_option:
+                    if argument.is_bool:
+                        val = (argument.longname,)
+                    else:
+                        val = (argument.longname, str(value))
+                    bisect.insort(options, val)
+                else:
+                    if type(value) is list:
+                        positional_dict[key] = [tuple(v,) for v in value]
+                    else:
+                        positional_dict[key] = [(str(value),)]
+        # Python built-in data structures suck.
+        # https://stackoverflow.com/questions/27726245/getting-the-key-index-in-a-python-ordereddict/27726534#27726534
+        positional = []
+        for key in self._arguments.keys():
+            if key in positional_dict:
+                positional.extend(positional_dict[key])
+        return options + positional
 
     @staticmethod
     def get_key(*args, **kwargs):
@@ -232,7 +290,6 @@ amazing function!
             self.add_argument('pos-optional', default=0, help='Help for pos-optional', type=int),
             self.add_argument('args-star', help='Help for args-star', nargs='*'),
         def main(self, **kwargs):
-            del kwargs['config_file']
             return kwargs
 
     one_cli_function = OneCliFunction()
@@ -259,7 +316,7 @@ amazing function!
     out = one_cli_function(pos_mandatory=1, asdf='B')
     assert out['asdf'] == 'B'
     out['asdf'] = default['asdf']
-    assert(out == default)
+    assert out == default
 
     # asdf and qwer
     out = one_cli_function(pos_mandatory=1, asdf='B', qwer='R')
@@ -267,7 +324,7 @@ amazing function!
     assert out['qwer'] == 'R'
     out['asdf'] = default['asdf']
     out['qwer'] = default['qwer']
-    assert(out == default)
+    assert out == default
 
     if '--bool':
         out = one_cli_function(pos_mandatory=1, bool=False)
@@ -275,22 +332,40 @@ amazing function!
         assert out == cli_out
         assert out['bool'] == False
         out['bool'] = default['bool']
-        assert(out == default)
+        assert out == default
 
     if '--bool-nargs':
-
         out = one_cli_function(pos_mandatory=1, bool_nargs=True)
         assert out['bool_nargs'] == True
         out['bool_nargs'] = default['bool_nargs']
-        assert(out == default)
+        assert out == default
 
         out = one_cli_function(pos_mandatory=1, bool_nargs='asdf')
         assert out['bool_nargs'] == 'asdf'
         out['bool_nargs'] = default['bool_nargs']
-        assert(out == default)
+        assert out == default
+
+    # Positional
+    out = one_cli_function(pos_mandatory=1, pos_optional=2, args_star=['3', '4'])
+    assert out['pos_mandatory'] == 1
+    assert out['pos_optional'] == 2
+    assert out['args_star'] == ['3', '4']
+    cli_out = one_cli_function.cli(['1', '2', '3', '4'])
+    assert out == cli_out
+    out['pos_mandatory'] = default['pos_mandatory']
+    out['pos_optional'] = default['pos_optional']
+    out['args_star'] = default['args_star']
+    assert out == default
 
     # Force a boolean value set on the config to be False on CLI.
     assert one_cli_function.cli(['--no-bool-cli', '1'])['bool_cli'] is False
 
-    # CLI call with argv command line arguments.
-    print(one_cli_function.cli())
+    # get_cli
+    assert one_cli_function.get_cli(pos_mandatory=1, asdf='B') == [('--asdf', 'B'), ('--bool-cli',), ('1',)]
+    assert one_cli_function.get_cli(pos_mandatory=1, asdf='B', qwer='R') == [('--asdf', 'B'), ('--bool-cli',), ('--qwer', 'R'), ('1',)]
+    assert one_cli_function.get_cli(pos_mandatory=1, bool=False) == [('--bool',), ('--bool-cli',), ('1',)]
+    assert one_cli_function.get_cli(pos_mandatory=1, pos_optional=2, args_star=['3', '4']) == [('--bool-cli',), ('1',), ('2',), ('3',), ('4',)]
+
+    if len(sys.argv) > 1:
+        # CLI call with argv command line arguments.
+        print(one_cli_function.cli())

common.py

@@ -7,6 +7,7 @@ import copy
 import datetime
 import glob
 import imp
+import inspect
 import json
 import multiprocessing
 import os
@@ -289,6 +290,17 @@ Use gem5 instead of QEMU. Shortcut for `--emulator gem5`.
 '''
         )
 
+    def __call__(self, **kwargs):
+        '''
+        For Python code calls, print the CLI equivalent of the call.
+        '''
+        print_cmd = ['./' + inspect.getfile(self.__class__), LF]
+        for line in self.get_cli(**kwargs):
+            print_cmd.extend(line)
+            print_cmd.append(LF)
+        shell_helpers.ShellHelpers.print_cmd(print_cmd)
+        return super().__call__(**kwargs)
+
     def _init_env(self, env):
         '''
         Update the kwargs from the command line with values derived from them.
@@ -683,13 +695,14 @@ Use gem5 instead of QEMU. Shortcut for `--emulator gem5`.
         '''
         Time the main of the derived class.
         '''
-        if not kwargs['dry_run']:
+        myargs = kwargs.copy()
+        if not myargs['dry_run']:
             start_time = time.time()
-        kwargs.update(consts)
-        self._init_env(kwargs)
+        myargs.update(consts)
+        self._init_env(myargs)
         self.sh = shell_helpers.ShellHelpers(dry_run=self.env['dry_run'])
         ret = self.timed_main()
-        if not kwargs['dry_run']:
+        if not myargs['dry_run']:
             end_time = time.time()
             self._print_time(end_time - start_time)
         return ret

run

@@ -618,7 +618,8 @@ Run QEMU with VNC instead of the default SDL. Connect to it with:
             if not self.env['userland']:
                 if os.path.exists(self.env['guest_terminal_file']):
                     with open(self.env['guest_terminal_file'], 'br') as logfile:
-                        if logfile.readlines()[-1].rstrip() == self.env['magic_fail_string']:
+                        lines = logfile.readlines()
+                        if lines and lines[-1].rstrip() == self.env['magic_fail_string']:
                             exit_status = 1
         if exit_status != 0:
             self.log_error('simulation error detected by parsing logs')

shell_helpers.py

@@ -28,6 +28,9 @@ class ShellHelpers:
     Attempt to print shell equivalents of all commands to make things
     easy to debug and understand what is going on.
     '''
+
+    _print_lock = threading.Lock()
+
     def __init__(self, dry_run=False):
         '''
         :param dry_run: don't run the commands, just potentially print them. Debug aid.
@@ -35,6 +38,16 @@ class ShellHelpers:
         '''
         self.dry_run = dry_run
 
+    @classmethod
+    def _print_thread_safe(cls, string):
+        # Python sucks: a naive print adds a bunch of random spaces to stdout,
+        # and then copy pasting the command fails.
+        # https://stackoverflow.com/questions/3029816/how-do-i-get-a-thread-safe-print-in-python-2-6
+        cls._print_lock.acquire()
+        sys.stdout.write(string + '\n')
+        sys.stdout.flush()
+        cls._print_lock.release()
+
     def add_newlines(self, cmd):
         out = []
         for arg in cmd:
@@ -46,7 +59,8 @@ class ShellHelpers:
         if not self.dry_run:
             shutil.copy2(src, dest)
 
-    def cmd_to_string(self, cmd, cwd=None, extra_env=None, extra_paths=None):
+    @staticmethod
+    def cmd_to_string(cmd, cwd=None, extra_env=None, extra_paths=None):
         '''
         Format a command given as a list of strings so that it can
         be viewed nicely and executed by bash directly and print it to stdout.
@@ -93,7 +107,8 @@ class ShellHelpers:
                         update=1,
                     )
 
-    def print_cmd(self, cmd, cwd=None, cmd_file=None, extra_env=None, extra_paths=None):
+    @classmethod
+    def print_cmd(cls, cmd, cwd=None, cmd_file=None, extra_env=None, extra_paths=None):
         '''
         Print cmd_to_string to stdout.
 
@@ -106,8 +121,8 @@ class ShellHelpers:
         if type(cmd) is str:
             cmd_string = cmd
         else:
-            cmd_string = self.cmd_to_string(cmd, cwd=cwd, extra_env=extra_env, extra_paths=extra_paths)
-        print('+ ' + cmd_string)
+            cmd_string = cls.cmd_to_string(cmd, cwd=cwd, extra_env=extra_env, extra_paths=extra_paths)
+        cls._print_thread_safe('+ ' + cmd_string)
         if cmd_file is not None:
             with open(cmd_file, 'w') as f:
                 f.write('#!/usr/bin/env bash\n')

test-userland

@@ -13,8 +13,7 @@ class Main(common.LkmcCliFunction):
             'hello_cpp.cpp',
             'print_argv.c',
         ]
-        # for emulator in self.env['emulators']:
-        for emulator in ['gem5']:
+        for emulator in self.env['emulators']:
             if emulator == 'gem5':
                 extra_args = {
                     'userland_build_id': 'static',