gem5: expose syscall emulation with --user

Then also expose QEMU user mode with --user. Docs not perfect yet,
would require a build alternative for userland/ for -static and or
passing options before the QEMU userland executable with a new CLI.

README.adoc

@@ -7647,7 +7647,7 @@ The reason this is cool, is that `ls` is not statically compiled, but since we h
 In other words, much cooler than:
 
 ....
-./run-toolchain --arch arm gcc -- -static "$(./getvar kernel_modules_src_dir)/user/hello.c"
+./run-toolchain --arch arm gcc -- -static "$(./getvar userland_src_dir)/hello.c"
 qemu-arm a.out
 ....
 
@@ -7686,9 +7686,22 @@ FATAL: kernel too old
 
 but it must be using the kernel version given by glibc, since we didn't hit that error on uclibc.
 
+==== QEMU user mode integration
+
+We have a nice user mode integration under way.
+
+It is already usable with:
+
+....
+./build-qemu --user
+./run --user path/to/static/executable -- arg1 "arg 2" arg3
+....
+
+TODO: factor <<qemu-user-mode>> with this:
+
 ==== gem5 syscall emulation mode
 
-Analogous to <<qemu-user-mode>>, but less usable.
+Analogous to <<qemu-user-mode>>, but less usable:
 
 * https://stackoverflow.com/questions/48986597/when-should-you-use-full-system-fs-vs-syscall-emulation-se-with-userland-program
 * https://stackoverflow.com/questions/48959349/how-to-solve-fatal-kernel-too-old-when-running-gem5-in-syscall-emulation-se-m
@@ -7698,12 +7711,12 @@ First we try some `-static` sanity checks.
 Works and prints `hello`:
 
 ....
-./run-toolchain --arch x86_64  gcc -- -static -o x86_64.out  "$(./getvar kernel_modules_src_dir)/user/hello.c"
+./run-toolchain --arch x86_64  gcc -- -static -o x86_64.out  "$(./getvar userland_src_dir)/hello.c"
-./run-toolchain --arch arm     gcc -- -static -o arm.out     "$(./getvar kernel_modules_src_dir)/user/hello.c"
+./run-toolchain --arch arm     gcc -- -static -o arm.out     "$(./getvar userland_src_dir)/hello.c"
-./run-toolchain --arch aarch64 gcc -- -static -o aarch64.out "$(./getvar kernel_modules_src_dir)/user/hello.c"
+./run-toolchain --arch aarch64 gcc -- -static -o aarch64.out "$(./getvar userland_src_dir)/hello.c"
-./run --arch x86_64  --gem5 --gem5-script se --exec-image ./x86_64.out
+./run --arch x86_64  --gem5 --user ./x86_64.out
-./run --arch arm     --gem5 --gem5-script se --exec-image ./arm.out
+./run --arch arm     --gem5 --user ./arm.out
-./run --arch aarch64 --gem5 --gem5-script se --exec-image ./aarch64.out
+./run --arch aarch64 --gem5 --user ./aarch64.out
 ....
 
 But I think this is unreliable, and only works because we are using uclibc which does not check the kernel version as glibc does: https://stackoverflow.com/questions/48959349/how-to-solve-fatal-kernel-too-old-when-running-gem5-in-syscall-emulation-se-m/50542301#50542301
@@ -7711,9 +7724,9 @@ But I think this is unreliable, and only works because we are using uclibc which
 Ignoring that insanity, we then try it with dynamically linked executables:
 
 ....
-./run --arch x86_64  --gem5 --gem5-script se --exec-image "$(./getvar --arch x86_64  --gem5 target_dir)/hello.out"
+./run --arch x86_64  --gem5 --user "$(./getvar --arch x86_64  --gem5 target_dir)/hello.out"
-./run --arch arm     --gem5 --gem5-script se --exec-image "$(./getvar --arch arm     --gem5 target_dir)/hello.out"
+./run --arch arm     --gem5 --user "$(./getvar --arch arm     --gem5 target_dir)/hello.out"
-./run --arch aarch64 --gem5 --gem5-script se --exec-image "$(./getvar --arch aarch64 --gem5 target_dir)/hello.out"
+./run --arch aarch64 --gem5 --user "$(./getvar --arch aarch64 --gem5 target_dir)/hello.out"
 ....
 
 But at 185c2730cc78d5adda683d76c0e3b35e7cb534f0 they fail with:
@@ -7736,6 +7749,12 @@ Then I was told that it is has never been tested outside of x86_64:
 * https://stackoverflow.com/questions/50542222/how-to-run-a-dynamically-linked-executable-syscall-emulation-mode-se-py-in-gem5
 * https://www.mail-archive.com/gem5-users@gem5.org/msg15585.html
 
+===== gem5 syscall emulation mode CLI options
+
+....
+./run --gem5 --user ./aarch64.out -- --options 'op1 "op 2" op3'
+....
+
 ==== User mode vs full system benchmark
 
 Let's see if user mode runs considerably faster than full system or not.
@@ -7743,16 +7762,21 @@ Let's see if user mode runs considerably faster than full system or not.
 gem5 user mode:
 
 ....
+./build-buildroot --config 'BR2_PACKAGE_DHRYSTONE=y' --arch arm
 make \
+  -B \
   -C "$(./getvar --arch arm build_dir)/dhrystone-2" \
   CC="$(./run-toolchain --arch arm --dry gcc)" \
   CFLAGS=-static \
 ;
 time \
-  "$(./getvar --arch arm --gem5 exec)" \
+  ./run \
-  "$(./getvar --arch arm gem5_se_file)" \
+  --arch arm \
-  -c "$(./getvar --arch arm build_dir)/dhrystone-2/dhrystone" \
+  --gem5 \
-  -o 100000 \
+  --user \
+  "$(./getvar --arch arm build_dir)/dhrystone-2/dhrystone" \
+  -- \
+  --options 100000 \
 ;
 ....
 

build-qemu

@@ -6,6 +6,12 @@ import common
 
 class QemuComponent(common.Component):
     def add_parser_arguments(self, parser):
+        parser.add_argument(
+            '--user',
+            default=False,
+            action='store_true',
+            help='Build QEMU user mode instead of system.',
+        )
         parser.add_argument(
             'extra_config_args',
             default=[],
@@ -20,12 +26,16 @@ class QemuComponent(common.Component):
             verbose = ['V=1']
         else:
             verbose = []
+        if args.user:
+            target_list = '{}-linux-user'.format(args.arch)
+        else:
+            target_list = '{}-softmmu'.format(args.arch)
         common.run_cmd(
             [
                 os.path.join(common.qemu_src_dir, 'configure'),
                 '--enable-debug',
                 '--enable-trace-backends=simple',
-                '--target-list={}-softmmu'.format(args.arch),
+                '--target-list={}'.format(target_list),
                 '--enable-sdl',
                 '--with-sdlabi=2.0',
             ] +

common.py

@@ -792,7 +792,6 @@ def setup(parser):
     # Kernel modules.
     this_module.kernel_modules_build_base_dir = os.path.join(this_module.out_dir, 'kernel_modules')
     this_module.kernel_modules_build_dir = os.path.join(this_module.kernel_modules_build_base_dir, args.arch)
-    this_module.kernel_modules_build_subdir = os.path.join(this_module.kernel_modules_build_dir, kernel_modules_subdir)
     this_module.kernel_modules_build_host_dir = os.path.join(this_module.kernel_modules_build_base_dir, 'host')
     this_module.userland_build_dir = os.path.join(this_module.out_dir, 'userland', args.arch)
     this_module.out_rootfs_overlay_dir = os.path.join(this_module.out_dir, 'rootfs_overlay', args.arch)

run

@@ -16,11 +16,10 @@ defaults = {
     'debug_vm': False,
     'eval': None,
     'extra_emulator_args': [],
-    'exec_image': None,
     'gem5_exe_args': '',
+    'gem5_script': 'fs',
     'gem5_readfile': '',
     'gem5_restore': None,
-    'gem5_script': 'fs',
     'graphic': False,
     'initramfs': False,
     'initrd': False,
@@ -36,6 +35,7 @@ defaults = {
     'tmux': False,
     'tmux_args': '',
     'trace': None,
+    'user': None,
     'vnc': False,
 }
 
@@ -104,6 +104,7 @@ def main(args, extra_args=None):
                         'Tried to use: ' + common.image)
     if common.image is None:
         raise Exception('Baremetal ELF file not found. Tried:\n' + '\n'.join(paths))
+    cmd = debug_vm.copy()
     if args.gem5:
         if args.baremetal is None:
             if not os.path.exists(common.rootfs_raw_file):
@@ -128,8 +129,7 @@ def main(args, extra_args=None):
         extra_env['M5_PATH'] = common.gem5_system_dir
         # https://stackoverflow.com/questions/52312070/how-to-modify-a-file-under-src-python-and-run-it-without-rebuilding-in-gem5/52312071#52312071
         extra_env['M5_OVERRIDE_PY_SOURCE'] = 'true'
-        cmd = (
+        cmd.extend(
-            debug_vm +
             [
                 common.executable,
                 '--debug-file=trace.txt',
@@ -139,169 +139,186 @@ def main(args, extra_args=None):
                 '-d', common.m5out_dir
             ]
         )
-        if args.gem5_script == 'fs':
+        if args.user is not None:
-            # TODO port
+            cmd.extend([common.gem5_se_file, '-c', args.user])
-            if args.gem5_restore is not None:
+        else:
-                cpt_dirs = common.gem_list_checkpoint_dirs()
+            if args.gem5_script == 'fs':
-                cpt_dir = cpt_dirs[-args.gem5_restore]
+                # TODO port
-                extra_emulator_args.extend(['-r', str(sorted(cpt_dirs).index(cpt_dir) + 1)])
+                if args.gem5_restore is not None:
-            cmd.extend([
+                    cpt_dirs = common.gem_list_checkpoint_dirs()
-                common.gem5_fs_file,
+                    cpt_dir = cpt_dirs[-args.gem5_restore]
-                '--disk-image', common.disk_image,
+                    extra_emulator_args.extend(['-r', str(sorted(cpt_dirs).index(cpt_dir) + 1)])
-                '--kernel', common.image,
-                '--mem-size', memory,
-                '--num-cpus', str(args.cpus),
-                '--script', common.gem5_readfile,
-            ])
-            if args.arch == 'x86_64':
-                if args.kvm:
-                    cmd.extend(['--cpu-type', 'X86KvmCPU'])
-                cmd.extend(['--command-line', 'earlyprintk=ttyS0 console=ttyS0 lpj=7999923 root=/dev/sda {}'.format(kernel_cli)])
-            elif args.arch == 'arm' or args.arch == 'aarch64':
-                # TODO why is it mandatory to pass mem= here? Not true for QEMU.
-                # Anything smaller than physical blows up as expected, but why can't it auto-detect the right value?
                 cmd.extend([
-                    '--command-line', 'earlyprintk=pl011,0x1c090000 console=ttyAMA0 lpj=19988480 rw loglevel=8 mem={} root=/dev/sda {}'.format(memory, kernel_cli),
+                    common.gem5_fs_file,
-                    '--dtb-filename', os.path.join(common.gem5_system_dir, 'arm', 'dt', 'armv{}_gem5_v1_{}cpu.dtb'.format(common.armv, args.cpus)),
+                    '--disk-image', common.disk_image,
-                    '--machine-type', common.machine,
+                    '--kernel', common.image,
-                    '--param', 'system.panic_on_panic = True',
+                    '--mem-size', memory,
+                    '--num-cpus', str(args.cpus),
+                    '--script', common.gem5_readfile,
+                ])
+                if args.arch == 'x86_64':
+                    if args.kvm:
+                        cmd.extend(['--cpu-type', 'X86KvmCPU'])
+                    cmd.extend(['--command-line', 'earlyprintk=ttyS0 console=ttyS0 lpj=7999923 root=/dev/sda {}'.format(kernel_cli)])
+                elif args.arch == 'arm' or args.arch == 'aarch64':
+                    # TODO why is it mandatory to pass mem= here? Not true for QEMU.
+                    # Anything smaller than physical blows up as expected, but why can't it auto-detect the right value?
+                    cmd.extend([
+                        '--command-line', 'earlyprintk=pl011,0x1c090000 console=ttyAMA0 lpj=19988480 rw loglevel=8 mem={} root=/dev/sda {}'.format(memory, kernel_cli),
+                        '--dtb-filename', os.path.join(common.gem5_system_dir, 'arm', 'dt', 'armv{}_gem5_v1_{}cpu.dtb'.format(common.armv, args.cpus)),
+                        '--machine-type', common.machine,
+                        '--param', 'system.panic_on_panic = True',
+                    ])
+                    if not args.baremetal is None:
+                        cmd.append('--bare-metal')
+                        if args.arch == 'aarch64':
+                            # https://stackoverflow.com/questions/43682311/uart-communication-in-gem5-with-arm-bare-metal/50983650#50983650
+                            cmd.extend(['--param', 'system.highest_el_is_64 = True'])
+                            cmd.extend(['--param', 'system.auto_reset_addr_64 = True'])
+            elif args.gem5_script == 'biglittle':
+                if args.gem5_restore is not None:
+                    cpt_dir = common.gem_list_checkpoint_dirs()[-args.gem5_restore]
+                    extra_emulator_args.extend(['--restore-from', os.path.join(common.m5out_dir, cpt_dir)])
+                cmd.extend([
+                    os.path.join(common.gem5_src_dir, 'configs', 'example', 'arm', 'fs_bigLITTLE.py'),
+                    '--big-cpus', '2',
+                    '--cpu-type', 'atomic',
+                    '--disk', common.disk_image,
+                    '--dtb', os.path.join(common.gem5_system_dir, 'arm', 'dt', 'armv8_gem5_v1_big_little_2_2.dtb'),
+                    '--kernel', common.image,
+                    '--little-cpus', '2'
                 ])
-                if not args.baremetal is None:
-                    cmd.append('--bare-metal')
-                    if args.arch == 'aarch64':
-                        # https://stackoverflow.com/questions/43682311/uart-communication-in-gem5-with-arm-bare-metal/50983650#50983650
-                        cmd.extend(['--param', 'system.highest_el_is_64 = True'])
-                        cmd.extend(['--param', 'system.auto_reset_addr_64 = True'])
-        elif args.gem5_script == 'biglittle':
-            if args.gem5_restore is not None:
-                cpt_dir = common.gem_list_checkpoint_dirs()[-args.gem5_restore]
-                extra_emulator_args.extend(['--restore-from', os.path.join(common.m5out_dir, cpt_dir)])
-            cmd.extend([
-                os.path.join(common.gem5_src_dir, 'configs', 'example', 'arm', 'fs_bigLITTLE.py'),
-                '--big-cpus', '2',
-                '--cpu-type', 'atomic',
-                '--disk', common.disk_image,
-                '--dtb', os.path.join(common.gem5_system_dir, 'arm', 'dt', 'armv8_gem5_v1_big_little_2_2.dtb'),
-                '--kernel', common.image,
-                '--little-cpus', '2'
-            ])
-        elif args.gem5_script == 'se':
-            assert(args.exec_image is not None)
-            cmd.extend([common.gem5_se_file, '-c', args.exec_image])
         if args.debug_guest:
             # https://stackoverflow.com/questions/49296092/how-to-make-gem5-wait-for-gdb-to-connect-to-reliably-break-at-start-kernel-of-th
             cmd.extend(['--param', 'system.cpu[0].wait_for_remote_gdb = True'])
     else:
-        if not os.path.exists(common.image):
+        qemu_user_and_system_options = [
-            raise_image_not_found()
+            '-trace', 'enable={},file={}'.format(trace_type, common.qemu_trace_file),
-        extra_emulator_args.extend(extra_qemu_args)
+        ]
-        common.make_run_dirs()
+        if args.user is not None:
-        if args.prebuilt:
+            if args.debug_guest:
-            qemu_executable = common.qemu_executable_basename
+                debug_args = ['-g', str(common.gdb_port)]
-            qemu_found = shutil.which(qemu_executable) is not None
-        else:
-            qemu_executable = common.qemu_executable
-            qemu_found = os.path.exists(qemu_executable)
-        if not qemu_found:
-            raise Exception('QEMU executable not found, did you forget to build or install it?\n' \
-                            'Tried to use: ' + qemu_executable)
-        if args.debug_vm:
-            serial_monitor = []
-        else:
-            serial_monitor = ['-serial', 'mon:stdio']
-        if args.kvm:
-            extra_emulator_args.append('-enable-kvm')
-        if args.kgdb:
-            extra_emulator_args.extend(['-serial', 'tcp::{},server,nowait'.format(common.gdb_port)])
-        cmd = (
-            debug_vm +
-            [
-                qemu_executable,
-                '-device', 'rtl8139,netdev=net0',
-                '-gdb', 'tcp::{}'.format(common.gdb_port),
-                '-kernel', common.image,
-                '-m', args.memory,
-                '-monitor', 'telnet::{},server,nowait'.format(common.qemu_monitor_port),
-                '-netdev', 'user,hostfwd=tcp::{}-:{},hostfwd=tcp::{}-:22,id=net0'.format(common.qemu_hostfwd_generic_port, common.qemu_hostfwd_generic_port, common.qemu_hostfwd_ssh_port),
-                '-no-reboot',
-                '-smp', str(args.cpus),
-                '-trace', 'enable={},file={}'.format(trace_type, common.qemu_trace_file),
-                '-virtfs', 'local,path={},mount_tag=host_data,security_model=mapped,id=host_data'.format(common.p9_dir),
-                '-virtfs', 'local,path={},mount_tag=host_out,security_model=mapped,id=host_out'.format(common.out_dir),
-                '-virtfs', 'local,path={},mount_tag=host_out_rootfs_overlay,security_model=mapped,id=host_out_rootfs_overlay'.format(common.out_rootfs_overlay_dir),
-                '-virtfs', 'local,path={},mount_tag=host_rootfs_overlay,security_model=mapped,id=host_rootfs_overlay'.format(common.rootfs_overlay_dir),
-            ] +
-            serial_monitor +
-            vnc
-        )
-        if args.initrd:
-            extra_emulator_args.extend(['-initrd', os.path.join(common.buildroot_images_dir, 'rootfs.cpio')])
-        rr = args.record or args.replay
-        if ramfs:
-            # TODO why is this needed, and why any string works.
-            root = 'root=/dev/anything'
-        else:
-            if rr:
-                driveif = 'none'
-                rrid = ',id=img-direct'
-                root = 'root=/dev/sda'
-                snapshot = ''
             else:
-                driveif = 'virtio'
+                debug_args = []
-                root = 'root=/dev/vda'
+            cmd.extend(
-                rrid = ''
-                snapshot = ',snapshot'
-            if args.baremetal is None:
-                if not os.path.exists(common.qcow2_file):
-                    if not os.path.exists(common.rootfs_raw_file):
-                        raise_rootfs_not_found()
-                    common.raw_to_qcow2(prebuilt=args.prebuilt)
-                extra_emulator_args.extend([
-                    '-drive',
-                    'file={},format=qcow2,if={}{}{}'.format(common.disk_image, driveif, snapshot, rrid)
-                ])
-                if rr:
-                    extra_emulator_args.extend([
-                    '-drive', 'driver=blkreplay,if=none,image=img-direct,id=img-blkreplay',
-                    '-device', 'ide-hd,drive=img-blkreplay'
-                ])
-        if rr:
-            extra_emulator_args.extend([
-                '-object', 'filter-replay,id=replay,netdev=net0',
-                '-icount', 'shift=7,rr={},rrfile={}'.format('record' if args.record else 'replay', common.qemu_rrfile),
-            ])
-            virtio_gpu_pci = []
-        else:
-            virtio_gpu_pci = ['-device', 'virtio-gpu-pci']
-        if args.arch == 'x86_64':
-            if args.kgdb:
-                kernel_cli += ' kgdboc=ttyS0,115200'
-            append = ['-append', '{} nopat {}'.format(root, kernel_cli)]
-            cmd.extend([
-                '-M', common.machine,
-                '-device', 'edu',
-            ])
-        elif args.arch == 'arm' or args.arch == 'aarch64':
-            extra_emulator_args.append('-semihosting')
-            if args.kgdb:
-                kernel_cli += ' kgdboc=ttyAMA0,115200'
-            if args.arch == 'arm':
-                cpu = 'cortex-a15'
-            else:
-                cpu = 'cortex-a57'
-            append = ['-append', '{} {}'.format(root, kernel_cli)]
-            cmd = (
-                cmd +
                 [
-                    # highmem=off needed since v3.0.0 due to:
+                    os.path.join(common.qemu_build_dir, '{}-linux-user'.format(args.arch), 'qemu-{}'.format(args.arch)),
-                    # http://lists.nongnu.org/archive/html/qemu-discuss/2018-08/msg00034.html
-                    '-M', '{},highmem=off'.format(common.machine),
-                    '-cpu', cpu,
                 ] +
-                virtio_gpu_pci
+                qemu_user_and_system_options +
+                debug_args +
+                [
+                    args.user
+                ]
             )
-        if args.baremetal is None:
+        else:
-            cmd.extend(append)
+            if not os.path.exists(common.image):
+                raise_image_not_found()
+            extra_emulator_args.extend(extra_qemu_args)
+            common.make_run_dirs()
+            if args.prebuilt:
+                qemu_executable = common.qemu_executable_basename
+                qemu_found = shutil.which(qemu_executable) is not None
+            else:
+                qemu_executable = common.qemu_executable
+                qemu_found = os.path.exists(qemu_executable)
+            if not qemu_found:
+                raise Exception('QEMU executable not found, did you forget to build or install it?\n' \
+                                'Tried to use: ' + qemu_executable)
+            if args.debug_vm:
+                serial_monitor = []
+            else:
+                serial_monitor = ['-serial', 'mon:stdio']
+            if args.kvm:
+                extra_emulator_args.append('-enable-kvm')
+            if args.kgdb:
+                extra_emulator_args.extend(['-serial', 'tcp::{},server,nowait'.format(common.gdb_port)])
+            cmd.extend(
+                [
+                    qemu_executable,
+                    '-device', 'rtl8139,netdev=net0',
+                    '-gdb', 'tcp::{}'.format(common.gdb_port),
+                    '-kernel', common.image,
+                    '-m', args.memory,
+                    '-monitor', 'telnet::{},server,nowait'.format(common.qemu_monitor_port),
+                    '-netdev', 'user,hostfwd=tcp::{}-:{},hostfwd=tcp::{}-:22,id=net0'.format(common.qemu_hostfwd_generic_port, common.qemu_hostfwd_generic_port, common.qemu_hostfwd_ssh_port),
+                    '-no-reboot',
+                    '-smp', str(args.cpus),
+                    '-virtfs', 'local,path={},mount_tag=host_data,security_model=mapped,id=host_data'.format(common.p9_dir),
+                    '-virtfs', 'local,path={},mount_tag=host_out,security_model=mapped,id=host_out'.format(common.out_dir),
+                    '-virtfs', 'local,path={},mount_tag=host_out_rootfs_overlay,security_model=mapped,id=host_out_rootfs_overlay'.format(common.out_rootfs_overlay_dir),
+                    '-virtfs', 'local,path={},mount_tag=host_rootfs_overlay,security_model=mapped,id=host_rootfs_overlay'.format(common.rootfs_overlay_dir),
+                ] +
+                qemu_user_and_system_options +
+                serial_monitor +
+                vnc
+            )
+            if args.initrd:
+                extra_emulator_args.extend(['-initrd', os.path.join(common.buildroot_images_dir, 'rootfs.cpio')])
+            rr = args.record or args.replay
+            if ramfs:
+                # TODO why is this needed, and why any string works.
+                root = 'root=/dev/anything'
+            else:
+                if rr:
+                    driveif = 'none'
+                    rrid = ',id=img-direct'
+                    root = 'root=/dev/sda'
+                    snapshot = ''
+                else:
+                    driveif = 'virtio'
+                    root = 'root=/dev/vda'
+                    rrid = ''
+                    snapshot = ',snapshot'
+                if args.baremetal is None:
+                    if not os.path.exists(common.qcow2_file):
+                        if not os.path.exists(common.rootfs_raw_file):
+                            raise_rootfs_not_found()
+                        common.raw_to_qcow2(prebuilt=args.prebuilt)
+                    extra_emulator_args.extend([
+                        '-drive',
+                        'file={},format=qcow2,if={}{}{}'.format(common.disk_image, driveif, snapshot, rrid)
+                    ])
+                    if rr:
+                        extra_emulator_args.extend([
+                        '-drive', 'driver=blkreplay,if=none,image=img-direct,id=img-blkreplay',
+                        '-device', 'ide-hd,drive=img-blkreplay'
+                    ])
+            if rr:
+                extra_emulator_args.extend([
+                    '-object', 'filter-replay,id=replay,netdev=net0',
+                    '-icount', 'shift=7,rr={},rrfile={}'.format('record' if args.record else 'replay', common.qemu_rrfile),
+                ])
+                virtio_gpu_pci = []
+            else:
+                virtio_gpu_pci = ['-device', 'virtio-gpu-pci']
+            if args.arch == 'x86_64':
+                if args.kgdb:
+                    kernel_cli += ' kgdboc=ttyS0,115200'
+                append = ['-append', '{} nopat {}'.format(root, kernel_cli)]
+                cmd.extend([
+                    '-M', common.machine,
+                    '-device', 'edu',
+                ])
+            elif args.arch == 'arm' or args.arch == 'aarch64':
+                extra_emulator_args.append('-semihosting')
+                if args.kgdb:
+                    kernel_cli += ' kgdboc=ttyAMA0,115200'
+                if args.arch == 'arm':
+                    cpu = 'cortex-a15'
+                else:
+                    cpu = 'cortex-a57'
+                append = ['-append', '{} {}'.format(root, kernel_cli)]
+                cmd.extend(
+                    [
+                        # highmem=off needed since v3.0.0 due to:
+                        # http://lists.nongnu.org/archive/html/qemu-discuss/2018-08/msg00034.html
+                        '-M', '{},highmem=off'.format(common.machine),
+                        '-cpu', cpu,
+                    ] +
+                    virtio_gpu_pci
+                )
+            if args.baremetal is None:
+                cmd.extend(append)
     if args.tmux:
         if args.gem5:
             subprocess.Popen([os.path.join(common.root_dir, 'tmu'),
@@ -360,15 +377,6 @@ def get_argparse():
         help='''\
 Replace the normal init with a minimal init that just evals the given string.
 See: https://github.com/cirosantilli/linux-kernel-module-cheat#replace-init
-'''
-    )
-    parser.add_argument(
-        '--exec-image', default=defaults['gem5_script'],
-        help='''\
-Set which executable image to use. Currently only used for gem5 se.py
-which has no sensible default. Maybe we could also select the --baremetal
-executable with this, might be a nicer interface. We could definitely use
-this to override the default Linux kernel image.
 '''
     )
     parser.add_argument(
@@ -410,7 +418,7 @@ gem.op5 --debug-flags=Exec fs.py --cpu-type=HPI --caches
 '''
     )
     parser.add_argument(
-        '--gem5-script', default=defaults['gem5_script'], choices=['fs', 'se', 'biglittle'],
+        '--gem5-script', default=defaults['gem5_script'], choices=['fs', 'biglittle'],
         help='Which gem5 script to use'
     )
     parser.add_argument(
@@ -488,6 +496,13 @@ to use this option:
 * on the split:
 ** if on QEMU and `-d` is given, GDB
 ** if on gem5, the gem5 terminal
+'''
+    )
+    parser.add_argument(
+        '--user', default=defaults['user'],
+        help='''\
+Run the given userland executable in user mode instead of full system mode.
+In gem5, user mode is called Syscall Emulation (SE) mode and uses se.py.
 '''
     )
     parser.add_argument(