userland: freestanding gem5 checkpoint restore examples

README.adoc

@@ -11211,8 +11211,10 @@ Analogous to QEMU's <<snapshot>>, but better since it can be started from inside
 
 Documentation: http://gem5.org/Checkpoints
 
+To see it in action try:
+
 ....
-./run --arch arm --emulator gem5
+./run --arch aarch64 --emulator gem5
 ....
 
 In the guest, wait for the boot to end and run:
@@ -11268,6 +11270,50 @@ Then there is no need to pass the kernel command line again to gem5 for replay:
 
 since boot has already happened, and the parameters are already in the RAM of the snapshot.
 
+==== gem5 checkpoint userland minimal example
+
+In order to debug checkpoint restore bugs, this minimal setup using link:userland/freestanding/gem5_checkpoint_restore.S[] can be handy:
+
+....
+./build-userland --arch aarch64 --static
+./run --arch aarch64 --emulator gem5 --static --userland userland/freestanding/gem5_checkpoint_restore.S --trace-insts-stdout
+./run --arch aarch64 --emulator gem5 --static --userland userland/freestanding/gem5_checkpoint_restore.S --trace-insts-stdout --gem5-restore 1
+./run --arch aarch64 --emulator gem5 --static --userland userland/freestanding/gem5_checkpoint_restore.S --trace-insts-stdout --gem5-restore 1 -- --cpu-type=DerivO3CPU --restore-with-cpu=DerivO3CPU --caches
+....
+
+On the initial run, we see that all instructions are executed and the checkpoint is taken:
+
+....
+      0: system.cpu: A0 T0 : @asm_main_after_prologue    :   movz   x0, #0, #0        : IntAlu :  D=0x0000000000000000  flags=(IsInteger)
+    500: system.cpu: A0 T0 : @asm_main_after_prologue+4    :   movz   x1, #0, #0        : IntAlu :  D=0x0000000000000000  flags=(IsInteger)
+   1000: system.cpu: A0 T0 : @asm_main_after_prologue+8    :   m5checkpoint             : IntAlu :   flags=(IsInteger|IsNonSpeculative|IsUnverifiable)
+Writing checkpoint
+warn: Checkpoints for file descriptors currently do not work.
+info: Entering event queue @ 1000.  Starting simulation...
+   1500: system.cpu: A0 T0 : @asm_main_after_prologue+12    :   movz   x0, #0, #0        : IntAlu :  D=0x0000000000000000  flags=(IsInteger)
+   2000: system.cpu: A0 T0 : @asm_main_after_prologue+16    :   m5exit                   : No_OpClass :   flags=(IsInteger|IsNonSpeculative)
+Exiting @ tick 2000 because m5_exit instruction encountered
+....
+
+Then, on the first restore run, the checkpoint is restored, and only instructions after the checkpoint are executed:
+
+....
+info: Entering event queue @ 1000.  Starting simulation...
+   1500: system.cpu: A0 T0 : @asm_main_after_prologue+12    :   movz   x0, #0, #0        : IntAlu :  D=0x0000000000000000  flags=(IsInteger)
+   2000: system.cpu: A0 T0 : @asm_main_after_prologue+16    :   m5exit                   : No_OpClass :   flags=(IsInteger|IsNonSpeculative)
+Exiting @ tick 2000 because m5_exit instruction encountered
+....
+
+and a similar thing happens for the restore with a different CPU type:
+
+....
+info: Entering event queue @ 1000.  Starting simulation...
+  79000: system.cpu: A0 T0 : @asm_main_after_prologue+12    :   movz   x0, #0, #0        : IntAlu :  D=0x0000000000000000  FetchSeq=1  CPSeq=1  flags=(IsInteger)
+Exiting @ tick 84500 because m5_exit instruction encountered
+....
+
+Here we don't see the last `m5 exit` instruction on the log, but it must just be something to do with the O3 logging.
+
 ==== gem5 checkpoint internals
 
 Checkpoints are stored inside the <<m5out-directory>> at:
@@ -19905,7 +19951,7 @@ Summary of manually collected results on <<p51>> at LKMC a18f28e263c91362519ef55
 
 |gem5 optimized build immediate exit on first instruction to benchmark the simulator startup time
 |ab6f7331406b22f8ab6e2df5f8b8e464fb35b611
-|immediate exit link:userland/arch/aarch64/freestanding/linux/gem5_exit.S[] `-O0`
+|immediate exit link:userland/freestanding/gem5_exit.S[] `-O0`
 |`gem5 --arch aarch64`
 |1
 |1
@@ -19914,7 +19960,7 @@ Summary of manually collected results on <<p51>> at LKMC a18f28e263c91362519ef55
 
 |same as above but debug build
 |ab6f7331406b22f8ab6e2df5f8b8e464fb35b611
-|link:userland/arch/aarch64/freestanding/linux/gem5_exit.S[] `-O0`
+|link:userland/freestanding/gem5_exit.S[] `-O0`
 |`gem5 --arch aarch64 --gem5-build-type debug`
 |1
 |1

lkmc/m5ops.h

@@ -14,11 +14,11 @@
  * D means RDI which is the first argument of a Linux C function call,
  * and S means RSI is the second one: these are also used by gem5 as arguments of the m5ops.
  */
-#define LKMC_M5OPS_CHECKPOINT_ASM mov $0, %%rdi; mov $0, %%rsi; .word 0x040F; .word 0x0043
+#define LKMC_M5OPS_CHECKPOINT_ASM mov $0, %rdi; mov $0, %rsi; .word 0x040F; .word 0x0043
-#define LKMC_M5OPS_DUMPSTATS_ASM  mov $0, %%rdi; mov $0, %%rsi; .word 0x040F; .word 0x0041
+#define LKMC_M5OPS_DUMPSTATS_ASM  mov $0, %rdi; mov $0, %rsi; .word 0x040F; .word 0x0041
-#define LKMC_M5OPS_EXIT_ASM       mov $0, %%rdi;                .word 0x040F; .word 0x0021
+#define LKMC_M5OPS_EXIT_ASM       mov $0, %rdi;               .word 0x040F; .word 0x0021
-#define LKMC_M5OPS_FAIL_1_ASM     mov $0, %%rdi; mov $1, %%rsi; .word 0x040F; .word 0x0022
+#define LKMC_M5OPS_FAIL_1_ASM     mov $0, %rdi; mov $1, %rsi; .word 0x040F; .word 0x0022
-#define LKMC_M5OPS_RESETSTATS_ASM mov $0, %%rdi; mov $0, %%rsi; .word 0x040F; .word 0x0040
+#define LKMC_M5OPS_RESETSTATS_ASM mov $0, %rdi; mov $0, %rsi; .word 0x040F; .word 0x0040
 
 #define LKMC_M5OPS_CHECKPOINT __asm__ __volatile__ (".word 0x040F; .word 0x0043;" : : "D" (0), "S" (0) :)
 #define LKMC_M5OPS_DUMPSTATS  __asm__ __volatile__ (".word 0x040F; .word 0x0041;" : : "D" (0), "S" (0) :)

path_properties.py

@@ -543,7 +543,6 @@ path_properties_tuples = (
                                         'linux': (
                                             {},
                                             {
-                                                'gem5_exit.S': {'allowed_emulators': {'gem5'}},
                                                 'wfe.S': {'more_than_1s': True},
                                                 'wfe_wfe.S': {'more_than_1s': True},
                                             }
@@ -678,6 +677,13 @@ path_properties_tuples = (
                         'thread_return_value.cpp': {'test_run_args': {'cpus': 2}},
                     },
                 ),
+                'freestanding': (
+                    freestanding_properties,
+                    {
+                        'gem5_checkpoint_restore.S': {'allowed_emulators': {'gem5'}},
+                        'gem5_exit.S': {'allowed_emulators': {'gem5'}},
+                    }
+                ),
                 'gcc': (
                     {**gnu_extension_properties, **{'cc_pedantic': False}},
                     {

run

@@ -497,6 +497,12 @@ Extra options to append at the end of the emulator command line.
                 ] +
                 gem5_exe_args
             )
+            if self.env['gem5_restore'] is not None:
+                # https://cirosantilli.com/linux-kernel-module-cheat#gem5-checkpoint-internals
+                cpt_dirs = self.gem5_list_checkpoint_dirs()
+                cpt_dir = cpt_dirs[-self.env['gem5_restore']]
+                cpt_dirs_sorted_by_tick = sorted(cpt_dirs, key=lambda x: int(x.split('.')[1]))
+                extra_emulator_args.extend(['-r', str(cpt_dirs_sorted_by_tick.index(cpt_dir) + 1)])
             if self.env['userland'] is not None:
                 cmd.extend([
                     self.env['gem5_se_file'], LF,
@@ -524,12 +530,6 @@ Extra options to append at the end of the emulator command line.
                     cmd.extend(['--interp-dir', self.env['userland_library_dir'], LF])
             else:
                 if self.env['gem5_script'] == 'fs':
-                    if self.env['gem5_restore'] is not None:
-                        # https://cirosantilli.com/linux-kernel-module-cheat#gem5-checkpoint-internals
-                        cpt_dirs = self.gem5_list_checkpoint_dirs()
-                        cpt_dir = cpt_dirs[-self.env['gem5_restore']]
-                        cpt_dirs_sorted_by_tick = sorted(cpt_dirs, key=lambda x: int(x.split('.')[1]))
-                        extra_emulator_args.extend(['-r', str(cpt_dirs_sorted_by_tick.index(cpt_dir) + 1)])
                     cmd.extend([
                         self.env['gem5_fs_file'], LF,
                         '--kernel', self.env['image'], LF,

userland/freestanding/gem5_checkpoint_restore.S

@@ -0,0 +1,11 @@
+/* https://cirosantilli.com/linux-kernel-module-cheat#gem5-checkpoint-userland-minimal-example */
+
+#define LKMC_M5OPS_ENABLE 1
+#include "lkmc/m5ops.h"
+
+.text
+.global _start
+_start:
+asm_main_after_prologue:
+    LKMC_M5OPS_CHECKPOINT_ASM
+    LKMC_M5OPS_EXIT_ASM

userland/freestanding/gem5_exit.S

@@ -0,0 +1,10 @@
+/* https://cirosantilli.com/linux-kernel-module-cheat#benchmark-emulators-on-userland-executables */
+
+#define LKMC_M5OPS_ENABLE 1
+#include "lkmc/m5ops.h"
+
+.text
+.global _start
+_start:
+asm_main_after_prologue:
+    LKMC_M5OPS_EXIT_ASM