From 8447a9a8a7fad0f0bdfd1e6039721b7ff7f54c9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ciro=20Santilli=20=E5=85=AD=E5=9B=9B=E4=BA=8B=E4=BB=B6=20?= =?UTF-8?q?=E6=B3=95=E8=BD=AE=E5=8A=9F?= Date: Thu, 20 Feb 2020 00:00:01 +0000 Subject: [PATCH] userland: freestanding gem5 checkpoint restore examples --- README.adoc | 52 +++++++++++++++++-- lkmc/m5ops.h | 10 ++-- path_properties.py | 8 ++- run | 12 ++--- .../freestanding/gem5_checkpoint_restore.S | 11 ++++ userland/freestanding/gem5_exit.S | 10 ++++ 6 files changed, 88 insertions(+), 15 deletions(-) create mode 100644 userland/freestanding/gem5_checkpoint_restore.S create mode 100644 userland/freestanding/gem5_exit.S diff --git a/README.adoc b/README.adoc index 28e7879..4872f31 100644 --- a/README.adoc +++ b/README.adoc @@ -11211,8 +11211,10 @@ Analogous to QEMU's <>, but better since it can be started from inside Documentation: http://gem5.org/Checkpoints +To see it in action try: + .... -./run --arch arm --emulator gem5 +./run --arch aarch64 --emulator gem5 .... In the guest, wait for the boot to end and run: @@ -11268,6 +11270,50 @@ Then there is no need to pass the kernel command line again to gem5 for replay: since boot has already happened, and the parameters are already in the RAM of the snapshot. +==== gem5 checkpoint userland minimal example + +In order to debug checkpoint restore bugs, this minimal setup using link:userland/freestanding/gem5_checkpoint_restore.S[] can be handy: + +.... +./build-userland --arch aarch64 --static +./run --arch aarch64 --emulator gem5 --static --userland userland/freestanding/gem5_checkpoint_restore.S --trace-insts-stdout +./run --arch aarch64 --emulator gem5 --static --userland userland/freestanding/gem5_checkpoint_restore.S --trace-insts-stdout --gem5-restore 1 +./run --arch aarch64 --emulator gem5 --static --userland userland/freestanding/gem5_checkpoint_restore.S --trace-insts-stdout --gem5-restore 1 -- --cpu-type=DerivO3CPU --restore-with-cpu=DerivO3CPU --caches +.... + +On the initial run, we see that all instructions are executed and the checkpoint is taken: + +.... + 0: system.cpu: A0 T0 : @asm_main_after_prologue : movz x0, #0, #0 : IntAlu : D=0x0000000000000000 flags=(IsInteger) + 500: system.cpu: A0 T0 : @asm_main_after_prologue+4 : movz x1, #0, #0 : IntAlu : D=0x0000000000000000 flags=(IsInteger) + 1000: system.cpu: A0 T0 : @asm_main_after_prologue+8 : m5checkpoint : IntAlu : flags=(IsInteger|IsNonSpeculative|IsUnverifiable) +Writing checkpoint +warn: Checkpoints for file descriptors currently do not work. +info: Entering event queue @ 1000. Starting simulation... + 1500: system.cpu: A0 T0 : @asm_main_after_prologue+12 : movz x0, #0, #0 : IntAlu : D=0x0000000000000000 flags=(IsInteger) + 2000: system.cpu: A0 T0 : @asm_main_after_prologue+16 : m5exit : No_OpClass : flags=(IsInteger|IsNonSpeculative) +Exiting @ tick 2000 because m5_exit instruction encountered +.... + +Then, on the first restore run, the checkpoint is restored, and only instructions after the checkpoint are executed: + +.... +info: Entering event queue @ 1000. Starting simulation... + 1500: system.cpu: A0 T0 : @asm_main_after_prologue+12 : movz x0, #0, #0 : IntAlu : D=0x0000000000000000 flags=(IsInteger) + 2000: system.cpu: A0 T0 : @asm_main_after_prologue+16 : m5exit : No_OpClass : flags=(IsInteger|IsNonSpeculative) +Exiting @ tick 2000 because m5_exit instruction encountered +.... + +and a similar thing happens for the restore with a different CPU type: + +.... +info: Entering event queue @ 1000. Starting simulation... + 79000: system.cpu: A0 T0 : @asm_main_after_prologue+12 : movz x0, #0, #0 : IntAlu : D=0x0000000000000000 FetchSeq=1 CPSeq=1 flags=(IsInteger) +Exiting @ tick 84500 because m5_exit instruction encountered +.... + +Here we don't see the last `m5 exit` instruction on the log, but it must just be something to do with the O3 logging. + ==== gem5 checkpoint internals Checkpoints are stored inside the <> at: @@ -19905,7 +19951,7 @@ Summary of manually collected results on <> at LKMC a18f28e263c91362519ef55 |gem5 optimized build immediate exit on first instruction to benchmark the simulator startup time |ab6f7331406b22f8ab6e2df5f8b8e464fb35b611 -|immediate exit link:userland/arch/aarch64/freestanding/linux/gem5_exit.S[] `-O0` +|immediate exit link:userland/freestanding/gem5_exit.S[] `-O0` |`gem5 --arch aarch64` |1 |1 @@ -19914,7 +19960,7 @@ Summary of manually collected results on <> at LKMC a18f28e263c91362519ef55 |same as above but debug build |ab6f7331406b22f8ab6e2df5f8b8e464fb35b611 -|link:userland/arch/aarch64/freestanding/linux/gem5_exit.S[] `-O0` +|link:userland/freestanding/gem5_exit.S[] `-O0` |`gem5 --arch aarch64 --gem5-build-type debug` |1 |1 diff --git a/lkmc/m5ops.h b/lkmc/m5ops.h index 4222103..2d7bbd6 100644 --- a/lkmc/m5ops.h +++ b/lkmc/m5ops.h @@ -14,11 +14,11 @@ * D means RDI which is the first argument of a Linux C function call, * and S means RSI is the second one: these are also used by gem5 as arguments of the m5ops. */ -#define LKMC_M5OPS_CHECKPOINT_ASM mov $0, %%rdi; mov $0, %%rsi; .word 0x040F; .word 0x0043 -#define LKMC_M5OPS_DUMPSTATS_ASM mov $0, %%rdi; mov $0, %%rsi; .word 0x040F; .word 0x0041 -#define LKMC_M5OPS_EXIT_ASM mov $0, %%rdi; .word 0x040F; .word 0x0021 -#define LKMC_M5OPS_FAIL_1_ASM mov $0, %%rdi; mov $1, %%rsi; .word 0x040F; .word 0x0022 -#define LKMC_M5OPS_RESETSTATS_ASM mov $0, %%rdi; mov $0, %%rsi; .word 0x040F; .word 0x0040 +#define LKMC_M5OPS_CHECKPOINT_ASM mov $0, %rdi; mov $0, %rsi; .word 0x040F; .word 0x0043 +#define LKMC_M5OPS_DUMPSTATS_ASM mov $0, %rdi; mov $0, %rsi; .word 0x040F; .word 0x0041 +#define LKMC_M5OPS_EXIT_ASM mov $0, %rdi; .word 0x040F; .word 0x0021 +#define LKMC_M5OPS_FAIL_1_ASM mov $0, %rdi; mov $1, %rsi; .word 0x040F; .word 0x0022 +#define LKMC_M5OPS_RESETSTATS_ASM mov $0, %rdi; mov $0, %rsi; .word 0x040F; .word 0x0040 #define LKMC_M5OPS_CHECKPOINT __asm__ __volatile__ (".word 0x040F; .word 0x0043;" : : "D" (0), "S" (0) :) #define LKMC_M5OPS_DUMPSTATS __asm__ __volatile__ (".word 0x040F; .word 0x0041;" : : "D" (0), "S" (0) :) diff --git a/path_properties.py b/path_properties.py index dd91257..ef03c23 100644 --- a/path_properties.py +++ b/path_properties.py @@ -543,7 +543,6 @@ path_properties_tuples = ( 'linux': ( {}, { - 'gem5_exit.S': {'allowed_emulators': {'gem5'}}, 'wfe.S': {'more_than_1s': True}, 'wfe_wfe.S': {'more_than_1s': True}, } @@ -678,6 +677,13 @@ path_properties_tuples = ( 'thread_return_value.cpp': {'test_run_args': {'cpus': 2}}, }, ), + 'freestanding': ( + freestanding_properties, + { + 'gem5_checkpoint_restore.S': {'allowed_emulators': {'gem5'}}, + 'gem5_exit.S': {'allowed_emulators': {'gem5'}}, + } + ), 'gcc': ( {**gnu_extension_properties, **{'cc_pedantic': False}}, { diff --git a/run b/run index 5b88ae4..fe779c5 100755 --- a/run +++ b/run @@ -497,6 +497,12 @@ Extra options to append at the end of the emulator command line. ] + gem5_exe_args ) + if self.env['gem5_restore'] is not None: + # https://cirosantilli.com/linux-kernel-module-cheat#gem5-checkpoint-internals + cpt_dirs = self.gem5_list_checkpoint_dirs() + cpt_dir = cpt_dirs[-self.env['gem5_restore']] + cpt_dirs_sorted_by_tick = sorted(cpt_dirs, key=lambda x: int(x.split('.')[1])) + extra_emulator_args.extend(['-r', str(cpt_dirs_sorted_by_tick.index(cpt_dir) + 1)]) if self.env['userland'] is not None: cmd.extend([ self.env['gem5_se_file'], LF, @@ -524,12 +530,6 @@ Extra options to append at the end of the emulator command line. cmd.extend(['--interp-dir', self.env['userland_library_dir'], LF]) else: if self.env['gem5_script'] == 'fs': - if self.env['gem5_restore'] is not None: - # https://cirosantilli.com/linux-kernel-module-cheat#gem5-checkpoint-internals - cpt_dirs = self.gem5_list_checkpoint_dirs() - cpt_dir = cpt_dirs[-self.env['gem5_restore']] - cpt_dirs_sorted_by_tick = sorted(cpt_dirs, key=lambda x: int(x.split('.')[1])) - extra_emulator_args.extend(['-r', str(cpt_dirs_sorted_by_tick.index(cpt_dir) + 1)]) cmd.extend([ self.env['gem5_fs_file'], LF, '--kernel', self.env['image'], LF, diff --git a/userland/freestanding/gem5_checkpoint_restore.S b/userland/freestanding/gem5_checkpoint_restore.S new file mode 100644 index 0000000..1e66d5f --- /dev/null +++ b/userland/freestanding/gem5_checkpoint_restore.S @@ -0,0 +1,11 @@ +/* https://cirosantilli.com/linux-kernel-module-cheat#gem5-checkpoint-userland-minimal-example */ + +#define LKMC_M5OPS_ENABLE 1 +#include "lkmc/m5ops.h" + +.text +.global _start +_start: +asm_main_after_prologue: + LKMC_M5OPS_CHECKPOINT_ASM + LKMC_M5OPS_EXIT_ASM diff --git a/userland/freestanding/gem5_exit.S b/userland/freestanding/gem5_exit.S new file mode 100644 index 0000000..ba4842d --- /dev/null +++ b/userland/freestanding/gem5_exit.S @@ -0,0 +1,10 @@ +/* https://cirosantilli.com/linux-kernel-module-cheat#benchmark-emulators-on-userland-executables */ + +#define LKMC_M5OPS_ENABLE 1 +#include "lkmc/m5ops.h" + +.text +.global _start +_start: +asm_main_after_prologue: + LKMC_M5OPS_EXIT_ASM