failed xen attempt, refactor timer, failed svc attempt, aarch64 use gicv3

.gitmodules

@@ -21,3 +21,9 @@
 [submodule "submodules/qemu"]
 	path = submodules/qemu
 	url = https://github.com/cirosantilli/qemu
+[submodule "submodules/xen"]
+	path = submodules/xen
+	url = git://xenbits.xen.org/xen.git
+[submodule "submodules/boot-wrapper-aarch64"]
+	path = submodules/boot-wrapper-aarch64
+	url = git://git.kernel.org/pub/scm/linux/kernel/git/mark/boot-wrapper-aarch64.git

README.adoc

@@ -2826,9 +2826,17 @@ QEMU automatically adds a second CPU to the DTB!
 
 The action seems to be happening at: `hw/arm/virt.c`.
 
+You can dump the DTB QEMU generated with:
+
+....
+./run --arch aarch64 -- -machine dumpdtb=dtb.dtb
+....
+
+as mentioned at: https://lists.gnu.org/archive/html/qemu-discuss/2017-02/msg00051.html
+
 <<gem5-fs_biglittle>> 2a9573f5942b5416fb0570cf5cb6cdecba733392 can also generate its own DTB.
 
-gem5 can generate DTBs on ARM with `--generate-dtb`, but we don't use that feature as of f8c0502bb2680f2dbe7c1f3d7958f60265347005 because it was buggy.
+gem5 can generate DTBs on ARM with `--generate-dtb`. The generated DTB is placed in the <<m5out-directory>> named as `system.dtb`.
 
 == KVM
 

baremetal/arch/aarch64/common_aarch64.h

@@ -0,0 +1,24 @@
+#ifndef COMMON_AARCH64_H
+#define COMMON_AARCH64_H
+
+#include <inttypes.h>
+
+#define SYSREG_READ(type, name) \
+    type sysreg_ ## name ## _read(void) { \
+        type name; \
+        __asm__ __volatile__("mrs %0, " #name : "=r" (name) : : ); \
+        return name; \
+    }
+
+#define SYSREG_WRITE(type, name) \
+    void sysreg_ ## name ## _write(type name) { \
+        __asm__ __volatile__("msr " #name ", %0" : : "r" (name) : ); \
+    }
+
+#define SYSREG_READ_WRITE(name, type) \
+    SYSREG_READ(name, type) \
+    SYSREG_WRITE(name, type)
+
+#define SVC(immediate) __asm__ __volatile__("svc " #immediate : : : )
+
+#endif

baremetal/arch/aarch64/svc.c

@@ -0,0 +1,28 @@
+#include <stdio.h>
+#include <inttypes.h>
+
+#include "common_aarch64.h"
+
+/* Masks each of the 4 exception types: Synchronous, System error,
+ * IRQ and FIQ.
+ */
+SYSREG_READ_WRITE(uint32_t, daif)
+
+/* Determines if we use SP0 or SPx. Default: SP0.
+ * See also: https://stackoverflow.com/questions/29393677/armv8-exception-vector-significance-of-el0-sp
+ */
+SYSREG_READ_WRITE(uint32_t, spsel)
+
+/* Jump to this SP if spsel == SPx. */
+SYSREG_READ_WRITE(uint64_t, sp_el1)
+
+int main(void) {
+    printf("daif 0x%" PRIx32 "\n", sysreg_daif_read());
+    printf("spsel 0x%" PRIx32 "\n", sysreg_spsel_read());
+    /* TODO this breaks execution because reading system registers that end
+     * in ELx "trap", leading into an exception on the upper EL.
+     */
+    /*printf("sp_el1 0x%" PRIx64 "\n", sysreg_sp_el1_read());*/
+    /*SVC(0);*/
+    return 0;
+}

baremetal/arch/aarch64/timer.c

@@ -1,28 +1,12 @@
-/* https://github.com/cirosantilli/linux-kernel-module-cheat#arm-exception-level */
-
 #include <stdio.h>
 #include <inttypes.h>
 
+#include "common_aarch64.h"
+
 #define CNTV_CTL_ENABLE  (1 << 0)
 #define CNTV_CTL_IMASK   (1 << 1)
 #define CNTV_CTL_ISTATUS (1 << 2)
 
-#define SYSREG_READ(type, name) \
-    type name ## _read(void) { \
-        type name; \
-        __asm__ __volatile__("mrs %0, " #name : "=r" (name) : : ); \
-        return name; \
-    }
-
-#define SYSREG_WRITE(type, name) \
-    void name ## _write(type name) { \
-        __asm__ __volatile__("msr " #name ", %0" : : "r" (name) : ); \
-    }
-
-#define SYSREG_READ_WRITE(name, type) \
-    SYSREG_READ(name, type) \
-    SYSREG_WRITE(name, type)
-
 /* Frequency in Hz. ? */
 SYSREG_READ_WRITE(uint64_t, cntfrq_el0)
 
@@ -41,7 +25,7 @@ SYSREG_READ_WRITE(uint64_t, cntv_tval_el0)
 SYSREG_READ_WRITE(uint32_t, cntv_ctl_el0)
 
 void cntv_ctl_el0_disable(void) {
-    cntv_ctl_el0_write(cntv_ctl_el0_read() & ~CNTV_CTL_ENABLE);
+    sysreg_cntv_ctl_el0_write(sysreg_cntv_ctl_el0_read() & ~CNTV_CTL_ENABLE);
 }
 
 /* If enabled, when: cntv_ctl > cntv_cval then:
@@ -50,25 +34,25 @@ void cntv_ctl_el0_disable(void) {
  * * set CNTV_CTL_ISTATUS
  */
 void cntv_ctl_el0_enable(void) {
-    cntv_ctl_el0_write(cntv_ctl_el0_read() | CNTV_CTL_ENABLE);
+    sysreg_cntv_ctl_el0_write(sysreg_cntv_ctl_el0_read() | CNTV_CTL_ENABLE);
 }
 
 int main(void) {
     /* Initial state. */
-    printf("cntv_ctl_el0 0x%" PRIx32 "\n", cntv_ctl_el0_read());
+    printf("cntv_ctl_el0 0x%" PRIx32 "\n", sysreg_cntv_ctl_el0_read());
-    printf("cntfrq_el0 0x%" PRIx64 "\n", cntfrq_el0_read());
+    printf("cntfrq_el0 0x%" PRIx64 "\n", sysreg_cntfrq_el0_read());
-    printf("cntv_cval_el0 0x%" PRIx64 "\n", cntv_cval_el0_read());
+    printf("cntv_cval_el0 0x%" PRIx64 "\n", sysreg_cntv_cval_el0_read());
 
     /* Get the counter value many times to watch the time pass. */
-    printf("cntvct_el0 0x%" PRIx64 "\n", cntvct_el0_read());
+    printf("cntvct_el0 0x%" PRIx64 "\n", sysreg_cntvct_el0_read());
-    printf("cntvct_el0 0x%" PRIx64 "\n", cntvct_el0_read());
+    printf("cntvct_el0 0x%" PRIx64 "\n", sysreg_cntvct_el0_read());
-    printf("cntvct_el0 0x%" PRIx64 "\n", cntvct_el0_read());
+    printf("cntvct_el0 0x%" PRIx64 "\n", sysreg_cntvct_el0_read());
 
 #if 0
     /* TODO crashes gem5. */
     puts("cntfrq_el0 = 1");
-    cntfrq_el0_write(1);
+    sysreg_cntfrq_el0_write(1);
-    printf("cntfrq_el0 0x%" PRIx64 "\n", cntfrq_el0_read());
+    printf("cntfrq_el0 0x%" PRIx64 "\n", sysreg_cntfrq_el0_read());
 #endif
 
     return 0;

build-xen

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+# TODO get working, aarch64 Xen integration attempt.
+# Current state: prints to Boot-wrapper v0.2 to screen and hangs.
+# Bibliography:
+# https://wiki.xenproject.org/wiki/Xen_ARM_with_Virtualization_Extensions/qemu-system-aarch64
+# https://blog.xenproject.org/2014/04/01/virtualization-on-arm-with-xen/
+cd submodules/xen
+make \
+  -j`nproc` \
+  dist-xen \
+  CONFIG_DEBUG=y \
+  CONFIG_EARLY_PRINTK=vexpress \
+  CROSS_COMPILE=aarch64-linux-gnu- \
+  XEN_TARGET_ARCH=arm64 \
+;
+cd ../boot-wraper-aarch64
+autoreconf -i
+# DTB dumped from QEMU with: -machine dumpdtb=dtb.dtb
+./configure \
+  --enable-gicv3 \
+  --enable-psci \
+  --host=aarch64-linux-gnu \
+  --with-cmdline="console=hvc0 root=/dev/vda rw mem=1G" \
+  --with-dtb=dtb.dtb \
+  --with-kernel-dir=../../out/linux/default/aarch64 \
+  --with-xen-cmdline="dtuart=/uart@1c090000 console=dtuart no-bootscrub dom0_mem=1G loglvl=all guest_loglvl=all" \
+  --with-xen=../xen/xen/xen \
+;
+dtb.dtb -j`nproc`
+../../out/qemu/default/aarch64-softmmu/qemu-system-aarch64 \
+  -M virt \
+  -M virtualization=on \
+  -cpu cortex-a57 \
+  -kernel xen-system.axf \
+  -serial mon:stdio \
+  -nographic \
+;

common.py

@@ -439,7 +439,11 @@ Valid emulators: {}
                         env['machine'] = 'VExpress_GEM5_V1'
             else:
                 if not env['_args_given']['machine']:
-                    env['machine'] = 'virt'
+                    # highmem=off needed since v3.0.0 due to:
+                    # http://lists.nongnu.org/archive/html/qemu-discuss/2018-08/msg00034.html
+                    env['machine'] = 'virt,highmem=off'
+                    if env['arch'] == 'aarch64':
+                        env['machine'] += ',gic_version=3'
 
         # Buildroot
         env['buildroot_build_dir'] = join(env['buildroot_out_dir'], 'build', env['buildroot_build_id'], env['arch'])

run

@@ -162,7 +162,7 @@ Output trace to stdout instead of a file. Only works for gem5 currently.
 Output to the terminal, don't pipe to tee as the default.
 Does not save the output to a file, but allows you to use debuggers.
 Set automatically by --debug-vm, but you still need this option to debug
-gem5 Python scripts.
+gem5 Python scripts with pdb.
 '''
         )
         self.add_argument(
@@ -475,6 +475,7 @@ Run QEMU with VNC instead of the default SDL. Connect to it with:
                 cmd.extend(
                     [
                         qemu_executable, LF,
+                        '-machine', self.env['machine'], LF,
                         '-device', 'rtl8139,netdev=net0', LF,
                         '-gdb', 'tcp::{}'.format(self.env['gdb_port']), LF,
                         '-kernel', self.env['image'], LF,
@@ -535,7 +536,6 @@ Run QEMU with VNC instead of the default SDL. Connect to it with:
                 if self.env['arch'] == 'x86_64':
                     append = ['-append', '{} nopat {}'.format(root, kernel_cli), LF]
                     cmd.extend([
-                        '-M', self.env['machine'], LF,
                         '-device', 'edu', LF,
                     ])
                 elif self.env['is_arm']:
@@ -547,9 +547,6 @@ Run QEMU with VNC instead of the default SDL. Connect to it with:
                     append = ['-append', '{} {}'.format(root, kernel_cli), LF]
                     cmd.extend(
                         [
-                            # highmem=off needed since v3.0.0 due to:
-                            # http://lists.nongnu.org/archive/html/qemu-discuss/2018-08/msg00034.html
-                            '-M', '{},highmem=off'.format(self.env['machine']), LF,
                             '-cpu', cpu, LF,
                         ] +
                         virtio_gpu_pci