From 171a3381b1ce408d2369c25dc75cc4891871e90a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ciro=20Santilli=20=E5=85=AD=E5=9B=9B=E4=BA=8B=E4=BB=B6=20?=
 =?UTF-8?q?=E6=B3=95=E8=BD=AE=E5=8A=9F?= <ciro.santilli@gmail.com>
Date: Thu, 9 Jan 2020 00:00:00 +0000
Subject: [PATCH] 657c59249e56d861bb0a437a1b1c757797281910

---
 index.html | 516 ++++++++++++++++++++++++++++++++++-------------------
 1 file changed, 329 insertions(+), 187 deletions(-)

diff --git a/index.html b/index.html
index b387ba5..d31cc33 100644
--- a/index.html
+++ b/index.html
@@ -1236,7 +1236,11 @@ body.book #toc,body.book #preamble,body.book h1.sect0,body.book .sect1>h2{page-b
 <li><a href="#login-as-a-non-root-user-without-password">20.4.1. Login as a non-root user without password</a></li>
 </ul>
 </li>
-<li><a href="#add-new-buildroot-packages">20.5. Add new Buildroot packages</a></li>
+<li><a href="#add-new-files-to-the-buildroot-image">20.5. Add new files to the Buildroot image</a>
+<ul class="sectlevel3">
+<li><a href="#add-new-buildroot-packages">20.5.1. Add new Buildroot packages</a></li>
+</ul>
+</li>
 <li><a href="#remove-buildroot-packages">20.6. Remove Buildroot packages</a></li>
 <li><a href="#br2-target-rootfs-ext2-size">20.7. BR2_TARGET_ROOTFS_EXT2_SIZE</a>
 <ul class="sectlevel3">
@@ -1892,71 +1896,76 @@ body.book #toc,body.book #preamble,body.book h1.sect0,body.book .sect1>h2{page-b
 <li><a href="#clean-the-build">33.7. Clean the build</a></li>
 <li><a href="#custom-build-directory">33.8. Custom build directory</a></li>
 <li><a href="#ccache">33.9. ccache</a></li>
-<li><a href="#rebuild-buildroot-while-running">33.10. Rebuild Buildroot while running</a></li>
-<li><a href="#simultaneous-runs">33.11. Simultaneous runs</a></li>
-<li><a href="#build-variants">33.12. Build variants</a>
+<li><a href="#getvar">33.10. getvar</a>
 <ul class="sectlevel3">
-<li><a href="#linux-kernel-build-variants">33.12.1. Linux kernel build variants</a></li>
-<li><a href="#qemu-build-variants">33.12.2. QEMU build variants</a></li>
-<li><a href="#gem5-build-variants">33.12.3. gem5 build variants</a>
-<ul class="sectlevel4">
-<li><a href="#gem5-worktree">33.12.3.1. gem5 worktree</a></li>
-<li><a href="#gem5-private-source-trees">33.12.3.2. gem5 private source trees</a></li>
+<li><a href="#run-toolchain">33.10.1. run-toolchain</a></li>
 </ul>
 </li>
-<li><a href="#buildroot-build-variants">33.12.4. Buildroot build variants</a></li>
-</ul>
-</li>
-<li><a href="#directory-structure">33.13. Directory structure</a>
+<li><a href="#rebuild-buildroot-while-running">33.11. Rebuild Buildroot while running</a></li>
+<li><a href="#simultaneous-runs">33.12. Simultaneous runs</a></li>
+<li><a href="#build-variants">33.13. Build variants</a>
 <ul class="sectlevel3">
-<li><a href="#lkmc-directory">33.13.1. lkmc directory</a>
+<li><a href="#linux-kernel-build-variants">33.13.1. Linux kernel build variants</a></li>
+<li><a href="#qemu-build-variants">33.13.2. QEMU build variants</a></li>
+<li><a href="#gem5-build-variants">33.13.3. gem5 build variants</a>
 <ul class="sectlevel4">
-<li><a href="#userland-objects-vs-header-only">33.13.1.1. Userland objects vs header-only</a></li>
+<li><a href="#gem5-worktree">33.13.3.1. gem5 worktree</a></li>
+<li><a href="#gem5-private-source-trees">33.13.3.2. gem5 private source trees</a></li>
 </ul>
 </li>
-<li><a href="#buildroot_packages-directory">33.13.2. buildroot_packages directory</a>
-<ul class="sectlevel4">
-<li><a href="#kernel_modules-buildroot-package">33.13.2.1. kernel_modules buildroot package</a></li>
+<li><a href="#buildroot-build-variants">33.13.4. Buildroot build variants</a></li>
 </ul>
 </li>
-<li><a href="#patches-directory">33.13.3. patches directory</a>
-<ul class="sectlevel4">
-<li><a href="#patches-global-directory">33.13.3.1. patches/global directory</a></li>
-<li><a href="#patches-manual-directory">33.13.3.2. patches/manual directory</a></li>
-</ul>
-</li>
-<li><a href="#rootfs_overlay">33.13.4. rootfs_overlay</a>
-<ul class="sectlevel4">
-<li><a href="#out_rootfs_overlay_dir">33.13.4.1. out_rootfs_overlay_dir</a></li>
-</ul>
-</li>
-<li><a href="#lkmc-c">33.13.5. lkmc.c</a></li>
-<li><a href="#rand_check-out">33.13.6. rand_check.out</a></li>
-<li><a href="#lkmc_home">33.13.7. lkmc_home</a></li>
-</ul>
-</li>
-<li><a href="#test-this-repo">33.14. Test this repo</a>
+<li><a href="#directory-structure">33.14. Directory structure</a>
 <ul class="sectlevel3">
-<li><a href="#automated-tests">33.14.1. Automated tests</a>
+<li><a href="#lkmc-directory">33.14.1. lkmc directory</a>
 <ul class="sectlevel4">
-<li><a href="#test-arch-and-emulator-selection">33.14.1.1. Test arch and emulator selection</a></li>
-<li><a href="#quit-on-fail">33.14.1.2. Quit on fail</a></li>
-<li><a href="#test-userland-in-full-system">33.14.1.3. Test userland in full system</a></li>
-<li><a href="#gdb-tests">33.14.1.4. GDB tests</a></li>
-<li><a href="#magic-failure-string">33.14.1.5. Magic failure string</a></li>
+<li><a href="#userland-objects-vs-header-only">33.14.1.1. Userland objects vs header-only</a></li>
 </ul>
 </li>
-<li><a href="#non-automated-tests">33.14.2. Non-automated tests</a>
+<li><a href="#buildroot_packages-directory">33.14.2. buildroot_packages directory</a>
 <ul class="sectlevel4">
-<li><a href="#test-gdb-linux-kernel">33.14.2.1. Test GDB Linux kernel</a></li>
-<li><a href="#test-the-internet">33.14.2.2. Test the Internet</a></li>
-<li><a href="#cli-script-tests">33.14.2.3. CLI script tests</a></li>
+<li><a href="#kernel_modules-buildroot-package">33.14.2.1. kernel_modules buildroot package</a></li>
+</ul>
+</li>
+<li><a href="#patches-directory">33.14.3. patches directory</a>
+<ul class="sectlevel4">
+<li><a href="#patches-global-directory">33.14.3.1. patches/global directory</a></li>
+<li><a href="#patches-manual-directory">33.14.3.2. patches/manual directory</a></li>
+</ul>
+</li>
+<li><a href="#rootfs_overlay">33.14.4. rootfs_overlay</a>
+<ul class="sectlevel4">
+<li><a href="#out_rootfs_overlay_dir">33.14.4.1. out_rootfs_overlay_dir</a></li>
+</ul>
+</li>
+<li><a href="#lkmc-c">33.14.5. lkmc.c</a></li>
+<li><a href="#lkmc_home">33.14.6. lkmc_home</a></li>
+<li><a href="#path-properties">33.14.7. path_properties.py</a></li>
+<li><a href="#rand_check-out">33.14.8. rand_check.out</a></li>
+</ul>
+</li>
+<li><a href="#test-this-repo">33.15. Test this repo</a>
+<ul class="sectlevel3">
+<li><a href="#automated-tests">33.15.1. Automated tests</a>
+<ul class="sectlevel4">
+<li><a href="#test-arch-and-emulator-selection">33.15.1.1. Test arch and emulator selection</a></li>
+<li><a href="#quit-on-fail">33.15.1.2. Quit on fail</a></li>
+<li><a href="#test-userland-in-full-system">33.15.1.3. Test userland in full system</a></li>
+<li><a href="#gdb-tests">33.15.1.4. GDB tests</a></li>
+<li><a href="#magic-failure-string">33.15.1.5. Magic failure string</a></li>
+</ul>
+</li>
+<li><a href="#non-automated-tests">33.15.2. Non-automated tests</a>
+<ul class="sectlevel4">
+<li><a href="#test-gdb-linux-kernel">33.15.2.1. Test GDB Linux kernel</a></li>
+<li><a href="#test-the-internet">33.15.2.2. Test the Internet</a></li>
+<li><a href="#cli-script-tests">33.15.2.3. CLI script tests</a></li>
 </ul>
 </li>
 </ul>
 </li>
-<li><a href="#bisection">33.15. Bisection</a></li>
-<li><a href="#path-properties">33.16. path_properties</a></li>
+<li><a href="#bisection">33.16. Bisection</a></li>
 <li><a href="#update-a-forked-submodule">33.17. Update a forked submodule</a></li>
 <li><a href="#release">33.18. Release</a>
 <ul class="sectlevel3">
@@ -2908,13 +2917,16 @@ j = 0</pre>
 <div class="ulist">
 <ul>
 <li>
-<p><a href="#gem5-run-benchmark">gem5 run benchmark</a></p>
+<p><a href="#gem5-run-benchmark">gem5 run benchmark</a>: how to run a benchmark in gem5 full system, including how to boot Linux, checkpoint and restore to skip the boot on a fast CPU</p>
 </li>
 <li>
-<p><a href="#m5out-directory">m5out directory</a></p>
+<p><a href="#m5out-directory">m5out directory</a>: understand the output files that gem5 produces, which contain information about your run</p>
 </li>
 <li>
-<p><a href="#m5ops">m5ops</a></p>
+<p><a href="#m5ops">m5ops</a>: magic guest instructions used to control gem5</p>
+</li>
+<li>
+<p><a href="#add-new-files-to-the-buildroot-image">Add new files to the Buildroot image</a>: how to add your own files to the image if you have a benchmark that we don&#8217;t already support out of the box (also send a pull request!)</p>
 </li>
 </ul>
 </div>
@@ -3769,7 +3781,7 @@ cd userland
 <p>The examples that work include most <a href="#c">C examples</a> that don&#8217;t rely on complicated syscalls such as threads, and almost all the <a href="#userland-assembly">Userland assembly</a> examples.</p>
 </div>
 <div class="paragraph">
-<p>The exact list of userland programs that work in baremetal is specified in <a href="#path-properties">path_properties</a> with the <code>baremetal</code> property, but you can also easily find it out with a <a href="#baremetal-tests">baremetal test dry run</a>:</p>
+<p>The exact list of userland programs that work in baremetal is specified in <a href="#path-properties">path_properties.py</a> with the <code>baremetal</code> property, but you can also easily find it out with a <a href="#baremetal-tests">baremetal test dry run</a>:</p>
 </div>
 <div class="literalblock">
 <div class="content">
@@ -7391,7 +7403,7 @@ qw er</pre>
 <p>The gem5 tests require building statically with build id <code>static</code>, see also: <a href="#gem5-syscall-emulation-mode">Section 10.7, &#8220;gem5 syscall emulation mode&#8221;</a>. TODO automate this better.</p>
 </div>
 <div class="paragraph">
-<p>See: <a href="#test-this-repo">Section 33.14, &#8220;Test this repo&#8221;</a> for more useful testing tips.</p>
+<p>See: <a href="#test-this-repo">Section 33.15, &#8220;Test this repo&#8221;</a> for more useful testing tips.</p>
 </div>
 </div>
 <div class="sect2">
@@ -8029,7 +8041,7 @@ hello
 <div class="ulist">
 <ul>
 <li>
-<p>modules built with Buildroot, see: <a href="#kernel_modules-buildroot-package">Section 33.13.2.1, &#8220;kernel_modules buildroot package&#8221;</a></p>
+<p>modules built with Buildroot, see: <a href="#kernel_modules-buildroot-package">Section 33.14.2.1, &#8220;kernel_modules buildroot package&#8221;</a></p>
 </li>
 <li>
 <p>modules built from the kernel tree itself, see: <a href="#dummy-irq">Section 15.12.2, &#8220;dummy-irq&#8221;</a></p>
@@ -9785,7 +9797,7 @@ git log | grep -E '    Linux [0-9]+\.' | head</pre>
 <p>This also makes this repo the perfect setup to develop the Linux kernel.</p>
 </div>
 <div class="paragraph">
-<p>In case something breaks while updating the Linux kernel, you can try to bisect it to understand the root cause, see: <a href="#bisection">Section 33.15, &#8220;Bisection&#8221;</a>.</p>
+<p>In case something breaks while updating the Linux kernel, you can try to bisect it to understand the root cause, see: <a href="#bisection">Section 33.16, &#8220;Bisection&#8221;</a>.</p>
 </div>
 <div class="sect4">
 <h5 id="update-the-linux-kernel-lkmc-procedure"><a class="anchor" href="#update-the-linux-kernel-lkmc-procedure"></a><a class="link" href="#update-the-linux-kernel-lkmc-procedure">15.2.2.1. Update the Linux kernel LKMC procedure</a></h5>
@@ -9793,7 +9805,7 @@ git log | grep -E '    Linux [0-9]+\.' | head</pre>
 <p>First, use use the branching procedure described at: <a href="#update-a-forked-submodule">Section 33.17, &#8220;Update a forked submodule&#8221;</a></p>
 </div>
 <div class="paragraph">
-<p>Because the kernel is so central to this repository, almost all tests must be re-run, so basically just follow the full testing procedure described at: <a href="#test-this-repo">Section 33.14, &#8220;Test this repo&#8221;</a>. The only tests that can be skipped are essentially the <a href="#baremetal">Baremetal</a> tests.</p>
+<p>Because the kernel is so central to this repository, almost all tests must be re-run, so basically just follow the full testing procedure described at: <a href="#test-this-repo">Section 33.15, &#8220;Test this repo&#8221;</a>. The only tests that can be skipped are essentially the <a href="#baremetal">Baremetal</a> tests.</p>
 </div>
 <div class="paragraph">
 <p>Before comitting, don&#8217;t forget to update:</p>
@@ -21999,7 +22011,60 @@ make menuconfig</pre>
 </div>
 </div>
 <div class="sect2">
-<h3 id="add-new-buildroot-packages"><a class="anchor" href="#add-new-buildroot-packages"></a><a class="link" href="#add-new-buildroot-packages">20.5. Add new Buildroot packages</a></h3>
+<h3 id="add-new-files-to-the-buildroot-image"><a class="anchor" href="#add-new-files-to-the-buildroot-image"></a><a class="link" href="#add-new-files-to-the-buildroot-image">20.5. Add new files to the Buildroot image</a></h3>
+<div class="paragraph">
+<p>There are basically two choices:</p>
+</div>
+<div class="ulist">
+<ul>
+<li>
+<p>create a Buildroot package: <a href="#add-new-buildroot-packages">Add new Buildroot packages</a></p>
+</li>
+<li>
+<p>drop your files directly in <a href="#rootfs_overlay">rootfs_overlay</a> and follow instructions from that section</p>
+</li>
+</ul>
+</div>
+<div class="paragraph">
+<p>If you need to cross compile input files such as C for the guest, then Buildroot packages are definitely the cleaner option as they make cross compilation easy.</p>
+</div>
+<div class="paragraph">
+<p>However, for a quick initial prototype, it should be fine to just manually compile your files and drop them in <a href="#rootfs_overlay">rootfs_overlay</a>.</p>
+</div>
+<div class="paragraph">
+<p>Ideally, you should still use the Buildroot cross compiler for this which ensures compatibility.</p>
+</div>
+<div class="paragraph">
+<p>The best way to do that is to use either <a href="#run-toolchain">run-toolchain</a> or <a href="#getvar">getvar</a>.</p>
+</div>
+<div class="paragraph">
+<p>In case you can&#8217;t for some reason, e.g. if you need to use your own custom toolchain, you should:</p>
+</div>
+<div class="ulist">
+<ul>
+<li>
+<p>make sure that you have built your toolchain to match the our kernel version. It often just works even if they are not perfectly matched however, partly because the Linux kernel is highly <a href="#update-the-linux-kernel">backwards compatible</a></p>
+</li>
+<li>
+<p>build statically with <code>-static</code> to avoid binary compatibility issues with our own glibc</p>
+</li>
+</ul>
+</div>
+<div class="paragraph">
+<p>Related threads:</p>
+</div>
+<div class="ulist">
+<ul>
+<li>
+<p><a href="https://github.com/cirosantilli/linux-kernel-module-cheat/issues/22" class="bare">https://github.com/cirosantilli/linux-kernel-module-cheat/issues/22</a></p>
+</li>
+<li>
+<p><a href="https://github.com/cirosantilli/linux-kernel-module-cheat/issues/50" class="bare">https://github.com/cirosantilli/linux-kernel-module-cheat/issues/50</a></p>
+</li>
+</ul>
+</div>
+<div class="sect3">
+<h4 id="add-new-buildroot-packages"><a class="anchor" href="#add-new-buildroot-packages"></a><a class="link" href="#add-new-buildroot-packages">20.5.1. Add new Buildroot packages</a></h4>
 <div class="paragraph">
 <p>First, see if you can&#8217;t get away without actually adding a new package, for example:</p>
 </div>
@@ -22017,12 +22082,13 @@ make menuconfig</pre>
 <p>If none of those methods are flexible enough for you, you can just fork or hack up <a href="https://github.com/cirosantilli/linux-kernel-module-cheat/blob/master/buildroot_packages/sample_package">buildroot_packages/sample_package</a> the sample package to do what you want.</p>
 </div>
 <div class="paragraph">
-<p>For how to use that package, see: <a href="#buildroot_packages-directory">Section 33.13.2, &#8220;buildroot_packages directory&#8221;</a>.</p>
+<p>For how to use that package, see: <a href="#buildroot_packages-directory">Section 33.14.2, &#8220;buildroot_packages directory&#8221;</a>.</p>
 </div>
 <div class="paragraph">
 <p>Then iterate trying to do what you want and reading the manual until it works: <a href="https://buildroot.org/downloads/manual/manual.html" class="bare">https://buildroot.org/downloads/manual/manual.html</a></p>
 </div>
 </div>
+</div>
 <div class="sect2">
 <h3 id="remove-buildroot-packages"><a class="anchor" href="#remove-buildroot-packages"></a><a class="link" href="#remove-buildroot-packages">20.6. Remove Buildroot packages</a></h3>
 <div class="paragraph">
@@ -22399,7 +22465,7 @@ cd ../..
 <p>We already even had a C SVE test in-tree, but it was disabled because the old toolchain does not support it.</p>
 </div>
 <div class="paragraph">
-<p>So once the new GCC 8 toolchain was built, we can first enable that test by editing the <a href="#path-properties">path_properties</a> file to not skip C SVE tests anymore:</p>
+<p>So once the new GCC 8 toolchain was built, we can first enable that test by editing the <a href="#path-properties">path_properties.py</a> file to not skip C SVE tests anymore:</p>
 </div>
 <div class="literalblock">
 <div class="content">
@@ -31872,7 +31938,7 @@ ISB</pre>
 <p>In baremetal, we detect if tests failed by parsing logs for the <a href="#magic-failure-string">Magic failure string</a>.</p>
 </div>
 <div class="paragraph">
-<p>See: <a href="#test-this-repo">Section 33.14, &#8220;Test this repo&#8221;</a> for more useful testing tips.</p>
+<p>See: <a href="#test-this-repo">Section 33.15, &#8220;Test this repo&#8221;</a> for more useful testing tips.</p>
 </div>
 </div>
 </div>
@@ -33942,7 +34008,83 @@ export CCACHE_MAXSIZE="20G"</pre>
 </div>
 </div>
 <div class="sect2">
-<h3 id="rebuild-buildroot-while-running"><a class="anchor" href="#rebuild-buildroot-while-running"></a><a class="link" href="#rebuild-buildroot-while-running">33.10. Rebuild Buildroot while running</a></h3>
+<h3 id="getvar"><a class="anchor" href="#getvar"></a><a class="link" href="#getvar">33.10. getvar</a></h3>
+<div class="paragraph">
+<p>The <a href="https://github.com/cirosantilli/linux-kernel-module-cheat/blob/master/getvar">getvar</a> helper script can print the values of internal LKMC variables.</p>
+</div>
+<div class="paragraph">
+<p>Within our Python scripts such as <a href="https://github.com/cirosantilli/linux-kernel-module-cheat/blob/master/common.py">common.py</a>, those variable are visible as <code>self.env[&lt;var&gt;]</code>.</p>
+</div>
+<div class="paragraph">
+<p>For example, to find the Buildroot output directory for an <code>aarch64</code> build, you could use:</p>
+</div>
+<div class="literalblock">
+<div class="content">
+<pre>./getvar --arch aarch64 buildroot_build_dir</pre>
+</div>
+</div>
+<div class="paragraph">
+<p>which as of LKMC b15a0e455d691afa49f3b813ad9b09394dfb02b7 outputs:</p>
+</div>
+<div class="literalblock">
+<div class="content">
+<pre>/path/to/linux-kernel-module-cheat/out/buildroot/build/default/aarch64</pre>
+</div>
+</div>
+<div class="paragraph">
+<p>You can also list all available variables in one go with just:</p>
+</div>
+<div class="literalblock">
+<div class="content">
+<pre>./getvar</pre>
+</div>
+</div>
+<div class="paragraph">
+<p>Using <a href="https://github.com/cirosantilli/linux-kernel-module-cheat/blob/master/getvar">getvar</a> makes it possible to make Bash scripts more portable if for example directory structure changes across LKMC versions.</p>
+</div>
+<div class="paragraph">
+<p>For this reason, we use it in particular often in this README to reduce the need for refactoring.</p>
+</div>
+<div class="sect3">
+<h4 id="run-toolchain"><a class="anchor" href="#run-toolchain"></a><a class="link" href="#run-toolchain">33.10.1. run-toolchain</a></h4>
+<div class="paragraph">
+<p>While you could just manually find/learn the path to toolchain tools, e.g. in LKMC b15a0e455d691afa49f3b813ad9b09394dfb02b7 they are</p>
+</div>
+<div class="literalblock">
+<div class="content">
+<pre>./out/buildroot/build/default/aarch64/host/bin/aarch64-buildroot-linux-gnu-gcc userland/c/hello.c
+./out/buildroot/build/default/aarch64/host/bin/aarch64-buildroot-linux-gnu-objdump -D a.out</pre>
+</div>
+</div>
+<div class="paragraph">
+<p>you can save some typing and get portability across directory structure changes with our <a href="https://github.com/cirosantilli/linux-kernel-module-cheat/blob/master/run-toolchain">run-toolchain</a> helper:</p>
+</div>
+<div class="literalblock">
+<div class="content">
+<pre>./run-toolchain --arch aarch64 gcc -- userland/c/hello.c
+./run-toolchain --arch aarch64 objdump -- -D a.out</pre>
+</div>
+</div>
+<div class="paragraph">
+<p>Alternatively, if you just need a variable to feed into your own Build system, you can also use <a href="#getvar">getvar</a>:</p>
+</div>
+<div class="literalblock">
+<div class="content">
+<pre>./getvar --arch aarch64 toolchain_prefix</pre>
+</div>
+</div>
+<div class="paragraph">
+<p>which outputs as of LKMC b15a0e455d691afa49f3b813ad9b09394dfb02b7:</p>
+</div>
+<div class="literalblock">
+<div class="content">
+<pre>/path/to/linux-kernel-module-cheat/out/buildroot/build/default/aarch64/host/usr/bin/aarch64-buildroot-linux-gnu</pre>
+</div>
+</div>
+</div>
+</div>
+<div class="sect2">
+<h3 id="rebuild-buildroot-while-running"><a class="anchor" href="#rebuild-buildroot-while-running"></a><a class="link" href="#rebuild-buildroot-while-running">33.11. Rebuild Buildroot while running</a></h3>
 <div class="paragraph">
 <p>It is not possible to rebuild the root filesystem while running QEMU because QEMU holds the file qcow2 file:</p>
 </div>
@@ -33953,7 +34095,7 @@ export CCACHE_MAXSIZE="20G"</pre>
 </div>
 </div>
 <div class="sect2">
-<h3 id="simultaneous-runs"><a class="anchor" href="#simultaneous-runs"></a><a class="link" href="#simultaneous-runs">33.11. Simultaneous runs</a></h3>
+<h3 id="simultaneous-runs"><a class="anchor" href="#simultaneous-runs"></a><a class="link" href="#simultaneous-runs">33.12. Simultaneous runs</a></h3>
 <div class="paragraph">
 <p>When doing long simulations sweeping across multiple system parameters, it becomes fundamental to do multiple simulations in parallel.</p>
 </div>
@@ -34049,7 +34191,7 @@ less "$(./getvar --arch aarch64 --emulator gem5 --run-id 1 termout_file)"</pre>
 </div>
 </div>
 <div class="paragraph">
-<p>To run multiple gem5 checkouts, see: <a href="#gem5-worktree">Section 33.12.3.1, &#8220;gem5 worktree&#8221;</a>.</p>
+<p>To run multiple gem5 checkouts, see: <a href="#gem5-worktree">Section 33.13.3.1, &#8220;gem5 worktree&#8221;</a>.</p>
 </div>
 <div class="paragraph">
 <p>Implementation note: we create multiple namespaces for two things:</p>
@@ -34088,7 +34230,7 @@ less "$(./getvar --arch aarch64 --emulator gem5 --run-id 1 termout_file)"</pre>
 </div>
 </div>
 <div class="sect2">
-<h3 id="build-variants"><a class="anchor" href="#build-variants"></a><a class="link" href="#build-variants">33.12. Build variants</a></h3>
+<h3 id="build-variants"><a class="anchor" href="#build-variants"></a><a class="link" href="#build-variants">33.13. Build variants</a></h3>
 <div class="paragraph">
 <p>It often happens that you are comparing two versions of the build, a good and a bad one, and trying to figure out why the bad one is bad.</p>
 </div>
@@ -34096,7 +34238,7 @@ less "$(./getvar --arch aarch64 --emulator gem5 --run-id 1 termout_file)"</pre>
 <p>Our build variants system allows you to keep multiple built versions of all major components, so that you can easily switching between running one or the other.</p>
 </div>
 <div class="sect3">
-<h4 id="linux-kernel-build-variants"><a class="anchor" href="#linux-kernel-build-variants"></a><a class="link" href="#linux-kernel-build-variants">33.12.1. Linux kernel build variants</a></h4>
+<h4 id="linux-kernel-build-variants"><a class="anchor" href="#linux-kernel-build-variants"></a><a class="link" href="#linux-kernel-build-variants">33.13.1. Linux kernel build variants</a></h4>
 <div class="paragraph">
 <p>If you want to keep two builds around, one for the latest Linux version, and the other for Linux <code>v4.16</code>:</p>
 </div>
@@ -34132,11 +34274,11 @@ git -C "$(./getvar linux_source_dir)" checkout -
 </div>
 </div>
 <div class="paragraph">
-<p>To run both kernels simultaneously, one on each QEMU instance, see: <a href="#simultaneous-runs">Section 33.11, &#8220;Simultaneous runs&#8221;</a>.</p>
+<p>To run both kernels simultaneously, one on each QEMU instance, see: <a href="#simultaneous-runs">Section 33.12, &#8220;Simultaneous runs&#8221;</a>.</p>
 </div>
 </div>
 <div class="sect3">
-<h4 id="qemu-build-variants"><a class="anchor" href="#qemu-build-variants"></a><a class="link" href="#qemu-build-variants">33.12.2. QEMU build variants</a></h4>
+<h4 id="qemu-build-variants"><a class="anchor" href="#qemu-build-variants"></a><a class="link" href="#qemu-build-variants">33.13.2. QEMU build variants</a></h4>
 <div class="paragraph">
 <p>Analogous to the <a href="#linux-kernel-build-variants">Linux kernel build variants</a> but with the <code>--qemu-build-id</code> option instead:</p>
 </div>
@@ -34152,7 +34294,7 @@ git -C "$(./getvar qemu_source_dir)" checkout -
 </div>
 </div>
 <div class="sect3">
-<h4 id="gem5-build-variants"><a class="anchor" href="#gem5-build-variants"></a><a class="link" href="#gem5-build-variants">33.12.3. gem5 build variants</a></h4>
+<h4 id="gem5-build-variants"><a class="anchor" href="#gem5-build-variants"></a><a class="link" href="#gem5-build-variants">33.13.3. gem5 build variants</a></h4>
 <div class="paragraph">
 <p>Analogous to the <a href="#linux-kernel-build-variants">Linux kernel build variants</a> but with the <code>--gem5-build-id</code> option instead:</p>
 </div>
@@ -34183,7 +34325,7 @@ git -C "$(./getvar gem5_source_dir)" checkout some-branch
 <p>Therefore, you can&#8217;t forget to checkout to the sources to that of the corresponding build before running, unless you explicitly tell gem5 to use a non-default source tree with <a href="#gem5-worktree">gem5 worktree</a>. This becomes inevitable when you want to launch multiple simultaneous runs at different checkouts.</p>
 </div>
 <div class="sect4">
-<h5 id="gem5-worktree"><a class="anchor" href="#gem5-worktree"></a><a class="link" href="#gem5-worktree">33.12.3.1. gem5 worktree</a></h5>
+<h5 id="gem5-worktree"><a class="anchor" href="#gem5-worktree"></a><a class="link" href="#gem5-worktree">33.13.3.1. gem5 worktree</a></h5>
 <div class="paragraph">
 <p><a href="#gem5-build-variants"><code>--gem5-build-id</code></a> goes a long way, but if you want to seamlessly switch between two gem5 tress without checking out multiple times, then <code>--gem5-worktree</code> is for you.</p>
 </div>
@@ -34236,7 +34378,7 @@ cd -
 </div>
 </div>
 <div class="sect4">
-<h5 id="gem5-private-source-trees"><a class="anchor" href="#gem5-private-source-trees"></a><a class="link" href="#gem5-private-source-trees">33.12.3.2. gem5 private source trees</a></h5>
+<h5 id="gem5-private-source-trees"><a class="anchor" href="#gem5-private-source-trees"></a><a class="link" href="#gem5-private-source-trees">33.13.3.2. gem5 private source trees</a></h5>
 <div class="paragraph">
 <p>Suppose that you are working on a private fork of gem5, but you want to use this repository to develop it as well.</p>
 </div>
@@ -34280,7 +34422,7 @@ gem5_internal="$(pwd)/gem5-internal"</pre>
 </div>
 </div>
 <div class="sect3">
-<h4 id="buildroot-build-variants"><a class="anchor" href="#buildroot-build-variants"></a><a class="link" href="#buildroot-build-variants">33.12.4. Buildroot build variants</a></h4>
+<h4 id="buildroot-build-variants"><a class="anchor" href="#buildroot-build-variants"></a><a class="link" href="#buildroot-build-variants">33.13.4. Buildroot build variants</a></h4>
 <div class="paragraph">
 <p>Allows you to have multiple versions of the GCC toolchain or root filesystem.</p>
 </div>
@@ -34300,9 +34442,9 @@ git -C "$(./getvar buildroot_source_dir)" checkout -
 </div>
 </div>
 <div class="sect2">
-<h3 id="directory-structure"><a class="anchor" href="#directory-structure"></a><a class="link" href="#directory-structure">33.13. Directory structure</a></h3>
+<h3 id="directory-structure"><a class="anchor" href="#directory-structure"></a><a class="link" href="#directory-structure">33.14. Directory structure</a></h3>
 <div class="sect3">
-<h4 id="lkmc-directory"><a class="anchor" href="#lkmc-directory"></a><a class="link" href="#lkmc-directory">33.13.1. lkmc directory</a></h4>
+<h4 id="lkmc-directory"><a class="anchor" href="#lkmc-directory"></a><a class="link" href="#lkmc-directory">33.14.1. lkmc directory</a></h4>
 <div class="paragraph">
 <p><a href="https://github.com/cirosantilli/linux-kernel-module-cheat/blob/master/lkmc/">lkmc/</a> contains sources and headers that are shared across kernel modules, userland and baremetal examples.</p>
 </div>
@@ -34313,7 +34455,7 @@ git -C "$(./getvar buildroot_source_dir)" checkout -
 <p>Another option would have been to name it as <code>includes/lkmc</code>, but that would make paths longer, and we might want to store source code in that directory as well in the future.</p>
 </div>
 <div class="sect4">
-<h5 id="userland-objects-vs-header-only"><a class="anchor" href="#userland-objects-vs-header-only"></a><a class="link" href="#userland-objects-vs-header-only">33.13.1.1. Userland objects vs header-only</a></h5>
+<h5 id="userland-objects-vs-header-only"><a class="anchor" href="#userland-objects-vs-header-only"></a><a class="link" href="#userland-objects-vs-header-only">33.14.1.1. Userland objects vs header-only</a></h5>
 <div class="paragraph">
 <p>When factoring out functionality across userland examples, there are two main options:</p>
 </div>
@@ -34372,7 +34514,7 @@ git -C "$(./getvar buildroot_source_dir)" checkout -
 </div>
 </div>
 <div class="sect3">
-<h4 id="buildroot_packages-directory"><a class="anchor" href="#buildroot_packages-directory"></a><a class="link" href="#buildroot_packages-directory">33.13.2. buildroot_packages directory</a></h4>
+<h4 id="buildroot_packages-directory"><a class="anchor" href="#buildroot_packages-directory"></a><a class="link" href="#buildroot_packages-directory">33.14.2. buildroot_packages directory</a></h4>
 <div class="paragraph">
 <p>Source: <a href="https://github.com/cirosantilli/linux-kernel-module-cheat/blob/master/buildroot_packages/">buildroot_packages/</a>.</p>
 </div>
@@ -34421,7 +34563,7 @@ git -C "$(./getvar buildroot_source_dir)" checkout -
 <p>A custom build script can give you more flexibility: e.g. the package can be made work with other root filesystems more easily, have better <a href="#9p">9P</a> support, and rebuild faster as it evades some Buildroot boilerplate.</p>
 </div>
 <div class="sect4">
-<h5 id="kernel_modules-buildroot-package"><a class="anchor" href="#kernel_modules-buildroot-package"></a><a class="link" href="#kernel_modules-buildroot-package">33.13.2.1. kernel_modules buildroot package</a></h5>
+<h5 id="kernel_modules-buildroot-package"><a class="anchor" href="#kernel_modules-buildroot-package"></a><a class="link" href="#kernel_modules-buildroot-package">33.14.2.1. kernel_modules buildroot package</a></h5>
 <div class="paragraph">
 <p>Source: <a href="https://github.com/cirosantilli/linux-kernel-module-cheat/blob/master/buildroot_packages/kernel_modules/">buildroot_packages/kernel_modules/</a></p>
 </div>
@@ -34468,9 +34610,9 @@ git -C "$(./getvar buildroot_source_dir)" checkout -
 </div>
 </div>
 <div class="sect3">
-<h4 id="patches-directory"><a class="anchor" href="#patches-directory"></a><a class="link" href="#patches-directory">33.13.3. patches directory</a></h4>
+<h4 id="patches-directory"><a class="anchor" href="#patches-directory"></a><a class="link" href="#patches-directory">33.14.3. patches directory</a></h4>
 <div class="sect4">
-<h5 id="patches-global-directory"><a class="anchor" href="#patches-global-directory"></a><a class="link" href="#patches-global-directory">33.13.3.1. patches/global directory</a></h5>
+<h5 id="patches-global-directory"><a class="anchor" href="#patches-global-directory"></a><a class="link" href="#patches-global-directory">33.14.3.1. patches/global directory</a></h5>
 <div class="paragraph">
 <p>Has the following structure:</p>
 </div>
@@ -34487,7 +34629,7 @@ git -C "$(./getvar buildroot_source_dir)" checkout -
 </div>
 </div>
 <div class="sect4">
-<h5 id="patches-manual-directory"><a class="anchor" href="#patches-manual-directory"></a><a class="link" href="#patches-manual-directory">33.13.3.2. patches/manual directory</a></h5>
+<h5 id="patches-manual-directory"><a class="anchor" href="#patches-manual-directory"></a><a class="link" href="#patches-manual-directory">33.14.3.2. patches/manual directory</a></h5>
 <div class="paragraph">
 <p>Patches in this directory are never applied automatically: it is up to users to manually apply them before usage following the instructions in this documentation.</p>
 </div>
@@ -34497,7 +34639,7 @@ git -C "$(./getvar buildroot_source_dir)" checkout -
 </div>
 </div>
 <div class="sect3">
-<h4 id="rootfs_overlay"><a class="anchor" href="#rootfs_overlay"></a><a class="link" href="#rootfs_overlay">33.13.4. rootfs_overlay</a></h4>
+<h4 id="rootfs_overlay"><a class="anchor" href="#rootfs_overlay"></a><a class="link" href="#rootfs_overlay">33.14.4. rootfs_overlay</a></h4>
 <div class="paragraph">
 <p>Source: <a href="https://github.com/cirosantilli/linux-kernel-module-cheat/blob/master/rootfs_overlay">rootfs_overlay</a>.</p>
 </div>
@@ -34544,7 +34686,7 @@ git -C "$(./getvar buildroot_source_dir)" checkout -
 <p>This way you can just hack away the scripts and try them out immediately without any further operations.</p>
 </div>
 <div class="sect4">
-<h5 id="out_rootfs_overlay_dir"><a class="anchor" href="#out_rootfs_overlay_dir"></a><a class="link" href="#out_rootfs_overlay_dir">33.13.4.1. out_rootfs_overlay_dir</a></h5>
+<h5 id="out_rootfs_overlay_dir"><a class="anchor" href="#out_rootfs_overlay_dir"></a><a class="link" href="#out_rootfs_overlay_dir">33.14.4.1. out_rootfs_overlay_dir</a></h5>
 <div class="paragraph">
 <p>This path can be found with:</p>
 </div>
@@ -34570,7 +34712,7 @@ git -C "$(./getvar buildroot_source_dir)" checkout -
 <p>LKMC first collects all the files that it will dump into the guest there, and then in the very last step dumps everything into the final image.</p>
 </div>
 <div class="paragraph">
-<p>In Buildroot, this is done by pointing <code>BR2_ROOTFS_OVERLAY</code> to that directory.</p>
+<p>In Buildroot, this is done by pointing <code>BR2_ROOTFS_OVERLAY</code> to that directory, which is documented at: <a href="https://buildroot.org/downloads/manual/manual.html#rootfs-custom" class="bare">https://buildroot.org/downloads/manual/manual.html#rootfs-custom</a></p>
 </div>
 <div class="paragraph">
 <p>This does not include native image modification mechanisms such as <a href="#buildroot_packages-directory">Buildroot packages</a>, which we let Buildroot itself manage.</p>
@@ -34578,7 +34720,7 @@ git -C "$(./getvar buildroot_source_dir)" checkout -
 </div>
 </div>
 <div class="sect3">
-<h4 id="lkmc-c"><a class="anchor" href="#lkmc-c"></a><a class="link" href="#lkmc-c">33.13.5. lkmc.c</a></h4>
+<h4 id="lkmc-c"><a class="anchor" href="#lkmc-c"></a><a class="link" href="#lkmc-c">33.14.5. lkmc.c</a></h4>
 <div class="paragraph">
 <p>The files:</p>
 </div>
@@ -34608,34 +34750,7 @@ git -C "$(./getvar buildroot_source_dir)" checkout -
 </div>
 </div>
 <div class="sect3">
-<h4 id="rand_check-out"><a class="anchor" href="#rand_check-out"></a><a class="link" href="#rand_check-out">33.13.6. rand_check.out</a></h4>
-<div class="paragraph">
-<p>Print out several parameters that normally change randomly from boot to boot:</p>
-</div>
-<div class="literalblock">
-<div class="content">
-<pre>./run --eval-after './linux/rand_check.out;./linux/poweroff.out'</pre>
-</div>
-</div>
-<div class="paragraph">
-<p>Source: <a href="https://github.com/cirosantilli/linux-kernel-module-cheat/blob/master/userland/linux/rand_check.c">userland/linux/rand_check.c</a></p>
-</div>
-<div class="paragraph">
-<p>This can be used to check the determinism of:</p>
-</div>
-<div class="ulist">
-<ul>
-<li>
-<p><a href="#norandmaps">norandmaps</a></p>
-</li>
-<li>
-<p><a href="#qemu-record-and-replay">QEMU record and replay</a></p>
-</li>
-</ul>
-</div>
-</div>
-<div class="sect3">
-<h4 id="lkmc_home"><a class="anchor" href="#lkmc_home"></a><a class="link" href="#lkmc_home">33.13.7. lkmc_home</a></h4>
+<h4 id="lkmc_home"><a class="anchor" href="#lkmc_home"></a><a class="link" href="#lkmc_home">33.14.6. lkmc_home</a></h4>
 <div class="paragraph">
 <p><code>lkmc_home</code> refers to the target base directory in which we put all our custom built stuff, such as <a href="#userland-setup">userland executables</a> and <a href="#your-first-kernel-module-hack">kernel modules</a>.</p>
 </div>
@@ -34667,11 +34782,101 @@ git -C "$(./getvar buildroot_source_dir)" checkout -
 <p>Whenever a relative path is used inside a guest sample command, e.g. <code>insmod hello.ko</code> or <code>./hello.out</code>, it means that the path lives in <code>lkmc_home</code> unless stated otherwise.</p>
 </div>
 </div>
+<div class="sect3">
+<h4 id="path-properties"><a class="anchor" href="#path-properties"></a><a class="link" href="#path-properties">33.14.7. path_properties.py</a></h4>
+<div class="paragraph">
+<p>In order to build and run each userland and <a href="#baremetal-setup">baremetal</a> example properly, we need per-file metadata such as compiler flags and required number of cores.</p>
+</div>
+<div class="paragraph">
+<p>This data is stored is stored in <a href="https://github.com/cirosantilli/linux-kernel-module-cheat/blob/master/path_properties.py">path_properties.py</a> at <code>path_properties_tuples</code>.</p>
+</div>
+<div class="paragraph">
+<p>Maybe we should embed it magically into source files directories to make it easier to see? But one big Python dict was easier to implement so we started like this. And it allows factoring chunks out easily.</p>
+</div>
+<div class="paragraph">
+<p>The format is as follows:</p>
+</div>
+<div class="literalblock">
+<div class="content">
+<pre>'path_component': (
+    {'property': value},
+    {
+        'child_path_component':
+        {
+            {'child_property': },
+            {}
+        }
+    }
+)</pre>
+</div>
+</div>
+<div class="paragraph">
+<p>and as a shortcut, paths that don&#8217;t have any children can be written directly as:</p>
+</div>
+<div class="literalblock">
+<div class="content">
+<pre>'path_component': {'property': value}</pre>
+</div>
+</div>
+<div class="paragraph">
+<p>Properties of parent directories apply to all children.</p>
+</div>
+<div class="paragraph">
+<p>Lists coming from parent directories are extended instead of overwritten by children, this is especially useful for C compiler flags.</p>
+</div>
+<div class="paragraph">
+<p>To quickly determine which properties a path has, you can use <a href="https://github.com/cirosantilli/linux-kernel-module-cheat/blob/master/getprops">getprops</a>, e.g.:</p>
+</div>
+<div class="literalblock">
+<div class="content">
+<pre>./getprops userland/c/hello.c</pre>
+</div>
+</div>
+<div class="paragraph">
+<p>which outputs values such as:</p>
+</div>
+<div class="literalblock">
+<div class="content">
+<pre>allowed_archs=None
+allowed_emulators=None
+arm_aarch32=False
+arm_sve=False
+baremetal=True</pre>
+</div>
+</div>
+</div>
+<div class="sect3">
+<h4 id="rand_check-out"><a class="anchor" href="#rand_check-out"></a><a class="link" href="#rand_check-out">33.14.8. rand_check.out</a></h4>
+<div class="paragraph">
+<p>Print out several parameters that normally change randomly from boot to boot:</p>
+</div>
+<div class="literalblock">
+<div class="content">
+<pre>./run --eval-after './linux/rand_check.out;./linux/poweroff.out'</pre>
+</div>
+</div>
+<div class="paragraph">
+<p>Source: <a href="https://github.com/cirosantilli/linux-kernel-module-cheat/blob/master/userland/linux/rand_check.c">userland/linux/rand_check.c</a></p>
+</div>
+<div class="paragraph">
+<p>This can be used to check the determinism of:</p>
+</div>
+<div class="ulist">
+<ul>
+<li>
+<p><a href="#norandmaps">norandmaps</a></p>
+</li>
+<li>
+<p><a href="#qemu-record-and-replay">QEMU record and replay</a></p>
+</li>
+</ul>
+</div>
+</div>
 </div>
 <div class="sect2">
-<h3 id="test-this-repo"><a class="anchor" href="#test-this-repo"></a><a class="link" href="#test-this-repo">33.14. Test this repo</a></h3>
+<h3 id="test-this-repo"><a class="anchor" href="#test-this-repo"></a><a class="link" href="#test-this-repo">33.15. Test this repo</a></h3>
 <div class="sect3">
-<h4 id="automated-tests"><a class="anchor" href="#automated-tests"></a><a class="link" href="#automated-tests">33.14.1. Automated tests</a></h4>
+<h4 id="automated-tests"><a class="anchor" href="#automated-tests"></a><a class="link" href="#automated-tests">33.15.1. Automated tests</a></h4>
 <div class="paragraph">
 <p>Run almost all tests:</p>
 </div>
@@ -34727,7 +34932,7 @@ echo $?</pre>
 <p><a href="https://github.com/cirosantilli/linux-kernel-module-cheat/blob/master/test">test</a> does not all possible tests, because there are too many possible variations and that would take forever. The rationale is the same as for <code>./build all</code> and is explained in <code>./build --help</code>.</p>
 </div>
 <div class="sect4">
-<h5 id="test-arch-and-emulator-selection"><a class="anchor" href="#test-arch-and-emulator-selection"></a><a class="link" href="#test-arch-and-emulator-selection">33.14.1.1. Test arch and emulator selection</a></h5>
+<h5 id="test-arch-and-emulator-selection"><a class="anchor" href="#test-arch-and-emulator-selection"></a><a class="link" href="#test-arch-and-emulator-selection">33.15.1.1. Test arch and emulator selection</a></h5>
 <div class="paragraph">
 <p>You can select multiple archs and emulators of interest, as for an other command, with:</p>
 </div>
@@ -34760,7 +34965,7 @@ echo $?</pre>
 </div>
 </div>
 <div class="sect4">
-<h5 id="quit-on-fail"><a class="anchor" href="#quit-on-fail"></a><a class="link" href="#quit-on-fail">33.14.1.2. Quit on fail</a></h5>
+<h5 id="quit-on-fail"><a class="anchor" href="#quit-on-fail"></a><a class="link" href="#quit-on-fail">33.15.1.2. Quit on fail</a></h5>
 <div class="paragraph">
 <p>By default, continue running even after the first failure happens, and they show a summary at the end.</p>
 </div>
@@ -34774,9 +34979,9 @@ echo $?</pre>
 </div>
 </div>
 <div class="sect4">
-<h5 id="test-userland-in-full-system"><a class="anchor" href="#test-userland-in-full-system"></a><a class="link" href="#test-userland-in-full-system">33.14.1.3. Test userland in full system</a></h5>
+<h5 id="test-userland-in-full-system"><a class="anchor" href="#test-userland-in-full-system"></a><a class="link" href="#test-userland-in-full-system">33.15.1.3. Test userland in full system</a></h5>
 <div class="paragraph">
-<p>TODO: we really need a mechanism to automatically generate the test list automatically e.g. based on <a href="#path-properties">path_properties</a>, currently there are many tests missing, and we have to add everything manually which is very annoying.</p>
+<p>TODO: we really need a mechanism to automatically generate the test list automatically e.g. based on <a href="#path-properties">path_properties.py</a>, currently there are many tests missing, and we have to add everything manually which is very annoying.</p>
 </div>
 <div class="paragraph">
 <p>We could just generate it on the fly on the host, and forward it to guest through CLI arguments.</p>
@@ -34803,7 +35008,7 @@ echo $?</pre>
 </div>
 </div>
 <div class="sect4">
-<h5 id="gdb-tests"><a class="anchor" href="#gdb-tests"></a><a class="link" href="#gdb-tests">33.14.1.4. GDB tests</a></h5>
+<h5 id="gdb-tests"><a class="anchor" href="#gdb-tests"></a><a class="link" href="#gdb-tests">33.15.1.4. GDB tests</a></h5>
 <div class="paragraph">
 <p>We have some <a href="https://github.com/pexpect/pexpect">pexpect</a> automated tests for GDB for both userland and baremetal programs!</p>
 </div>
@@ -34876,7 +35081,7 @@ echo $?</pre>
 </div>
 </div>
 <div class="sect4">
-<h5 id="magic-failure-string"><a class="anchor" href="#magic-failure-string"></a><a class="link" href="#magic-failure-string">33.14.1.5. Magic failure string</a></h5>
+<h5 id="magic-failure-string"><a class="anchor" href="#magic-failure-string"></a><a class="link" href="#magic-failure-string">33.15.1.5. Magic failure string</a></h5>
 <div class="paragraph">
 <p>We do not know of any way to set the emulator exit status in QEMU arm full system.</p>
 </div>
@@ -34979,9 +35184,9 @@ echo $?</pre>
 </div>
 </div>
 <div class="sect3">
-<h4 id="non-automated-tests"><a class="anchor" href="#non-automated-tests"></a><a class="link" href="#non-automated-tests">33.14.2. Non-automated tests</a></h4>
+<h4 id="non-automated-tests"><a class="anchor" href="#non-automated-tests"></a><a class="link" href="#non-automated-tests">33.15.2. Non-automated tests</a></h4>
 <div class="sect4">
-<h5 id="test-gdb-linux-kernel"><a class="anchor" href="#test-gdb-linux-kernel"></a><a class="link" href="#test-gdb-linux-kernel">33.14.2.1. Test GDB Linux kernel</a></h5>
+<h5 id="test-gdb-linux-kernel"><a class="anchor" href="#test-gdb-linux-kernel"></a><a class="link" href="#test-gdb-linux-kernel">33.15.2.1. Test GDB Linux kernel</a></h5>
 <div class="paragraph">
 <p>For the Linux kernel, do the following manual tests for now.</p>
 </div>
@@ -35019,7 +35224,7 @@ echo $?</pre>
 </div>
 </div>
 <div class="sect4">
-<h5 id="test-the-internet"><a class="anchor" href="#test-the-internet"></a><a class="link" href="#test-the-internet">33.14.2.2. Test the Internet</a></h5>
+<h5 id="test-the-internet"><a class="anchor" href="#test-the-internet"></a><a class="link" href="#test-the-internet">33.15.2.2. Test the Internet</a></h5>
 <div class="paragraph">
 <p>You should also test that the Internet works:</p>
 </div>
@@ -35030,7 +35235,7 @@ echo $?</pre>
 </div>
 </div>
 <div class="sect4">
-<h5 id="cli-script-tests"><a class="anchor" href="#cli-script-tests"></a><a class="link" href="#cli-script-tests">33.14.2.3. CLI script tests</a></h5>
+<h5 id="cli-script-tests"><a class="anchor" href="#cli-script-tests"></a><a class="link" href="#cli-script-tests">33.15.2.3. CLI script tests</a></h5>
 <div class="paragraph">
 <p><code>build-userland</code> and <code>test-executables</code> have a wide variety of target selection modes, and it was hard to keep them all working without some tests:</p>
 </div>
@@ -35048,7 +35253,7 @@ echo $?</pre>
 </div>
 </div>
 <div class="sect2">
-<h3 id="bisection"><a class="anchor" href="#bisection"></a><a class="link" href="#bisection">33.15. Bisection</a></h3>
+<h3 id="bisection"><a class="anchor" href="#bisection"></a><a class="link" href="#bisection">33.16. Bisection</a></h3>
 <div class="paragraph">
 <p>When updating the Linux kernel, QEMU and gem5, things sometimes break.</p>
 </div>
@@ -35104,69 +35309,6 @@ git submodule update
 </div>
 </div>
 <div class="sect2">
-<h3 id="path-properties"><a class="anchor" href="#path-properties"></a><a class="link" href="#path-properties">33.16. path_properties</a></h3>
-<div class="paragraph">
-<p>In order to build and run each userland and <a href="#baremetal-setup">baremetal</a> example properly, we need per-file metadata such as compiler flags and required number of cores.</p>
-</div>
-<div class="paragraph">
-<p>This data is stored is stored in <a href="https://github.com/cirosantilli/linux-kernel-module-cheat/blob/master/path_properties.py">path_properties.py</a> at <code>path_properties_tuples</code>.</p>
-</div>
-<div class="paragraph">
-<p>Maybe we should embed it magically into source files directories to make it easier to see? But one big Python dict was easier to implement so we started like this. And it allows factoring chunks out easily.</p>
-</div>
-<div class="paragraph">
-<p>The format is as follows:</p>
-</div>
-<div class="literalblock">
-<div class="content">
-<pre>'path_component': (
-    {'property': value},
-    {
-        'child_path_component':
-        {
-            {'child_property': },
-            {}
-        }
-    }
-)</pre>
-</div>
-</div>
-<div class="paragraph">
-<p>and as a shortcut, paths that don&#8217;t have any children can be written directly as:</p>
-</div>
-<div class="literalblock">
-<div class="content">
-<pre>'path_component': {'property': value}</pre>
-</div>
-</div>
-<div class="paragraph">
-<p>Properties of parent directories apply to all children.</p>
-</div>
-<div class="paragraph">
-<p>Lists coming from parent directories are extended instead of overwritten by children, this is especially useful for C compiler flags.</p>
-</div>
-<div class="paragraph">
-<p>To quickly determine which properties a path has, you can use <a href="https://github.com/cirosantilli/linux-kernel-module-cheat/blob/master/getprops">getprops</a>, e.g.:</p>
-</div>
-<div class="literalblock">
-<div class="content">
-<pre>./getprops userland/c/hello.c</pre>
-</div>
-</div>
-<div class="paragraph">
-<p>which outputs values such as:</p>
-</div>
-<div class="literalblock">
-<div class="content">
-<pre>allowed_archs=None
-allowed_emulators=None
-arm_aarch32=False
-arm_sve=False
-baremetal=True</pre>
-</div>
-</div>
-</div>
-<div class="sect2">
 <h3 id="update-a-forked-submodule"><a class="anchor" href="#update-a-forked-submodule"></a><a class="link" href="#update-a-forked-submodule">33.17. Update a forked submodule</a></h3>
 <div class="paragraph">
 <p>This is a template update procedure for submodules for which we have some patches on on top of mainline.</p>