userland: convert make to python

2026-01-23 02:05:57 +01:00 · 2019-03-11 00:00:00 +00:00
parent 0deab8f8f7
commit 1a0d15ca86
19 changed files with 445 additions and 290 deletions
--- a/README.adoc
+++ b/README.adoc
@@ -502,8 +502,8 @@ Now rebuild GCC, the program and re-run it:
 and the new ouptut is now:

 ....
-j = 0
 i = 2
+j = 0
 ....

 We need to use the ugly `-final` thing because GCC has to packages in Buildroot, `-initial` and `-final`: https://stackoverflow.com/questions/54992977/how-to-select-an-override-srcdir-source-for-gcc-when-building-buildroot No one is able to example precisely with a minimal example why this is required:
@@ -5992,7 +5992,7 @@ Bibliography: https://stackoverflow.com/questions/8516021/proc-create-example-fo

 ===== /proc/version

-Its data is shared with `uname()`, which is a POSIX C function and has a Linux syscall to back it up.
+Its data is shared with `uname()`, which is a <<posix,POSIX C>> function and has a Linux syscall to back it up.

 Where the data comes from and how to modify it:

@@ -8195,7 +8195,7 @@ TODO a large chunk of tests, the Open POSIX test suite, is disabled with a comme

 ==== stress

-POSIX userland stress. Two versions:
+<<posix>> userland stress. Two versions:

 ....
 ./build-buildroot \
@@ -13117,6 +13117,28 @@ TODO: OpenMP does not like `-static`:

 See: https://stackoverflow.com/questions/23869981/linking-openmp-statically-with-gcc

+===== userland cheats
+
+We have accumulated considerable material in the following userland subjects.
+
+====== C
+
+Programs under link:userland/c/[] are examples of link:https://en.wikipedia.org/wiki/ANSI_C[ANSI C] programming.
+
+[[cpp]]
+====== C++
+
+Programs under link:userland/cpp/[] are examples of link:https://en.wikipedia.org/wiki/C%2B%2B#Standardization[ISO C] programming.
+
+====== POSIX
+
+Programs under link:userland/posix/[] are examples of POSIX C programming.
+
+What is POSIX:
+
+* https://stackoverflow.com/questions/1780599/what-is-the-meaning-of-posix/31865755#31865755
+* https://unix.stackexchange.com/questions/11983/what-exactly-is-posix/220877#220877
+
 ==== buildroot_packages directory

 Source: link:buildroot_packages/[]
--- a/9
+++ b/9
@@ -45,7 +45,6 @@ Build the baremetal examples with crosstool-NG.
            '-nostartfiles', LF,
        ]
        cflags_after = ['-lm']
-        gcc = self.get_toolchain_tool('gcc')
        if self.env['emulator'] == 'gem5':
            if self.env['machine'] == 'VExpress_GEM5_V1':
                entry_address = 0x80000000
@@ -70,7 +69,7 @@ Build the baremetal examples with crosstool-NG.
        )
        if self.need_rebuild([src], bootloader_obj):
            self.sh.run_cmd(
-                [gcc,  LF] +
+                [self.env['gcc'],  LF] +
                cflags +
                [
                    '-c', LF,
@@ -85,7 +84,7 @@ Build the baremetal examples with crosstool-NG.
        ]:
            if self.need_rebuild([src, self.env['common_h']], obj):
                self.sh.run_cmd(
-                    [gcc,  LF] +
+                    [self.env['gcc'],  LF] +
                    cflags +
                    [
                        '-c', LF,
@@ -125,7 +124,7 @@ Build the baremetal examples with crosstool-NG.
                        src = os.path.join(self.env['baremetal_source_dir'], in_path)
                        if self.need_rebuild([src, self.env['common_h']], main_obj):
                            self.sh.run_cmd(
-                                [gcc,  LF] +
+                                [self.env['gcc'],  LF] +
                                cflags +
                                [
                                    '-c', LF,
@@ -138,7 +137,7 @@ Build the baremetal examples with crosstool-NG.
                        out = os.path.join(out_dir, in_name + self.env['baremetal_build_ext'])
                        if self.need_rebuild(objs + [self.env['baremetal_link_script']], out):
                            self.sh.run_cmd(
-                                [gcc,  LF] +
+                                [self.env['gcc'],  LF] +
                                cflags +
                                [
                                    '-Wl,--section-start=.text={:#x}'.format(entry_address), LF,
--- a/6
+++ b/6
@@ -8,8 +8,6 @@ from shell_helpers import LF
 class Main(common.BuildCliFunction):
    def _get_make_cmd(self):
        allowed_toolchains = ['buildroot']
-        cc = self.get_toolchain_tool('gcc')
-        ld = self.get_toolchain_tool('ld')
        if self.env['arch'] == 'x86_64':
            arch = 'x86'
        else:
@@ -18,8 +16,8 @@ class Main(common.BuildCliFunction):
            'make', LF,
            '-j', str(self.env['nproc']), LF,
            '-f', 'Makefile.{}'.format(arch), LF,
-            'CC={}'.format(cc), LF,
-            'LD={}'.format(ld), LF,
+            'CC={}'.format(self.env['gcc']), LF,
+            'LD={}'.format(self.env['ld']), LF,
            'PWD={}'.format(self.env['gem5_m5_source_dir']), LF,
        ]

--- a/228
+++ b/228
@@ -4,6 +4,7 @@ import os
 import shlex

 import common
+import subprocess
 from shell_helpers import LF

 class Main(common.BuildCliFunction):
@@ -22,18 +23,6 @@ Indicate that a given package is present in the root filesystem, which
 allows us to build examples that rely on it.
 ''',
        )
-        self.add_argument(
-            '--host',
-            default=False,
-            help='''\
-Build the userland programs for the host instead of guest.
-Use the host packaged cross toolchain.
-''',
-        )
-        self.add_argument(
-            '--make-args',
-            default='',
-        )
        self.add_argument(
            'targets',
            default=[],
@@ -46,43 +35,196 @@ has the OpenBLAS libraries and headers installed.
            nargs='*',
        )

+    def _build_one(
+        self,
+        in_path,
+        out_path,
+        ccflags,
+        extra_deps=None,
+        link=True,
+        extra_objs=None,
+        std=None,
+        ccflags_after=None,
+    ):
+        if extra_deps is None:
+            extra_deps = []
+        if extra_objs is None:
+            extra_objs = []
+        if ccflags_after is None:
+            ccflags_after = []
+        if self.need_rebuild([in_path] + extra_objs + extra_deps, out_path):
+            ccflags = ccflags.copy()
+            if not link:
+                ccflags.extend(['-c', LF])
+            in_ext = os.path.splitext(in_path)[1]
+            if in_ext == self.env['c_ext']:
+                cc = self.env['gcc']
+                if std is None:
+                    std = 'c11'
+                ccflags.extend([
+                    '-fopenmp', LF,
+                ])
+            elif in_ext == self.env['cxx_ext']:
+                cc = self.env['gxx']
+                if std is None:
+                    std = 'c++17'
+            else:
+                return
+            self.sh.run_cmd(
+                [
+                    cc, LF,
+                ] +
+                ccflags +
+                [
+                    '-std={}'.format(std), LF,
+                    '-o', out_path, LF,
+                    in_path, LF,
+                ] +
+                extra_objs +
+                [
+                    '-lm', LF,
+                    '-pthread', LF,
+                ] +
+                ccflags_after
+            )
+
    def build(self):
        build_dir = self.get_build_dir()
        os.makedirs(build_dir, exist_ok=True)
-        make_args = shlex.split(self.env['make_args'])
+        has_packages = set(self.env['has_package'])
+        ccflags = [
+            '-I', self.env['root_dir'], LF,
+            '-I', self.env['userland_source_dir'], LF,
+            '-O0', LF,
+            '-Wall', LF,
+            '-Werror', LF,
+            '-Wextra', LF,
+            '-Wno-unused-function', LF,
+            '-pedantic', LF,
+            '-ggdb3', LF,
+        ]
        if self.env['static']:
-            make_args.extend(['CCFLAGS_EXTRA=-static', LF])
-        self.sh.run_cmd(
-            (
-                [
-                    'make', LF,
-                    '-j', str(self.env['nproc']), LF,
-                    'ARCH={}'.format(self.env['arch']), LF,
-                    'CCFLAGS_SCRIPT={} {}'.format('-I', self.env['userland_source_dir']), LF,
-                    'COMMON_DIR={}'.format(self.env['root_dir']), LF,
-                    'CC={}'.format(self.get_toolchain_tool('gcc')), LF,
-                    'CXX={}'.format(self.get_toolchain_tool('g++')), LF,
-                    'PKG_CONFIG={}'.format(self.env['buildroot_pkg_config']), LF,
-                    'STAGING_DIR={}'.format(self.env['buildroot_staging_dir']), LF,
-                    'OUT_DIR={}'.format(build_dir), LF,
-                ] +
-                self.sh.add_newlines([
-                    'HAS_{}=y'.format(package.upper())
-                    for package in
-                    self.env['has_package']
-                ]) +
-                make_args +
-                self.sh.add_newlines([
-                    os.path.join(
+            ccflags.extend(['-static', LF])
+        common_obj = os.path.join(
            build_dir,
-                        os.path.splitext(os.path.split(target)[1])[0]
-                    ) + self.env['userland_build_ext']
-                    for target in self.env['targets']
-                ])
-            ),
-            cwd=self.env['userland_source_dir'],
-            extra_paths=[self.env['ccache_dir']],
+            self.env['common_basename_noext'] + self.env['obj_ext']
        )
+        self._build_one(
+            in_path=self.env['common_c'],
+            out_path=common_obj,
+            ccflags=ccflags,
+            extra_deps=[self.env['common_h']],
+            link=False,
+        )
+        pkgs = {
+            'eigen': {
+                # TODO: was failing with:
+                # fatal error: Eigen/Dense: No such file or directory as of
+                # 975ce0723ee3fa1fea1766e6683e2f3acb8558d6
+                # http://lists.busybox.net/pipermail/buildroot/2018-June/222914.html
+                'ccflags': [
+                    '-I',
+                    os.path.join(
+                        self.env['buildroot_staging_dir'],
+                        'usr',
+                        'include',
+                        'eigen3'
+                    ),
+                    LF
+                ],
+                # Header only.
+                'ccflags_after': [],
+            },
+            'libdrm': {},
+            'openblas': {},
+        }
+        rootdir_abs_len = len(self.env['userland_source_dir'])
+        for path, in_dirnames, in_filenames in os.walk(self.env['userland_source_dir']):
+            in_dirnames.sort()
+            dirpath_relative_root = path[rootdir_abs_len + 1:]
+            dirpath_relative_root_components = dirpath_relative_root.split(os.sep)
+            if (
+                len(dirpath_relative_root_components) < 2 or
+                dirpath_relative_root_components[0] != 'arch' or
+                dirpath_relative_root_components[1] == self.env['arch']
+            ):
+                out_dir = os.path.join(
+                    self.env['userland_build_dir'],
+                    dirpath_relative_root
+                )
+                os.makedirs(out_dir, exist_ok=True)
+                for in_filename in in_filenames:
+                    in_path = os.path.join(path, in_filename)
+                    in_name, in_ext = os.path.splitext(in_filename)
+                    out_path = os.path.join(
+                        out_dir,
+                        in_name + self.env['userland_build_ext']
+                    )
+                    pkg_key = in_name.split('_')[0]
+                    ccflags_file = ccflags.copy()
+                    ccflags_after = []
+                    if pkg_key in pkgs:
+                        if pkg_key not in has_packages:
+                            continue
+                        pkg = pkgs[pkg_key]
+                        if 'ccflags' in pkg:
+                            ccflags_file.extend(pkg['ccflags'])
+                        else:
+                            pkg_config_output = subprocess.check_output([
+                                self.env['buildroot_pkg_config'],
+                                '--cflags',
+                                pkg_key
+                            ]).decode()
+                            ccflags_file.extend(self.sh.shlex_split(pkg_config_output))
+                        if 'ccflags_after' in pkg:
+                            ccflags_file.extend(pkg['ccflags_after'])
+                        else:
+                            pkg_config_output = subprocess.check_output([
+                                self.env['buildroot_pkg_config'],
+                                '--libs',
+                                pkg_key
+                            ]).decode()
+                            ccflags_after.extend(self.sh.shlex_split(pkg_config_output))
+                    self._build_one(
+                        in_path=in_path,
+                        out_path=out_path,
+                        ccflags=ccflags_file,
+                        extra_objs=[common_obj],
+                        ccflags_after=ccflags_after,
+                    )
+
+        #self.sh.run_cmd(
+        #    (
+        #        [
+        #            'make', LF,
+        #            '-j', str(self.env['nproc']), LF,
+        #            'ARCH={}'.format(self.env['arch']), LF,
+        #            'CCFLAGS_SCRIPT={} {}'.format('-I', self.env['userland_source_dir']), LF,
+        #            'COMMON_DIR={}'.format(self.env['root_dir']), LF,
+        #            'CC={}'.format(self.env['gcc']), LF,
+        #            'CXX={}'.format(self.env['gxx']), LF,
+        #            'PKG_CONFIG={}'.format(self.env['buildroot_pkg_config']), LF,
+        #            'STAGING_DIR={}'.format(self.env['buildroot_staging_dir']), LF,
+        #            'OUT_DIR={}'.format(build_dir), LF,
+        #        ] +
+        #        self.sh.add_newlines([
+        #            'HAS_{}=y'.format(package.upper())
+        #            for package in
+        #            self.env['has_package']
+        #        ]) +
+        #        make_args +
+        #        self.sh.add_newlines([
+        #            os.path.join(
+        #                build_dir,
+        #                os.path.splitext(os.path.split(target)[1])[0]
+        #            ) + self.env['userland_build_ext']
+        #            for target in self.env['targets']
+        #        ])
+        #    ),
+        #    cwd=self.env['userland_source_dir'],
+        #    extra_paths=[self.env['ccache_dir']],
+        #)
+
        self.sh.copy_dir_if_update(
            srcdir=build_dir,
            destdir=self.env['out_rootfs_overlay_dir'],
--- a/common.py
+++ b/common.py
@@ -94,6 +94,7 @@ consts['release_zip_file'] = os.path.join(consts['release_dir'], 'lkmc-{}.zip'.f
 consts['github_repo_id'] = 'cirosantilli/linux-kernel-module-cheat'
 consts['asm_ext'] = '.S'
 consts['c_ext'] = '.c'
+consts['cxx_ext'] = '.cpp'
 consts['header_ext'] = '.h'
 consts['kernel_module_ext'] = '.ko'
 consts['obj_ext'] = '.o'
@@ -796,7 +797,7 @@ Valid emulators: {}
 lunch aosp_{}-eng
 '''.format(self.env['android_arch'])

-        # Toolchain choice.
+        # Toolchain.
        if not env['_args_given']['gcc_which']:
            if env['baremetal']:
                env['gcc_which'] = 'crosstool-ng'
@@ -826,6 +827,8 @@ lunch aosp_{}-eng
                raise Exception('There is no host baremetal chain for arch: ' + env['arch'])
        else:
            raise Exception('Unknown toolchain: ' + env['gcc_which'])
+        env['gcc'] = self.get_toolchain_tool('gcc')
+        env['gxx'] = self.get_toolchain_tool('g++')

    def add_argument(self, *args, **kwargs):
        '''
--- a/userland/README.adoc
+++ b/userland/README.adoc
@@ -1 +1 @@
-https://github.com/cirosantilli/linux-kernel-module-cheat#userland-directory
+https://github.com/cirosantilli/linux-kernel-module-cheat#ansi-c
--- a/userland/count.c
+++ b/userland/count.c
@@ -1,3 +1,4 @@
+#define _XOPEN_SOURCE 700
 #include <limits.h>
 #include <stdio.h>
 #include <stdlib.h>
--- a/userland/cpp/README.adoc
+++ b/userland/cpp/README.adoc
@@ -0,0 +1 @@
+https://github.com/cirosantilli/linux-kernel-module-cheat#ansi-cpp
--- a/userland/gcc/README.adoc
+++ b/userland/gcc/README.adoc
@@ -0,0 +1 @@
+Programs in this directory use GCC extensions.
--- a/userland/kernel_modules/README.adoc
+++ b/userland/kernel_modules/README.adoc
@@ -0,0 +1 @@
+Programs in this directory rely on / test functionality provided by kernel modules.
--- a/userland/linux/README.adoc
+++ b/userland/linux/README.adoc
@@ -0,0 +1,3 @@
+Programs in this directory rely on Linux kernel specific functionality such as `/proc`.
+
+These programs may also conform to ANSI C or POSIX, but we put them here because you can only observe them at work under Linux.
--- a/userland/m5ops.c
+++ b/userland/m5ops.c
@@ -6,16 +6,14 @@

 #include "m5ops.h"

-int main(int argc, char **argv)
-{
+int main(int argc, char **argv) {
    char action;
    if (argc > 1) {
        action = argv[1][0];
    } else {
        action = 'e';
    }
-	switch (action)
-	{
+    switch (action) {
        case 'c':
            m5_checkpoint();
            break;
--- a/userland/m5ops.h
+++ b/userland/m5ops.h
@@ -2,53 +2,43 @@
 #define M5OPS_H

 #if defined(__arm__)
-static void m5_checkpoint(void)
-{
+static void m5_checkpoint(void) {
    __asm__ __volatile__ ("mov r0, #0; mov r1, #0; mov r2, #0; mov r3, #0; .inst 0xEE000110 | (0x43 << 16);" : : : "r0", "r1", "r2", "r3");
-};
-static void m5_dumpstats(void)
-{
+}
+static void m5_dumpstats(void) {
    __asm__ __volatile__ ("mov r0, #0; mov r1, #0; mov r2, #0; mov r3, #0; .inst 0xEE000110 | (0x41 << 16);" : : : "r0", "r1", "r2", "r3");
-};
-static void m5_exit()
-{
+}
+static void m5_exit() {
    __asm__ __volatile__ ("mov r0, #0; mov r1, #0; .inst 0xEE000110 | (0x21 << 16);" : : : "r0", "r1");
-};
-static void m5_fail_1(void)
-{
+}
+static void m5_fail_1(void) {
    __asm__ __volatile__ ("mov r0, #0; mov r1, #0; mov r2, #1; mov r3, #0; .inst 0xEE000110 | (0x22 << 16);" : : : "r0", "r1", "r2", "r3");
-};
-static void m5_resetstats(void)
-{
+}
+static void m5_resetstats(void) {
    __asm__ __volatile__ ("mov r0, #0; mov r1, #0; mov r2, #0; mov r3, #0; .inst 0xEE000110 | (0x40 << 16);" : : : "r0", "r1", "r2", "r3");
-};
+}
 #elif defined(__aarch64__)
-static void m5_checkpoint(void)
-{
+static void m5_checkpoint(void) {
    __asm__ __volatile__ ("mov x0, #0; mov x1, #0; .inst 0xFF000110 | (0x43 << 16);" : : : "x0", "x1");
-};
-static void m5_dumpstats(void)
-{
+}
+static void m5_dumpstats(void) {
    __asm__ __volatile__ ("mov x0, #0; mov x1, #0; .inst 0xFF000110 | (0x41 << 16);" : : : "x0", "x1");
-};
-static void m5_exit(void)
-{
+}
+static void m5_exit(void) {
    __asm__ __volatile__ ("mov x0, #0; .inst 0XFF000110 | (0x21 << 16);" : : : "x0");
-};
-static void m5_fail_1(void)
-{
+}
+static void m5_fail_1(void) {
    __asm__ __volatile__ ("mov x0, #0; mov x1, #1; .inst 0xFF000110 | (0x22 << 16);" : : : "x0", "x1");
-};
-static void m5_resetstats(void)
-{
+}
+static void m5_resetstats(void) {
    __asm__ __volatile__ ("mov x0, #0; mov x1, #0; .inst 0XFF000110 | (0x40 << 16);" : : : "x0", "x1");
-};
+}
 #else
-static void m5_checkpoint(void) {};
-static void m5_dumpstats(void) {};
-static void m5_exit(void) {};
-static void m5_fail_1(void) {};
-static void m5_resetstats(void) {};
+static void m5_checkpoint(void) {}
+static void m5_dumpstats(void) {}
+static void m5_exit(void) {}
+static void m5_fail_1(void) {}
+static void m5_resetstats(void) {}
 #endif

 #endif
--- a/userland/netlink.c
+++ b/userland/netlink.c
@@ -20,8 +20,7 @@ struct msghdr msg;
 struct nlmsghdr *nlh;
 struct sockaddr_nl src_addr, dest_addr;

-int main(void)
-{
+int main(void) {
    sock_fd = socket(PF_NETLINK, SOCK_RAW, NETLINK_USER);
    if (sock_fd < 0) {
        perror("socket");
--- a/userland/params.mk
+++ b/userland/params.mk
@@ -1 +0,0 @@
-SUBDIRS := interactive arch/$(ARCH)/
--- a/userland/posix/README.adoc
+++ b/userland/posix/README.adoc
@@ -0,0 +1 @@
+https://github.com/cirosantilli/linux-kernel-module-cheat#posix
--- a/userland/proc_events.c
+++ b/userland/proc_events.c
@@ -24,8 +24,7 @@ int main(void) {}

 static volatile bool need_exit = false;

-static int nl_connect()
-{
+static int nl_connect() {
    int rc;
    int nl_sock;
    struct sockaddr_nl sa_nl;
@@ -47,8 +46,7 @@ static int nl_connect()
    return nl_sock;
 }

-static int set_proc_ev_listen(int nl_sock, bool enable)
-{
+static int set_proc_ev_listen(int nl_sock, bool enable) {
    int rc;
    struct __attribute__ ((aligned(NLMSG_ALIGNTO))) {
        struct nlmsghdr nl_hdr;
@@ -78,8 +76,7 @@ static int set_proc_ev_listen(int nl_sock, bool enable)
    return 0;
 }

-static int handle_proc_ev(int nl_sock)
-{
+static int handle_proc_ev(int nl_sock) {
    int rc;
    struct __attribute__ ((aligned(NLMSG_ALIGNTO))) {
        struct nlmsghdr nl_hdr;
@@ -142,13 +139,11 @@ static int handle_proc_ev(int nl_sock)
    return 0;
 }

-static void on_sigint(__attribute__ ((unused)) int unused)
-{
+static void on_sigint(__attribute__ ((unused)) int unused) {
    need_exit = true;
 }

-int main(void)
-{
+int main(void) {
    int nl_sock;
    int rc = EXIT_SUCCESS;

--- a/userland/sleep_forever.c
+++ b/userland/sleep_forever.c
@@ -1,5 +1,6 @@
 /* https://github.com/cirosantilli/linux-kernel-module-cheat#sleep_forever-out */

+#define _XOPEN_SOURCE 700
 #include <stdio.h>
 #include <unistd.h>

--- a/userland/time_boot.c
+++ b/userland/time_boot.c
@@ -1,5 +1,6 @@
 /* https://github.com/cirosantilli/linux-kernel-module-cheat#time_boot-out */

+#define _XOPEN_SOURCE 700
 #include <stdio.h>
 #include <unistd.h>
				`@@ -0,0 +1 @@`
				`https://github.com/cirosantilli/linux-kernel-module-cheat#ansi-cpp`
				`@@ -0,0 +1 @@`
				`Programs in this directory use GCC extensions.`
				`@@ -0,0 +1 @@`
				`Programs in this directory rely on / test functionality provided by kernel modules.`