From 627623157cba7a6e36b61760fd6ad2db3ca42ad4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ciro=20Santilli=20=E5=85=AD=E5=9B=9B=E4=BA=8B=E4=BB=B6=20?= =?UTF-8?q?=E6=B3=95=E8=BD=AE=E5=8A=9F?= Date: Fri, 15 Nov 2019 00:00:00 +0000 Subject: [PATCH] atomic.cpp: split into several versions one per executable --- README.adoc | 31 ++++++-- path_properties.py | 14 +++- userland/cpp/atomic/README.adoc | 1 + userland/cpp/atomic/aarch64_add.cpp | 2 + userland/cpp/atomic/aarch64_ldadd.cpp | 2 + userland/cpp/atomic/build | 1 + userland/cpp/atomic/fail.cpp | 2 + userland/cpp/atomic/main.hpp | 101 ++++++++++++++++++++++++ userland/cpp/atomic/mutex.cpp | 2 + userland/cpp/atomic/std_atomic.cpp | 2 + userland/cpp/atomic/test | 1 + userland/cpp/atomic/x86_64_inc.cpp | 2 + userland/cpp/atomic/x86_64_lock_inc.cpp | 2 + 13 files changed, 154 insertions(+), 9 deletions(-) create mode 100644 userland/cpp/atomic/README.adoc create mode 100644 userland/cpp/atomic/aarch64_add.cpp create mode 100644 userland/cpp/atomic/aarch64_ldadd.cpp create mode 120000 userland/cpp/atomic/build create mode 100644 userland/cpp/atomic/fail.cpp create mode 100644 userland/cpp/atomic/main.hpp create mode 100644 userland/cpp/atomic/mutex.cpp create mode 100644 userland/cpp/atomic/std_atomic.cpp create mode 120000 userland/cpp/atomic/test create mode 100644 userland/cpp/atomic/x86_64_inc.cpp create mode 100644 userland/cpp/atomic/x86_64_lock_inc.cpp diff --git a/README.adoc b/README.adoc index c22fdd4..fcedd1e 100644 --- a/README.adoc +++ b/README.adoc @@ -3899,25 +3899,28 @@ it fails with: ld: cannot find -lopenblas .... +[[cpp-static-and-pthreads]] +===== C++ static and pthreads + `g++` and pthreads also causes issues: * https://stackoverflow.com/questions/35116327/when-g-static-link-pthread-cause-segmentation-fault-why * https://stackoverflow.com/questions/58848694/gcc-whole-archive-recipe-for-static-linking-to-pthread-stopped-working-in-rec -As a consequence, the following fails: +As a consequence, the following just hangs as of LKMC ca0403849e03844a328029d70c08556155dc1cd0 + 1 the example link:userland/cpp/atomic/std_atomic.cpp[]: .... -./run --userland userland/cpp/atomic.cpp --static +./run --userland userland/cpp/atomic/std_atomic.cpp --static .... -with error: +And before that, it used to fail with other randomly different errors, e.g.: .... qemu-x86_64: /path/to/linux-kernel-module-cheat/submodules/qemu/accel/tcg/cpu-exec.c:700: cpu_exec: Assertion `!have_mmap_lock()' failed. qemu-x86_64: /path/to/linux-kernel-module-cheat/submodules/qemu/accel/tcg/cpu-exec.c:700: cpu_exec: Assertion `!have_mmap_lock()' failed. .... -and if we manually build and run natively on host it segfaults. +And a native Ubuntu 18.04 AMD64 run with static compilation segfaults. The workaround: @@ -13894,7 +13897,21 @@ Programs under link:userland/cpp/[] are examples of https://en.wikipedia.org/wik ** link:userland/cpp/thread_get_id.cpp[] `std::thread::get_id` ** link:userland/cpp/thread_return_value.cpp[]: how to return a value from a thread * https://en.cppreference.com/w/cpp/header/atomic[``]: <> 32 "Atomic operations library" -** link:userland/cpp/atomic.cpp[] + +===== atomic.cpp + +link:userland/cpp/atomic/[] + +In this set of examples, we exemplify various synchronization mechanisms, including assembly specific ones, by using the convenience of C++ multithreading: + +* link:userland/cpp/atomic/main.hpp[]: contains all the code which is then specialized in spearated cpp files with macros +* link:userland/cpp/atomic/aarch64_add.cpp[]: non synchronized aarch64 inline assembly +* link:userland/cpp/atomic/aarch64_ldadd.cpp[]: synchronized aarch64 inline assembly +* link:userland/cpp/atomic/fail.cpp[]: non synchronized C++ operator `++` +* link:userland/cpp/atomic/mutex.cpp[]: synchronized `std::mutex` +* link:userland/cpp/atomic/std_atomic.cpp[]: synchronized `std::atomic_ulong` +* link:userland/cpp/atomic/x86_64_inc.cpp[]: non synchronized x86_64 inline assembly +* link:userland/cpp/atomic/x86_64_lock_inc.cpp[]: synchronized x86_64 inline assembly [[cpp-standards]] ==== C++ standards @@ -15456,7 +15473,7 @@ TODO We didn't manage to find a working ARM analogue to < ==== x86 LOCK prefix -Inline assembly example at: link:userland/cpp/atomic.cpp[] +Inline assembly example at: link:userland/cpp/atomic/x86_64_lock_inc.cpp[], see also: <>. Ensures that memory modifications are visible across all CPUs, which is fundamental for thread synchronization. @@ -16529,7 +16546,7 @@ That document then describes the SVE instructions and registers. <> "ARMv8.1-LSE, ARMv8.1 Large System Extensions" -* LDADD: link:userland/cpp/atomic.cpp[] +* LDADD: link:userland/cpp/atomic/aarch64_ldadd.cpp[], see also: <>. Bibliography: diff --git a/path_properties.py b/path_properties.py index 3aa727d..93e9992 100644 --- a/path_properties.py +++ b/path_properties.py @@ -213,7 +213,7 @@ class PathProperties: os.path.splitext(self.path_components[-1])[1] == '.c' and self['arm_sve'] ) and not ( # C++ multithreading in static does not seem to work: - # https://cirosantilli.com/linux-kernel-module-cheat#user-mode-static-executables-with-dynamic-libraries + # https://cirosantilli.com/linux-kernel-module-cheat#cpp-static-and-pthreads os.path.splitext(self.path_components[-1])[1] == '.cpp' and ( # TODO the better check here would be for 'static' # to factor out with test-executable logic, but lazy. @@ -581,7 +581,17 @@ path_properties_tuples = ( 'cpp': ( {}, { - 'atomic.cpp': {'test_run_args': {'cpus': 3}}, + 'atomic': ( + { + 'test_run_args': {'cpus': 3}, + }, + { + 'aarch64_add.cpp': {'allowed_archs': {'aarch64'}}, + 'aarch64_ldadd.cpp': {'allowed_archs': {'aarch64'}}, + 'x86_64_add.cpp': {'allowed_archs': {'x86_64'}}, + 'x86_64_ldadd.cpp': {'allowed_archs': {'x86_64'}}, + }, + ), 'count.cpp': {'more_than_1s': True}, 'sleep_for.cpp': { 'more_than_1s': True, diff --git a/userland/cpp/atomic/README.adoc b/userland/cpp/atomic/README.adoc new file mode 100644 index 0000000..d84d9de --- /dev/null +++ b/userland/cpp/atomic/README.adoc @@ -0,0 +1 @@ +// https://cirosantilli.com/linux-kernel-module-cheat#atomic-cpp diff --git a/userland/cpp/atomic/aarch64_add.cpp b/userland/cpp/atomic/aarch64_add.cpp new file mode 100644 index 0000000..ecdd2f8 --- /dev/null +++ b/userland/cpp/atomic/aarch64_add.cpp @@ -0,0 +1,2 @@ +#define LKMC_USERLAND_ATOMIC_AARCH64_ADD +#include "main.hpp" diff --git a/userland/cpp/atomic/aarch64_ldadd.cpp b/userland/cpp/atomic/aarch64_ldadd.cpp new file mode 100644 index 0000000..830200d --- /dev/null +++ b/userland/cpp/atomic/aarch64_ldadd.cpp @@ -0,0 +1,2 @@ +#define LKMC_USERLAND_ATOMIC_AARCH64_LDADD +#include "main.hpp" diff --git a/userland/cpp/atomic/build b/userland/cpp/atomic/build new file mode 120000 index 0000000..ab18017 --- /dev/null +++ b/userland/cpp/atomic/build @@ -0,0 +1 @@ +../build \ No newline at end of file diff --git a/userland/cpp/atomic/fail.cpp b/userland/cpp/atomic/fail.cpp new file mode 100644 index 0000000..1d9d023 --- /dev/null +++ b/userland/cpp/atomic/fail.cpp @@ -0,0 +1,2 @@ +#define LKMC_USERLAND_ATOMIC_FAIL +#include "main.hpp" diff --git a/userland/cpp/atomic/main.hpp b/userland/cpp/atomic/main.hpp new file mode 100644 index 0000000..1600ac5 --- /dev/null +++ b/userland/cpp/atomic/main.hpp @@ -0,0 +1,101 @@ +// https://cirosantilli.com/linux-kernel-module-cheat#atomic-cpp + +#if __cplusplus >= 201103L +#include +#include +#include +#include +#include +#include + +size_t niters; + +#if defined(LKMC_USERLAND_ATOMIC_STD_ATOMIC) +std::atomic_ulong global(0); +#else +uint64_t global = 0; +#endif + +#if defined(LKMC_USERLAND_ATOMIC_MUTEX) +std::mutex mutex; +#endif + +void threadMain() { + for (size_t i = 0; i < niters; ++i) { +#if defined(LKMC_USERLAND_ATOMIC_MUTEX) + mutex.lock(); +#endif +#if defined(LKMC_USERLAND_ATOMIC_X86_64_INC) + __asm__ __volatile__ ( + "incq %0;" + : "+g" (global) + : + : + ); +#elif defined(LKMC_USERLAND_ATOMIC_X86_64_LOCK_INC) + // https://cirosantilli.com/linux-kernel-module-cheat#x86-lock-prefix + __asm__ __volatile__ ( + "lock;" + "incq %0;" + : "+m" (global) + : + : + ); +#elif defined(LKMC_USERLAND_ATOMIC_AARCH64_ADD) + __asm__ __volatile__ ( + "add %0, %0, 1;" + : "+r" (global) + : + : + ); +#elif defined(LKMC_USERLAND_ATOMIC_AARCH64_LDADD) + // https://cirosantilli.com/linux-kernel-module-cheat#arm-lse + __asm__ __volatile__ ( + "ldadd %[inc], xzr, [%[addr]];" + : "=m" (global) + : [inc] "r" (1), + [addr] "r" (&global) + : + ); +#else + global++; +#endif +#if defined(LKMC_USERLAND_ATOMIC_MUTEX) + mutex.unlock(); +#endif +#if 0 +#endif + } +} +#endif + +int main(int argc, char **argv) { +#if __cplusplus >= 201103L + size_t nthreads; + if (argc > 1) { + nthreads = std::stoull(argv[1], NULL, 0); + } else { + nthreads = 2; + } + if (argc > 2) { + niters = std::stoull(argv[2], NULL, 0); + } else { + niters = 10; + } + std::vector threads(nthreads); + for (size_t i = 0; i < nthreads; ++i) + threads[i] = std::thread(threadMain); + for (size_t i = 0; i < nthreads; ++i) + threads[i].join(); + uint64_t expect = nthreads * niters; +#if defined(LKMC_USERLAND_ATOMIC_FAIL) || \ + defined(LKMC_USERLAND_ATOMIC_X86_64_INC) || \ + defined(LKMC_USERLAND_ATOMIC_AARCH64_INC) + // These fail, so we just print the outcomes. + std::cout << "expect " << expect << std::endl; + std::cout << "global " << global << std::endl; +#else + assert(global == expect); +#endif +#endif +} diff --git a/userland/cpp/atomic/mutex.cpp b/userland/cpp/atomic/mutex.cpp new file mode 100644 index 0000000..27c2946 --- /dev/null +++ b/userland/cpp/atomic/mutex.cpp @@ -0,0 +1,2 @@ +#define LKMC_USERLAND_ATOMIC_MUTEX +#include "main.hpp" diff --git a/userland/cpp/atomic/std_atomic.cpp b/userland/cpp/atomic/std_atomic.cpp new file mode 100644 index 0000000..0569abc --- /dev/null +++ b/userland/cpp/atomic/std_atomic.cpp @@ -0,0 +1,2 @@ +#define LKMC_USERLAND_ATOMIC_STD_ATOMIC +#include "main.hpp" diff --git a/userland/cpp/atomic/test b/userland/cpp/atomic/test new file mode 120000 index 0000000..419df4f --- /dev/null +++ b/userland/cpp/atomic/test @@ -0,0 +1 @@ +../test \ No newline at end of file diff --git a/userland/cpp/atomic/x86_64_inc.cpp b/userland/cpp/atomic/x86_64_inc.cpp new file mode 100644 index 0000000..2eeb793 --- /dev/null +++ b/userland/cpp/atomic/x86_64_inc.cpp @@ -0,0 +1,2 @@ +#define LKMC_USERLAND_ATOMIC_X86_64_INC +#include "main.hpp" diff --git a/userland/cpp/atomic/x86_64_lock_inc.cpp b/userland/cpp/atomic/x86_64_lock_inc.cpp new file mode 100644 index 0000000..4431411 --- /dev/null +++ b/userland/cpp/atomic/x86_64_lock_inc.cpp @@ -0,0 +1,2 @@ +#define LKMC_USERLAND_ATOMIC_X86_64_LOCK_INC +#include "main.hpp"