Changeset eef8dfb for libcfa/src

Timestamp:

Jan 7, 2021, 2:55:57 PM (5 years ago)

Author:

Thierry Delisle <tdelisle@…>

Branches:

ADT, arm-eh, ast-experimental, enum, forall-pointer-decay, jacob/cs343-translation, master, new-ast-unique-expr, pthread-emulation, qualifiedEnum

Children:

58fe85a

Parents:

bdfc032 (diff), 44e37ef (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the (diff) links above to see all the changes relative to each parent.

Message:

Merge branch 'master' into dkobets-vector

Location:

libcfa/src

Files:

• Makefile.am (modified) (6 diffs)
• Makefile.in (deleted)
• assert.cfa (modified) (3 diffs)
• bitmanip.hfa (added)
• bits/collection.hfa (added)
• bits/containers.hfa (modified) (8 diffs)
• bits/debug.cfa (modified) (3 diffs)
• bits/debug.hfa (modified) (6 diffs)
• bits/defs.hfa (modified) (3 diffs)
• bits/locks.hfa (modified) (7 diffs)
• bits/queue.hfa (added)
• bits/random.hfa (added)
• bits/sequence.hfa (added)
• bits/signal.hfa (modified) (3 diffs)
• bits/stack.hfa (added)
• common.hfa (modified) (2 diffs)
• concurrency/CtxSwitch-arm.S (deleted)
• concurrency/CtxSwitch-arm32.S (added)
• concurrency/CtxSwitch-arm64.S (added)
• concurrency/CtxSwitch-i386.S (modified) (3 diffs)
• concurrency/CtxSwitch-x86_64.S (modified) (3 diffs)
• concurrency/alarm.cfa (modified) (6 diffs)
• concurrency/alarm.hfa (modified) (2 diffs)
• concurrency/clib/cfathread.cfa (added)
• concurrency/clib/cfathread.h (added)
• concurrency/coroutine.cfa (modified) (13 diffs)
• concurrency/coroutine.hfa (modified) (10 diffs)
• concurrency/exception.cfa (added)
• concurrency/exception.hfa (added)
• concurrency/future.hfa (added)
• concurrency/invoke.c (modified) (10 diffs)
• concurrency/invoke.h (modified) (12 diffs)
• concurrency/io.cfa (added)
• concurrency/io/call.cfa.in (added)
• concurrency/io/setup.cfa (added)
• concurrency/io/types.hfa (added)
• concurrency/iofwd.hfa (added)
• concurrency/kernel.cfa (modified) (13 diffs)
• concurrency/kernel.hfa (modified) (11 diffs)
• concurrency/kernel/fwd.hfa (added)
• concurrency/kernel/startup.cfa (added)
• concurrency/kernel_private.hfa (modified) (4 diffs)
• concurrency/locks.cfa (added)
• concurrency/locks.hfa (added)
• concurrency/monitor.cfa (modified) (41 diffs)
• concurrency/monitor.hfa (modified) (7 diffs)
• concurrency/mutex.cfa (modified) (10 diffs)
• concurrency/mutex.hfa (modified) (3 diffs)
• concurrency/preemption.cfa (modified) (22 diffs)
• concurrency/preemption.hfa (modified) (1 diff)
• concurrency/ready_queue.cfa (added)
• concurrency/ready_subqueue.hfa (added)
• concurrency/snzi.hfa (added)
• concurrency/stats.cfa (added)
• concurrency/stats.hfa (added)
• concurrency/thread.cfa (modified) (4 diffs)
• concurrency/thread.hfa (modified) (3 diffs)
• containers/list.hfa (added)
• containers/stackLockFree.hfa (added)
• containers/vector.hfa (modified) (1 diff)
• exception.c (modified) (16 diffs)
• exception.h (modified) (2 diffs)
• exception.hfa (added)
• executor.cfa (modified) (1 diff)
• fstream.cfa (modified) (14 diffs)
• fstream.hfa (modified) (5 diffs)
• gmp.hfa (modified) (5 diffs)
• heap.cfa (modified) (59 diffs)
• heap.hfa (added)
• interpose.cfa (modified) (10 diffs)
• iostream.cfa (modified) (44 diffs)
• iostream.hfa (modified) (20 diffs)
• math.hfa (modified) (2 diffs)
• memory.cfa (added)
• memory.hfa (added)
• parseargs.cfa (added)
• parseargs.hfa (added)
• rational.cfa (modified) (2 diffs)
• startup.cfa (modified) (3 diffs)
• stdhdr/assert.h (modified) (3 diffs)
• stdhdr/bfdlink.h (modified) (1 diff)
• stdhdr/hwloc.h (modified) (1 diff)
• stdhdr/krb5.h (modified) (1 diff)
• stdhdr/malloc.h (modified) (1 diff)
• stdhdr/math.h (modified) (1 diff)
• stdhdr/sys/mman.h (added)
• stdhdr/sys/time.h (added)
• stdhdr/sys/ucontext.h (deleted)
• stdhdr/unwind.h (added)
• stdlib.cfa (modified) (9 diffs)
• stdlib.hfa (modified) (9 diffs)
• time.cfa (modified) (2 diffs)
• time.hfa (modified) (4 diffs)
• vec/vec.hfa (modified) (1 diff)
• vec/vec2.hfa (modified) (1 diff)
• vec/vec3.hfa (modified) (1 diff)
• vec/vec4.hfa (modified) (1 diff)

libcfa/src/Makefile.am

@@ -11 +11 @@
 ## Created On       : Sun May 31 08:54:01 2015
 ## Last Modified By : Peter A. Buhr
-## Last Modified On : Mon Jul 15 22:43:27 2019
-## Update Count     : 241
+## Last Modified On : Wed Dec  9 22:46:14 2020
+## Update Count     : 250
 ###############################################################################
 
@@ -19 +19 @@
 ACLOCAL_AMFLAGS  = -I automake
 
-include $(srcdir)/../../src/cfa.make
+include $(top_srcdir)/../tools/build/cfa.make
 
 libdir = ${CFA_LIBDIR}
@@ -31 +31 @@
 # AM_CFAFLAGS for only cfa source
 # use -no-include-stdhdr to prevent rebuild cycles
-# The built sources must not depend on the installed headers
-AM_CFAFLAGS = -quiet -cfalib -I$(srcdir)/stdhdr $(if $(findstring ${gdbwaittarget}, ${@}), -XCFA --gdb) @CONFIG_CFAFLAGS@
-AM_CFLAGS = -g -Wall -Wno-unused-function -fPIC -pthread @ARCH_FLAGS@ @CONFIG_CFLAGS@
+# The built sources must not depend on the installed inst_headers_src
+AM_CFAFLAGS = -quiet -cfalib -I$(srcdir)/stdhdr -I$(srcdir)/concurrency $(if $(findstring ${gdbwaittarget}, ${@}), -XCFA --gdb) @CONFIG_CFAFLAGS@
+AM_CFLAGS = -g -Wall -Wno-unused-function -fPIC -fexceptions -pthread @ARCH_FLAGS@ @CONFIG_CFLAGS@
 AM_CCASFLAGS = -g -Wall -Wno-unused-function @ARCH_FLAGS@ @CONFIG_CFLAGS@
 CFACC = @CFACC@
@@ -39 +39 @@
 #----------------------------------------------------------------------------------------------------------------
 if BUILDLIB
-headers_nosrc = math.hfa gmp.hfa time_t.hfa bits/align.hfa bits/containers.hfa bits/defs.hfa bits/debug.hfa bits/locks.hfa
-headers = fstream.hfa iostream.hfa iterator.hfa limits.hfa rational.hfa time.hfa stdlib.hfa common.hfa \
-          containers/maybe.hfa containers/pair.hfa containers/result.hfa containers/vector.hfa \
-          vec/vec.hfa vec/vec2.hfa vec/vec3.hfa vec/vec4.hfa
-
-libsrc = startup.cfa interpose.cfa bits/debug.cfa assert.cfa exception.c virtual.c heap.cfa ${headers:.hfa=.cfa}
+inst_headers_nosrc = \
+        bitmanip.hfa \
+        clock.hfa \
+        exception.hfa \
+        exception.h \
+        gmp.hfa \
+        math.hfa \
+        time_t.hfa \
+        bits/align.hfa \
+        bits/containers.hfa \
+        bits/debug.hfa \
+        bits/defs.hfa \
+        bits/locks.hfa \
+        bits/collection.hfa \
+        bits/stack.hfa \
+        bits/queue.hfa \
+        bits/sequence.hfa \
+        concurrency/iofwd.hfa \
+        containers/list.hfa \
+        containers/stackLockFree.hfa \
+        vec/vec.hfa \
+        vec/vec2.hfa \
+        vec/vec3.hfa \
+        vec/vec4.hfa
+
+inst_headers_src = \
+        common.hfa \
+        fstream.hfa \
+        heap.hfa \
+        iostream.hfa \
+        iterator.hfa \
+        limits.hfa \
+        memory.hfa \
+        parseargs.hfa \
+        rational.hfa \
+        stdlib.hfa \
+        time.hfa \
+        containers/maybe.hfa \
+        containers/pair.hfa \
+        containers/result.hfa \
+        containers/vector.hfa
+
+libsrc = ${inst_headers_src} ${inst_headers_src:.hfa=.cfa} \
+        assert.cfa \
+        bits/algorithm.hfa \
+        bits/debug.cfa \
+        exception.c \
+        interpose.cfa \
+        lsda.h \
+        startup.cfa \
+        startup.hfa \
+        virtual.c \
+        virtual.h
 
 # not all platforms support concurrency, add option do disable it
-thread_headers_nosrc = concurrency/invoke.h
-thread_headers = concurrency/coroutine.hfa concurrency/thread.hfa concurrency/kernel.hfa concurrency/monitor.hfa concurrency/mutex.hfa
-thread_libsrc = concurrency/CtxSwitch-@ARCHITECTURE@.S concurrency/alarm.cfa concurrency/invoke.c concurrency/preemption.cfa ${thread_headers:.hfa=.cfa}
+inst_thread_headers_nosrc = \
+        bits/random.hfa \
+        concurrency/clib/cfathread.h \
+        concurrency/invoke.h \
+        concurrency/future.hfa \
+        concurrency/kernel/fwd.hfa
+
+inst_thread_headers_src = \
+        concurrency/coroutine.hfa \
+        concurrency/exception.hfa \
+        concurrency/kernel.hfa \
+        concurrency/locks.hfa \
+        concurrency/monitor.hfa \
+        concurrency/mutex.hfa \
+        concurrency/thread.hfa
+
+thread_libsrc = ${inst_thread_headers_src} ${inst_thread_headers_src:.hfa=.cfa} \
+        bits/signal.hfa \
+        concurrency/alarm.cfa \
+        concurrency/alarm.hfa \
+        concurrency/clib/cfathread.cfa \
+        concurrency/CtxSwitch-@ARCHITECTURE@.S \
+        concurrency/invoke.c \
+        concurrency/io.cfa \
+        concurrency/io/setup.cfa \
+        concurrency/io/types.hfa \
+        concurrency/io/call.cfa \
+        concurrency/iofwd.hfa \
+        concurrency/kernel_private.hfa \
+        concurrency/kernel/startup.cfa \
+        concurrency/preemption.cfa \
+        concurrency/preemption.hfa \
+        concurrency/ready_queue.cfa \
+        concurrency/ready_subqueue.hfa \
+        concurrency/snzi.hfa \
+        concurrency/stats.cfa \
+        concurrency/stats.hfa \
+        concurrency/stats.hfa
+
 else
-headers =
-thread_headers =
-headers_nosrc =
-thread_headers_nosrc =
+inst_headers_src =
+inst_thread_headers_src =
+inst_headers_nosrc =
+inst_thread_headers_nosrc =
 libsrc =
 endif
@@ -97 +180 @@
 
 prelude.o : prelude.cfa extras.cf gcc-builtins.cf builtins.cf @LOCAL_CFACC@ @CFACPP@
-        ${AM_V_GEN}$(CFACOMPILE) -quiet -XCFA -l ${<} -c -o ${@}
+        ${AM_V_GEN}$(CFACOMPILE) -quiet -XCFA,-l ${<} -c -o ${@}
 
 prelude.lo: prelude.cfa extras.cf gcc-builtins.cf builtins.cf @LOCAL_CFACC@ @CFACPP@
         ${AM_V_GEN}$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile \
-        $(CFACOMPILE) -quiet -XCFA -l ${<} -c -o ${@}
-
-#----------------------------------------------------------------------------------------------------------------
-libcfa_la_SOURCES = prelude.cfa ${libsrc}
+        $(CFACOMPILE) -quiet -XCFA,-l ${<} -c -o ${@}
+
+#----------------------------------------------------------------------------------------------------------------
+libcfa_la_SOURCES = ${libsrc}
+nodist_libcfa_la_SOURCES = prelude.cfa
 libcfa_la_LDFLAGS = -version-info @CFA_VERSION@
 
@@ -113 +197 @@
 
 cfa_includedir = $(CFA_INCDIR)
-nobase_cfa_include_HEADERS = ${stdhdr} ${headers} ${headers_nosrc} ${thread_headers} ${thread_headers_nosrc}
+nobase_cfa_include_HEADERS = ${stdhdr} ${inst_headers_src} ${inst_headers_nosrc} ${inst_thread_headers_src} ${inst_thread_headers_nosrc}
+EXTRA_DIST = stdhdr
 
 #----------------------------------------------------------------------------------------------------------------
 maintainer-clean-local:
         -rm -rf ${CFA_INCDIR} ${CFA_LIBDIR}
+
+distclean-local:
+        find ${builddir} -path '*.Plo' -delete
 
 

libcfa/src/assert.cfa

@@ -10 +10 @@
 // Created On       : Mon Nov 28 12:27:26 2016
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Thu Nov 21 17:09:26 2019
-// Update Count     : 5
+// Last Modified On : Tue Feb  4 13:00:18 2020
+// Update Count     : 6
 //
 
@@ -26 +26 @@
 
         // called by macro assert in assert.h
-        void __assert_fail( const char *assertion, const char *file, unsigned int line, const char *function ) {
+        void __assert_fail( const char assertion[], const char file[], unsigned int line, const char function[] ) {
                 __cfaabi_bits_print_safe( STDERR_FILENO, CFA_ASSERT_FMT ".\n", assertion, __progname, function, line, file );
                 abort();
@@ -32 +32 @@
 
         // called by macro assertf
-        void __assert_fail_f( const char *assertion, const char *file, unsigned int line, const char *function, const char *fmt, ... ) {
+        void __assert_fail_f( const char assertion[], const char file[], unsigned int line, const char function[], const char fmt[], ... ) {
                 __cfaabi_bits_acquire();
                 __cfaabi_bits_print_nolock( STDERR_FILENO, CFA_ASSERT_FMT ": ", assertion, __progname, function, line, file );

libcfa/src/bits/containers.hfa

@@ -17 +17 @@
 #include "bits/align.hfa"
 #include "bits/defs.hfa"
-
+#include <stdio.h>
 //-----------------------------------------------------------------------------
 // Array
@@ -36 +36 @@
         #define __small_array_t(T) __small_array(T)
 #else
-        #define __small_array_t(T) struct __small_array
+        #define __small_array_t(T) __small_array
 #endif
 
@@ -146 +146 @@
         static inline forall( dtype T | is_node(T) ) {
                 void ?{}( __queue(T) & this ) with( this ) {
-                        head{ 0p };
-                        tail{ &head };
+                        (this.head){ 1p };
+                        (this.tail){ &this.head };
+                        verify(*this.tail == 1p);
                 }
 
                 void append( __queue(T) & this, T * val ) with( this ) {
-                        verify(tail != 0p);
-                        *tail = val;
-                        tail = &get_next( *val );
+                        verify(this.tail != 0p);
+                        verify(*this.tail == 1p);
+                        *this.tail = val;
+                        this.tail = &get_next( *val );
+                        *this.tail = 1p;
+                }
+
+                T * peek( __queue(T) & this ) {
+                        verify(*this.tail == 1p);
+                        T * front = this.head;
+                        if( front != 1p ) {
+                                verify(*this.tail == 1p);
+                                return front;
+                        }
+                        verify(*this.tail == 1p);
+                        return 0p;
                 }
 
                 T * pop_head( __queue(T) & this ) {
-                        T * head = this.head;
-                        if( head ) {
-                                this.head = get_next( *head );
-                                if( !get_next( *head ) ) {
+                        verify(*this.tail == 1p);
+                        T * _head = this.head;
+                        if( _head != 1p ) {
+                                this.head = get_next( *_head );
+                                if( get_next( *_head ) == 1p ) {
                                         this.tail = &this.head;
                                 }
-                                get_next( *head ) = 0p;
-                        }
-                        return head;
+                                get_next( *_head ) = 0p;
+                                verify(*this.tail == 1p);
+                                verify( get_next(*_head) == 0p );
+                                return _head;
+                        }
+                        verify(*this.tail == 1p);
+                        return 0p;
                 }
 
@@ -174 +193 @@
                         (*it) = get_next( *val );
 
-                        if( tail == &get_next( *val ) ) {
-                                tail = it;
+                        if( this.tail == &get_next( *val ) ) {
+                                this.tail = it;
                         }
 
                         get_next( *val ) = 0p;
 
-                        verify( (head == 0p) == (&head == tail) );
-                        verify( *tail == 0p );
+                        verify( (this.head == 1p) == (&this.head == this.tail) );
+                        verify( *this.tail == 1p );
                         return val;
                 }
 
                 int ?!=?( const __queue(T) & this, __attribute__((unused)) zero_t zero ) {
-                        return this.head != 0;
+                        return this.head != 1p;
                 }
         }
@@ -220 +239 @@
         forall(dtype T )
         static inline [void] ?{}( __dllist(T) & this, * [T * & next, T * & prev] ( T & ) __get ) {
-                this.head{ 0p };
+                (this.head){ 0p };
                 this.__get = __get;
         }
@@ -229 +248 @@
                 void push_front( __dllist(T) & this, T & node ) with( this ) {
                         verify(__get);
-                        if ( head ) {
-                                __get( node ).next = head;
-                                __get( node ).prev = __get( *head ).prev;
+                        if ( this.head ) {
+                                __get( node ).next = this.head;
+                                __get( node ).prev = __get( *this.head ).prev;
                                 // inserted node must be consistent before it is seen
                                 // prevent code movement across barrier
                                 asm( "" : : : "memory" );
-                                __get( *head ).prev = &node;
+                                __get( *this.head ).prev = &node;
                                 T & _prev = *__get( node ).prev;
                                 __get( _prev ).next = &node;
@@ -245 +264 @@
                         // prevent code movement across barrier
                         asm( "" : : : "memory" );
-                        head = &node;
+                        this.head = &node;
                 }
 
                 void remove( __dllist(T) & this, T & node ) with( this ) {
                         verify(__get);
-                        if ( &node == head ) {
-                                if ( __get( *head ).next == head ) {
-                                        head = 0p;
+                        if ( &node == this.head ) {
+                                if ( __get( *this.head ).next == this.head ) {
+                                        this.head = 0p;
                                 } else {
-                                        head = __get( *head ).next;
+                                        this.head = __get( *this.head ).next;
                                 }
                         }
@@ -266 +285 @@
                         return this.head != 0;
                 }
+
+                void move_to_front( __dllist(T) & src, __dllist(T) & dst, T & node ) {
+                        remove    (src, node);
+                        push_front(dst, node);
+                }
         }
         #undef next

libcfa/src/bits/debug.cfa

@@ -10 +10 @@
 // Created On       : Thu Mar 30 12:30:01 2017
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Thu Nov 21 17:16:30 2019
-// Update Count     : 10
+// Last Modified On : Wed Jun 17 11:07:13 2020
+// Update Count     : 12
 //
 
-extern "C" {
 #include <stdio.h>
 #include <stdlib.h>
@@ -21 +20 @@
 #include <stdarg.h>
 #include <unistd.h>
-}
 
 enum { buffer_size = 4096 };
@@ -27 +25 @@
 
 extern "C" {
-
-        void __cfaabi_bits_write( int fd, const char *in_buffer, int len ) {
+        void __cfaabi_bits_write( int fd, const char in_buffer[], int len ) {
                 // ensure all data is written
                 for ( int count = 0, retcode; count < len; count += retcode ) {

libcfa/src/bits/debug.hfa

@@ -9 +9 @@
 // Author           : Thierry Delisle
 // Created On       : Mon Nov 28 12:27:26 2016
-// Last Modified By : Peter A. Buhr
-// Last Modified On : Thu Nov 21 17:06:58 2019
-// Update Count     : 8
+// Last Modified By : Andrew Beach
+// Last Modified On : Mon Apr 27 10:15:00 2020
+// Update Count     : 10
 //
 
 #pragma once
+
+#include <assert.h>
 
 #ifdef __CFA_DEBUG__
@@ -21 +23 @@
         #define __cfaabi_dbg_ctx __PRETTY_FUNCTION__
         #define __cfaabi_dbg_ctx2 , __PRETTY_FUNCTION__
-        #define __cfaabi_dbg_ctx_param const char * caller
-        #define __cfaabi_dbg_ctx_param2 , const char * caller
+        #define __cfaabi_dbg_ctx_param const char caller[]
+        #define __cfaabi_dbg_ctx_param2 , const char caller[]
+        #define __cfaabi_dbg_ctx_fwd caller
+        #define __cfaabi_dbg_ctx_fwd2 , caller
 #else
         #define __cfaabi_dbg_debug_do(...)
@@ -30 +34 @@
         #define __cfaabi_dbg_ctx_param
         #define __cfaabi_dbg_ctx_param2
+        #define __cfaabi_dbg_ctx_fwd
+        #define __cfaabi_dbg_ctx_fwd2
 #endif
 
@@ -36 +42 @@
 #endif
         #include <stdarg.h>
-        #include <stdio.h>
 
-        extern void __cfaabi_bits_write( int fd, const char *buffer, int len );
+        extern void __cfaabi_bits_write( int fd, const char buffer[], int len );
         extern void __cfaabi_bits_acquire();
         extern void __cfaabi_bits_release();
@@ -45 +50 @@
         extern void __cfaabi_bits_print_vararg( int fd, const char fmt[], va_list arg );
         extern void __cfaabi_bits_print_buffer( int fd, char buffer[], int buffer_size, const char fmt[], ... ) __attribute__(( format(printf, 4, 5) ));
+
+#if defined(__CFA_DEBUG_PRINT__) \
+                || defined(__CFA_DEBUG_PRINT_IO__) || defined(__CFA_DEBUG_PRINT_IO_CORE__) \
+                || defined(__CFA_DEBUG_PRINT_MONITOR__) || defined(__CFA_DEBUG_PRINT_PREEMPTION__) \
+                || defined(__CFA_DEBUG_PRINT_RUNTIME_CORE__) || defined(__CFA_DEBUG_PRINT_EXCEPTION__) \
+                || defined(__CFA_DEBUG_PRINT_READY_QUEUE__)
+        #include <stdio.h>
+        #include <unistd.h>
+#endif
 #ifdef __cforall
 }
 #endif
 
+// Deprecated: Use the versions with the new module names.
 #ifdef __CFA_DEBUG_PRINT__
         #define __cfaabi_dbg_write( buffer, len )         __cfaabi_bits_write( STDERR_FILENO, buffer, len )
         #define __cfaabi_dbg_acquire()                    __cfaabi_bits_acquire()
         #define __cfaabi_dbg_release()                    __cfaabi_bits_release()
-        #define __cfaabi_dbg_print_safe(...)              __cfaabi_bits_print_safe   (__VA_ARGS__)
-        #define __cfaabi_dbg_print_nolock(...)            __cfaabi_bits_print_nolock (__VA_ARGS__)
-        #define __cfaabi_dbg_print_buffer(...)            __cfaabi_bits_print_buffer (__VA_ARGS__)
-        #define __cfaabi_dbg_print_buffer_decl(...)       char __dbg_text[256]; int __dbg_len = snprintf( __dbg_text, 256, __VA_ARGS__ ); __cfaabi_bits_write( __dbg_text, __dbg_len );
-        #define __cfaabi_dbg_print_buffer_local(...)      __dbg_len = snprintf( __dbg_text, 256, __VA_ARGS__ ); __cfaabi_dbg_write( __dbg_text, __dbg_len );
+        #define __cfaabi_dbg_print_safe(...)              __cfaabi_bits_print_safe   ( STDERR_FILENO, __VA_ARGS__ )
+        #define __cfaabi_dbg_print_nolock(...)            __cfaabi_bits_print_nolock ( STDERR_FILENO, __VA_ARGS__ )
+        #define __cfaabi_dbg_print_buffer(...)            __cfaabi_bits_print_buffer ( STDERR_FILENO, __VA_ARGS__ )
+        #define __cfaabi_dbg_print_buffer_decl(...)       char __dbg_text[256]; int __dbg_len = snprintf( __dbg_text, 256, __VA_ARGS__ ); __cfaabi_bits_write( STDERR_FILENO, __dbg_text, __dbg_len );
+        #define __cfaabi_dbg_print_buffer_local(...)      __dbg_len = snprintf( __dbg_text, 256, __VA_ARGS__ ); __cfaabi_dbg_write( STDERR_FILENO, __dbg_text, __dbg_len );
 #else
         #define __cfaabi_dbg_write(...)               ((void)0)
@@ -69 +84 @@
 #endif
 
+// Debug print functions and statements:
+// Most are wrappers around the bits printing function but are not always used.
+// If they are used depends if the group (first argument) is active or not. The group must be one
+// defined belowe. The other arguments depend on the wrapped function.
+#define __cfadbg_write(group, buffer, len) \
+        __CFADBG_PRINT_GROUP_##group(__cfaabi_bits_write(STDERR_FILENO, buffer, len))
+#define __cfadbg_acquire(group) \
+        __CFADBG_PRINT_GROUP_##group(__cfaabi_bits_acquire())
+#define __cfadbg_release(group) \
+        __CFADBG_PRINT_GROUP_##group(__cfaabi_bits_release())
+#define __cfadbg_print_safe(group, ...) \
+        __CFADBG_PRINT_GROUP_##group(__cfaabi_bits_print_safe(STDERR_FILENO, __VA_ARGS__))
+#define __cfadbg_print_nolock(group, ...) \
+        __CFADBG_PRINT_GROUP_##group(__cfaabi_bits_print_nolock(STDERR_FILENO, __VA_ARGS__))
+#define __cfadbg_print_buffer(group, ...) \
+        __CFADBG_PRINT_GROUP_##group(__cfaabi_bits_print_buffer(STDERR_FILENO, __VA_ARGS__))
+#define __cfadbg_print_buffer_decl(group, ...) \
+        __CFADBG_PRINT_GROUP_##group(char __dbg_text[256]; int __dbg_len = snprintf( __dbg_text, 256, __VA_ARGS__ ); __cfaabi_bits_write( __dbg_text, __dbg_len ))
+#define __cfadbg_print_buffer_local(group, ...) \
+        __CFADBG_PRINT_GROUP_##group(__dbg_len = snprintf( __dbg_text, 256, __VA_ARGS__ ); __cfaabi_bits_write(STDERR_FILENO, __dbg_text, __dbg_len))
+
+// The debug print groups:
+#if defined(__CFA_DEBUG_PRINT__) || defined(__CFA_DEBUG_PRINT_IO__)
+#       define __CFADBG_PRINT_GROUP_io(...) __VA_ARGS__
+#else
+#       define __CFADBG_PRINT_GROUP_io(...) ((void)0)
+#endif
+#if defined(__CFA_DEBUG_PRINT__) || defined(__CFA_DEBUG_PRINT_IO__) || defined(__CFA_DEBUG_PRINT_IO_CORE__)
+#       define __CFADBG_PRINT_GROUP_io_core(...) __VA_ARGS__
+#else
+#       define __CFADBG_PRINT_GROUP_io_core(...) ((void)0)
+#endif
+#if defined(__CFA_DEBUG_PRINT__) || defined(__CFA_DEBUG_PRINT_MONITOR__)
+#       define __CFADBG_PRINT_GROUP_monitor(...) __VA_ARGS__
+#else
+#       define __CFADBG_PRINT_GROUP_monitor(...) ((void)0)
+#endif
+#if defined(__CFA_DEBUG_PRINT__) || defined(__CFA_DEBUG_PRINT_PREEMPTION__)
+#       define __CFADBG_PRINT_GROUP_preemption(...) __VA_ARGS__
+#else
+#       define __CFADBG_PRINT_GROUP_preemption(...) ((void)0)
+#endif
+#if defined(__CFA_DEBUG_PRINT__) || defined(__CFA_DEBUG_PRINT_RUNTIME_CORE__)
+#       define __CFADBG_PRINT_GROUP_runtime_core(...) __VA_ARGS__
+#else
+#       define __CFADBG_PRINT_GROUP_runtime_core(...) ((void)0)
+#endif
+#if defined(__CFA_DEBUG_PRINT__) || defined(__CFA_DEBUG_PRINT_READY_QUEUE__)
+#       define __CFADBG_PRINT_GROUP_ready_queue(...) __VA_ARGS__
+#else
+#       define __CFADBG_PRINT_GROUP_ready_queue(...) ((void)0)
+#endif
+#if defined(__CFA_DEBUG_PRINT__) || defined(__CFA_DEBUG_PRINT_EXCEPTION__)
+#       define __CFADBG_PRINT_GROUP_exception(...) __VA_ARGS__
+#else
+#       define __CFADBG_PRINT_GROUP_exception(...) ((void)0)
+#endif
+
 // Local Variables: //
 // mode: c //

libcfa/src/bits/defs.hfa

@@ -10 +10 @@
 // Created On       : Thu Nov  9 13:24:10 2017
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Tue Jan 28 22:38:27 2020
-// Update Count     : 9
+// Last Modified On : Sat Oct 24 10:53:15 2020
+// Update Count     : 21
 //
 
 #pragma once
 
-#include <stdbool.h>
-#include <stddef.h>
 #include <stdint.h>
+#include <assert.h>
 
 #define likely(x)   __builtin_expect(!!(x), 1)
@@ -30 +29 @@
 #define __cfa_anonymous_object(x) inline struct x
 #else
-#define __cfa_anonymous_object(x) x __cfa_anonymous_object
+#define __cfa_anonymous_object(x) struct x __cfa_anonymous_object
 #endif
 
@@ -49 +48 @@
 #endif
 
-static inline long long rdtscl(void) {
-    unsigned int lo, hi;
-    __asm__ __volatile__ ("rdtsc" : "=a"(lo), "=d"(hi));
-    return ( (unsigned long long)lo)|( ((unsigned long long)hi)<<32 );
+static inline long long int rdtscl(void) {
+        #if defined( __i386 ) || defined( __x86_64 )
+        unsigned int lo, hi;
+        __asm__ __volatile__ ("rdtsc" : "=a"(lo), "=d"(hi));
+        return ( (unsigned long long)lo)|( ((unsigned long long)hi)<<32 );
+        #elif defined( __aarch64__ ) || defined( __arm__ )
+        // https://github.com/google/benchmark/blob/v1.1.0/src/cycleclock.h#L116
+        long long int virtual_timer_value;
+        asm volatile("mrs %0, cntvct_el0" : "=r"(virtual_timer_value));
+        return virtual_timer_value;
+        #else
+                #error unsupported hardware architecture
+        #endif
 }

libcfa/src/bits/locks.hfa

@@ -10 +10 @@
 // Created On       : Tue Oct 31 15:14:38 2017
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Sat Aug 11 15:42:24 2018
-// Update Count     : 10
+// Last Modified On : Wed Aug 12 14:18:07 2020
+// Update Count     : 13
 //
 
@@ -27 +27 @@
 
 // pause to prevent excess processor bus usage
-#if defined( __sparc )
-        #define Pause() __asm__ __volatile__ ( "rd %ccr,%g0" )
-#elif defined( __i386 ) || defined( __x86_64 )
+#if defined( __i386 ) || defined( __x86_64 )
         #define Pause() __asm__ __volatile__ ( "pause" : : : )
 #elif defined( __ARM_ARCH )
-        #define Pause() __asm__ __volatile__ ( "nop" : : : )
+        #define Pause() __asm__ __volatile__ ( "YIELD" : : : )
 #else
         #error unsupported architecture
@@ -54 +52 @@
 
                 #ifdef __CFA_DEBUG__
-                        void __cfaabi_dbg_record(__spinlock_t & this, const char * prev_name);
+                        void __cfaabi_dbg_record_lock(__spinlock_t & this, const char prev_name[]);
                 #else
-                        #define __cfaabi_dbg_record(x, y)
+                        #define __cfaabi_dbg_record_lock(x, y)
                 #endif
         }
-
-        extern void yield( unsigned int );
 
         static inline void ?{}( __spinlock_t & this ) {
@@ -68 +64 @@
         // Lock the spinlock, return false if already acquired
         static inline bool try_lock  ( __spinlock_t & this __cfaabi_dbg_ctx_param2 ) {
+                disable_interrupts();
                 bool result = (this.lock == 0) && (__atomic_test_and_set( &this.lock, __ATOMIC_ACQUIRE ) == 0);
                 if( result ) {
-                        disable_interrupts();
-                        __cfaabi_dbg_record( this, caller );
+                        __cfaabi_dbg_record_lock( this, caller );
+                } else {
+                        enable_interrupts_noPoll();
                 }
                 return result;
@@ -83 +81 @@
                 #endif
 
+                disable_interrupts();
                 for ( unsigned int i = 1;; i += 1 ) {
                         if ( (this.lock == 0) && (__atomic_test_and_set( &this.lock, __ATOMIC_ACQUIRE ) == 0) ) break;
@@ -98 +97 @@
                         #endif
                 }
-                disable_interrupts();
-                __cfaabi_dbg_record( this, caller );
+                __cfaabi_dbg_record_lock( this, caller );
         }
 
         static inline void unlock( __spinlock_t & this ) {
+                __atomic_clear( &this.lock, __ATOMIC_RELEASE );
                 enable_interrupts_noPoll();
-                __atomic_clear( &this.lock, __ATOMIC_RELEASE );
         }
 
@@ -112 +110 @@
         #endif
 
+        extern "C" {
+                char * strerror(int);
+        }
+        #define CHECKED(x) { int err = x; if( err != 0 ) abort("KERNEL ERROR: Operation \"" #x "\" return error %d - %s\n", err, strerror(err)); }
+
         struct __bin_sem_t {
-                bool                    signaled;
                 pthread_mutex_t         lock;
                 pthread_cond_t          cond;
+                int                     val;
         };
 
         static inline void ?{}(__bin_sem_t & this) with( this ) {
-                signaled = false;
-                pthread_mutex_init(&lock, NULL);
-                pthread_cond_init (&cond, NULL);
+                // Create the mutex with error checking
+                pthread_mutexattr_t mattr;
+                pthread_mutexattr_init( &mattr );
+                pthread_mutexattr_settype( &mattr, PTHREAD_MUTEX_ERRORCHECK_NP);
+                pthread_mutex_init(&lock, &mattr);
+
+                pthread_cond_init (&cond, (const pthread_condattr_t *)0p);  // workaround trac#208: cast should not be required
+                val = 0;
         }
 
         static inline void ^?{}(__bin_sem_t & this) with( this ) {
-                pthread_mutex_destroy(&lock);
-                pthread_cond_destroy (&cond);
+                CHECKED( pthread_mutex_destroy(&lock) );
+                CHECKED( pthread_cond_destroy (&cond) );
         }
 
         static inline void wait(__bin_sem_t & this) with( this ) {
                 verify(__cfaabi_dbg_in_kernel());
-                pthread_mutex_lock(&lock);
-                        if(!signaled) {   // this must be a loop, not if!
+                CHECKED( pthread_mutex_lock(&lock) );
+                        while(val < 1) {
                                 pthread_cond_wait(&cond, &lock);
                         }
-                        signaled = false;
-                pthread_mutex_unlock(&lock);
-        }
-
-        static inline void post(__bin_sem_t & this) with( this ) {
-                verify(__cfaabi_dbg_in_kernel());
-
-                pthread_mutex_lock(&lock);
-                        bool needs_signal = !signaled;
-                        signaled = true;
-                pthread_mutex_unlock(&lock);
-
-                if (needs_signal)
-                        pthread_cond_signal(&cond);
+                        val -= 1;
+                CHECKED( pthread_mutex_unlock(&lock) );
+        }
+
+        static inline bool post(__bin_sem_t & this) with( this ) {
+                bool needs_signal = false;
+
+                CHECKED( pthread_mutex_lock(&lock) );
+                        if(val < 1) {
+                                val += 1;
+                                pthread_cond_signal(&cond);
+                                needs_signal = true;
+                        }
+                CHECKED( pthread_mutex_unlock(&lock) );
+
+                return needs_signal;
+        }
+
+        #undef CHECKED
+
+        struct $thread;
+        extern void park( void );
+        extern void unpark( struct $thread * this );
+        static inline struct $thread * active_thread ();
+
+        // Semaphore which only supports a single thread
+        struct single_sem {
+                struct $thread * volatile ptr;
+        };
+
+        static inline {
+                void  ?{}(single_sem & this) {
+                        this.ptr = 0p;
+                }
+
+                void ^?{}(single_sem &) {}
+
+                bool wait(single_sem & this) {
+                        for() {
+                                struct $thread * expected = this.ptr;
+                                if(expected == 1p) {
+                                        if(__atomic_compare_exchange_n(&this.ptr, &expected, 0p, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) {
+                                                return false;
+                                        }
+                                }
+                                else {
+                                        /* paranoid */ verify( expected == 0p );
+                                        if(__atomic_compare_exchange_n(&this.ptr, &expected, active_thread(), false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) {
+                                                park();
+                                                return true;
+                                        }
+                                }
+
+                        }
+                }
+
+                bool post(single_sem & this) {
+                        for() {
+                                struct $thread * expected = this.ptr;
+                                if(expected == 1p) return false;
+                                if(expected == 0p) {
+                                        if(__atomic_compare_exchange_n(&this.ptr, &expected, 1p, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) {
+                                                return false;
+                                        }
+                                }
+                                else {
+                                        if(__atomic_compare_exchange_n(&this.ptr, &expected, 0p, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) {
+                                                unpark( expected );
+                                                return true;
+                                        }
+                                }
+                        }
+                }
+        }
+
+        // Synchronozation primitive which only supports a single thread and one post
+        // Similar to a binary semaphore with a 'one shot' semantic
+        // is expected to be discarded after each party call their side
+        struct oneshot {
+                // Internal state :
+                //     0p     : is initial state (wait will block)
+                //     1p     : fulfilled (wait won't block)
+                // any thread : a thread is currently waiting
+                struct $thread * volatile ptr;
+        };
+
+        static inline {
+                void  ?{}(oneshot & this) {
+                        this.ptr = 0p;
+                }
+
+                void ^?{}(oneshot &) {}
+
+                // Wait for the post, return immidiately if it already happened.
+                // return true if the thread was parked
+                bool wait(oneshot & this) {
+                        for() {
+                                struct $thread * expected = this.ptr;
+                                if(expected == 1p) return false;
+                                /* paranoid */ verify( expected == 0p );
+                                if(__atomic_compare_exchange_n(&this.ptr, &expected, active_thread(), false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) {
+                                        park();
+                                        /* paranoid */ verify( this.ptr == 1p );
+                                        return true;
+                                }
+                        }
+                }
+
+                // Mark as fulfilled, wake thread if needed
+                // return true if a thread was unparked
+                bool post(oneshot & this) {
+                        struct $thread * got = __atomic_exchange_n( &this.ptr, 1p, __ATOMIC_SEQ_CST);
+                        if( got == 0p ) return false;
+                        unpark( got );
+                        return true;
+                }
+        }
+
+        // base types for future to build upon
+        // It is based on the 'oneshot' type to allow multiple futures
+        // to block on the same instance, permitting users to block a single
+        // thread on "any of" [a given set of] futures.
+        // does not support multiple threads waiting on the same future
+        struct future_t {
+                // Internal state :
+                //     0p      : is initial state (wait will block)
+                //     1p      : fulfilled (wait won't block)
+                //     2p      : in progress ()
+                //     3p      : abandoned, server should delete
+                // any oneshot : a context has been setup to wait, a thread could wait on it
+                struct oneshot * volatile ptr;
+        };
+
+        static inline {
+                void  ?{}(future_t & this) {
+                        this.ptr = 0p;
+                }
+
+                void ^?{}(future_t &) {}
+
+                void reset(future_t & this) {
+                        // needs to be in 0p or 1p
+                        __atomic_exchange_n( &this.ptr, 0p, __ATOMIC_SEQ_CST);
+                }
+
+                // check if the future is available
+                bool available( future_t & this ) {
+                        return this.ptr == 1p;
+                }
+
+                // Prepare the future to be waited on
+                // intented to be use by wait, wait_any, waitfor, etc. rather than used directly
+                bool setup( future_t & this, oneshot & wait_ctx ) {
+                        /* paranoid */ verify( wait_ctx.ptr == 0p );
+                        // The future needs to set the wait context
+                        for() {
+                                struct oneshot * expected = this.ptr;
+                                // Is the future already fulfilled?
+                                if(expected == 1p) return false; // Yes, just return false (didn't block)
+
+                                // The future is not fulfilled, try to setup the wait context
+                                /* paranoid */ verify( expected == 0p );
+                                if(__atomic_compare_exchange_n(&this.ptr, &expected, &wait_ctx, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) {
+                                        return true;
+                                }
+                        }
+                }
+
+                // Stop waiting on a future
+                // When multiple futures are waited for together in "any of" pattern
+                // futures that weren't fulfilled before the thread woke up
+                // should retract the wait ctx
+                // intented to be use by wait, wait_any, waitfor, etc. rather than used directly
+                void retract( future_t & this, oneshot & wait_ctx ) {
+                        // Remove the wait context
+                        struct oneshot * got = __atomic_exchange_n( &this.ptr, 0p, __ATOMIC_SEQ_CST);
+
+                        // got == 0p: future was never actually setup, just return
+                        if( got == 0p ) return;
+
+                        // got == wait_ctx: since fulfil does an atomic_swap,
+                        // if we got back the original then no one else saw context
+                        // It is safe to delete (which could happen after the return)
+                        if( got == &wait_ctx ) return;
+
+                        // got == 1p: the future is ready and the context was fully consumed
+                        // the server won't use the pointer again
+                        // It is safe to delete (which could happen after the return)
+                        if( got == 1p ) return;
+
+                        // got == 2p: the future is ready but the context hasn't fully been consumed
+                        // spin until it is safe to move on
+                        if( got == 2p ) {
+                                while( this.ptr != 1p ) Pause();
+                                return;
+                        }
+
+                        // got == any thing else, something wen't wrong here, abort
+                        abort("Future in unexpected state");
+                }
+
+                // Mark the future as abandoned, meaning it will be deleted by the server
+                bool abandon( future_t & this ) {
+                        /* paranoid */ verify( this.ptr != 3p );
+
+                        // Mark the future as abandonned
+                        struct oneshot * got = __atomic_exchange_n( &this.ptr, 3p, __ATOMIC_SEQ_CST);
+
+                        // If the future isn't already fulfilled, let the server delete it
+                        if( got == 0p ) return false;
+
+                        // got == 2p: the future is ready but the context hasn't fully been consumed
+                        // spin until it is safe to move on
+                        if( got == 2p ) {
+                                while( this.ptr != 1p ) Pause();
+                                got = 1p;
+                        }
+
+                        // The future is completed delete it now
+                        /* paranoid */ verify( this.ptr != 1p );
+                        free( &this );
+                        return true;
+                }
+
+                // from the server side, mark the future as fulfilled
+                // delete it if needed
+                bool fulfil( future_t & this ) {
+                        for() {
+                                struct oneshot * expected = this.ptr;
+                                // was this abandoned?
+                                #if defined(__GNUC__) && __GNUC__ >= 7
+                                        #pragma GCC diagnostic push
+                                        #pragma GCC diagnostic ignored "-Wfree-nonheap-object"
+                                #endif
+                                        if( expected == 3p ) { free( &this ); return false; }
+                                #if defined(__GNUC__) && __GNUC__ >= 7
+                                        #pragma GCC diagnostic pop
+                                #endif
+
+                                /* paranoid */ verify( expected != 1p ); // Future is already fulfilled, should not happen
+                                /* paranoid */ verify( expected != 2p ); // Future is bein fulfilled by someone else, this is even less supported then the previous case.
+
+                                // If there is a wait context, we need to consume it and mark it as consumed after
+                                // If there is no context then we can skip the in progress phase
+                                struct oneshot * want = expected == 0p ? 1p : 2p;
+                                if(__atomic_compare_exchange_n(&this.ptr, &expected, want, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) {
+                                        if( expected == 0p ) { /* paranoid */ verify( this.ptr == 1p); return false; }
+                                        bool ret = post( *expected );
+                                        __atomic_store_n( &this.ptr, 1p, __ATOMIC_SEQ_CST);
+                                        return ret;
+                                }
+                        }
+
+                }
+
+                // Wait for the future to be fulfilled
+                bool wait( future_t & this ) {
+                        oneshot temp;
+                        if( !setup(this, temp) ) return false;
+
+                        // Wait context is setup, just wait on it
+                        bool ret = wait( temp );
+
+                        // Wait for the future to tru
+                        while( this.ptr == 2p ) Pause();
+                        // Make sure the state makes sense
+                        // Should be fulfilled, could be in progress but it's out of date if so
+                        // since if that is the case, the oneshot was fulfilled (unparking this thread)
+                        // and the oneshot should not be needed any more
+                        __attribute__((unused)) struct oneshot * was = this.ptr;
+                        /* paranoid */ verifyf( was == 1p, "Expected this.ptr to be 1p, was %p\n", was );
+
+                        // Mark the future as fulfilled, to be consistent
+                        // with potential calls to avail
+                        // this.ptr = 1p;
+                        return ret;
+                }
         }
 #endif

libcfa/src/bits/signal.hfa

@@ -19 +19 @@
 #include "bits/defs.hfa"
 
-extern "C" {
 #include <errno.h>
 #define __USE_GNU
@@ -26 +25 @@
 #include <stdlib.h>
 #include <string.h>
-}
 
 // Short hands for signal context information
@@ -54 +52 @@
                         sig, handler, flags, errno, strerror( errno )
                 );
-                _exit( EXIT_FAILURE );
+                _Exit( EXIT_FAILURE );
         } // if
 }

libcfa/src/common.hfa

@@ -10 +10 @@
 // Created On       : Wed Jul 11 17:54:36 2018
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Thu Jul 12 08:02:18 2018
-// Update Count     : 5
+// Last Modified On : Sat Aug 15 08:51:29 2020
+// Update Count     : 14
 // 
 
@@ -67 +67 @@
 
 static inline {
+        char min( char t1, char t2 ) { return t1 < t2 ? t1 : t2; } // optimization
+        intptr_t min( intptr_t t1, intptr_t t2 ) { return t1 < t2 ? t1 : t2; } // optimization
+        uintptr_t min( uintptr_t t1, uintptr_t t2 ) { return t1 < t2 ? t1 : t2; } // optimization
         forall( otype T | { int ?<?( T, T ); } )
         T min( T t1, T t2 ) { return t1 < t2 ? t1 : t2; }
 
+        char max( char t1, char t2 ) { return t1 > t2 ? t1 : t2; } // optimization
+        intptr_t max( intptr_t t1, intptr_t t2 ) { return t1 > t2 ? t1 : t2; } // optimization
+        uintptr_t max( uintptr_t t1, uintptr_t t2 ) { return t1 > t2 ? t1 : t2; } // optimization
         forall( otype T | { int ?>?( T, T ); } )
         T max( T t1, T t2 ) { return t1 > t2 ? t1 : t2; }

libcfa/src/concurrency/CtxSwitch-i386.S

@@ -10 +10 @@
 // Created On       : Tue Dec 6 12:27:26 2016
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Fri Jul 21 22:29:25 2017
-// Update Count     : 1
-//
-// This  library is free  software; you  can redistribute  it and/or  modify it
-// under the terms of the GNU Lesser General Public License as published by the
-// Free Software  Foundation; either  version 2.1 of  the License, or  (at your
-// option) any later version.
-//
-// This library is distributed in the  hope that it will be useful, but WITHOUT
-// ANY  WARRANTY;  without even  the  implied  warranty  of MERCHANTABILITY  or
-// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License
-// for more details.
-//
-// You should  have received a  copy of the  GNU Lesser General  Public License
-// along  with this library.
+// Last Modified On : Sun Sep  6 18:23:37 2020
+// Update Count     : 5
 //
 
-// This context switch routine depends on the fact that the stack of a new
-// thread has been set up to look like the thread has saved its context in
-// the normal manner.
-//
-// void CtxSwitch( machine_context *from, machine_context *to );
+// The context switch routine requires the initial the stack of a thread to
+// look like the thread has saved its context in the normal manner.
 
-// Offsets in the context structure. This needs to be synchronized with the
-// high level code a little better.
+// Offsets must synchronized with the __stack_context_t in invoke.h.
 
 #define PTR_BYTE        4
 #define SP_OFFSET       ( 0 * PTR_BYTE )
 #define FP_OFFSET       ( 1 * PTR_BYTE )
-#define PC_OFFSET       ( 2 * PTR_BYTE )
 
+// Context switch between coroutines/tasks.
+//   void __cfactx_switch( struct __stack_context_t * from, struct __stack_context_t * to ) ;
+// Arguments "from" in register 4(%esp), "to" in register 20(%esp)
+
+        .file "CtxSwitch-i386.S"
         .text
         .align 2
-        .globl CtxSwitch
-        .type  CtxSwitch, @function
-CtxSwitch:
+        .global __cfactx_switch
+        .type __cfactx_switch, @function
+__cfactx_switch:
 
         // Copy the "from" context argument from the stack to register eax
-        // Return address is at 0(%esp), with parameters following
+        // Return address is at 0(%esp), with parameters following.
 
         movl 4(%esp),%eax
@@ -63 +50 @@
         movl %ebp,FP_OFFSET(%eax)
 
-        // Copy the "to" context argument from the stack to register eax
-        // Having pushed three words (= 12 bytes) on the stack, the
-        // argument is now at 8 + 12 = 20(%esp)
+        // Copy the "to" context argument from the stack to register eax. Having
+        // pushed 3 words (= 12 bytes) on the stack, the argument is now at
+        // 8 + 12 = 20(%esp).
 
         movl 20(%esp),%eax
@@ -83 +70 @@
 
         ret
-        .size  CtxSwitch, .-CtxSwitch
+        .size __cfactx_switch, .-__cfactx_switch
 
 // Local Variables: //

libcfa/src/concurrency/CtxSwitch-x86_64.S

@@ -7 +7 @@
 // CtxSwitch-x86_64.S --
 //
-// Author           : Thierry Delisle
-// Created On       : Mon Nov 28 12:27:26 2016
+// Author           : Peter A. Buhr
+// Created On       : Mon Aug 10 08:10:26 2020
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Fri Jul 21 22:28:11 2017
-// Update Count     : 1
-//
-// This  library is free  software; you  can redistribute  it and/or  modify it
-// under the terms of the GNU Lesser General Public License as published by the
-// Free Software  Foundation; either  version 2.1 of  the License, or  (at your
-// option) any later version.
-//
-// This library is distributed in the  hope that it will be useful, but WITHOUT
-// ANY  WARRANTY;  without even  the  implied  warranty  of MERCHANTABILITY  or
-// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License
-// for more details.
-//
-// You should  have received a  copy of the  GNU Lesser General  Public License
-// along  with this library.
+// Last Modified On : Sat Oct 24 14:36:25 2020
+// Update Count     : 10
 //
 
-// This context switch routine depends on the fact that the stack of a new
-// thread has been set up to look like the thread has saved its context in
-// the normal manner.
-//
-// void CtxSwitch( machine_context *from, machine_context *to );
+// The context switch routine requires the initial the stack of a thread to
+// look like the thread has saved its context in the normal manner.
 
-// Offsets in the context structure. This needs to be synchronized with the
-// high level code a little better.
+// Offsets must synchronized with the __stack_context_t in invoke.h.
 
 #define PTR_BYTE        8
@@ -40 +23 @@
 #define FP_OFFSET       ( 1 * PTR_BYTE )
 
-//-----------------------------------------------------------------------------
-// Regular context switch routine which enables switching from one context to anouther
+// Context switch between coroutines/tasks.
+//   void __cfactx_switch( struct __stack_context_t * from, struct __stack_context_t * to ) ;
+// Arguments "from" in register rdi, "to" in register rsi.
+
+        .file "CtxSwitch-x86_64.S"
         .text
         .align 2
-        .globl CtxSwitch
-        .type  CtxSwitch, @function
-CtxSwitch:
+        .global __cfactx_switch
+        .type __cfactx_switch, @function
+__cfactx_switch:
 
         // Save volatile registers on the stack.
@@ -77 +63 @@
 
         ret
-        .size  CtxSwitch, .-CtxSwitch
+        .size __cfactx_switch, .-__cfactx_switch
 
-//-----------------------------------------------------------------------------
-// Stub used to create new stacks which are ready to be context switched to
+// Stub to create new stacks which can be context switched to
+//   void __cfactx_invoke_stub( void );
+
         .text
         .align 2
-        .globl CtxInvokeStub
-        .type    CtxInvokeStub, @function
-CtxInvokeStub:
-        movq %rbx, %rdi
+        .global __cfactx_invoke_stub
+        .type __cfactx_invoke_stub, @function
+__cfactx_invoke_stub:
+        movq %rbx, %rdi                                         // move main and this to first two arguments
         movq %r12, %rsi
-        jmp *%r13
-        .size  CtxInvokeStub, .-CtxInvokeStub
+        jmp *%r13                                                       // jmp to invoke
+        .size __cfactx_invoke_stub, .-__cfactx_invoke_stub
 
 // Local Variables: //
-// mode: c //
+// mode: asm //
 // tab-width: 4 //
 // End: //

libcfa/src/concurrency/alarm.cfa

@@ -10 +10 @@
 // Created On       : Fri Jun 2 11:31:25 2017
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Sun Jan  5 08:41:36 2020
-// Update Count     : 69
+// Last Modified On : Wed Jun 17 16:11:35 2020
+// Update Count     : 75
 //
 
 #define __cforall_thread__
 
-extern "C" {
 #include <errno.h>
 #include <stdio.h>
+#include <unistd.h>
 #include <string.h>
-#include <unistd.h>
 #include <sys/time.h>
-}
 
 #include "alarm.hfa"
-#include "kernel_private.hfa"
+#include "kernel/fwd.hfa"
 #include "preemption.hfa"
 
@@ -47 +45 @@
 //=============================================================================================
 
-void ?{}( alarm_node_t & this, thread_desc * thrd, Time alarm, Duration period ) with( this ) {
+void ?{}( alarm_node_t & this, $thread * thrd, Time alarm, Duration period) with( this ) {
         this.thrd = thrd;
         this.alarm = alarm;
         this.period = period;
-        next = 0;
         set = false;
-        kernel_alarm = false;
+        type = User;
 }
 
-void ?{}( alarm_node_t & this, processor   * proc, Time alarm, Duration period ) with( this ) {
+void ?{}( alarm_node_t & this, processor * proc, Time alarm, Duration period ) with( this ) {
         this.proc = proc;
         this.alarm = alarm;
         this.period = period;
-        next = 0;
         set = false;
-        kernel_alarm = true;
+        type = Kernel;
+}
+void ?{}( alarm_node_t & this, Alarm_Callback callback, Time alarm, Duration period ) with( this ) {
+        this.alarm = alarm;
+        this.period = period;
+        this.callback = callback;
+        set = false;
+        type = Callback;
 }
 
@@ -71 +74 @@
 }
 
-#if !defined(NDEBUG) && (defined(__CFA_DEBUG__) || defined(__CFA_VERIFY__))
-bool validate( alarm_list_t * this ) {
-        alarm_node_t ** it = &this->head;
-        while( (*it) ) {
-                it = &(*it)->next;
+void insert( alarm_list_t * this, alarm_node_t * n ) {
+        alarm_node_t * it = & (*this)`first;
+        while( it && (n->alarm > it->alarm) ) {
+                it = & (*it)`next;
+        }
+        if ( it ) {
+                insert_before( *it, *n );
+        } else {
+                insert_last(*this, *n);
         }
 
-        return it == this->tail;
-}
-#endif
-
-static inline void insert_at( alarm_list_t * this, alarm_node_t * n, __alarm_it_t p ) {
-        verify( !n->next );
-        if( p == this->tail ) {
-                this->tail = &n->next;
-        }
-        else {
-                n->next = *p;
-        }
-        *p = n;
-
-        verify( validate( this ) );
-}
-
-void insert( alarm_list_t * this, alarm_node_t * n ) {
-        alarm_node_t ** it = &this->head;
-        while( (*it) && (n->alarm > (*it)->alarm) ) {
-                it = &(*it)->next;
-        }
-
-        insert_at( this, n, it );
-
-        verify( validate( this ) );
+        verify( validate( *this ) );
 }
 
 alarm_node_t * pop( alarm_list_t * this ) {
-        alarm_node_t * head = this->head;
+        verify( validate( *this ) );
+        alarm_node_t * head = & (*this)`first;
         if( head ) {
-                this->head = head->next;
-                if( !head->next ) {
-                        this->tail = &this->head;
-                }
-                head->next = 0p;
+                remove(*head);
         }
-        verify( validate( this ) );
+        verify( validate( *this ) );
         return head;
 }
 
-static inline void remove_at( alarm_list_t * this, alarm_node_t * n, __alarm_it_t it ) {
-        verify( it );
-        verify( (*it) == n );
-
-        (*it) = n->next;
-        if( !n-> next ) {
-                this->tail = it;
-        }
-        n->next = 0p;
-
-        verify( validate( this ) );
-}
-
-static inline void remove( alarm_list_t * this, alarm_node_t * n ) {
-        alarm_node_t ** it = &this->head;
-        while( (*it) && (*it) != n ) {
-                it = &(*it)->next;
-        }
-
-        verify( validate( this ) );
-
-        if( *it ) { remove_at( this, n, it ); }
-
-        verify( validate( this ) );
-}
-
 void register_self( alarm_node_t * this ) {
-        alarm_list_t * alarms = &event_kernel->alarms;
+        alarm_list_t & alarms = event_kernel->alarms;
 
         disable_interrupts();
@@ -152 +105 @@
         {
                 verify( validate( alarms ) );
-                bool first = !alarms->head;
+                bool first = ! & alarms`first;
 
-                insert( alarms, this );
+                insert( &alarms, this );
                 if( first ) {
-                        __kernel_set_timer( alarms->head->alarm - __kernel_get_time() );
+                        __kernel_set_timer( alarms`first.alarm - __kernel_get_time() );
                 }
         }
@@ -168 +121 @@
         lock( event_kernel->lock __cfaabi_dbg_ctx2 );
         {
-                verify( validate( &event_kernel->alarms ) );
-                remove( &event_kernel->alarms, this );
+                verify( validate( event_kernel->alarms ) );
+                remove( *this );
         }
         unlock( event_kernel->lock );
@@ -176 +129 @@
 }
 
+//=============================================================================================
+// Utilities
+//=============================================================================================
+
+void sleep( Duration duration ) {
+        alarm_node_t node = { active_thread(), __kernel_get_time() + duration, 0`s };
+
+        register_self( &node );
+        park();
+
+        /* paranoid */ verify( !node.set );
+        /* paranoid */ verify( & node`next == 0p );
+        /* paranoid */ verify( & node`prev == 0p );
+}
+
 // Local Variables: //
 // mode: c //

libcfa/src/concurrency/alarm.hfa

@@ -23 +23 @@
 #include "time.hfa"
 
-struct thread_desc;
+#include "containers/list.hfa"
+
+struct $thread;
 struct processor;
 
@@ -37 +39 @@
 //=============================================================================================
 
+enum alarm_type{ Kernel = 0, User = 1, Callback = 2 };
+
+struct alarm_node_t;
+
+typedef void (*Alarm_Callback)(alarm_node_t & );
+
 struct alarm_node_t {
         Time alarm;                             // time when alarm goes off
         Duration period;                        // if > 0 => period of alarm
-        alarm_node_t * next;            // intrusive link list field
+
+        DLISTED_MGD_IMPL_IN(alarm_node_t)
 
         union {
-                thread_desc * thrd;     // thrd who created event
-                processor * proc;               // proc who created event
+                $thread * thrd;                                 // thrd who created event
+                processor * proc;                               // proc who created event
+                Alarm_Callback callback;                // callback to handle event
         };
 
         bool set                :1;             // whether or not the alarm has be registered
-        bool kernel_alarm       :1;             // true if this is not a user defined alarm
+        enum alarm_type type;           // true if this is not a user defined alarm
 };
+DLISTED_MGD_IMPL_OUT(alarm_node_t)
 
-typedef alarm_node_t ** __alarm_it_t;
-
-void ?{}( alarm_node_t & this, thread_desc * thrd, Time alarm, Duration period );
+void ?{}( alarm_node_t & this, $thread * thrd, Time alarm, Duration period );
 void ?{}( alarm_node_t & this, processor   * proc, Time alarm, Duration period );
+void ?{}( alarm_node_t & this, Alarm_Callback callback, Time alarm, Duration period );
 void ^?{}( alarm_node_t & this );
 
-struct alarm_list_t {
-        alarm_node_t * head;
-        __alarm_it_t tail;
-};
-
-static inline void ?{}( alarm_list_t & this ) with( this ) {
-        head = 0;
-        tail = &head;
-}
+typedef dlist(alarm_node_t, alarm_node_t) alarm_list_t;
 
 void insert( alarm_list_t * this, alarm_node_t * n );

libcfa/src/concurrency/coroutine.cfa

@@ -10 +10 @@
 // Created On       : Mon Nov 28 12:27:26 2016
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Thu Dec  5 14:37:29 2019
-// Update Count     : 15
+// Last Modified On : Tue Dec 15 12:06:04 2020
+// Update Count     : 23
 //
 
@@ -18 +18 @@
 #include "coroutine.hfa"
 
-extern "C" {
 #include <stddef.h>
 #include <malloc.h>
@@ -24 +23 @@
 #include <string.h>
 #include <unistd.h>
-// use this define to make unwind.h play nice, definetely a hack
-#define HIDE_EXPORTS
+#include <sys/mman.h>                                                                   // mprotect
 #include <unwind.h>
-#undef HIDE_EXPORTS
-#include <sys/mman.h>
-}
 
 #include "kernel_private.hfa"
+#include "exception.hfa"
+#include "math.hfa"
+
+#define CFA_COROUTINE_USE_MMAP 0
 
 #define __CFA_INVOKE_PRIVATE__
@@ -37 +36 @@
 
 extern "C" {
-        void _CtxCoroutine_Unwind(struct _Unwind_Exception * storage, struct coroutine_desc *) __attribute__ ((__noreturn__));
+        void _CtxCoroutine_Unwind(struct _Unwind_Exception * storage, struct $coroutine *) __attribute__ ((__noreturn__));
         static void _CtxCoroutine_UnwindCleanup(_Unwind_Reason_Code, struct _Unwind_Exception *) __attribute__ ((__noreturn__));
         static void _CtxCoroutine_UnwindCleanup(_Unwind_Reason_Code, struct _Unwind_Exception *) {
@@ -47 +46 @@
 
 //-----------------------------------------------------------------------------
+FORALL_DATA_INSTANCE(CoroutineCancelled, (dtype coroutine_t), (coroutine_t))
+
+forall(dtype T)
+void mark_exception(CoroutineCancelled(T) *) {}
+
+forall(dtype T)
+void copy(CoroutineCancelled(T) * dst, CoroutineCancelled(T) * src) {
+        dst->virtual_table = src->virtual_table;
+        dst->the_coroutine = src->the_coroutine;
+        dst->the_exception = src->the_exception;
+}
+
+forall(dtype T)
+const char * msg(CoroutineCancelled(T) *) {
+        return "CoroutineCancelled(...)";
+}
+
+// This code should not be inlined. It is the error path on resume.
+forall(dtype T | is_coroutine(T))
+void __cfaehm_cancelled_coroutine( T & cor, $coroutine * desc ) {
+        verify( desc->cancellation );
+        desc->state = Cancelled;
+        exception_t * except = __cfaehm_cancellation_exception( desc->cancellation );
+
+        // TODO: Remove explitate vtable set once trac#186 is fixed.
+        CoroutineCancelled(T) except;
+        except.virtual_table = &get_exception_vtable(&except);
+        except.the_coroutine = &cor;
+        except.the_exception = except;
+        throwResume except;
+
+        except->virtual_table->free( except );
+        free( desc->cancellation );
+        desc->cancellation = 0p;
+}
+
+//-----------------------------------------------------------------------------
 // Global state variables
 
 // minimum feasible stack size in bytes
-#define MinStackSize 1000
+static const size_t MinStackSize = 1000;
 extern size_t __page_size;                              // architecture pagesize HACK, should go in proper runtime singleton
+extern int __map_prot;
 
 void __stack_prepare( __stack_info_t * this, size_t create_size );
+void __stack_clean  ( __stack_info_t * this );
 
 //-----------------------------------------------------------------------------
@@ -74 +112 @@
         bool userStack = ((intptr_t)this.storage & 0x1) != 0;
         if ( ! userStack && this.storage ) {
-                __attribute__((may_alias)) intptr_t * istorage = (intptr_t *)&this.storage;
-                *istorage &= (intptr_t)-1;
-
-                void * storage = this.storage->limit;
-                __cfaabi_dbg_debug_do(
-                        storage = (char*)(storage) - __page_size;
-                        if ( mprotect( storage, __page_size, PROT_READ | PROT_WRITE ) == -1 ) {
-                                abort( "(coStack_t *)%p.^?{}() : internal error, mprotect failure, error(%d) %s.", &this, errno, strerror( errno ) );
-                        }
-                );
-                __cfaabi_dbg_print_safe("Kernel : Deleting stack %p\n", storage);
-                free( storage );
-        }
-}
-
-void ?{}( coroutine_desc & this, const char * name, void * storage, size_t storageSize ) with( this ) {
+                __stack_clean( &this );
+        }
+}
+
+void ?{}( $coroutine & this, const char name[], void * storage, size_t storageSize ) with( this ) {
         (this.context){0p, 0p};
         (this.stack){storage, storageSize};
@@ -99 +126 @@
 }
 
-void ^?{}(coroutine_desc& this) {
+void ^?{}($coroutine& this) {
         if(this.state != Halted && this.state != Start && this.state != Primed) {
-                coroutine_desc * src = TL_GET( this_thread )->curr_cor;
-                coroutine_desc * dst = &this;
+                $coroutine * src = active_coroutine();
+                $coroutine * dst = &this;
 
                 struct _Unwind_Exception storage;
@@ -115 +142 @@
                 }
 
-                CoroutineCtxSwitch( src, dst );
+                $ctx_switch( src, dst );
         }
 }
@@ -123 +150 @@
 forall(dtype T | is_coroutine(T))
 void prime(T& cor) {
-        coroutine_desc* this = get_coroutine(cor);
+        $coroutine* this = get_coroutine(cor);
         assert(this->state == Start);
 
@@ -134 +161 @@
         assert(__page_size != 0l);
         size_t size = libCeiling( storageSize, 16 ) + stack_data_size;
+        size = ceiling(size, __page_size);
 
         // If we are running debug, we also need to allocate a guardpage to catch stack overflows.
         void * storage;
-        __cfaabi_dbg_debug_do(
-                storage = memalign( __page_size, size + __page_size );
-        );
-        __cfaabi_dbg_no_debug_do(
-                storage = (void*)malloc(size);
-        );
-
+        #if CFA_COROUTINE_USE_MMAP
+                storage = mmap(0p, size + __page_size, PROT_EXEC | PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
+                if(storage == ((void*)-1)) {
+                        abort( "coroutine stack creation : internal error, mmap failure, error(%d) %s.", errno, strerror( errno ) );
+                }
+                if ( mprotect( storage, __page_size, PROT_NONE ) == -1 ) {
+                        abort( "coroutine stack creation : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
+                } // if
+                storage = (void *)(((intptr_t)storage) + __page_size);
+        #else
+                __cfaabi_dbg_debug_do(
+                        storage = memalign( __page_size, size + __page_size );
+                );
+                __cfaabi_dbg_no_debug_do(
+                        storage = (void*)malloc(size);
+                );
+
+                __cfaabi_dbg_debug_do(
+                        if ( mprotect( storage, __page_size, PROT_NONE ) == -1 ) {
+                                abort( "__stack_alloc : internal error, mprotect failure, error(%d) %s.", (int)errno, strerror( (int)errno ) );
+                        }
+                        storage = (void *)(((intptr_t)storage) + __page_size);
+                );
+        #endif
         __cfaabi_dbg_print_safe("Kernel : Created stack %p of size %zu\n", storage, size);
-        __cfaabi_dbg_debug_do(
-                if ( mprotect( storage, __page_size, PROT_NONE ) == -1 ) {
-                        abort( "__stack_alloc : internal error, mprotect failure, error(%d) %s.", (int)errno, strerror( (int)errno ) );
-                }
-                storage = (void *)(((intptr_t)storage) + __page_size);
-        );
 
         verify( ((intptr_t)storage & (libAlign() - 1)) == 0ul );
         return [storage, size];
+}
+
+void __stack_clean  ( __stack_info_t * this ) {
+        size_t size = ((intptr_t)this->storage->base) - ((intptr_t)this->storage->limit) + sizeof(__stack_t);
+        void * storage = this->storage->limit;
+
+        #if CFA_COROUTINE_USE_MMAP
+                storage = (void *)(((intptr_t)storage) - __page_size);
+                if(munmap(storage, size + __page_size) == -1) {
+                        abort( "coroutine stack destruction : internal error, munmap failure, error(%d) %s.", errno, strerror( errno ) );
+                }
+        #else
+                __cfaabi_dbg_debug_do(
+                        storage = (char*)(storage) - __page_size;
+                        if ( mprotect( storage, __page_size, __map_prot ) == -1 ) {
+                                abort( "(coStack_t *)%p.^?{}() : internal error, mprotect failure, error(%d) %s.", &this, errno, strerror( errno ) );
+                        }
+                );
+
+                free( storage );
+        #endif
+        __cfaabi_dbg_print_safe("Kernel : Deleting stack %p\n", storage);
 }
 
@@ -175 +236 @@
                 size = libFloor(create_size - stack_data_size - diff, libAlign());
         } // if
-        assertf( size >= MinStackSize, "Stack size %zd provides less than minimum of %d bytes for a stack.", size, MinStackSize );
-
-        this->storage = (__stack_t *)((intptr_t)storage + size);
+        assertf( size >= MinStackSize, "Stack size %zd provides less than minimum of %zd bytes for a stack.", size, MinStackSize );
+
+        this->storage = (__stack_t *)((intptr_t)storage + size - sizeof(__stack_t));
         this->storage->limit = storage;
-        this->storage->base  = (void*)((intptr_t)storage + size);
+        this->storage->base  = (void*)((intptr_t)storage + size - sizeof(__stack_t));
+        this->storage->exception_context.top_resume = 0p;
+        this->storage->exception_context.current_exception = 0p;
         __attribute__((may_alias)) intptr_t * istorage = (intptr_t*)&this->storage;
         *istorage |= userStack ? 0x1 : 0x0;
@@ -187 +250 @@
 // is not inline (We can't inline Cforall in C)
 extern "C" {
-        void __leave_coroutine( struct coroutine_desc * src ) {
-                coroutine_desc * starter = src->cancellation != 0 ? src->last : src->starter;
+        void __cfactx_cor_leave( struct $coroutine * src ) {
+                $coroutine * starter = src->cancellation != 0 ? src->last : src->starter;
 
                 src->state = Halted;
@@ -201 +264 @@
                         src->name, src, starter->name, starter );
 
-                CoroutineCtxSwitch( src, starter );
-        }
-
-        struct coroutine_desc * __finish_coroutine(void) {
-                struct coroutine_desc * cor = kernelTLS.this_thread->curr_cor;
+                $ctx_switch( src, starter );
+        }
+
+        struct $coroutine * __cfactx_cor_finish(void) {
+                struct $coroutine * cor = active_coroutine();
 
                 if(cor->state == Primed) {
-                        suspend();
+                        __cfactx_suspend();
                 }
 

libcfa/src/concurrency/coroutine.hfa

@@ -10 +10 @@
 // Created On       : Mon Nov 28 12:27:26 2016
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Tue Dec  3 22:47:58 2019
-// Update Count     : 10
+// Last Modified On : Tue Feb  4 12:29:26 2020
+// Update Count     : 11
 //
 
@@ -18 +18 @@
 #include <assert.h>
 #include "invoke.h"
+#include "../exception.hfa"
+
+//-----------------------------------------------------------------------------
+// Exception thrown from resume when a coroutine stack is cancelled.
+FORALL_DATA_EXCEPTION(CoroutineCancelled, (dtype coroutine_t), (coroutine_t)) (
+        coroutine_t * the_coroutine;
+        exception_t * the_exception;
+);
+
+forall(dtype T)
+void copy(CoroutineCancelled(T) * dst, CoroutineCancelled(T) * src);
+
+forall(dtype T)
+const char * msg(CoroutineCancelled(T) *);
 
 //-----------------------------------------------------------------------------
@@ -23 +37 @@
 // Anything that implements this trait can be resumed.
 // Anything that is resumed is a coroutine.
-trait is_coroutine(dtype T) {
-      void main(T & this);
-      coroutine_desc * get_coroutine(T & this);
+trait is_coroutine(dtype T | IS_RESUMPTION_EXCEPTION(CoroutineCancelled, (T))) {
+        void main(T & this);
+        $coroutine * get_coroutine(T & this);
 };
 
-#define DECL_COROUTINE(X) static inline coroutine_desc* get_coroutine(X& this) { return &this.__cor; } void main(X& this)
+#define DECL_COROUTINE(X) static inline $coroutine* get_coroutine(X& this) { return &this.__cor; } void main(X& this)
 
 //-----------------------------------------------------------------------------
@@ -35 +49 @@
 // void ^?{}( coStack_t & this );
 
-void ?{}( coroutine_desc & this, const char * name, void * storage, size_t storageSize );
-void ^?{}( coroutine_desc & this );
+void  ?{}( $coroutine & this, const char name[], void * storage, size_t storageSize );
+void ^?{}( $coroutine & this );
 
-static inline void ?{}( coroutine_desc & this)                                       { this{ "Anonymous Coroutine", 0p, 0 }; }
-static inline void ?{}( coroutine_desc & this, size_t stackSize)                     { this{ "Anonymous Coroutine", 0p, stackSize }; }
-static inline void ?{}( coroutine_desc & this, void * storage, size_t storageSize )  { this{ "Anonymous Coroutine", storage, storageSize }; }
-static inline void ?{}( coroutine_desc & this, const char * name)                    { this{ name, 0p, 0 }; }
-static inline void ?{}( coroutine_desc & this, const char * name, size_t stackSize ) { this{ name, 0p, stackSize }; }
+static inline void ?{}( $coroutine & this)                                       { this{ "Anonymous Coroutine", 0p, 0 }; }
+static inline void ?{}( $coroutine & this, size_t stackSize)                     { this{ "Anonymous Coroutine", 0p, stackSize }; }
+static inline void ?{}( $coroutine & this, void * storage, size_t storageSize )  { this{ "Anonymous Coroutine", storage, storageSize }; }
+static inline void ?{}( $coroutine & this, const char name[])                    { this{ name, 0p, 0 }; }
+static inline void ?{}( $coroutine & this, const char name[], size_t stackSize ) { this{ name, 0p, stackSize }; }
 
 //-----------------------------------------------------------------------------
 // Public coroutine API
-static inline void suspend(void);
-
-forall(dtype T | is_coroutine(T))
-static inline T & resume(T & cor);
-
 forall(dtype T | is_coroutine(T))
 void prime(T & cor);
 
-static inline struct coroutine_desc * active_coroutine() { return TL_GET( this_thread )->curr_cor; }
+static inline struct $coroutine * active_coroutine() { return active_thread()->curr_cor; }
 
 //-----------------------------------------------------------------------------
@@ -61 +70 @@
 // Start coroutine routines
 extern "C" {
-        void CtxInvokeCoroutine(void (*main)(void *), void * this);
+        void __cfactx_invoke_coroutine(void (*main)(void *), void * this);
 
         forall(dtype T)
-        void CtxStart(void (*main)(T &), struct coroutine_desc * cor, T & this, void (*invoke)(void (*main)(void *), void *));
+        void __cfactx_start(void (*main)(T &), struct $coroutine * cor, T & this, void (*invoke)(void (*main)(void *), void *));
 
-        extern void _CtxCoroutine_Unwind(struct _Unwind_Exception * storage, struct coroutine_desc *) __attribute__ ((__noreturn__));
+        extern void __cfactx_coroutine_unwind(struct _Unwind_Exception * storage, struct $coroutine *) __attribute__ ((__noreturn__));
 
-        extern void CtxSwitch( struct __stack_context_t * from, struct __stack_context_t * to ) asm ("CtxSwitch");
+        extern void __cfactx_switch( struct __stack_context_t * from, struct __stack_context_t * to ) asm ("__cfactx_switch");
 }
 
 // Private wrappers for context switch and stack creation
 // Wrapper for co
-static inline void CoroutineCtxSwitch(coroutine_desc* src, coroutine_desc* dst) {
+static inline void $ctx_switch( $coroutine * src, $coroutine * dst ) __attribute__((nonnull (1, 2))) {
         // set state of current coroutine to inactive
-        src->state = src->state == Halted ? Halted : Inactive;
+        src->state = src->state == Halted ? Halted : Blocked;
 
         // set new coroutine that task is executing
-        TL_GET( this_thread )->curr_cor = dst;
+        active_thread()->curr_cor = dst;
 
         // context switch to specified coroutine
         verify( dst->context.SP );
-        CtxSwitch( &src->context, &dst->context );
-        // when CtxSwitch returns we are back in the src coroutine
+        __cfactx_switch( &src->context, &dst->context );
+        // when __cfactx_switch returns we are back in the src coroutine
 
         // set state of new coroutine to active
@@ -89 +98 @@
 
         if( unlikely(src->cancellation != 0p) ) {
-                _CtxCoroutine_Unwind(src->cancellation, src);
+                __cfactx_coroutine_unwind(src->cancellation, src);
         }
 }
 
-extern void __stack_prepare   ( __stack_info_t * this, size_t size /* ignored if storage already allocated */);
+extern void __stack_prepare( __stack_info_t * this, size_t size /* ignored if storage already allocated */);
+extern void __stack_clean  ( __stack_info_t * this );
+
 
 // Suspend implementation inlined for performance
-static inline void suspend(void) {
-        // optimization : read TLS once and reuse it
-        // Safety note: this is preemption safe since if
-        // preemption occurs after this line, the pointer
-        // will also migrate which means this value will
-        // stay in syn with the TLS
-        coroutine_desc * src = TL_GET( this_thread )->curr_cor;
+extern "C" {
+        static inline void __cfactx_suspend(void) {
+                // optimization : read TLS once and reuse it
+                // Safety note: this is preemption safe since if
+                // preemption occurs after this line, the pointer
+                // will also migrate which means this value will
+                // stay in syn with the TLS
+                $coroutine * src = active_coroutine();
 
-        assertf( src->last != 0,
-                "Attempt to suspend coroutine \"%.256s\" (%p) that has never been resumed.\n"
-                "Possible cause is a suspend executed in a member called by a coroutine user rather than by the coroutine main.",
-                src->name, src );
-        assertf( src->last->state != Halted,
-                "Attempt by coroutine \"%.256s\" (%p) to suspend back to terminated coroutine \"%.256s\" (%p).\n"
-                "Possible cause is terminated coroutine's main routine has already returned.",
-                src->name, src, src->last->name, src->last );
+                assertf( src->last != 0,
+                        "Attempt to suspend coroutine \"%.256s\" (%p) that has never been resumed.\n"
+                        "Possible cause is a suspend executed in a member called by a coroutine user rather than by the coroutine main.",
+                        src->name, src );
+                assertf( src->last->state != Halted,
+                        "Attempt by coroutine \"%.256s\" (%p) to suspend back to terminated coroutine \"%.256s\" (%p).\n"
+                        "Possible cause is terminated coroutine's main routine has already returned.",
+                        src->name, src, src->last->name, src->last );
 
-        CoroutineCtxSwitch( src, src->last );
+                $ctx_switch( src, src->last );
+        }
 }
+
+forall(dtype T | is_coroutine(T))
+void __cfaehm_cancelled_coroutine( T & cor, $coroutine * desc );
 
 // Resume implementation inlined for performance
@@ -124 +140 @@
         // will also migrate which means this value will
         // stay in syn with the TLS
-        coroutine_desc * src = TL_GET( this_thread )->curr_cor;
-        coroutine_desc * dst = get_coroutine(cor);
+        $coroutine * src = active_coroutine();
+        $coroutine * dst = get_coroutine(cor);
 
         if( unlikely(dst->context.SP == 0p) ) {
-                TL_GET( this_thread )->curr_cor = dst;
                 __stack_prepare(&dst->stack, 65000);
-                CtxStart(main, dst, cor, CtxInvokeCoroutine);
-                TL_GET( this_thread )->curr_cor = src;
+                __cfactx_start(main, dst, cor, __cfactx_invoke_coroutine);
         }
 
@@ -147 +161 @@
 
         // always done for performance testing
-        CoroutineCtxSwitch( src, dst );
+        $ctx_switch( src, dst );
+        if ( unlikely(dst->cancellation) ) {
+                __cfaehm_cancelled_coroutine( cor, dst );
+        }
 
         return cor;
 }
 
-static inline void resume(coroutine_desc * dst) {
+static inline void resume( $coroutine * dst ) __attribute__((nonnull (1))) {
         // optimization : read TLS once and reuse it
         // Safety note: this is preemption safe since if
@@ -158 +175 @@
         // will also migrate which means this value will
         // stay in syn with the TLS
-        coroutine_desc * src = TL_GET( this_thread )->curr_cor;
+        $coroutine * src = active_coroutine();
 
         // not resuming self ?
@@ -172 +189 @@
 
         // always done for performance testing
-        CoroutineCtxSwitch( src, dst );
+        $ctx_switch( src, dst );
 }
 

libcfa/src/concurrency/invoke.c

@@ -10 +10 @@
 // Created On       : Tue Jan 17 12:27:26 2016
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Fri Feb  9 16:37:42 2018
-// Update Count     : 5
+// Last Modified On : Sat Oct 24 14:35:28 2020
+// Update Count     : 32
 //
 
@@ -29 +29 @@
 // Called from the kernel when starting a coroutine or task so must switch back to user mode.
 
-extern void __leave_coroutine ( struct coroutine_desc * );
-extern struct coroutine_desc * __finish_coroutine(void);
-extern void __leave_thread_monitor();
+extern struct $coroutine * __cfactx_cor_finish(void);
+extern void __cfactx_cor_leave ( struct $coroutine * );
+extern void __cfactx_thrd_leave();
+
 extern void disable_interrupts() OPTIONAL_THREAD;
 extern void enable_interrupts( __cfaabi_dbg_ctx_param );
 
-void CtxInvokeCoroutine(
+void __cfactx_invoke_coroutine(
         void (*main)(void *),
         void *this
 ) {
         // Finish setting up the coroutine by setting its state
-        struct coroutine_desc * cor = __finish_coroutine();
+        struct $coroutine * cor = __cfactx_cor_finish();
 
         // Call the main of the coroutine
@@ -46 +47 @@
 
         //Final suspend, should never return
-        __leave_coroutine( cor );
+        __cfactx_cor_leave( cor );
         __cabi_abort( "Resumed dead coroutine" );
 }
 
-static _Unwind_Reason_Code _CtxCoroutine_UnwindStop(
+static _Unwind_Reason_Code __cfactx_coroutine_unwindstop(
         __attribute((__unused__)) int version,
         _Unwind_Action actions,
@@ -61 +62 @@
                 // We finished unwinding the coroutine,
                 // leave it
-                __leave_coroutine( param );
+                __cfactx_cor_leave( param );
                 __cabi_abort( "Resumed dead coroutine" );
         }
@@ -69 +70 @@
 }
 
-void _CtxCoroutine_Unwind(struct _Unwind_Exception * storage, struct coroutine_desc * cor) __attribute__ ((__noreturn__));
-void _CtxCoroutine_Unwind(struct _Unwind_Exception * storage, struct coroutine_desc * cor) {
-        _Unwind_Reason_Code ret = _Unwind_ForcedUnwind( storage, _CtxCoroutine_UnwindStop, cor );
+void __cfactx_coroutine_unwind(struct _Unwind_Exception * storage, struct $coroutine * cor) __attribute__ ((__noreturn__));
+void __cfactx_coroutine_unwind(struct _Unwind_Exception * storage, struct $coroutine * cor) {
+        _Unwind_Reason_Code ret = _Unwind_ForcedUnwind( storage, __cfactx_coroutine_unwindstop, cor );
         printf("UNWIND ERROR %d after force unwind\n", ret);
         abort();
 }
 
-void CtxInvokeThread(
+void __cfactx_invoke_thread(
         void (*main)(void *),
         void *this
@@ -93 +94 @@
         // The order of these 4 operations is very important
         //Final suspend, should never return
-        __leave_thread_monitor();
+        __cfactx_thrd_leave();
         __cabi_abort( "Resumed dead thread" );
 }
 
-void CtxStart(
+void __cfactx_start(
         void (*main)(void *),
-        struct coroutine_desc * cor,
+        struct $coroutine * cor,
         void *this,
         void (*invoke)(void *)
@@ -108 +109 @@
 
         struct FakeStack {
-            void *fixedRegisters[3];              // fixed registers ebx, edi, esi (popped on 1st uSwitch, values unimportant)
-            void *rturn;                          // where to go on return from uSwitch
-            void *dummyReturn;                    // fake return compiler would have pushed on call to uInvoke
-            void *argument[3];                    // for 16-byte ABI, 16-byte alignment starts here
-            void *padding;                        // padding to force 16-byte alignment, as "base" is 16-byte aligned
+            void *fixedRegisters[3];                                            // fixed registers ebx, edi, esi (popped on 1st uSwitch, values unimportant)
+            void *rturn;                                                                        // where to go on return from uSwitch
+            void *dummyReturn;                                                          // fake return compiler would have pushed on call to uInvoke
+            void *argument[3];                                                          // for 16-byte ABI, 16-byte alignment starts here
+            void *padding;                                                                      // padding to force 16-byte alignment, as "base" is 16-byte aligned
         };
 
@@ -121 +122 @@
 
         fs->dummyReturn = NULL;
-        fs->argument[0] = main;     // argument to invoke
-        fs->argument[1] = this;     // argument to invoke
+        fs->argument[0] = main;                                                         // argument to invoke
+        fs->argument[1] = this;                                                         // argument to invoke
         fs->rturn = invoke;
 
@@ -128 +129 @@
 
         struct FakeStack {
-                void *fixedRegisters[5];            // fixed registers rbx, r12, r13, r14, r15
-                void *rturn;                        // where to go on return from uSwitch
-                void *dummyReturn;                  // NULL return address to provide proper alignment
+                void *fixedRegisters[5];                                                // fixed registers rbx, r12, r13, r14, r15
+                void *rturn;                                                                    // where to go on return from uSwitch
+                void *dummyReturn;                                                              // NULL return address to provide proper alignment
         };
 
         cor->context.SP = (char *)stack->base - sizeof( struct FakeStack );
-        cor->context.FP = NULL;         // terminate stack with NULL fp
+        cor->context.FP = NULL;                                                         // terminate stack with NULL fp
 
         struct FakeStack *fs = (struct FakeStack *)cor->context.SP;
 
         fs->dummyReturn = NULL;
-        fs->rturn = CtxInvokeStub;
-        fs->fixedRegisters[0] = main;
-        fs->fixedRegisters[1] = this;
+        fs->rturn = __cfactx_invoke_stub;
+        fs->fixedRegisters[0] = main;                                           // argument to invoke
+        fs->fixedRegisters[1] = this;                                           // argument to invoke
         fs->fixedRegisters[2] = invoke;
 
-#elif defined( __ARM_ARCH )
-#error ARM needs to be upgrade to use to parameters like X86/X64 (A.K.A. : I broke this and do not know how to fix it)
+#elif defined( __ARM_ARCH_32 )
+#error ARM needs to be upgrade to use two parameters like X86/X64 (A.K.A. : I broke this and do not know how to fix it)
+        // More details about the error:
+        // To avoid the thunk problem, I changed the invoke routine to pass the main explicitly
+        // instead of relying on an assertion. This effectively hoists any required thunk one level
+        // which was enough to get to global scope in most cases.
+        // This means that __cfactx_invoke_... now takes two parameters and the FakeStack needs
+        // to be adjusted as a consequence of that.
+        // I don't know how to do that for ARM, hence the #error
+
         struct FakeStack {
-                float fpRegs[16];                       // floating point registers
-                void *intRegs[9];                       // integer/pointer registers
-                void *arg[2];                           // placeholder for this pointer
+                float fpRegs[16];                                                               // floating point registers
+                void * intRegs[9];                                                              // integer/pointer registers
+                void * arg[2];                                                                  // placeholder for this pointer
         };
 
@@ -157 +166 @@
         struct FakeStack *fs = (struct FakeStack *)cor->context.SP;
 
-        fs->intRegs[8] = CtxInvokeStub;
+        fs->intRegs[8] = __cfactx_invoke_stub;
         fs->arg[0] = this;
         fs->arg[1] = invoke;
 
+#elif defined( __ARM_ARCH )
+        struct FakeStack {
+                void * intRegs[12];                                                             // x19-x30 integer registers
+                double fpRegs[8];                                                               // v8-v15 floating point
+        };
+
+        cor->context.SP = (char *)stack->base - sizeof( struct FakeStack );
+        cor->context.FP = NULL;
+
+        struct FakeStack *fs = (struct FakeStack *)cor->context.SP;
+
+        fs->intRegs[0] = main;                                                          // argument to invoke x19 => x0
+        fs->intRegs[1] = this;                                                          // argument to invoke x20 => x1
+        fs->intRegs[2] = invoke;
+        fs->intRegs[11] = __cfactx_invoke_stub;                         // link register x30 => ret moves to pc
 #else
         #error uknown hardware architecture

libcfa/src/concurrency/invoke.h

@@ -17 +17 @@
 #include "bits/defs.hfa"
 #include "bits/locks.hfa"
+#include "kernel/fwd.hfa"
 
 #ifdef __cforall
@@ -26 +27 @@
 #define _INVOKE_H_
 
-#ifdef __ARM_ARCH
-        // function prototypes are only really used by these macros on ARM
-        void disable_global_interrupts();
-        void enable_global_interrupts();
-
-        #define TL_GET( member ) ( { __typeof__( kernelTLS.member ) target; \
-                disable_global_interrupts(); \
-                target = kernelTLS.member; \
-                enable_global_interrupts(); \
-                target; } )
-        #define TL_SET( member, value ) disable_global_interrupts(); \
-                kernelTLS.member = value; \
-                enable_global_interrupts();
-#else
-        #define TL_GET( member ) kernelTLS.member
-        #define TL_SET( member, value ) kernelTLS.member = value;
-#endif
-
-        #ifdef __cforall
-        extern "Cforall" {
-                extern __attribute__((aligned(128))) thread_local struct KernelThreadData {
-                        struct thread_desc    * volatile this_thread;
-                        struct processor      * volatile this_processor;
-
-                        struct {
-                                volatile unsigned short disable_count;
-                                volatile bool enabled;
-                                volatile bool in_progress;
-                        } preemption_state;
-
-                        uint32_t rand_seed;
-                } kernelTLS __attribute__ ((tls_model ( "initial-exec" )));
-        }
-        #endif
+        struct __cfaehm_try_resume_node;
+        struct __cfaehm_base_exception_t;
+        struct exception_context_t {
+                struct __cfaehm_try_resume_node * top_resume;
+                struct __cfaehm_base_exception_t * current_exception;
+        };
 
         struct __stack_context_t {
@@ -85 +58 @@
                 // base of stack
                 void * base;
+
+                // Information for exception handling.
+                struct exception_context_t exception_context;
         };
 
@@ -92 +68 @@
         };
 
-        enum coroutine_state { Halted, Start, Inactive, Active, Primed };
-
-        struct coroutine_desc {
-                // context that is switch during a CtxSwitch
+        enum __Coroutine_State { Halted, Start, Primed, Blocked, Ready, Active, Cancelled, Halting };
+
+        struct $coroutine {
+                // context that is switch during a __cfactx_switch
                 struct __stack_context_t context;
 
@@ -105 +81 @@
 
                 // current execution status for coroutine
-                enum coroutine_state state;
+                enum __Coroutine_State state;
 
                 // first coroutine to resume this one
-                struct coroutine_desc * starter;
+                struct $coroutine * starter;
 
                 // last coroutine to resume this one
-                struct coroutine_desc * last;
+                struct $coroutine * last;
 
                 // If non-null stack must be unwound with this exception
@@ -117 +93 @@
 
         };
+        // Wrapper for gdb
+        struct cfathread_coroutine_t { struct $coroutine debug; };
+
+        static inline struct __stack_t * __get_stack( struct $coroutine * cor ) {
+                return (struct __stack_t*)(((uintptr_t)cor->stack.storage) & ((uintptr_t)-2));
+        }
 
         // struct which calls the monitor is accepting
@@ -127 +109 @@
         };
 
-        struct monitor_desc {
+        struct $monitor {
                 // spinlock to protect internal data
                 struct __spinlock_t lock;
 
                 // current owner of the monitor
-                struct thread_desc * owner;
+                struct $thread * owner;
 
                 // queue of threads that are blocked waiting for the monitor
-                __queue_t(struct thread_desc) entry_queue;
+                __queue_t(struct $thread) entry_queue;
 
                 // stack of conditions to run next once we exit the monitor
@@ -149 +131 @@
                 struct __condition_node_t * dtor_node;
         };
+        // Wrapper for gdb
+        struct cfathread_monitor_t { struct $monitor debug; };
 
         struct __monitor_group_t {
                 // currently held monitors
-                __cfa_anonymous_object( __small_array_t(monitor_desc*) );
+                __cfa_anonymous_object( __small_array_t($monitor*) );
 
                 // last function that acquired monitors
@@ -158 +142 @@
         };
 
-        struct thread_desc {
+        // Link lists fields
+        // instrusive link field for threads
+        struct __thread_desc_link {
+                struct $thread * next;
+                struct $thread * prev;
+                volatile unsigned long long ts;
+                int preferred;
+        };
+
+        struct $thread {
                 // Core threading fields
-                // context that is switch during a CtxSwitch
+                // context that is switch during a __cfactx_switch
                 struct __stack_context_t context;
 
                 // current execution status for coroutine
-                enum coroutine_state state;
+                // Possible values are:
+                //    - TICKET_BLOCKED (-1) thread is blocked
+                //    - TICKET_RUNNING ( 0) thread is running
+                //    - TICKET_UNBLOCK ( 1) thread should ignore next block
+                volatile int ticket;
+                enum __Coroutine_State state:8;
+                enum __Preemption_Reason preempted:8;
 
                 //SKULLDUGGERY errno is not save in the thread data structure because returnToKernel appears to be the only function to require saving and restoring it
-
-                // coroutine body used to store context
-                struct coroutine_desc  self_cor;
-
-                // current active context
-                struct coroutine_desc * curr_cor;
-
-                // monitor body used for mutual exclusion
-                struct monitor_desc    self_mon;
-
-                // pointer to monitor with sufficient lifetime for current monitors
-                struct monitor_desc *  self_mon_p;
 
                 // pointer to the cluster on which the thread is running
                 struct cluster * curr_cluster;
 
+                // Link lists fields
+                // instrusive link field for threads
+                struct __thread_desc_link link;
+
+                // coroutine body used to store context
+                struct $coroutine  self_cor;
+
+                // current active context
+                struct $coroutine * curr_cor;
+
+                // monitor body used for mutual exclusion
+                struct $monitor    self_mon;
+
+                // pointer to monitor with sufficient lifetime for current monitors
+                struct $monitor *  self_mon_p;
+
                 // monitors currently held by this thread
                 struct __monitor_group_t monitors;
 
-                // Link lists fields
-                // instrusive link field for threads
-                struct thread_desc * next;
-
+                // used to put threads on user data structures
                 struct {
-                        struct thread_desc * next;
-                        struct thread_desc * prev;
+                        struct $thread * next;
+                        struct $thread * back;
+                } seqable;
+
+                struct {
+                        struct $thread * next;
+                        struct $thread * prev;
                 } node;
-        };
+
+                #if defined( __CFA_WITH_VERIFY__ )
+                        void * canary;
+                #endif
+        };
+        // Wrapper for gdb
+        struct cfathread_thread_t { struct $thread debug; };
+
+        #ifdef __CFA_DEBUG__
+                void __cfaabi_dbg_record_thrd($thread & this, bool park, const char prev_name[]);
+        #else
+                #define __cfaabi_dbg_record_thrd(x, y, z)
+        #endif
 
         #ifdef __cforall
         extern "Cforall" {
-                static inline thread_desc *& get_next( thread_desc & this ) {
-                        return this.next;
-                }
-
-                static inline [thread_desc *&, thread_desc *& ] __get( thread_desc & this ) {
+
+                static inline $thread *& get_next( $thread & this ) __attribute__((const)) {
+                        return this.link.next;
+                }
+
+                static inline [$thread *&, $thread *& ] __get( $thread & this ) __attribute__((const)) {
                         return this.node.[next, prev];
+                }
+
+                static inline $thread *& Back( $thread * this ) __attribute__((const)) {
+                        return this->seqable.back;
+                }
+
+                static inline $thread *& Next( $thread * this ) __attribute__((const)) {
+                        return this->seqable.next;
+                }
+
+                static inline bool listed( $thread * this ) {
+                        return this->seqable.next != 0p;
                 }
 
@@ -212 +242 @@
                 }
 
-                static inline void ?{}(__monitor_group_t & this, struct monitor_desc ** data, __lock_size_t size, fptr_t func) {
+                static inline void ?{}(__monitor_group_t & this, struct $monitor ** data, __lock_size_t size, fptr_t func) {
                         (this.data){data};
                         (this.size){size};
@@ -218 +248 @@
                 }
 
-                static inline bool ?==?( const __monitor_group_t & lhs, const __monitor_group_t & rhs ) {
+                static inline bool ?==?( const __monitor_group_t & lhs, const __monitor_group_t & rhs ) __attribute__((const)) {
                         if( (lhs.data != 0) != (rhs.data != 0) ) return false;
                         if( lhs.size != rhs.size ) return false;
@@ -252 +282 @@
 
         // assembler routines that performs the context switch
-        extern void CtxInvokeStub( void );
-        extern void CtxSwitch( struct __stack_context_t * from, struct __stack_context_t * to ) asm ("CtxSwitch");
+        extern void __cfactx_invoke_stub( void );
+        extern void __cfactx_switch( struct __stack_context_t * from, struct __stack_context_t * to ) asm ("__cfactx_switch");
         // void CtxStore ( void * this ) asm ("CtxStore");
         // void CtxRet   ( void * dst  ) asm ("CtxRet");

libcfa/src/concurrency/kernel.cfa

@@ -10 +10 @@
 // Created On       : Tue Jan 17 12:27:26 2017
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Thu Jan 30 22:55:50 2020
-// Update Count     : 56
+// Last Modified On : Mon Aug 31 07:08:20 2020
+// Update Count     : 71
 //
 
 #define __cforall_thread__
+// #define __CFA_DEBUG_PRINT_RUNTIME_CORE__
 
 //C Includes
-#include <stddef.h>
 #include <errno.h>
-#include <string.h>
-extern "C" {
 #include <stdio.h>
-#include <fenv.h>
-#include <sys/resource.h>
 #include <signal.h>
 #include <unistd.h>
-#include <limits.h>                                                                             // PTHREAD_STACK_MIN
-#include <sys/mman.h>                                                                   // mprotect
-}
 
 //CFA Includes
-#include "time.hfa"
 #include "kernel_private.hfa"
 #include "preemption.hfa"
-#include "startup.hfa"
 
 //Private includes
@@ -40 +31 @@
 #include "invoke.h"
 
+
 //-----------------------------------------------------------------------------
 // Some assembly required
 #if defined( __i386 )
-        #define CtxGet( ctx )        \
-                __asm__ volatile (     \
-                        "movl %%esp,%0\n"\
-                        "movl %%ebp,%1\n"\
-                        : "=rm" (ctx.SP),\
-                                "=rm" (ctx.FP) \
-                )
-
         // mxcr : SSE Status and Control bits (control bits are preserved across function calls)
         // fcw  : X87 FPU control word (preserved across function calls)
@@ -72 +56 @@
 
 #elif defined( __x86_64 )
-        #define CtxGet( ctx )        \
-                __asm__ volatile (     \
-                        "movq %%rsp,%0\n"\
-                        "movq %%rbp,%1\n"\
-                        : "=rm" (ctx.SP),\
-                                "=rm" (ctx.FP) \
-                )
-
         #define __x87_store         \
                 uint32_t __mxcr;      \
@@ -98 +74 @@
                 )
 
-
-#elif defined( __ARM_ARCH )
-#define CtxGet( ctx ) __asm__ ( \
-                "mov %0,%%sp\n"   \
-                "mov %1,%%r11\n"   \
-        : "=rm" (ctx.SP), "=rm" (ctx.FP) )
+#elif defined( __arm__ )
+        #define __x87_store
+        #define __x87_load
+
+#elif defined( __aarch64__ )
+        #define __x87_store              \
+                uint32_t __fpcntl[2];    \
+                __asm__ volatile (    \
+                        "mrs x9, FPCR\n" \
+                        "mrs x10, FPSR\n"  \
+                        "stp x9, x10, %0\n"  \
+                        : "=m" (__fpcntl) : : "x9", "x10" \
+                )
+
+        #define __x87_load         \
+                __asm__ volatile (    \
+                        "ldp x9, x10, %0\n"  \
+                        "msr FPSR, x10\n"  \
+                        "msr FPCR, x9\n" \
+                : "=m" (__fpcntl) : : "x9", "x10" \
+                )
+
 #else
-        #error unknown hardware architecture
+        #error unsupported hardware architecture
 #endif
 
+extern $thread * mainThread;
+extern processor * mainProcessor;
+
 //-----------------------------------------------------------------------------
-//Start and stop routine for the kernel, declared first to make sure they run first
-static void kernel_startup(void)  __attribute__(( constructor( STARTUP_PRIORITY_KERNEL ) ));
-static void kernel_shutdown(void) __attribute__(( destructor ( STARTUP_PRIORITY_KERNEL ) ));
-
-//-----------------------------------------------------------------------------
-// Kernel storage
-KERNEL_STORAGE(cluster,         mainCluster);
-KERNEL_STORAGE(processor,       mainProcessor);
-KERNEL_STORAGE(thread_desc,     mainThread);
-KERNEL_STORAGE(__stack_t,       mainThreadCtx);
-
-cluster     * mainCluster;
-processor   * mainProcessor;
-thread_desc * mainThread;
-
-extern "C" {
-        struct { __dllist_t(cluster) list; __spinlock_t lock; } __cfa_dbg_global_clusters;
-}
-
-size_t __page_size = 0;
-
-//-----------------------------------------------------------------------------
-// Global state
-thread_local struct KernelThreadData kernelTLS __attribute__ ((tls_model ( "initial-exec" ))) = {
-        NULL,                                                                                           // cannot use 0p
-        NULL,
-        { 1, false, false },
-        6u //this should be seeded better but due to a bug calling rdtsc doesn't work
-};
-
-//-----------------------------------------------------------------------------
-// Struct to steal stack
-struct current_stack_info_t {
-        __stack_t * storage;                                                            // pointer to stack object
-        void * base;                                                                            // base of stack
-        void * limit;                                                                           // stack grows towards stack limit
-        void * context;                                                                         // address of cfa_context_t
-};
-
-void ?{}( current_stack_info_t & this ) {
-        __stack_context_t ctx;
-        CtxGet( ctx );
-        this.base = ctx.FP;
-
-        rlimit r;
-        getrlimit( RLIMIT_STACK, &r);
-        size_t size = r.rlim_cur;
-
-        this.limit = (void *)(((intptr_t)this.base) - size);
-        this.context = &storage_mainThreadCtx;
-}
-
-//-----------------------------------------------------------------------------
-// Main thread construction
-
-void ?{}( coroutine_desc & this, current_stack_info_t * info) with( this ) {
-        stack.storage = info->storage;
-        with(*stack.storage) {
-                limit     = info->limit;
-                base      = info->base;
-        }
-        __attribute__((may_alias)) intptr_t * istorage = (intptr_t*) &stack.storage;
-        *istorage |= 0x1;
-        name = "Main Thread";
-        state = Start;
-        starter = 0p;
-        last = 0p;
-        cancellation = 0p;
-}
-
-void ?{}( thread_desc & this, current_stack_info_t * info) with( this ) {
-        state = Start;
-        self_cor{ info };
-        curr_cor = &self_cor;
-        curr_cluster = mainCluster;
-        self_mon.owner = &this;
-        self_mon.recursion = 1;
-        self_mon_p = &self_mon;
-        next = 0p;
-
-        node.next = 0p;
-        node.prev = 0p;
-        doregister(curr_cluster, this);
-
-        monitors{ &self_mon_p, 1, (fptr_t)0 };
-}
-
-//-----------------------------------------------------------------------------
-// Processor coroutine
-void ?{}(processorCtx_t & this) {
-
-}
-
-// Construct the processor context of non-main processors
-static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info) {
-        (this.__cor){ info };
-        this.proc = proc;
-}
-
-static void start(processor * this);
-void ?{}(processor & this, const char * name, cluster & cltr) with( this ) {
-        this.name = name;
-        this.cltr = &cltr;
-        terminated{ 0 };
-        do_terminate = false;
-        preemption_alarm = 0p;
-        pending_preemption = false;
-        runner.proc = &this;
-
-        idleLock{};
-
-        start( &this );
-}
-
-void ^?{}(processor & this) with( this ){
-        if( ! __atomic_load_n(&do_terminate, __ATOMIC_ACQUIRE) ) {
-                __cfaabi_dbg_print_safe("Kernel : core %p signaling termination\n", &this);
-
-                __atomic_store_n(&do_terminate, true, __ATOMIC_RELAXED);
-                wake( &this );
-
-                P( terminated );
-                verify( kernelTLS.this_processor != &this);
-        }
-
-        pthread_join( kernel_thread, 0p );
-        free( this.stack );
-}
-
-void ?{}(cluster & this, const char * name, Duration preemption_rate) with( this ) {
-        this.name = name;
-        this.preemption_rate = preemption_rate;
-        ready_queue{};
-        ready_queue_lock{};
-
-        procs{ __get };
-        idles{ __get };
-        threads{ __get };
-
-        doregister(this);
-}
-
-void ^?{}(cluster & this) {
-        unregister(this);
-}
+// Kernel Scheduling logic
+static $thread * __next_thread(cluster * this);
+static $thread * __next_thread_slow(cluster * this);
+static void __run_thread(processor * this, $thread * dst);
+static void __wake_one(cluster * cltr);
+
+static void push  (__cluster_idles & idles, processor & proc);
+static void remove(__cluster_idles & idles, processor & proc);
+static [unsigned idle, unsigned total, * processor] query( & __cluster_idles idles );
+
 
 //=============================================================================================
 // Kernel Scheduling logic
 //=============================================================================================
-static void runThread(processor * this, thread_desc * dst);
-static void finishRunning(processor * this);
-static void halt(processor * this);
-
 //Main of the processor contexts
 void main(processorCtx_t & runner) {
         // Because of a bug, we couldn't initialized the seed on construction
         // Do it here
-        kernelTLS.rand_seed ^= rdtscl();
+        __cfaabi_tls.rand_seed ^= rdtscl();
+        __cfaabi_tls.ready_rng.fwd_seed = 25214903917_l64u * (rdtscl() ^ (uintptr_t)&runner);
+        __tls_rand_advance_bck();
 
         processor * this = runner.proc;
         verify(this);
 
-        __cfaabi_dbg_print_safe("Kernel : core %p starting\n", this);
-
-        doregister(this->cltr, this);
+        __cfadbg_print_safe(runtime_core, "Kernel : core %p starting\n", this);
+        #if !defined(__CFA_NO_STATISTICS__)
+                if( this->print_halts ) {
+                        __cfaabi_bits_print_safe( STDOUT_FILENO, "Processor : %d - %s (%p)\n", this->id, this->name, (void*)this);
+                }
+        #endif
 
         {
@@ -279 +140 @@
                 preemption_scope scope = { this };
 
-                __cfaabi_dbg_print_safe("Kernel : core %p started\n", this);
-
-                thread_desc * readyThread = 0p;
-                for( unsigned int spin_count = 0; ! __atomic_load_n(&this->do_terminate, __ATOMIC_SEQ_CST); spin_count++ ) {
-                        readyThread = nextThread( this->cltr );
-
-                        if(readyThread) {
-                                verify( ! kernelTLS.preemption_state.enabled );
-
-                                runThread(this, readyThread);
-
-                                verify( ! kernelTLS.preemption_state.enabled );
-
-                                //Some actions need to be taken from the kernel
-                                finishRunning(this);
-
-                                spin_count = 0;
-                        } else {
-                                // spin(this, &spin_count);
-                                halt(this);
+                __cfadbg_print_safe(runtime_core, "Kernel : core %p started\n", this);
+
+                $thread * readyThread = 0p;
+                MAIN_LOOP:
+                for() {
+                        // Try to get the next thread
+                        readyThread = __next_thread( this->cltr );
+
+                        if( !readyThread ) {
+                                readyThread = __next_thread_slow( this->cltr );
                         }
-                }
-
-                __cfaabi_dbg_print_safe("Kernel : core %p stopping\n", this);
-        }
-
-        unregister(this->cltr, this);
+
+                        HALT:
+                        if( !readyThread ) {
+                                // Don't block if we are done
+                                if( __atomic_load_n(&this->do_terminate, __ATOMIC_SEQ_CST) ) break MAIN_LOOP;
+
+                                #if !defined(__CFA_NO_STATISTICS__)
+                                        __tls_stats()->ready.sleep.halts++;
+                                #endif
+
+                                // Push self to idle stack
+                                push(this->cltr->idles, * this);
+
+                                // Confirm the ready-queue is empty
+                                readyThread = __next_thread_slow( this->cltr );
+                                if( readyThread ) {
+                                        // A thread was found, cancel the halt
+                                        remove(this->cltr->idles, * this);
+
+                                        #if !defined(__CFA_NO_STATISTICS__)
+                                                __tls_stats()->ready.sleep.cancels++;
+                                        #endif
+
+                                        // continue the mai loop
+                                        break HALT;
+                                }
+
+                                #if !defined(__CFA_NO_STATISTICS__)
+                                        if(this->print_halts) {
+                                                __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 0\n", this->id, rdtscl());
+                                        }
+                                #endif
+
+                                wait( this->idle );
+
+                                #if !defined(__CFA_NO_STATISTICS__)
+                                        if(this->print_halts) {
+                                                __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 1\n", this->id, rdtscl());
+                                        }
+                                #endif
+
+                                // We were woken up, remove self from idle
+                                remove(this->cltr->idles, * this);
+
+                                // DON'T just proceed, start looking again
+                                continue MAIN_LOOP;
+                        }
+
+                        /* paranoid */ verify( readyThread );
+
+                        // We found a thread run it
+                        __run_thread(this, readyThread);
+
+                        // Are we done?
+                        if( __atomic_load_n(&this->do_terminate, __ATOMIC_SEQ_CST) ) break MAIN_LOOP;
+                }
+
+                __cfadbg_print_safe(runtime_core, "Kernel : core %p stopping\n", this);
+        }
 
         V( this->terminated );
 
-        __cfaabi_dbg_print_safe("Kernel : core %p terminated\n", this);
+        if(this == mainProcessor) {
+                // HACK : the coroutine context switch expects this_thread to be set
+                // and it make sense for it to be set in all other cases except here
+                // fake it
+                __cfaabi_tls.this_thread = mainThread;
+        }
+
+        __cfadbg_print_safe(runtime_core, "Kernel : core %p terminated\n", this);
 }
 
@@ -318 +229 @@
 // runThread runs a thread by context switching
 // from the processor coroutine to the target thread
-static void runThread(processor * this, thread_desc * thrd_dst) {
-        coroutine_desc * proc_cor = get_coroutine(this->runner);
-
-        // Reset the terminating actions here
-        this->finish.action_code = No_Action;
-
-        // Update global state
-        kernelTLS.this_thread = thrd_dst;
-
-        // set state of processor coroutine to inactive and the thread to active
-        proc_cor->state = proc_cor->state == Halted ? Halted : Inactive;
-        thrd_dst->state = Active;
-
-        // set context switch to the thread that the processor is executing
-        verify( thrd_dst->context.SP );
-        CtxSwitch( &proc_cor->context, &thrd_dst->context );
-        // when CtxSwitch returns we are back in the processor coroutine
-
-        // set state of processor coroutine to active and the thread to inactive
-        thrd_dst->state = thrd_dst->state == Halted ? Halted : Inactive;
+static void __run_thread(processor * this, $thread * thrd_dst) {
+        /* paranoid */ verify( ! __preemption_enabled() );
+        /* paranoid */ verifyf( thrd_dst->state == Ready || thrd_dst->preempted != __NO_PREEMPTION, "state : %d, preempted %d\n", thrd_dst->state, thrd_dst->preempted);
+        /* paranoid */ verifyf( thrd_dst->link.next == 0p, "Expected null got %p", thrd_dst->link.next );
+        __builtin_prefetch( thrd_dst->context.SP );
+
+        $coroutine * proc_cor = get_coroutine(this->runner);
+
+        // set state of processor coroutine to inactive
+        verify(proc_cor->state == Active);
+        proc_cor->state = Blocked;
+
+        // Actually run the thread
+        RUNNING:  while(true) {
+                thrd_dst->preempted = __NO_PREEMPTION;
+                thrd_dst->state = Active;
+
+                // Update global state
+                kernelTLS().this_thread = thrd_dst;
+
+                /* paranoid */ verify( ! __preemption_enabled() );
+                /* paranoid */ verify( kernelTLS().this_thread == thrd_dst );
+                /* paranoid */ verify( thrd_dst->curr_cluster == this->cltr );
+                /* paranoid */ verify( thrd_dst->context.SP );
+                /* paranoid */ verify( thrd_dst->state != Halted );
+                /* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) < ((uintptr_t)__get_stack(thrd_dst->curr_cor)->base ) || thrd_dst->curr_cor == proc_cor, "ERROR : Destination $thread %p has been corrupted.\n StackPointer too small.\n", thrd_dst ); // add escape condition if we are setting up the processor
+                /* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) > ((uintptr_t)__get_stack(thrd_dst->curr_cor)->limit) || thrd_dst->curr_cor == proc_cor, "ERROR : Destination $thread %p has been corrupted.\n StackPointer too large.\n", thrd_dst ); // add escape condition if we are setting up the processor
+                /* paranoid */ verify( 0x0D15EA5E0D15EA5Ep == thrd_dst->canary );
+
+
+
+                // set context switch to the thread that the processor is executing
+                __cfactx_switch( &proc_cor->context, &thrd_dst->context );
+                // when __cfactx_switch returns we are back in the processor coroutine
+
+                /* paranoid */ verify( 0x0D15EA5E0D15EA5Ep == thrd_dst->canary );
+                /* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) > ((uintptr_t)__get_stack(thrd_dst->curr_cor)->limit), "ERROR : Destination $thread %p has been corrupted.\n StackPointer too large.\n", thrd_dst );
+                /* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) < ((uintptr_t)__get_stack(thrd_dst->curr_cor)->base ), "ERROR : Destination $thread %p has been corrupted.\n StackPointer too small.\n", thrd_dst );
+                /* paranoid */ verify( thrd_dst->context.SP );
+                /* paranoid */ verify( thrd_dst->curr_cluster == this->cltr );
+                /* paranoid */ verify( kernelTLS().this_thread == thrd_dst );
+                /* paranoid */ verify( ! __preemption_enabled() );
+
+                // Reset global state
+                kernelTLS().this_thread = 0p;
+
+                // We just finished running a thread, there are a few things that could have happened.
+                // 1 - Regular case : the thread has blocked and now one has scheduled it yet.
+                // 2 - Racy case    : the thread has blocked but someone has already tried to schedule it.
+                // 4 - Preempted
+                // In case 1, we may have won a race so we can't write to the state again.
+                // In case 2, we lost the race so we now own the thread.
+
+                if(unlikely(thrd_dst->preempted != __NO_PREEMPTION)) {
+                        // The thread was preempted, reschedule it and reset the flag
+                        __schedule_thread( thrd_dst );
+                        break RUNNING;
+                }
+
+                if(unlikely(thrd_dst->state == Halting)) {
+                        // The thread has halted, it should never be scheduled/run again
+                        // finish the thread
+                        __thread_finish( thrd_dst );
+                        break RUNNING;
+                }
+
+                /* paranoid */ verify( thrd_dst->state == Active );
+                thrd_dst->state = Blocked;
+
+                // set state of processor coroutine to active and the thread to inactive
+                int old_ticket = __atomic_fetch_sub(&thrd_dst->ticket, 1, __ATOMIC_SEQ_CST);
+                switch(old_ticket) {
+                        case TICKET_RUNNING:
+                                // This is case 1, the regular case, nothing more is needed
+                                break RUNNING;
+                        case TICKET_UNBLOCK:
+                                // This is case 2, the racy case, someone tried to run this thread before it finished blocking
+                                // In this case, just run it again.
+                                continue RUNNING;
+                        default:
+                                // This makes no sense, something is wrong abort
+                                abort();
+                }
+        }
+
+        // Just before returning to the processor, set the processor coroutine to active
         proc_cor->state = Active;
+
+        /* paranoid */ verify( ! __preemption_enabled() );
 }
 
 // KERNEL_ONLY
-static void returnToKernel() {
-        coroutine_desc * proc_cor = get_coroutine(kernelTLS.this_processor->runner);
-        thread_desc * thrd_src = kernelTLS.this_thread;
-
-        // set state of current coroutine to inactive
-        thrd_src->state = thrd_src->state == Halted ? Halted : Inactive;
-        proc_cor->state = Active;
-        int local_errno = *__volatile_errno();
-        #if defined( __i386 ) || defined( __x86_64 )
-                __x87_store;
+void returnToKernel() {
+        /* paranoid */ verify( ! __preemption_enabled() );
+        $coroutine * proc_cor = get_coroutine(kernelTLS().this_processor->runner);
+        $thread * thrd_src = kernelTLS().this_thread;
+
+        #if !defined(__CFA_NO_STATISTICS__)
+                struct processor * last_proc = kernelTLS().this_processor;
         #endif
 
-        // set new coroutine that the processor is executing
-        // and context switch to it
-        verify( proc_cor->context.SP );
-        CtxSwitch( &thrd_src->context, &proc_cor->context );
-
-        // set state of new coroutine to active
-        proc_cor->state = proc_cor->state == Halted ? Halted : Inactive;
-        thrd_src->state = Active;
-
-        #if defined( __i386 ) || defined( __x86_64 )
-                __x87_load;
+        // Run the thread on this processor
+        {
+                int local_errno = *__volatile_errno();
+                #if defined( __i386 ) || defined( __x86_64 )
+                        __x87_store;
+                #endif
+                /* paranoid */ verify( proc_cor->context.SP );
+                /* paranoid */ verify( 0x0D15EA5E0D15EA5Ep == thrd_src->canary );
+                __cfactx_switch( &thrd_src->context, &proc_cor->context );
+                /* paranoid */ verify( 0x0D15EA5E0D15EA5Ep == thrd_src->canary );
+                #if defined( __i386 ) || defined( __x86_64 )
+                        __x87_load;
+                #endif
+                *__volatile_errno() = local_errno;
+        }
+
+        #if !defined(__CFA_NO_STATISTICS__)
+                if(last_proc != kernelTLS().this_processor) {
+                        __tls_stats()->ready.threads.migration++;
+                }
         #endif
-        *__volatile_errno() = local_errno;
-}
-
-// KERNEL_ONLY
-// Once a thread has finished running, some of
-// its final actions must be executed from the kernel
-static void finishRunning(processor * this) with( this->finish ) {
-        verify( ! kernelTLS.preemption_state.enabled );
-        choose( action_code ) {
-        case No_Action:
-                break;
-        case Release:
-                unlock( *lock );
-        case Schedule:
-                ScheduleThread( thrd );
-        case Release_Schedule:
-                unlock( *lock );
-                ScheduleThread( thrd );
-        case Release_Multi:
-                for(int i = 0; i < lock_count; i++) {
-                        unlock( *locks[i] );
-                }
-        case Release_Multi_Schedule:
-                for(int i = 0; i < lock_count; i++) {
-                        unlock( *locks[i] );
-                }
-                for(int i = 0; i < thrd_count; i++) {
-                        ScheduleThread( thrds[i] );
-                }
-        case Callback:
-                callback();
-        default:
-                abort("KERNEL ERROR: Unexpected action to run after thread");
-        }
-}
-
-// KERNEL_ONLY
-// Context invoker for processors
-// This is the entry point for processors (kernel threads)
-// It effectively constructs a coroutine by stealing the pthread stack
-static void * CtxInvokeProcessor(void * arg) {
-        processor * proc = (processor *) arg;
-        kernelTLS.this_processor = proc;
-        kernelTLS.this_thread    = 0p;
-        kernelTLS.preemption_state.[enabled, disable_count] = [false, 1];
-        // SKULLDUGGERY: We want to create a context for the processor coroutine
-        // which is needed for the 2-step context switch. However, there is no reason
-        // to waste the perfectly valid stack create by pthread.
-        current_stack_info_t info;
-        __stack_t ctx;
-        info.storage = &ctx;
-        (proc->runner){ proc, &info };
-
-        __cfaabi_dbg_print_safe("Coroutine : created stack %p\n", get_coroutine(proc->runner)->stack.storage);
-
-        //Set global state
-        kernelTLS.this_thread = 0p;
-
-        //We now have a proper context from which to schedule threads
-        __cfaabi_dbg_print_safe("Kernel : core %p created (%p, %p)\n", proc, &proc->runner, &ctx);
-
-        // SKULLDUGGERY: Since the coroutine doesn't have its own stack, we can't
-        // resume it to start it like it normally would, it will just context switch
-        // back to here. Instead directly call the main since we already are on the
-        // appropriate stack.
-        get_coroutine(proc->runner)->state = Active;
-        main( proc->runner );
-        get_coroutine(proc->runner)->state = Halted;
-
-        // Main routine of the core returned, the core is now fully terminated
-        __cfaabi_dbg_print_safe("Kernel : core %p main ended (%p)\n", proc, &proc->runner);
-
-        return 0p;
-}
-
-static void Abort( int ret, const char * func ) {
-        if ( ret ) {                                                                            // pthread routines return errno values
-                abort( "%s : internal error, error(%d) %s.", func, ret, strerror( ret ) );
-        } // if
-} // Abort
-
-void * create_pthread( pthread_t * pthread, void * (*start)(void *), void * arg ) {
-        pthread_attr_t attr;
-
-        Abort( pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
-
-        size_t stacksize;
-        // default stack size, normally defined by shell limit
-        Abort( pthread_attr_getstacksize( &attr, &stacksize ), "pthread_attr_getstacksize" );
-        assert( stacksize >= PTHREAD_STACK_MIN );
-
-        void * stack;
-        __cfaabi_dbg_debug_do(
-                stack = memalign( __page_size, stacksize + __page_size );
-                // pthread has no mechanism to create the guard page in user supplied stack.
-                if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
-                        abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
-                } // if
-        );
-        __cfaabi_dbg_no_debug_do(
-                stack = malloc( stacksize );
-        );
-
-        Abort( pthread_attr_setstack( &attr, stack, stacksize ), "pthread_attr_setstack" );
-
-        Abort( pthread_create( pthread, &attr, start, arg ), "pthread_create" );
-        return stack;
-}
-
-static void start(processor * this) {
-        __cfaabi_dbg_print_safe("Kernel : Starting core %p\n", this);
-
-        this->stack = create_pthread( &this->kernel_thread, CtxInvokeProcessor, (void *)this );
-
-        __cfaabi_dbg_print_safe("Kernel : core %p started\n", this);
-}
-
-// KERNEL_ONLY
-void kernel_first_resume( processor * this ) {
-        thread_desc * src = mainThread;
-        coroutine_desc * dst = get_coroutine(this->runner);
-
-        verify( ! kernelTLS.preemption_state.enabled );
-
-        kernelTLS.this_thread->curr_cor = dst;
-        __stack_prepare( &dst->stack, 65000 );
-        CtxStart(main, dst, this->runner, CtxInvokeCoroutine);
-
-        verify( ! kernelTLS.preemption_state.enabled );
-
-        dst->last = &src->self_cor;
-        dst->starter = dst->starter ? dst->starter : &src->self_cor;
-
-        // set state of current coroutine to inactive
-        src->state = src->state == Halted ? Halted : Inactive;
-
-        // context switch to specified coroutine
-        verify( dst->context.SP );
-        CtxSwitch( &src->context, &dst->context );
-        // when CtxSwitch returns we are back in the src coroutine
-
-        mainThread->curr_cor = &mainThread->self_cor;
-
-        // set state of new coroutine to active
-        src->state = Active;
-
-        verify( ! kernelTLS.preemption_state.enabled );
-}
-
-// KERNEL_ONLY
-void kernel_last_resume( processor * this ) {
-        coroutine_desc * src = &mainThread->self_cor;
-        coroutine_desc * dst = get_coroutine(this->runner);
-
-        verify( ! kernelTLS.preemption_state.enabled );
-        verify( dst->starter == src );
-        verify( dst->context.SP );
-
-        // context switch to the processor
-        CtxSwitch( &src->context, &dst->context );
+
+        /* paranoid */ verify( ! __preemption_enabled() );
+        /* paranoid */ verifyf( ((uintptr_t)thrd_src->context.SP) < ((uintptr_t)__get_stack(thrd_src->curr_cor)->base ), "ERROR : Returning $thread %p has been corrupted.\n StackPointer too small.\n", thrd_src );
+        /* paranoid */ verifyf( ((uintptr_t)thrd_src->context.SP) > ((uintptr_t)__get_stack(thrd_src->curr_cor)->limit), "ERROR : Returning $thread %p has been corrupted.\n StackPointer too large.\n", thrd_src );
 }
 
 //-----------------------------------------------------------------------------
 // Scheduler routines
-
 // KERNEL ONLY
-void ScheduleThread( thread_desc * thrd ) {
-        verify( thrd );
-        verify( thrd->state != Halted );
-
-        verify( ! kernelTLS.preemption_state.enabled );
-
-        verifyf( thrd->next == 0p, "Expected null got %p", thrd->next );
-
-        with( *thrd->curr_cluster ) {
-                lock  ( ready_queue_lock __cfaabi_dbg_ctx2 );
-                bool was_empty = !(ready_queue != 0);
-                append( ready_queue, thrd );
-                unlock( ready_queue_lock );
-
-                if(was_empty) {
-                        lock      (proc_list_lock __cfaabi_dbg_ctx2);
-                        if(idles) {
-                                wake_fast(idles.head);
+void __schedule_thread( $thread * thrd ) {
+        /* paranoid */ verify( ! __preemption_enabled() );
+        /* paranoid */ verify( kernelTLS().this_proc_id );
+        /* paranoid */ verify( thrd );
+        /* paranoid */ verify( thrd->state != Halted );
+        /* paranoid */ verify( thrd->curr_cluster );
+        /* paranoid */ #if defined( __CFA_WITH_VERIFY__ )
+        /* paranoid */  if( thrd->state == Blocked || thrd->state == Start ) assertf( thrd->preempted == __NO_PREEMPTION,
+                                        "Error inactive thread marked as preempted, state %d, preemption %d\n", thrd->state, thrd->preempted );
+        /* paranoid */  if( thrd->preempted != __NO_PREEMPTION ) assertf(thrd->state == Active,
+                                        "Error preempted thread marked as not currently running, state %d, preemption %d\n", thrd->state, thrd->preempted );
+        /* paranoid */ #endif
+        /* paranoid */ verifyf( thrd->link.next == 0p, "Expected null got %p", thrd->link.next );
+        /* paranoid */ verify( 0x0D15EA5E0D15EA5Ep == thrd->canary );
+
+
+        if (thrd->preempted == __NO_PREEMPTION) thrd->state = Ready;
+
+        ready_schedule_lock();
+                // Dereference the thread now because once we push it, there is not guaranteed it's still valid.
+                struct cluster * cl = thrd->curr_cluster;
+
+                // push the thread to the cluster ready-queue
+                push( cl, thrd );
+
+                // variable thrd is no longer safe to use
+
+                // wake the cluster using the save variable.
+                __wake_one( cl );
+        ready_schedule_unlock();
+
+        /* paranoid */ verify( ! __preemption_enabled() );
+}
+
+// KERNEL ONLY
+static inline $thread * __next_thread(cluster * this) with( *this ) {
+        /* paranoid */ verify( ! __preemption_enabled() );
+        /* paranoid */ verify( kernelTLS().this_proc_id );
+
+        ready_schedule_lock();
+                $thread * thrd = pop( this );
+        ready_schedule_unlock();
+
+        /* paranoid */ verify( kernelTLS().this_proc_id );
+        /* paranoid */ verify( ! __preemption_enabled() );
+        return thrd;
+}
+
+// KERNEL ONLY
+static inline $thread * __next_thread_slow(cluster * this) with( *this ) {
+        /* paranoid */ verify( ! __preemption_enabled() );
+        /* paranoid */ verify( kernelTLS().this_proc_id );
+
+        ready_schedule_lock();
+                $thread * thrd = pop_slow( this );
+        ready_schedule_unlock();
+
+        /* paranoid */ verify( kernelTLS().this_proc_id );
+        /* paranoid */ verify( ! __preemption_enabled() );
+        return thrd;
+}
+
+void unpark( $thread * thrd ) {
+        if( !thrd ) return;
+
+        int old_ticket = __atomic_fetch_add(&thrd->ticket, 1, __ATOMIC_SEQ_CST);
+        switch(old_ticket) {
+                case TICKET_RUNNING:
+                        // Wake won the race, the thread will reschedule/rerun itself
+                        break;
+                case TICKET_BLOCKED:
+                        /* paranoid */ verify( ! thrd->preempted != __NO_PREEMPTION );
+                        /* paranoid */ verify( thrd->state == Blocked );
+
+                        {
+                                /* paranoid */ verify( publicTLS_get(this_proc_id) );
+                                bool full = publicTLS_get(this_proc_id)->full_proc;
+                                if(full) disable_interrupts();
+
+                                /* paranoid */ verify( ! __preemption_enabled() );
+
+                                // Wake lost the race,
+                                __schedule_thread( thrd );
+
+                                /* paranoid */ verify( ! __preemption_enabled() );
+
+                                if(full) enable_interrupts( __cfaabi_dbg_ctx );
+                                /* paranoid */ verify( publicTLS_get(this_proc_id) );
                         }
-                        unlock    (proc_list_lock);
-                }
-                else if( struct processor * idle = idles.head ) {
-                        wake_fast(idle);
-                }
-
-        }
-
-        verify( ! kernelTLS.preemption_state.enabled );
+
+                        break;
+                default:
+                        // This makes no sense, something is wrong abort
+                        abort("Thread %p (%s) has mismatch park/unpark\n", thrd, thrd->self_cor.name);
+        }
+}
+
+void park( void ) {
+        /* paranoid */ verify( __preemption_enabled() );
+        disable_interrupts();
+        /* paranoid */ verify( ! __preemption_enabled() );
+        /* paranoid */ verify( kernelTLS().this_thread->preempted == __NO_PREEMPTION );
+
+        returnToKernel();
+
+        /* paranoid */ verify( ! __preemption_enabled() );
+        enable_interrupts( __cfaabi_dbg_ctx );
+        /* paranoid */ verify( __preemption_enabled() );
+
+}
+
+extern "C" {
+        // Leave the thread monitor
+        // last routine called by a thread.
+        // Should never return
+        void __cfactx_thrd_leave() {
+                $thread * thrd = active_thread();
+                $monitor * this = &thrd->self_mon;
+
+                // Lock the monitor now
+                lock( this->lock __cfaabi_dbg_ctx2 );
+
+                disable_interrupts();
+
+                /* paranoid */ verify( ! __preemption_enabled() );
+                /* paranoid */ verify( thrd->state == Active );
+                /* paranoid */ verify( 0x0D15EA5E0D15EA5Ep == thrd->canary );
+                /* paranoid */ verify( kernelTLS().this_thread == thrd );
+                /* paranoid */ verify( thrd->context.SP );
+                /* paranoid */ verifyf( ((uintptr_t)thrd->context.SP) > ((uintptr_t)__get_stack(thrd->curr_cor)->limit), "ERROR : $thread %p has been corrupted.\n StackPointer too large.\n", thrd );
+                /* paranoid */ verifyf( ((uintptr_t)thrd->context.SP) < ((uintptr_t)__get_stack(thrd->curr_cor)->base ), "ERROR : $thread %p has been corrupted.\n StackPointer too small.\n", thrd );
+
+                thrd->state = Halting;
+                if( TICKET_RUNNING != thrd->ticket ) { abort( "Thread terminated with pending unpark" ); }
+                if( thrd != this->owner ) { abort( "Thread internal monitor has incorrect owner" ); }
+                if( this->recursion != 1) { abort( "Thread internal monitor has unbalanced recursion" ); }
+
+                // Leave the thread
+                returnToKernel();
+
+                // Control flow should never reach here!
+                abort();
+        }
 }
 
 // KERNEL ONLY
-thread_desc * nextThread(cluster * this) with( *this ) {
-        verify( ! kernelTLS.preemption_state.enabled );
-        lock( ready_queue_lock __cfaabi_dbg_ctx2 );
-        thread_desc * head = pop_head( ready_queue );
-        unlock( ready_queue_lock );
-        verify( ! kernelTLS.preemption_state.enabled );
-        return head;
-}
-
-void BlockInternal() {
+bool force_yield( __Preemption_Reason reason ) {
+        /* paranoid */ verify( __preemption_enabled() );
         disable_interrupts();
-        verify( ! kernelTLS.preemption_state.enabled );
-        returnToKernel();
-        verify( ! kernelTLS.preemption_state.enabled );
+        /* paranoid */ verify( ! __preemption_enabled() );
+
+        $thread * thrd = kernelTLS().this_thread;
+        /* paranoid */ verify(thrd->state == Active);
+
+        // SKULLDUGGERY: It is possible that we are preempting this thread just before
+        // it was going to park itself. If that is the case and it is already using the
+        // intrusive fields then we can't use them to preempt the thread
+        // If that is the case, abandon the preemption.
+        bool preempted = false;
+        if(thrd->link.next == 0p) {
+                preempted = true;
+                thrd->preempted = reason;
+                returnToKernel();
+        }
+
+        /* paranoid */ verify( ! __preemption_enabled() );
+        enable_interrupts_noPoll();
+        /* paranoid */ verify( __preemption_enabled() );
+
+        return preempted;
+}
+
+//=============================================================================================
+// Kernel Idle Sleep
+//=============================================================================================
+// Wake a thread from the front if there are any
+static void __wake_one(cluster * this) {
+        /* paranoid */ verify( ! __preemption_enabled() );
+        /* paranoid */ verify( ready_schedule_islocked() );
+
+        // Check if there is a sleeping processor
+        processor * p;
+        unsigned idle;
+        unsigned total;
+        [idle, total, p] = query(this->idles);
+
+        // If no one is sleeping, we are done
+        if( idle == 0 ) return;
+
+        // We found a processor, wake it up
+        post( p->idle );
+
+        #if !defined(__CFA_NO_STATISTICS__)
+                __tls_stats()->ready.sleep.wakes++;
+        #endif
+
+        /* paranoid */ verify( ready_schedule_islocked() );
+        /* paranoid */ verify( ! __preemption_enabled() );
+
+        return;
+}
+
+// Unconditionnaly wake a thread
+void __wake_proc(processor * this) {
+        __cfadbg_print_safe(runtime_core, "Kernel : waking Processor %p\n", this);
+
+        disable_interrupts();
+                /* paranoid */ verify( ! __preemption_enabled() );
+                post( this->idle );
         enable_interrupts( __cfaabi_dbg_ctx );
 }
 
-void BlockInternal( __spinlock_t * lock ) {
-        disable_interrupts();
-        with( *kernelTLS.this_processor ) {
-                finish.action_code = Release;
-                finish.lock        = lock;
-        }
-
-        verify( ! kernelTLS.preemption_state.enabled );
-        returnToKernel();
-        verify( ! kernelTLS.preemption_state.enabled );
-
-        enable_interrupts( __cfaabi_dbg_ctx );
-}
-
-void BlockInternal( thread_desc * thrd ) {
-        disable_interrupts();
-        with( * kernelTLS.this_processor ) {
-                finish.action_code = Schedule;
-                finish.thrd        = thrd;
-        }
-
-        verify( ! kernelTLS.preemption_state.enabled );
-        returnToKernel();
-        verify( ! kernelTLS.preemption_state.enabled );
-
-        enable_interrupts( __cfaabi_dbg_ctx );
-}
-
-void BlockInternal( __spinlock_t * lock, thread_desc * thrd ) {
-        assert(thrd);
-        disable_interrupts();
-        with( * kernelTLS.this_processor ) {
-                finish.action_code = Release_Schedule;
-                finish.lock        = lock;
-                finish.thrd        = thrd;
-        }
-
-        verify( ! kernelTLS.preemption_state.enabled );
-        returnToKernel();
-        verify( ! kernelTLS.preemption_state.enabled );
-
-        enable_interrupts( __cfaabi_dbg_ctx );
-}
-
-void BlockInternal(__spinlock_t * locks [], unsigned short count) {
-        disable_interrupts();
-        with( * kernelTLS.this_processor ) {
-                finish.action_code = Release_Multi;
-                finish.locks       = locks;
-                finish.lock_count  = count;
-        }
-
-        verify( ! kernelTLS.preemption_state.enabled );
-        returnToKernel();
-        verify( ! kernelTLS.preemption_state.enabled );
-
-        enable_interrupts( __cfaabi_dbg_ctx );
-}
-
-void BlockInternal(__spinlock_t * locks [], unsigned short lock_count, thread_desc * thrds [], unsigned short thrd_count) {
-        disable_interrupts();
-        with( *kernelTLS.this_processor ) {
-                finish.action_code = Release_Multi_Schedule;
-                finish.locks       = locks;
-                finish.lock_count  = lock_count;
-                finish.thrds       = thrds;
-                finish.thrd_count  = thrd_count;
-        }
-
-        verify( ! kernelTLS.preemption_state.enabled );
-        returnToKernel();
-        verify( ! kernelTLS.preemption_state.enabled );
-
-        enable_interrupts( __cfaabi_dbg_ctx );
-}
-
-void BlockInternal(__finish_callback_fptr_t callback) {
-        disable_interrupts();
-        with( *kernelTLS.this_processor ) {
-                finish.action_code = Callback;
-                finish.callback    = callback;
-        }
-
-        verify( ! kernelTLS.preemption_state.enabled );
-        returnToKernel();
-        verify( ! kernelTLS.preemption_state.enabled );
-
-        enable_interrupts( __cfaabi_dbg_ctx );
-}
-
-// KERNEL ONLY
-void LeaveThread(__spinlock_t * lock, thread_desc * thrd) {
-        verify( ! kernelTLS.preemption_state.enabled );
-        with( * kernelTLS.this_processor ) {
-                finish.action_code = thrd ? Release_Schedule : Release;
-                finish.lock        = lock;
-                finish.thrd        = thrd;
-        }
-
-        returnToKernel();
-}
-
-//=============================================================================================
-// Kernel Setup logic
-//=============================================================================================
-//-----------------------------------------------------------------------------
-// Kernel boot procedures
-static void kernel_startup(void) {
-        verify( ! kernelTLS.preemption_state.enabled );
-        __cfaabi_dbg_print_safe("Kernel : Starting\n");
-
-        __page_size = sysconf( _SC_PAGESIZE );
-
-        __cfa_dbg_global_clusters.list{ __get };
-        __cfa_dbg_global_clusters.lock{};
-
-        // Initialize the main cluster
-        mainCluster = (cluster *)&storage_mainCluster;
-        (*mainCluster){"Main Cluster"};
-
-        __cfaabi_dbg_print_safe("Kernel : Main cluster ready\n");
-
-        // Start by initializing the main thread
-        // SKULLDUGGERY: the mainThread steals the process main thread
-        // which will then be scheduled by the mainProcessor normally
-        mainThread = (thread_desc *)&storage_mainThread;
-        current_stack_info_t info;
-        info.storage = (__stack_t*)&storage_mainThreadCtx;
-        (*mainThread){ &info };
-
-        __cfaabi_dbg_print_safe("Kernel : Main thread ready\n");
-
-
-
-        // Construct the processor context of the main processor
-        void ?{}(processorCtx_t & this, processor * proc) {
-                (this.__cor){ "Processor" };
-                this.__cor.starter = 0p;
-                this.proc = proc;
-        }
-
-        void ?{}(processor & this) with( this ) {
-                name = "Main Processor";
-                cltr = mainCluster;
-                terminated{ 0 };
-                do_terminate = false;
-                preemption_alarm = 0p;
-                pending_preemption = false;
-                kernel_thread = pthread_self();
-
-                runner{ &this };
-                __cfaabi_dbg_print_safe("Kernel : constructed main processor context %p\n", &runner);
-        }
-
-        // Initialize the main processor and the main processor ctx
-        // (the coroutine that contains the processing control flow)
-        mainProcessor = (processor *)&storage_mainProcessor;
-        (*mainProcessor){};
-
-        //initialize the global state variables
-        kernelTLS.this_processor = mainProcessor;
-        kernelTLS.this_thread    = mainThread;
-
-        // Enable preemption
-        kernel_start_preemption();
-
-        // Add the main thread to the ready queue
-        // once resume is called on mainProcessor->runner the mainThread needs to be scheduled like any normal thread
-        ScheduleThread(mainThread);
-
-        // SKULLDUGGERY: Force a context switch to the main processor to set the main thread's context to the current UNIX
-        // context. Hence, the main thread does not begin through CtxInvokeThread, like all other threads. The trick here is that
-        // mainThread is on the ready queue when this call is made.
-        kernel_first_resume( kernelTLS.this_processor );
-
-
-
-        // THE SYSTEM IS NOW COMPLETELY RUNNING
-        __cfaabi_dbg_print_safe("Kernel : Started\n--------------------------------------------------\n\n");
-
-        verify( ! kernelTLS.preemption_state.enabled );
-        enable_interrupts( __cfaabi_dbg_ctx );
-        verify( TL_GET( preemption_state.enabled ) );
-}
-
-static void kernel_shutdown(void) {
-        __cfaabi_dbg_print_safe("\n--------------------------------------------------\nKernel : Shutting down\n");
-
-        verify( TL_GET( preemption_state.enabled ) );
-        disable_interrupts();
-        verify( ! kernelTLS.preemption_state.enabled );
-
-        // SKULLDUGGERY: Notify the mainProcessor it needs to terminates.
-        // When its coroutine terminates, it return control to the mainThread
-        // which is currently here
-        __atomic_store_n(&mainProcessor->do_terminate, true, __ATOMIC_RELEASE);
-        kernel_last_resume( kernelTLS.this_processor );
-        mainThread->self_cor.state = Halted;
-
-        // THE SYSTEM IS NOW COMPLETELY STOPPED
-
-        // Disable preemption
-        kernel_stop_preemption();
-
-        // Destroy the main processor and its context in reverse order of construction
-        // These were manually constructed so we need manually destroy them
-        ^(mainProcessor->runner){};
-        ^(mainProcessor){};
-
-        // Final step, destroy the main thread since it is no longer needed
-        // Since we provided a stack to this taxk it will not destroy anything
-        ^(mainThread){};
-
-        ^(__cfa_dbg_global_clusters.list){};
-        ^(__cfa_dbg_global_clusters.lock){};
-
-        __cfaabi_dbg_print_safe("Kernel : Shutdown complete\n");
-}
-
-//=============================================================================================
-// Kernel Quiescing
-//=============================================================================================
-static void halt(processor * this) with( *this ) {
-        // verify( ! __atomic_load_n(&do_terminate, __ATOMIC_SEQ_CST) );
-
-        with( *cltr ) {
-                lock      (proc_list_lock __cfaabi_dbg_ctx2);
-                remove    (procs, *this);
-                push_front(idles, *this);
-                unlock    (proc_list_lock);
-        }
-
-        __cfaabi_dbg_print_safe("Kernel : Processor %p ready to sleep\n", this);
-
-        wait( idleLock );
-
-        __cfaabi_dbg_print_safe("Kernel : Processor %p woke up and ready to run\n", this);
-
-        with( *cltr ) {
-                lock      (proc_list_lock __cfaabi_dbg_ctx2);
-                remove    (idles, *this);
-                push_front(procs, *this);
-                unlock    (proc_list_lock);
+static void push  (__cluster_idles & this, processor & proc) {
+        /* paranoid */ verify( ! __preemption_enabled() );
+        lock( this );
+                this.idle++;
+                /* paranoid */ verify( this.idle <= this.total );
+
+                insert_first(this.list, proc);
+        unlock( this );
+        /* paranoid */ verify( ! __preemption_enabled() );
+}
+
+static void remove(__cluster_idles & this, processor & proc) {
+        /* paranoid */ verify( ! __preemption_enabled() );
+        lock( this );
+                this.idle--;
+                /* paranoid */ verify( this.idle >= 0 );
+
+                remove(proc);
+        unlock( this );
+        /* paranoid */ verify( ! __preemption_enabled() );
+}
+
+static [unsigned idle, unsigned total, * processor] query( & __cluster_idles this ) {
+        for() {
+                uint64_t l = __atomic_load_n(&this.lock, __ATOMIC_SEQ_CST);
+                if( 1 == (l % 2) ) { Pause(); continue; }
+                unsigned idle    = this.idle;
+                unsigned total   = this.total;
+                processor * proc = &this.list`first;
+                // Compiler fence is unnecessary, but gcc-8 and older incorrectly reorder code without it
+                asm volatile("": : :"memory");
+                if(l != __atomic_load_n(&this.lock, __ATOMIC_SEQ_CST)) { Pause(); continue; }
+                return [idle, total, proc];
         }
 }
@@ -836 +618 @@
         // the globalAbort flag is true.
         lock( kernel_abort_lock __cfaabi_dbg_ctx2 );
+
+        // disable interrupts, it no longer makes sense to try to interrupt this processor
+        disable_interrupts();
 
         // first task to abort ?
@@ -853 +638 @@
         }
 
-        return kernelTLS.this_thread;
+        return __cfaabi_tls.this_thread;
 }
 
 void kernel_abort_msg( void * kernel_data, char * abort_text, int abort_text_size ) {
-        thread_desc * thrd = kernel_data;
+        $thread * thrd = ( $thread * ) kernel_data;
 
         if(thrd) {
@@ -878 +663 @@
 
 int kernel_abort_lastframe( void ) __attribute__ ((__nothrow__)) {
-        return get_coroutine(kernelTLS.this_thread) == get_coroutine(mainThread) ? 4 : 2;
+        return get_coroutine(kernelTLS().this_thread) == get_coroutine(mainThread) ? 4 : 2;
 }
 
@@ -905 +690 @@
 void ^?{}(semaphore & this) {}
 
-void P(semaphore & this) with( this ){
+bool P(semaphore & this) with( this ){
         lock( lock __cfaabi_dbg_ctx2 );
         count -= 1;
         if ( count < 0 ) {
                 // queue current task
-                append( waiting, kernelTLS.this_thread );
+                append( waiting, active_thread() );
 
                 // atomically release spin lock and block
-                BlockInternal( &lock );
+                unlock( lock );
+                park();
+                return true;
         }
         else {
             unlock( lock );
-        }
-}
-
-void V(semaphore & this) with( this ) {
-        thread_desc * thrd = 0p;
+            return false;
+        }
+}
+
+bool V(semaphore & this) with( this ) {
+        $thread * thrd = 0p;
         lock( lock __cfaabi_dbg_ctx2 );
         count += 1;
@@ -932 +720 @@
 
         // make new owner
-        WakeThread( thrd );
-}
-
-//-----------------------------------------------------------------------------
-// Global Queues
-void doregister( cluster     & cltr ) {
-        lock      ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
-        push_front( __cfa_dbg_global_clusters.list, cltr );
-        unlock    ( __cfa_dbg_global_clusters.lock );
-}
-
-void unregister( cluster     & cltr ) {
-        lock  ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
-        remove( __cfa_dbg_global_clusters.list, cltr );
-        unlock( __cfa_dbg_global_clusters.lock );
-}
-
-void doregister( cluster * cltr, thread_desc & thrd ) {
-        lock      (cltr->thread_list_lock __cfaabi_dbg_ctx2);
-        cltr->nthreads += 1;
-        push_front(cltr->threads, thrd);
-        unlock    (cltr->thread_list_lock);
-}
-
-void unregister( cluster * cltr, thread_desc & thrd ) {
-        lock  (cltr->thread_list_lock __cfaabi_dbg_ctx2);
-        remove(cltr->threads, thrd );
-        cltr->nthreads -= 1;
-        unlock(cltr->thread_list_lock);
-}
-
-void doregister( cluster * cltr, processor * proc ) {
-        lock      (cltr->proc_list_lock __cfaabi_dbg_ctx2);
-        cltr->nprocessors += 1;
-        push_front(cltr->procs, *proc);
-        unlock    (cltr->proc_list_lock);
-}
-
-void unregister( cluster * cltr, processor * proc ) {
-        lock  (cltr->proc_list_lock __cfaabi_dbg_ctx2);
-        remove(cltr->procs, *proc );
-        cltr->nprocessors -= 1;
-        unlock(cltr->proc_list_lock);
+        unpark( thrd );
+
+        return thrd != 0p;
+}
+
+bool V(semaphore & this, unsigned diff) with( this ) {
+        $thread * thrd = 0p;
+        lock( lock __cfaabi_dbg_ctx2 );
+        int release = max(-count, (int)diff);
+        count += diff;
+        for(release) {
+                unpark( pop_head( waiting ) );
+        }
+
+        unlock( lock );
+
+        return thrd != 0p;
 }
 
@@ -981 +743 @@
 __cfaabi_dbg_debug_do(
         extern "C" {
-                void __cfaabi_dbg_record(__spinlock_t & this, const char * prev_name) {
+                void __cfaabi_dbg_record_lock(__spinlock_t & this, const char prev_name[]) {
                         this.prev_name = prev_name;
-                        this.prev_thrd = kernelTLS.this_thread;
+                        this.prev_thrd = kernelTLS().this_thread;
                 }
         }
@@ -990 +752 @@
 //-----------------------------------------------------------------------------
 // Debug
-bool threading_enabled(void) {
+bool threading_enabled(void) __attribute__((const)) {
         return true;
 }
+
+//-----------------------------------------------------------------------------
+// Statistics
+#if !defined(__CFA_NO_STATISTICS__)
+        void print_halts( processor & this ) {
+                this.print_halts = true;
+        }
+
+        void print_stats_now( cluster & this, int flags ) {
+                __print_stats( this.stats, this.print_stats, "Cluster", this.name, (void*)&this );
+        }
+
+        extern int __print_alarm_stats;
+        void print_alarm_stats() {
+                __print_alarm_stats = -1;
+        }
+#endif
 // Local Variables: //
 // mode: c //

libcfa/src/concurrency/kernel.hfa

@@ -10 +10 @@
 // Created On       : Tue Jan 17 12:27:26 2017
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Wed Dec  4 07:54:51 2019
-// Update Count     : 18
+// Last Modified On : Tue Feb  4 12:29:26 2020
+// Update Count     : 22
 //
 
 #pragma once
-
-#include <stdbool.h>
 
 #include "invoke.h"
@@ -22 +20 @@
 #include "coroutine.hfa"
 
+#include "containers/list.hfa"
+
 extern "C" {
-#include <pthread.h>
-#include <semaphore.h>
+        #include <bits/pthreadtypes.h>
+        #include <linux/types.h>
 }
 
@@ -32 +32 @@
         __spinlock_t lock;
         int count;
-        __queue_t(thread_desc) waiting;
+        __queue_t($thread) waiting;
 };
 
 void  ?{}(semaphore & this, int count = 1);
 void ^?{}(semaphore & this);
-void   P (semaphore & this);
-void   V (semaphore & this);
+bool   P (semaphore & this);
+bool   V (semaphore & this);
+bool   V (semaphore & this, unsigned count);
 
 
@@ -45 +46 @@
 extern struct cluster * mainCluster;
 
-enum FinishOpCode { No_Action, Release, Schedule, Release_Schedule, Release_Multi, Release_Multi_Schedule, Callback };
-
-typedef void (*__finish_callback_fptr_t)(void);
-
-//TODO use union, many of these fields are mutually exclusive (i.e. MULTI vs NOMULTI)
-struct FinishAction {
-        FinishOpCode action_code;
-        /*
-        // Union of possible actions
-        union {
-                // Option 1 : locks and threads
-                struct {
-                        // 1 thread or N thread
-                        union {
-                                thread_desc * thrd;
-                                struct {
-                                        thread_desc ** thrds;
-                                        unsigned short thrd_count;
-                                };
-                        };
-                        // 1 lock or N lock
-                        union {
-                                __spinlock_t * lock;
-                                struct {
-                                        __spinlock_t ** locks;
-                                        unsigned short lock_count;
-                                };
-                        };
-                };
-                // Option 2 : action pointer
-                __finish_callback_fptr_t callback;
-        };
-        /*/
-        thread_desc * thrd;
-        thread_desc ** thrds;
-        unsigned short thrd_count;
-        __spinlock_t * lock;
-        __spinlock_t ** locks;
-        unsigned short lock_count;
-        __finish_callback_fptr_t callback;
-        //*/
-};
-static inline void ?{}(FinishAction & this) {
-        this.action_code = No_Action;
-        this.thrd = 0p;
-        this.lock = 0p;
-}
-static inline void ^?{}(FinishAction &) {}
-
-// Processor
+// Processor id, required for scheduling threads
+struct __processor_id_t {
+        unsigned id:24;
+        bool full_proc:1;
+
+        #if !defined(__CFA_NO_STATISTICS__)
+                struct __stats_t * stats;
+        #endif
+};
+
 coroutine processorCtx_t {
         struct processor * proc;
@@ -100 +61 @@
 
 // Wrapper around kernel threads
-struct processor {
+struct __attribute__((aligned(128))) processor {
         // Main state
+        inline __processor_id_t;
+
+        // Cluster from which to get threads
+        struct cluster * cltr;
+
+        // Set to true to notify the processor should terminate
+        volatile bool do_terminate;
+
         // Coroutine ctx who does keeps the state of the processor
         struct processorCtx_t runner;
 
-        // Cluster from which to get threads
-        struct cluster * cltr;
-
         // Name of the processor
         const char * name;
@@ -113 +79 @@
         // Handle to pthreads
         pthread_t kernel_thread;
-
-        // RunThread data
-        // Action to do after a thread is ran
-        struct FinishAction finish;
 
         // Preemption data
@@ -125 +87 @@
         bool pending_preemption;
 
-        // Idle lock
-        __bin_sem_t idleLock;
-
-        // Termination
-        // Set to true to notify the processor should terminate
-        volatile bool do_terminate;
-
-        // Termination synchronisation
+        // Idle lock (kernel semaphore)
+        __bin_sem_t idle;
+
+        // Termination synchronisation (user semaphore)
         semaphore terminated;
 
@@ -139 +97 @@
 
         // Link lists fields
-        struct __dbg_node_proc {
-                struct processor * next;
-                struct processor * prev;
-        } node;
+        DLISTED_MGD_IMPL_IN(processor)
+
+        #if !defined(__CFA_NO_STATISTICS__)
+                int print_stats;
+                bool print_halts;
+        #endif
 
 #ifdef __CFA_DEBUG__
@@ -150 +110 @@
 };
 
-void  ?{}(processor & this, const char * name, struct cluster & cltr);
+void  ?{}(processor & this, const char name[], struct cluster & cltr);
 void ^?{}(processor & this);
 
 static inline void  ?{}(processor & this)                    { this{ "Anonymous Processor", *mainCluster}; }
 static inline void  ?{}(processor & this, struct cluster & cltr)    { this{ "Anonymous Processor", cltr}; }
-static inline void  ?{}(processor & this, const char * name) { this{name, *mainCluster }; }
-
-static inline [processor *&, processor *& ] __get( processor & this ) {
-        return this.node.[next, prev];
-}
+static inline void  ?{}(processor & this, const char name[]) { this{name, *mainCluster }; }
+
+DLISTED_MGD_IMPL_OUT(processor)
+
+//-----------------------------------------------------------------------------
+// I/O
+struct __io_data;
+
+// IO poller user-thread
+// Not using the "thread" keyword because we want to control
+// more carefully when to start/stop it
+struct $io_ctx_thread {
+        struct __io_data * ring;
+        single_sem sem;
+        volatile bool done;
+        $thread self;
+};
+
+
+struct io_context {
+        $io_ctx_thread thrd;
+};
+
+struct io_context_params {
+        int num_entries;
+        int num_ready;
+        int submit_aff;
+        bool eager_submits:1;
+        bool poller_submits:1;
+        bool poll_submit:1;
+        bool poll_complete:1;
+};
+
+void  ?{}(io_context_params & this);
+
+void  ?{}(io_context & this, struct cluster & cl);
+void  ?{}(io_context & this, struct cluster & cl, const io_context_params & params);
+void ^?{}(io_context & this);
+
+struct io_cancellation {
+        __u64 target;
+};
+
+static inline void  ?{}(io_cancellation & this) { this.target = -1u; }
+static inline void ^?{}(io_cancellation &) {}
+bool cancel(io_cancellation & this);
+
+//-----------------------------------------------------------------------------
+// Cluster Tools
+
+// Intrusives lanes which are used by the relaxed ready queue
+struct __attribute__((aligned(128))) __intrusive_lane_t;
+void  ?{}(__intrusive_lane_t & this);
+void ^?{}(__intrusive_lane_t & this);
+
+// Counter used for wether or not the lanes are all empty
+struct __attribute__((aligned(128))) __snzi_node_t;
+struct __snzi_t {
+        unsigned mask;
+        int root;
+        __snzi_node_t * nodes;
+};
+
+void  ?{}( __snzi_t & this, unsigned depth );
+void ^?{}( __snzi_t & this );
+
+//TODO adjust cache size to ARCHITECTURE
+// Structure holding the relaxed ready queue
+struct __ready_queue_t {
+        // Data tracking how many/which lanes are used
+        // Aligned to 128 for cache locality
+        __snzi_t snzi;
+
+        // Data tracking the actual lanes
+        // On a seperate cacheline from the used struct since
+        // used can change on each push/pop but this data
+        // only changes on shrink/grow
+        struct {
+                // Arary of lanes
+                __intrusive_lane_t * volatile data;
+
+                // Number of lanes (empty or not)
+                volatile size_t count;
+        } lanes;
+};
+
+void  ?{}(__ready_queue_t & this);
+void ^?{}(__ready_queue_t & this);
+
+// Idle Sleep
+struct __cluster_idles {
+        // Spin lock protecting the queue
+        volatile uint64_t lock;
+
+        // Total number of processors
+        unsigned total;
+
+        // Total number of idle processors
+        unsigned idle;
+
+        // List of idle processors
+        dlist(processor, processor) list;
+};
 
 //-----------------------------------------------------------------------------
 // Cluster
-struct cluster {
-        // Ready queue locks
-        __spinlock_t ready_queue_lock;
-
+struct __attribute__((aligned(128))) cluster {
         // Ready queue for threads
-        __queue_t(thread_desc) ready_queue;
+        __ready_queue_t ready_queue;
 
         // Name of the cluster
@@ -176 +231 @@
         Duration preemption_rate;
 
-        // List of processors
-        __spinlock_t proc_list_lock;
-        __dllist_t(struct processor) procs;
-        __dllist_t(struct processor) idles;
-        unsigned int nprocessors;
+        // List of idle processors
+        __cluster_idles idles;
 
         // List of threads
         __spinlock_t thread_list_lock;
-        __dllist_t(struct thread_desc) threads;
+        __dllist_t(struct $thread) threads;
         unsigned int nthreads;
 
@@ -192 +244 @@
                 cluster * prev;
         } node;
+
+        struct {
+                io_context * ctxs;
+                unsigned cnt;
+        } io;
+
+        #if !defined(__CFA_NO_STATISTICS__)
+                struct __stats_t * stats;
+                int print_stats;
+        #endif
 };
 extern Duration default_preemption();
 
-void ?{} (cluster & this, const char * name, Duration preemption_rate);
+void ?{} (cluster & this, const char name[], Duration preemption_rate, unsigned num_io, const io_context_params & io_params);
 void ^?{}(cluster & this);
 
-static inline void ?{} (cluster & this)                           { this{"Anonymous Cluster", default_preemption()}; }
-static inline void ?{} (cluster & this, Duration preemption_rate) { this{"Anonymous Cluster", preemption_rate}; }
-static inline void ?{} (cluster & this, const char * name)        { this{name, default_preemption()}; }
-
-static inline [cluster *&, cluster *& ] __get( cluster & this ) {
-        return this.node.[next, prev];
-}
-
-static inline struct processor * active_processor() { return TL_GET( this_processor ); } // UNSAFE
-static inline struct cluster   * active_cluster  () { return TL_GET( this_processor )->cltr; }
+static inline void ?{} (cluster & this)                                            { io_context_params default_params;    this{"Anonymous Cluster", default_preemption(), 1, default_params}; }
+static inline void ?{} (cluster & this, Duration preemption_rate)                  { io_context_params default_params;    this{"Anonymous Cluster", preemption_rate, 1, default_params}; }
+static inline void ?{} (cluster & this, const char name[])                         { io_context_params default_params;    this{name, default_preemption(), 1, default_params}; }
+static inline void ?{} (cluster & this, unsigned num_io)                           { io_context_params default_params;    this{"Anonymous Cluster", default_preemption(), num_io, default_params}; }
+static inline void ?{} (cluster & this, Duration preemption_rate, unsigned num_io) { io_context_params default_params;    this{"Anonymous Cluster", preemption_rate, num_io, default_params}; }
+static inline void ?{} (cluster & this, const char name[], unsigned num_io)        { io_context_params default_params;    this{name, default_preemption(), num_io, default_params}; }
+static inline void ?{} (cluster & this, const io_context_params & io_params)                                            { this{"Anonymous Cluster", default_preemption(), 1, io_params}; }
+static inline void ?{} (cluster & this, Duration preemption_rate, const io_context_params & io_params)                  { this{"Anonymous Cluster", preemption_rate, 1, io_params}; }
+static inline void ?{} (cluster & this, const char name[], const io_context_params & io_params)                         { this{name, default_preemption(), 1, io_params}; }
+static inline void ?{} (cluster & this, unsigned num_io, const io_context_params & io_params)                           { this{"Anonymous Cluster", default_preemption(), num_io, io_params}; }
+static inline void ?{} (cluster & this, Duration preemption_rate, unsigned num_io, const io_context_params & io_params) { this{"Anonymous Cluster", preemption_rate, num_io, io_params}; }
+static inline void ?{} (cluster & this, const char name[], unsigned num_io, const io_context_params & io_params)        { this{name, default_preemption(), num_io, io_params}; }
+
+static inline [cluster *&, cluster *& ] __get( cluster & this ) __attribute__((const)) { return this.node.[next, prev]; }
+
+static inline struct processor * active_processor() { return publicTLS_get( this_processor ); } // UNSAFE
+static inline struct cluster   * active_cluster  () { return publicTLS_get( this_processor )->cltr; }
+
+#if !defined(__CFA_NO_STATISTICS__)
+        void print_stats_now( cluster & this, int flags );
+
+        static inline void print_stats_at_exit( cluster & this, int flags ) {
+                this.print_stats |= flags;
+        }
+
+        static inline void print_stats_at_exit( processor & this, int flags ) {
+                this.print_stats |= flags;
+        }
+
+        void print_halts( processor & this );
+#endif
 
 // Local Variables: //

libcfa/src/concurrency/kernel_private.hfa

@@ -10 +10 @@
 // Created On       : Mon Feb 13 12:27:26 2017
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Sat Nov 30 19:25:02 2019
-// Update Count     : 8
+// Last Modified On : Wed Aug 12 08:21:33 2020
+// Update Count     : 9
 //
 
@@ -20 +20 @@
 
 #include "alarm.hfa"
-
+#include "stats.hfa"
 
 //-----------------------------------------------------------------------------
 // Scheduler
+
+struct __attribute__((aligned(128))) __scheduler_lock_id_t;
 
 extern "C" {
@@ -31 +33 @@
 }
 
-void ScheduleThread( thread_desc * );
-static inline void WakeThread( thread_desc * thrd ) {
-        if( !thrd ) return;
-
-        verify(thrd->state == Inactive);
-
-        disable_interrupts();
-        ScheduleThread( thrd );
-        enable_interrupts( __cfaabi_dbg_ctx );
-}
-thread_desc * nextThread(cluster * this);
-
-//Block current thread and release/wake-up the following resources
-void BlockInternal(void);
-void BlockInternal(__spinlock_t * lock);
-void BlockInternal(thread_desc * thrd);
-void BlockInternal(__spinlock_t * lock, thread_desc * thrd);
-void BlockInternal(__spinlock_t * locks [], unsigned short count);
-void BlockInternal(__spinlock_t * locks [], unsigned short count, thread_desc * thrds [], unsigned short thrd_count);
-void BlockInternal(__finish_callback_fptr_t callback);
-void LeaveThread(__spinlock_t * lock, thread_desc * thrd);
+void __schedule_thread( $thread * )
+#if defined(NDEBUG) || (!defined(__CFA_DEBUG__) && !defined(__CFA_VERIFY__))
+        __attribute__((nonnull (1)))
+#endif
+;
+
+extern bool __preemption_enabled();
+
+//release/wake-up the following resources
+void __thread_finish( $thread * thrd );
 
 //-----------------------------------------------------------------------------
@@ -57 +48 @@
 void main(processorCtx_t *);
 
-void * create_pthread( pthread_t *, void * (*)(void *), void * );
-
-static inline void wake_fast(processor * this) {
-        __cfaabi_dbg_print_safe("Kernel : Waking up processor %p\n", this);
-        post( this->idleLock );
-}
-
-static inline void wake(processor * this) {
-        disable_interrupts();
-        wake_fast(this);
-        enable_interrupts( __cfaabi_dbg_ctx );
-}
-
-struct event_kernel_t {
-        alarm_list_t alarms;
-        __spinlock_t lock;
-};
-
-extern event_kernel_t * event_kernel;
-
-struct __cfa_kernel_preemption_state_t {
-        bool enabled;
-        bool in_progress;
-        unsigned short disable_count;
-};
-
-extern volatile thread_local __cfa_kernel_preemption_state_t preemption_state __attribute__ ((tls_model ( "initial-exec" )));
+void * __create_pthread( pthread_t *, void * (*)(void *), void * );
+void __destroy_pthread( pthread_t pthread, void * stack, void ** retval );
+
+
+
+extern cluster * mainCluster;
 
 //-----------------------------------------------------------------------------
 // Threads
 extern "C" {
-      void CtxInvokeThread(void (*main)(void *), void * this);
-}
-
-extern void ThreadCtxSwitch(coroutine_desc * src, coroutine_desc * dst);
+      void __cfactx_invoke_thread(void (*main)(void *), void * this);
+}
 
 __cfaabi_dbg_debug_do(
-        extern void __cfaabi_dbg_thread_register  ( thread_desc * thrd );
-        extern void __cfaabi_dbg_thread_unregister( thread_desc * thrd );
+        extern void __cfaabi_dbg_thread_register  ( $thread * thrd );
+        extern void __cfaabi_dbg_thread_unregister( $thread * thrd );
 )
 
+#define TICKET_BLOCKED (-1) // thread is blocked
+#define TICKET_RUNNING ( 0) // thread is running
+#define TICKET_UNBLOCK ( 1) // thread should ignore next block
+
 //-----------------------------------------------------------------------------
 // Utils
-#define KERNEL_STORAGE(T,X) static char storage_##X[sizeof(T)]
-
-static inline uint32_t tls_rand() {
-        kernelTLS.rand_seed ^= kernelTLS.rand_seed << 6;
-        kernelTLS.rand_seed ^= kernelTLS.rand_seed >> 21;
-        kernelTLS.rand_seed ^= kernelTLS.rand_seed << 7;
-        return kernelTLS.rand_seed;
-}
-
-
-void doregister( struct cluster & cltr );
-void unregister( struct cluster & cltr );
-
-void doregister( struct cluster * cltr, struct thread_desc & thrd );
-void unregister( struct cluster * cltr, struct thread_desc & thrd );
-
-void doregister( struct cluster * cltr, struct processor * proc );
-void unregister( struct cluster * cltr, struct processor * proc );
+void doregister( struct cluster * cltr, struct $thread & thrd );
+void unregister( struct cluster * cltr, struct $thread & thrd );
+
+//-----------------------------------------------------------------------------
+// I/O
+void ^?{}(io_context & this, bool );
+
+//=======================================================================
+// Cluster lock API
+//=======================================================================
+// Cells use by the reader writer lock
+// while not generic it only relies on a opaque pointer
+struct __attribute__((aligned(128))) __scheduler_lock_id_t {
+        // Spin lock used as the underlying lock
+        volatile bool lock;
+
+        // Handle pointing to the proc owning this cell
+        // Used for allocating cells and debugging
+        __processor_id_t * volatile handle;
+
+        #ifdef __CFA_WITH_VERIFY__
+                // Debug, check if this is owned for reading
+                bool owned;
+        #endif
+};
+
+static_assert( sizeof(struct __scheduler_lock_id_t) <= __alignof(struct __scheduler_lock_id_t));
+
+// Lock-Free registering/unregistering of threads
+// Register a processor to a given cluster and get its unique id in return
+unsigned doregister( struct __processor_id_t * proc );
+
+// Unregister a processor from a given cluster using its id, getting back the original pointer
+void     unregister( struct __processor_id_t * proc );
+
+//-----------------------------------------------------------------------
+// Cluster idle lock/unlock
+static inline void lock(__cluster_idles & this) {
+        for() {
+                uint64_t l = this.lock;
+                if(
+                        (0 == (l % 2))
+                        && __atomic_compare_exchange_n(&this.lock, &l, l + 1, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)
+                ) return;
+                Pause();
+        }
+}
+
+static inline void unlock(__cluster_idles & this) {
+        /* paranoid */ verify( 1 == (this.lock % 2) );
+        __atomic_fetch_add( &this.lock, 1, __ATOMIC_SEQ_CST );
+}
+
+//=======================================================================
+// Reader-writer lock implementation
+// Concurrent with doregister/unregister,
+//    i.e., threads can be added at any point during or between the entry/exit
+
+//-----------------------------------------------------------------------
+// simple spinlock underlying the RWLock
+// Blocking acquire
+static inline void __atomic_acquire(volatile bool * ll) {
+        while( __builtin_expect(__atomic_exchange_n(ll, (bool)true, __ATOMIC_SEQ_CST), false) ) {
+                while(__atomic_load_n(ll, (int)__ATOMIC_RELAXED))
+                        Pause();
+        }
+        /* paranoid */ verify(*ll);
+}
+
+// Non-Blocking acquire
+static inline bool __atomic_try_acquire(volatile bool * ll) {
+        return !__atomic_exchange_n(ll, (bool)true, __ATOMIC_SEQ_CST);
+}
+
+// Release
+static inline void __atomic_unlock(volatile bool * ll) {
+        /* paranoid */ verify(*ll);
+        __atomic_store_n(ll, (bool)false, __ATOMIC_RELEASE);
+}
+
+//-----------------------------------------------------------------------
+// Reader-Writer lock protecting the ready-queues
+// while this lock is mostly generic some aspects
+// have been hard-coded to for the ready-queue for
+// simplicity and performance
+struct __scheduler_RWLock_t {
+        // total cachelines allocated
+        unsigned int max;
+
+        // cachelines currently in use
+        volatile unsigned int alloc;
+
+        // cachelines ready to itereate over
+        // (!= to alloc when thread is in second half of doregister)
+        volatile unsigned int ready;
+
+        // writer lock
+        volatile bool lock;
+
+        // data pointer
+        __scheduler_lock_id_t * data;
+};
+
+void  ?{}(__scheduler_RWLock_t & this);
+void ^?{}(__scheduler_RWLock_t & this);
+
+extern __scheduler_RWLock_t * __scheduler_lock;
+
+//-----------------------------------------------------------------------
+// Reader side : acquire when using the ready queue to schedule but not
+//  creating/destroying queues
+static inline void ready_schedule_lock(void) with(*__scheduler_lock) {
+        /* paranoid */ verify( ! __preemption_enabled() );
+        /* paranoid */ verify( kernelTLS().this_proc_id );
+
+        unsigned iproc = kernelTLS().this_proc_id->id;
+        /*paranoid*/ verify(data[iproc].handle == kernelTLS().this_proc_id);
+        /*paranoid*/ verify(iproc < ready);
+
+        // Step 1 : make sure no writer are in the middle of the critical section
+        while(__atomic_load_n(&lock, (int)__ATOMIC_RELAXED))
+                Pause();
+
+        // Fence needed because we don't want to start trying to acquire the lock
+        // before we read a false.
+        // Not needed on x86
+        // std::atomic_thread_fence(std::memory_order_seq_cst);
+
+        // Step 2 : acquire our local lock
+        __atomic_acquire( &data[iproc].lock );
+        /*paranoid*/ verify(data[iproc].lock);
+
+        #ifdef __CFA_WITH_VERIFY__
+                // Debug, check if this is owned for reading
+                data[iproc].owned = true;
+        #endif
+}
+
+static inline void ready_schedule_unlock(void) with(*__scheduler_lock) {
+        /* paranoid */ verify( ! __preemption_enabled() );
+        /* paranoid */ verify( kernelTLS().this_proc_id );
+
+        unsigned iproc = kernelTLS().this_proc_id->id;
+        /*paranoid*/ verify(data[iproc].handle == kernelTLS().this_proc_id);
+        /*paranoid*/ verify(iproc < ready);
+        /*paranoid*/ verify(data[iproc].lock);
+        /*paranoid*/ verify(data[iproc].owned);
+        #ifdef __CFA_WITH_VERIFY__
+                // Debug, check if this is owned for reading
+                data[iproc].owned = false;
+        #endif
+        __atomic_unlock(&data[iproc].lock);
+}
+
+#ifdef __CFA_WITH_VERIFY__
+        static inline bool ready_schedule_islocked(void) {
+                /* paranoid */ verify( ! __preemption_enabled() );
+                /*paranoid*/ verify( kernelTLS().this_proc_id );
+                __processor_id_t * proc = kernelTLS().this_proc_id;
+                return __scheduler_lock->data[proc->id].owned;
+        }
+
+        static inline bool ready_mutate_islocked() {
+                return __scheduler_lock->lock;
+        }
+#endif
+
+//-----------------------------------------------------------------------
+// Writer side : acquire when changing the ready queue, e.g. adding more
+//  queues or removing them.
+uint_fast32_t ready_mutate_lock( void );
+
+void ready_mutate_unlock( uint_fast32_t /* value returned by lock */ );
+
+//=======================================================================
+// Ready-Queue API
+//-----------------------------------------------------------------------
+// pop thread from the ready queue of a cluster
+// returns 0p if empty
+__attribute__((hot)) bool query(struct cluster * cltr);
+
+//-----------------------------------------------------------------------
+// push thread onto a ready queue for a cluster
+// returns true if the list was previously empty, false otherwise
+__attribute__((hot)) bool push(struct cluster * cltr, struct $thread * thrd);
+
+//-----------------------------------------------------------------------
+// pop thread from the ready queue of a cluster
+// returns 0p if empty
+// May return 0p spuriously
+__attribute__((hot)) struct $thread * pop(struct cluster * cltr);
+
+//-----------------------------------------------------------------------
+// pop thread from the ready queue of a cluster
+// returns 0p if empty
+// guaranteed to find any threads added before this call
+__attribute__((hot)) struct $thread * pop_slow(struct cluster * cltr);
+
+//-----------------------------------------------------------------------
+// remove thread from the ready queue of a cluster
+// returns bool if it wasn't found
+bool remove_head(struct cluster * cltr, struct $thread * thrd);
+
+//-----------------------------------------------------------------------
+// Increase the width of the ready queue (number of lanes) by 4
+void ready_queue_grow  (struct cluster * cltr, int target);
+
+//-----------------------------------------------------------------------
+// Decrease the width of the ready queue (number of lanes) by 4
+void ready_queue_shrink(struct cluster * cltr, int target);
+
 
 // Local Variables: //

libcfa/src/concurrency/monitor.cfa

@@ -5 +5 @@
 // file "LICENCE" distributed with Cforall.
 //
-// monitor_desc.c --
+// $monitor.c --
 //
 // Author           : Thierry Delisle
@@ -27 +27 @@
 //-----------------------------------------------------------------------------
 // Forward declarations
-static inline void set_owner ( monitor_desc * this, thread_desc * owner );
-static inline void set_owner ( monitor_desc * storage [], __lock_size_t count, thread_desc * owner );
-static inline void set_mask  ( monitor_desc * storage [], __lock_size_t count, const __waitfor_mask_t & mask );
-static inline void reset_mask( monitor_desc * this );
-
-static inline thread_desc * next_thread( monitor_desc * this );
-static inline bool is_accepted( monitor_desc * this, const __monitor_group_t & monitors );
+static inline void __set_owner ( $monitor * this, $thread * owner );
+static inline void __set_owner ( $monitor * storage [], __lock_size_t count, $thread * owner );
+static inline void set_mask  ( $monitor * storage [], __lock_size_t count, const __waitfor_mask_t & mask );
+static inline void reset_mask( $monitor * this );
+
+static inline $thread * next_thread( $monitor * this );
+static inline bool is_accepted( $monitor * this, const __monitor_group_t & monitors );
 
 static inline void lock_all  ( __spinlock_t * locks [], __lock_size_t count );
-static inline void lock_all  ( monitor_desc * source [], __spinlock_t * /*out*/ locks [], __lock_size_t count );
+static inline void lock_all  ( $monitor * source [], __spinlock_t * /*out*/ locks [], __lock_size_t count );
 static inline void unlock_all( __spinlock_t * locks [], __lock_size_t count );
-static inline void unlock_all( monitor_desc * locks [], __lock_size_t count );
-
-static inline void save   ( monitor_desc * ctx [], __lock_size_t count, __spinlock_t * locks [], unsigned int /*out*/ recursions [], __waitfor_mask_t /*out*/ masks [] );
-static inline void restore( monitor_desc * ctx [], __lock_size_t count, __spinlock_t * locks [], unsigned int /*in */ recursions [], __waitfor_mask_t /*in */ masks [] );
-
-static inline void init     ( __lock_size_t count, monitor_desc * monitors [], __condition_node_t & waiter, __condition_criterion_t criteria [] );
-static inline void init_push( __lock_size_t count, monitor_desc * monitors [], __condition_node_t & waiter, __condition_criterion_t criteria [] );
-
-static inline thread_desc *        check_condition   ( __condition_criterion_t * );
+static inline void unlock_all( $monitor * locks [], __lock_size_t count );
+
+static inline void save   ( $monitor * ctx [], __lock_size_t count, __spinlock_t * locks [], unsigned int /*out*/ recursions [], __waitfor_mask_t /*out*/ masks [] );
+static inline void restore( $monitor * ctx [], __lock_size_t count, __spinlock_t * locks [], unsigned int /*in */ recursions [], __waitfor_mask_t /*in */ masks [] );
+
+static inline void init     ( __lock_size_t count, $monitor * monitors [], __condition_node_t & waiter, __condition_criterion_t criteria [] );
+static inline void init_push( __lock_size_t count, $monitor * monitors [], __condition_node_t & waiter, __condition_criterion_t criteria [] );
+
+static inline $thread *        check_condition   ( __condition_criterion_t * );
 static inline void                 brand_condition   ( condition & );
-static inline [thread_desc *, int] search_entry_queue( const __waitfor_mask_t &, monitor_desc * monitors [], __lock_size_t count );
+static inline [$thread *, int] search_entry_queue( const __waitfor_mask_t &, $monitor * monitors [], __lock_size_t count );
 
 forall(dtype T | sized( T ))
 static inline __lock_size_t insert_unique( T * array [], __lock_size_t & size, T * val );
 static inline __lock_size_t count_max    ( const __waitfor_mask_t & mask );
-static inline __lock_size_t aggregate    ( monitor_desc * storage [], const __waitfor_mask_t & mask );
+static inline __lock_size_t aggregate    ( $monitor * storage [], const __waitfor_mask_t & mask );
 
 //-----------------------------------------------------------------------------
@@ -68 +68 @@
 
 #define monitor_ctx( mons, cnt )                                /* Define that create the necessary struct for internal/external scheduling operations */ \
-        monitor_desc ** monitors = mons;                          /* Save the targeted monitors                                                          */ \
+        $monitor ** monitors = mons;                          /* Save the targeted monitors                                                          */ \
         __lock_size_t count = cnt;                                /* Save the count to a local variable                                                  */ \
         unsigned int recursions[ count ];                         /* Save the current recursion levels to restore them later                             */ \
@@ -80 +80 @@
 //-----------------------------------------------------------------------------
 // Enter/Leave routines
-
-
-extern "C" {
-        // Enter single monitor
-        static void __enter_monitor_desc( monitor_desc * this, const __monitor_group_t & group ) {
-                // Lock the monitor spinlock
-                lock( this->lock __cfaabi_dbg_ctx2 );
-                // Interrupts disable inside critical section
-                thread_desc * thrd = kernelTLS.this_thread;
-
-                __cfaabi_dbg_print_safe( "Kernel : %10p Entering mon %p (%p)\n", thrd, this, this->owner);
-
-                if( !this->owner ) {
-                        // No one has the monitor, just take it
-                        set_owner( this, thrd );
-
-                        __cfaabi_dbg_print_safe( "Kernel :  mon is free \n" );
-                }
-                else if( this->owner == thrd) {
-                        // We already have the monitor, just note how many times we took it
-                        this->recursion += 1;
-
-                        __cfaabi_dbg_print_safe( "Kernel :  mon already owned \n" );
-                }
-                else if( is_accepted( this, group) ) {
-                        // Some one was waiting for us, enter
-                        set_owner( this, thrd );
-
-                        // Reset mask
-                        reset_mask( this );
-
-                        __cfaabi_dbg_print_safe( "Kernel :  mon accepts \n" );
-                }
-                else {
-                        __cfaabi_dbg_print_safe( "Kernel :  blocking \n" );
-
-                        // Some one else has the monitor, wait in line for it
-                        append( this->entry_queue, thrd );
-
-                        BlockInternal( &this->lock );
-
-                        __cfaabi_dbg_print_safe( "Kernel : %10p Entered  mon %p\n", thrd, this);
-
-                        // BlockInternal will unlock spinlock, no need to unlock ourselves
-                        return;
-                }
+// Enter single monitor
+static void __enter( $monitor * this, const __monitor_group_t & group ) {
+        $thread * thrd = active_thread();
+
+        // Lock the monitor spinlock
+        lock( this->lock __cfaabi_dbg_ctx2 );
+
+        __cfaabi_dbg_print_safe( "Kernel : %10p Entering mon %p (%p)\n", thrd, this, this->owner);
+
+        if( unlikely(0 != (0x1 & (uintptr_t)this->owner)) ) {
+                abort( "Attempt by thread \"%.256s\" (%p) to access joined monitor %p.", thrd->self_cor.name, thrd, this );
+        }
+        else if( !this->owner ) {
+                // No one has the monitor, just take it
+                __set_owner( this, thrd );
+
+                __cfaabi_dbg_print_safe( "Kernel :  mon is free \n" );
+        }
+        else if( this->owner == thrd) {
+                // We already have the monitor, just note how many times we took it
+                this->recursion += 1;
+
+                __cfaabi_dbg_print_safe( "Kernel :  mon already owned \n" );
+        }
+        else if( is_accepted( this, group) ) {
+                // Some one was waiting for us, enter
+                __set_owner( this, thrd );
+
+                // Reset mask
+                reset_mask( this );
+
+                __cfaabi_dbg_print_safe( "Kernel :  mon accepts \n" );
+        }
+        else {
+                __cfaabi_dbg_print_safe( "Kernel :  blocking \n" );
+
+                // Some one else has the monitor, wait in line for it
+                /* paranoid */ verify( thrd->link.next == 0p );
+                append( this->entry_queue, thrd );
+                /* paranoid */ verify( thrd->link.next == 1p );
+
+                unlock( this->lock );
+                park();
 
                 __cfaabi_dbg_print_safe( "Kernel : %10p Entered  mon %p\n", thrd, this);
 
-                // Release the lock and leave
+                /* paranoid */ verifyf( active_thread() == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", active_thread(), this->owner, this->recursion, this );
+                return;
+        }
+
+        __cfaabi_dbg_print_safe( "Kernel : %10p Entered  mon %p\n", thrd, this);
+
+        /* paranoid */ verifyf( active_thread() == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", active_thread(), this->owner, this->recursion, this );
+        /* paranoid */ verify( this->lock.lock );
+
+        // Release the lock and leave
+        unlock( this->lock );
+        return;
+}
+
+static void __dtor_enter( $monitor * this, fptr_t func, bool join ) {
+        $thread * thrd = active_thread();
+        #if defined( __CFA_WITH_VERIFY__ )
+                bool is_thrd = this == &thrd->self_mon;
+        #endif
+
+        // Lock the monitor spinlock
+        lock( this->lock __cfaabi_dbg_ctx2 );
+
+        __cfaabi_dbg_print_safe( "Kernel : %10p Entering dtor for mon %p (%p)\n", thrd, this, this->owner);
+
+
+        if( !this->owner ) {
+                __cfaabi_dbg_print_safe( "Kernel : Destroying free mon %p\n", this);
+
+                // No one has the monitor, just take it
+                __set_owner( this, thrd );
+
+                /* paranoid */ verifyf( active_thread() == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", active_thread(), this->owner, this->recursion, this );
+                /* paranoid */ verify( !is_thrd || thrd->state == Halted || thrd->state == Cancelled );
+
                 unlock( this->lock );
                 return;
         }
-
-        static void __enter_monitor_dtor( monitor_desc * this, fptr_t func ) {
-                // Lock the monitor spinlock
-                lock( this->lock __cfaabi_dbg_ctx2 );
-                // Interrupts disable inside critical section
-                thread_desc * thrd = kernelTLS.this_thread;
-
-                __cfaabi_dbg_print_safe( "Kernel : %10p Entering dtor for mon %p (%p)\n", thrd, this, this->owner);
-
-
-                if( !this->owner ) {
-                        __cfaabi_dbg_print_safe( "Kernel : Destroying free mon %p\n", this);
-
-                        // No one has the monitor, just take it
-                        set_owner( this, thrd );
-
-                        unlock( this->lock );
-                        return;
+        else if( this->owner == thrd && !join) {
+                // We already have the monitor... but where about to destroy it so the nesting will fail
+                // Abort!
+                abort( "Attempt to destroy monitor %p by thread \"%.256s\" (%p) in nested mutex.", this, thrd->self_cor.name, thrd );
+        }
+        // SKULLDUGGERY: join will act as a dtor so it would normally trigger to above check
+        // because join will not release the monitor after it executed.
+        // to avoid that it sets the owner to the special value thrd | 1p before exiting
+        else if( this->owner == ($thread*)(1 | (uintptr_t)thrd) ) {
+                // restore the owner and just return
+                __cfaabi_dbg_print_safe( "Kernel : Destroying free mon %p\n", this);
+
+                // No one has the monitor, just take it
+                __set_owner( this, thrd );
+
+                /* paranoid */ verifyf( active_thread() == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", active_thread(), this->owner, this->recursion, this );
+                /* paranoid */ verify( !is_thrd || thrd->state == Halted || thrd->state == Cancelled );
+
+                unlock( this->lock );
+                return;
+        }
+
+        // The monitor is busy, if this is a thread and the thread owns itself, it better be active
+        /* paranoid */ verify( !is_thrd || this->owner != thrd || (thrd->state != Halted && thrd->state != Cancelled) );
+
+        __lock_size_t count = 1;
+        $monitor ** monitors = &this;
+        __monitor_group_t group = { &this, 1, func };
+        if( is_accepted( this, group) ) {
+                __cfaabi_dbg_print_safe( "Kernel :  mon accepts dtor, block and signal it \n" );
+
+                // Wake the thread that is waiting for this
+                __condition_criterion_t * urgent = pop( this->signal_stack );
+                /* paranoid */ verify( urgent );
+
+                // Reset mask
+                reset_mask( this );
+
+                // Create the node specific to this wait operation
+                wait_ctx_primed( thrd, 0 )
+
+                // Some one else has the monitor, wait for him to finish and then run
+                unlock( this->lock );
+
+                // Release the next thread
+                /* paranoid */ verifyf( urgent->owner->waiting_thread == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", active_thread(), this->owner, this->recursion, this );
+                unpark( urgent->owner->waiting_thread );
+
+                // Park current thread waiting
+                park();
+
+                // Some one was waiting for us, enter
+                /* paranoid */ verifyf( active_thread() == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", active_thread(), this->owner, this->recursion, this );
+
+                __cfaabi_dbg_print_safe( "Kernel : Destroying %p\n", this);
+                return;
+        }
+        else {
+                __cfaabi_dbg_print_safe( "Kernel :  blocking \n" );
+
+                wait_ctx( thrd, 0 )
+                this->dtor_node = &waiter;
+
+                // Some one else has the monitor, wait in line for it
+                /* paranoid */ verify( thrd->link.next == 0p );
+                append( this->entry_queue, thrd );
+                /* paranoid */ verify( thrd->link.next == 1p );
+                unlock( this->lock );
+
+                // Park current thread waiting
+                park();
+
+                /* paranoid */ verifyf( active_thread() == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", active_thread(), this->owner, this->recursion, this );
+                return;
+        }
+}
+
+// Leave single monitor
+void __leave( $monitor * this ) {
+        // Lock the monitor spinlock
+        lock( this->lock __cfaabi_dbg_ctx2 );
+
+        __cfaabi_dbg_print_safe( "Kernel : %10p Leaving mon %p (%p)\n", active_thread(), this, this->owner);
+
+        /* paranoid */ verifyf( active_thread() == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", active_thread(), this->owner, this->recursion, this );
+
+        // Leaving a recursion level, decrement the counter
+        this->recursion -= 1;
+
+        // If we haven't left the last level of recursion
+        // it means we don't need to do anything
+        if( this->recursion != 0) {
+                __cfaabi_dbg_print_safe( "Kernel :  recursion still %d\n", this->recursion);
+                unlock( this->lock );
+                return;
+        }
+
+        // Get the next thread, will be null on low contention monitor
+        $thread * new_owner = next_thread( this );
+
+        // Check the new owner is consistent with who we wake-up
+        // new_owner might be null even if someone owns the monitor when the owner is still waiting for another monitor
+        /* paranoid */ verifyf( !new_owner || new_owner == this->owner, "Expected owner to be %p, got %p (m: %p)", new_owner, this->owner, this );
+
+        // We can now let other threads in safely
+        unlock( this->lock );
+
+        //We need to wake-up the thread
+        /* paranoid */ verifyf( !new_owner || new_owner == this->owner, "Expected owner to be %p, got %p (m: %p)", new_owner, this->owner, this );
+        unpark( new_owner );
+}
+
+// Leave single monitor for the last time
+void __dtor_leave( $monitor * this, bool join ) {
+        __cfaabi_dbg_debug_do(
+                if( active_thread() != this->owner ) {
+                        abort( "Destroyed monitor %p has inconsistent owner, expected %p got %p.\n", this, active_thread(), this->owner);
                 }
-                else if( this->owner == thrd) {
-                        // We already have the monitor... but where about to destroy it so the nesting will fail
-                        // Abort!
-                        abort( "Attempt to destroy monitor %p by thread \"%.256s\" (%p) in nested mutex.", this, thrd->self_cor.name, thrd );
+                if( this->recursion != 1  && !join ) {
+                        abort( "Destroyed monitor %p has %d outstanding nested calls.\n", this, this->recursion - 1);
                 }
-
-                __lock_size_t count = 1;
-                monitor_desc ** monitors = &this;
-                __monitor_group_t group = { &this, 1, func };
-                if( is_accepted( this, group) ) {
-                        __cfaabi_dbg_print_safe( "Kernel :  mon accepts dtor, block and signal it \n" );
-
-                        // Wake the thread that is waiting for this
-                        __condition_criterion_t * urgent = pop( this->signal_stack );
-                        verify( urgent );
-
-                        // Reset mask
-                        reset_mask( this );
-
-                        // Create the node specific to this wait operation
-                        wait_ctx_primed( thrd, 0 )
-
-                        // Some one else has the monitor, wait for him to finish and then run
-                        BlockInternal( &this->lock, urgent->owner->waiting_thread );
-
-                        // Some one was waiting for us, enter
-                        set_owner( this, thrd );
-                }
-                else {
-                        __cfaabi_dbg_print_safe( "Kernel :  blocking \n" );
-
-                        wait_ctx( thrd, 0 )
-                        this->dtor_node = &waiter;
-
-                        // Some one else has the monitor, wait in line for it
-                        append( this->entry_queue, thrd );
-                        BlockInternal( &this->lock );
-
-                        // BlockInternal will unlock spinlock, no need to unlock ourselves
-                        return;
-                }
-
-                __cfaabi_dbg_print_safe( "Kernel : Destroying %p\n", this);
-
-        }
-
-        // Leave single monitor
-        void __leave_monitor_desc( monitor_desc * this ) {
-                // Lock the monitor spinlock
-                lock( this->lock __cfaabi_dbg_ctx2 );
-
-                __cfaabi_dbg_print_safe( "Kernel : %10p Leaving mon %p (%p)\n", kernelTLS.this_thread, this, this->owner);
-
-                verifyf( kernelTLS.this_thread == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", kernelTLS.this_thread, this->owner, this->recursion, this );
-
-                // Leaving a recursion level, decrement the counter
-                this->recursion -= 1;
-
-                // If we haven't left the last level of recursion
-                // it means we don't need to do anything
-                if( this->recursion != 0) {
-                        __cfaabi_dbg_print_safe( "Kernel :  recursion still %d\n", this->recursion);
-                        unlock( this->lock );
-                        return;
-                }
-
-                // Get the next thread, will be null on low contention monitor
-                thread_desc * new_owner = next_thread( this );
-
-                // We can now let other threads in safely
-                unlock( this->lock );
-
-                //We need to wake-up the thread
-                WakeThread( new_owner );
-        }
-
-        // Leave single monitor for the last time
-        void __leave_dtor_monitor_desc( monitor_desc * this ) {
-                __cfaabi_dbg_debug_do(
-                        if( TL_GET( this_thread ) != this->owner ) {
-                                abort( "Destroyed monitor %p has inconsistent owner, expected %p got %p.\n", this, TL_GET( this_thread ), this->owner);
-                        }
-                        if( this->recursion != 1 ) {
-                                abort( "Destroyed monitor %p has %d outstanding nested calls.\n", this, this->recursion - 1);
-                        }
-                )
-        }
-
-        // Leave the thread monitor
-        // last routine called by a thread.
-        // Should never return
-        void __leave_thread_monitor() {
-                thread_desc * thrd = TL_GET( this_thread );
-                monitor_desc * this = &thrd->self_mon;
-
-                // Lock the monitor now
-                lock( this->lock __cfaabi_dbg_ctx2 );
-
-                disable_interrupts();
-
-                thrd->self_cor.state = Halted;
-
-                verifyf( thrd == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", thrd, this->owner, this->recursion, this );
-
-                // Leaving a recursion level, decrement the counter
-                this->recursion -= 1;
-
-                // If we haven't left the last level of recursion
-                // it must mean there is an error
-                if( this->recursion != 0) { abort( "Thread internal monitor has unbalanced recursion" ); }
-
-                // Fetch the next thread, can be null
-                thread_desc * new_owner = next_thread( this );
-
-                // Leave the thread, this will unlock the spinlock
-                // Use leave thread instead of BlockInternal which is
-                // specialized for this case and supports null new_owner
-                LeaveThread( &this->lock, new_owner );
-
-                // Control flow should never reach here!
-        }
+        )
+
+        this->owner = ($thread*)(1 | (uintptr_t)this->owner);
+}
+
+void __thread_finish( $thread * thrd ) {
+        $monitor * this = &thrd->self_mon;
+
+        // Lock the monitor now
+        /* paranoid */ verify( 0x0D15EA5E0D15EA5Ep == thrd->canary );
+        /* paranoid */ verify( this->lock.lock );
+        /* paranoid */ verify( thrd->context.SP );
+        /* paranoid */ verifyf( ((uintptr_t)thrd->context.SP) > ((uintptr_t)__get_stack(thrd->curr_cor)->limit), "ERROR : $thread %p has been corrupted.\n StackPointer too large.\n", thrd );
+        /* paranoid */ verifyf( ((uintptr_t)thrd->context.SP) < ((uintptr_t)__get_stack(thrd->curr_cor)->base ), "ERROR : $thread %p has been corrupted.\n StackPointer too small.\n", thrd );
+        /* paranoid */ verify( ! __preemption_enabled() );
+
+        /* paranoid */ verifyf( thrd == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", thrd, this->owner, this->recursion, this );
+        /* paranoid */ verify( thrd->state == Halting );
+        /* paranoid */ verify( this->recursion == 1 );
+
+        // Leaving a recursion level, decrement the counter
+        this->recursion -= 1;
+        this->owner = 0p;
+
+        // Fetch the next thread, can be null
+        $thread * new_owner = next_thread( this );
+
+        // Mark the state as fully halted
+        thrd->state = Halted;
+
+        // Release the monitor lock
+        unlock( this->lock );
+
+        // Unpark the next owner if needed
+        /* paranoid */ verifyf( !new_owner || new_owner == this->owner, "Expected owner to be %p, got %p (m: %p)", new_owner, this->owner, this );
+        /* paranoid */ verify( ! __preemption_enabled() );
+        /* paranoid */ verify( thrd->state == Halted );
+        unpark( new_owner );
 }
 
@@ -279 +329 @@
 static inline void enter( __monitor_group_t monitors ) {
         for( __lock_size_t i = 0; i < monitors.size; i++) {
-                __enter_monitor_desc( monitors[i], monitors );
+                __enter( monitors[i], monitors );
         }
 }
@@ -285 +335 @@
 // Leave multiple monitor
 // relies on the monitor array being sorted
-static inline void leave(monitor_desc * monitors [], __lock_size_t count) {
+static inline void leave($monitor * monitors [], __lock_size_t count) {
         for( __lock_size_t i = count - 1; i >= 0; i--) {
-                __leave_monitor_desc( monitors[i] );
+                __leave( monitors[i] );
         }
 }
@@ -293 +343 @@
 // Ctor for monitor guard
 // Sorts monitors before entering
-void ?{}( monitor_guard_t & this, monitor_desc * m [], __lock_size_t count, fptr_t func ) {
-        thread_desc * thrd = TL_GET( this_thread );
+void ?{}( monitor_guard_t & this, $monitor * m [], __lock_size_t count, fptr_t func ) {
+        $thread * thrd = active_thread();
 
         // Store current array
@@ -329 +379 @@
 
         // Restore thread context
-        TL_GET( this_thread )->monitors = this.prev;
+        active_thread()->monitors = this.prev;
 }
 
 // Ctor for monitor guard
 // Sorts monitors before entering
-void ?{}( monitor_dtor_guard_t & this, monitor_desc * m [], fptr_t func ) {
+void ?{}( monitor_dtor_guard_t & this, $monitor * m [], fptr_t func, bool join ) {
         // optimization
-        thread_desc * thrd = TL_GET( this_thread );
+        $thread * thrd = active_thread();
 
         // Store current array
@@ -344 +394 @@
         this.prev = thrd->monitors;
 
+        // Save whether we are in a join or not
+        this.join = join;
+
         // Update thread context (needed for conditions)
         (thrd->monitors){m, 1, func};
 
-        __enter_monitor_dtor( this.m, func );
+        __dtor_enter( this.m, func, join );
 }
 
@@ -353 +406 @@
 void ^?{}( monitor_dtor_guard_t & this ) {
         // Leave the monitors in order
-        __leave_dtor_monitor_desc( this.m );
+        __dtor_leave( this.m, this.join );
 
         // Restore thread context
-        TL_GET( this_thread )->monitors = this.prev;
+        active_thread()->monitors = this.prev;
 }
 
 //-----------------------------------------------------------------------------
 // Internal scheduling types
-void ?{}(__condition_node_t & this, thread_desc * waiting_thread, __lock_size_t count, uintptr_t user_info ) {
+void ?{}(__condition_node_t & this, $thread * waiting_thread, __lock_size_t count, uintptr_t user_info ) {
         this.waiting_thread = waiting_thread;
         this.count = count;
@@ -375 +428 @@
 }
 
-void ?{}(__condition_criterion_t & this, monitor_desc * target, __condition_node_t & owner ) {
+void ?{}(__condition_criterion_t & this, $monitor * target, __condition_node_t & owner ) {
         this.ready  = false;
         this.target = target;
@@ -396 +449 @@
 
         // Create the node specific to this wait operation
-        wait_ctx( TL_GET( this_thread ), user_info );
+        wait_ctx( active_thread(), user_info );
 
         // Append the current wait operation to the ones already queued on the condition
         // We don't need locks for that since conditions must always be waited on inside monitor mutual exclusion
+        /* paranoid */ verify( waiter.next == 0p );
         append( this.blocked, &waiter );
+        /* paranoid */ verify( waiter.next == 1p );
 
         // Lock all monitors (aggregates the locks as well)
@@ -407 +462 @@
         // Find the next thread(s) to run
         __lock_size_t thread_count = 0;
-        thread_desc * threads[ count ];
+        $thread * threads[ count ];
         __builtin_memset( threads, 0, sizeof( threads ) );
 
@@ -415 +470 @@
         // Remove any duplicate threads
         for( __lock_size_t i = 0; i < count; i++) {
-                thread_desc * new_owner = next_thread( monitors[i] );
+                $thread * new_owner = next_thread( monitors[i] );
                 insert_unique( threads, thread_count, new_owner );
         }
 
+        // Unlock the locks, we don't need them anymore
+        for(int i = 0; i < count; i++) {
+                unlock( *locks[i] );
+        }
+
+        // Wake the threads
+        for(int i = 0; i < thread_count; i++) {
+                unpark( threads[i] );
+        }
+
         // Everything is ready to go to sleep
-        BlockInternal( locks, count, threads, thread_count );
+        park();
 
         // We are back, restore the owners and recursions
@@ -435 +500 @@
         //Some more checking in debug
         __cfaabi_dbg_debug_do(
-                thread_desc * this_thrd = TL_GET( this_thread );
+                $thread * this_thrd = active_thread();
                 if ( this.monitor_count != this_thrd->monitors.size ) {
                         abort( "Signal on condition %p made with different number of monitor(s), expected %zi got %zi", &this, this.monitor_count, this_thrd->monitors.size );
@@ -483 +548 @@
 
         // Create the node specific to this wait operation
-        wait_ctx_primed( kernelTLS.this_thread, 0 )
+        wait_ctx_primed( active_thread(), 0 )
 
         //save contexts
@@ -489 +554 @@
 
         //Find the thread to run
-        thread_desc * signallee = pop_head( this.blocked )->waiting_thread;
-        set_owner( monitors, count, signallee );
+        $thread * signallee = pop_head( this.blocked )->waiting_thread;
+        __set_owner( monitors, count, signallee );
 
         __cfaabi_dbg_print_buffer_decl( "Kernel : signal_block condition %p (s: %p)\n", &this, signallee );
 
+        // unlock all the monitors
+        unlock_all( locks, count );
+
+        // unpark the thread we signalled
+        unpark( signallee );
+
         //Everything is ready to go to sleep
-        BlockInternal( locks, count, &signallee, 1 );
+        park();
 
 
@@ -536 +607 @@
         // Create one!
         __lock_size_t max = count_max( mask );
-        monitor_desc * mon_storage[max];
+        $monitor * mon_storage[max];
         __builtin_memset( mon_storage, 0, sizeof( mon_storage ) );
         __lock_size_t actual_count = aggregate( mon_storage, mask );
@@ -554 +625 @@
         {
                 // Check if the entry queue
-                thread_desc * next; int index;
+                $thread * next; int index;
                 [next, index] = search_entry_queue( mask, monitors, count );
 
@@ -564 +635 @@
                                 verifyf( accepted.size == 1,  "ERROR: Accepted dtor has more than 1 mutex parameter." );
 
-                                monitor_desc * mon2dtor = accepted[0];
+                                $monitor * mon2dtor = accepted[0];
                                 verifyf( mon2dtor->dtor_node, "ERROR: Accepted monitor has no dtor_node." );
 
@@ -576 +647 @@
 
                                 // Create the node specific to this wait operation
-                                wait_ctx_primed( kernelTLS.this_thread, 0 );
+                                wait_ctx_primed( active_thread(), 0 );
 
                                 // Save monitor states
@@ -590 +661 @@
 
                                 // Set the owners to be the next thread
-                                set_owner( monitors, count, next );
-
-                                // Everything is ready to go to sleep
-                                BlockInternal( locks, count, &next, 1 );
+                                __set_owner( monitors, count, next );
+
+                                // unlock all the monitors
+                                unlock_all( locks, count );
+
+                                // unpark the thread we signalled
+                                unpark( next );
+
+                                //Everything is ready to go to sleep
+                                park();
 
                                 // We are back, restore the owners and recursions
@@ -622 +699 @@
 
         // Create the node specific to this wait operation
-        wait_ctx_primed( kernelTLS.this_thread, 0 );
+        wait_ctx_primed( active_thread(), 0 );
 
         monitor_save;
@@ -628 +705 @@
 
         for( __lock_size_t i = 0; i < count; i++) {
-                verify( monitors[i]->owner == kernelTLS.this_thread );
-        }
+                verify( monitors[i]->owner == active_thread() );
+        }
+
+        // unlock all the monitors
+        unlock_all( locks, count );
 
         //Everything is ready to go to sleep
-        BlockInternal( locks, count );
+        park();
 
 
@@ -649 +729 @@
 // Utilities
 
-static inline void set_owner( monitor_desc * this, thread_desc * owner ) {
-        // __cfaabi_dbg_print_safe( "Kernal :   Setting owner of %p to %p ( was %p)\n", this, owner, this->owner );
+static inline void __set_owner( $monitor * this, $thread * owner ) {
+        /* paranoid */ verify( this->lock.lock );
 
         //Pass the monitor appropriately
@@ -659 +739 @@
 }
 
-static inline void set_owner( monitor_desc * monitors [], __lock_size_t count, thread_desc * owner ) {
-        monitors[0]->owner     = owner;
-        monitors[0]->recursion = 1;
+static inline void __set_owner( $monitor * monitors [], __lock_size_t count, $thread * owner ) {
+        /* paranoid */ verify ( monitors[0]->lock.lock );
+        /* paranoid */ verifyf( monitors[0]->owner == active_thread(), "Expected owner to be %p, got %p (r: %i, m: %p)", active_thread(), monitors[0]->owner, monitors[0]->recursion, monitors[0] );
+        monitors[0]->owner        = owner;
+        monitors[0]->recursion    = 1;
         for( __lock_size_t i = 1; i < count; i++ ) {
-                monitors[i]->owner     = owner;
-                monitors[i]->recursion = 0;
-        }
-}
-
-static inline void set_mask( monitor_desc * storage [], __lock_size_t count, const __waitfor_mask_t & mask ) {
+                /* paranoid */ verify ( monitors[i]->lock.lock );
+                /* paranoid */ verifyf( monitors[i]->owner == active_thread(), "Expected owner to be %p, got %p (r: %i, m: %p)", active_thread(), monitors[i]->owner, monitors[i]->recursion, monitors[i] );
+                monitors[i]->owner        = owner;
+                monitors[i]->recursion    = 0;
+        }
+}
+
+static inline void set_mask( $monitor * storage [], __lock_size_t count, const __waitfor_mask_t & mask ) {
         for( __lock_size_t i = 0; i < count; i++) {
                 storage[i]->mask = mask;
@@ -674 +758 @@
 }
 
-static inline void reset_mask( monitor_desc * this ) {
+static inline void reset_mask( $monitor * this ) {
         this->mask.accepted = 0p;
         this->mask.data = 0p;
@@ -680 +764 @@
 }
 
-static inline thread_desc * next_thread( monitor_desc * this ) {
+static inline $thread * next_thread( $monitor * this ) {
         //Check the signaller stack
         __cfaabi_dbg_print_safe( "Kernel :  mon %p AS-stack top %p\n", this, this->signal_stack.top);
@@ -688 +772 @@
                 //regardless of if we are ready to baton pass,
                 //we need to set the monitor as in use
-                set_owner( this,  urgent->owner->waiting_thread );
+                /* paranoid */ verifyf( !this->owner || active_thread() == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", active_thread(), this->owner, this->recursion, this );
+                __set_owner( this,  urgent->owner->waiting_thread );
 
                 return check_condition( urgent );
@@ -695 +780 @@
         // No signaller thread
         // Get the next thread in the entry_queue
-        thread_desc * new_owner = pop_head( this->entry_queue );
-        set_owner( this, new_owner );
+        $thread * new_owner = pop_head( this->entry_queue );
+        /* paranoid */ verifyf( !this->owner || active_thread() == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", active_thread(), this->owner, this->recursion, this );
+        /* paranoid */ verify( !new_owner || new_owner->link.next == 0p );
+        __set_owner( this, new_owner );
 
         return new_owner;
 }
 
-static inline bool is_accepted( monitor_desc * this, const __monitor_group_t & group ) {
+static inline bool is_accepted( $monitor * this, const __monitor_group_t & group ) {
         __acceptable_t * it = this->mask.data; // Optim
         __lock_size_t count = this->mask.size;
@@ -723 +810 @@
 }
 
-static inline void init( __lock_size_t count, monitor_desc * monitors [], __condition_node_t & waiter, __condition_criterion_t criteria [] ) {
+static inline void init( __lock_size_t count, $monitor * monitors [], __condition_node_t & waiter, __condition_criterion_t criteria [] ) {
         for( __lock_size_t i = 0; i < count; i++) {
                 (criteria[i]){ monitors[i], waiter };
@@ -731 +818 @@
 }
 
-static inline void init_push( __lock_size_t count, monitor_desc * monitors [], __condition_node_t & waiter, __condition_criterion_t criteria [] ) {
+static inline void init_push( __lock_size_t count, $monitor * monitors [], __condition_node_t & waiter, __condition_criterion_t criteria [] ) {
         for( __lock_size_t i = 0; i < count; i++) {
                 (criteria[i]){ monitors[i], waiter };
@@ -747 +834 @@
 }
 
-static inline void lock_all( monitor_desc * source [], __spinlock_t * /*out*/ locks [], __lock_size_t count ) {
+static inline void lock_all( $monitor * source [], __spinlock_t * /*out*/ locks [], __lock_size_t count ) {
         for( __lock_size_t i = 0; i < count; i++ ) {
                 __spinlock_t * l = &source[i]->lock;
@@ -761 +848 @@
 }
 
-static inline void unlock_all( monitor_desc * locks [], __lock_size_t count ) {
+static inline void unlock_all( $monitor * locks [], __lock_size_t count ) {
         for( __lock_size_t i = 0; i < count; i++ ) {
                 unlock( locks[i]->lock );
@@ -768 +855 @@
 
 static inline void save(
-        monitor_desc * ctx [],
+        $monitor * ctx [],
         __lock_size_t count,
         __attribute((unused)) __spinlock_t * locks [],
@@ -781 +868 @@
 
 static inline void restore(
-        monitor_desc * ctx [],
+        $monitor * ctx [],
         __lock_size_t count,
         __spinlock_t * locks [],
@@ -799 +886 @@
 // 2 - Checks if all the monitors are ready to run
 //     if so return the thread to run
-static inline thread_desc * check_condition( __condition_criterion_t * target ) {
+static inline $thread * check_condition( __condition_criterion_t * target ) {
         __condition_node_t * node = target->owner;
         unsigned short count = node->count;
@@ -817 +904 @@
         }
 
-        __cfaabi_dbg_print_safe( "Kernel :  Runing %i (%p)\n", ready2run, ready2run ? node->waiting_thread : 0p );
+        __cfaabi_dbg_print_safe( "Kernel :  Runing %i (%p)\n", ready2run, ready2run ? (thread*)node->waiting_thread : (thread*)0p );
         return ready2run ? node->waiting_thread : 0p;
 }
 
 static inline void brand_condition( condition & this ) {
-        thread_desc * thrd = TL_GET( this_thread );
+        $thread * thrd = active_thread();
         if( !this.monitors ) {
                 // __cfaabi_dbg_print_safe( "Branding\n" );
@@ -828 +915 @@
                 this.monitor_count = thrd->monitors.size;
 
-                this.monitors = (monitor_desc **)malloc( this.monitor_count * sizeof( *this.monitors ) );
+                this.monitors = ($monitor **)malloc( this.monitor_count * sizeof( *this.monitors ) );
                 for( int i = 0; i < this.monitor_count; i++ ) {
                         this.monitors[i] = thrd->monitors[i];
@@ -835 +922 @@
 }
 
-static inline [thread_desc *, int] search_entry_queue( const __waitfor_mask_t & mask, monitor_desc * monitors [], __lock_size_t count ) {
-
-        __queue_t(thread_desc) & entry_queue = monitors[0]->entry_queue;
+static inline [$thread *, int] search_entry_queue( const __waitfor_mask_t & mask, $monitor * monitors [], __lock_size_t count ) {
+
+        __queue_t($thread) & entry_queue = monitors[0]->entry_queue;
 
         // For each thread in the entry-queue
-        for(    thread_desc ** thrd_it = &entry_queue.head;
-                *thrd_it;
-                thrd_it = &(*thrd_it)->next
+        for(    $thread ** thrd_it = &entry_queue.head;
+                (*thrd_it) != 1p;
+                thrd_it = &(*thrd_it)->link.next
         ) {
                 // For each acceptable check if it matches
@@ -884 +971 @@
 }
 
-static inline __lock_size_t aggregate( monitor_desc * storage [], const __waitfor_mask_t & mask ) {
+static inline __lock_size_t aggregate( $monitor * storage [], const __waitfor_mask_t & mask ) {
         __lock_size_t size = 0;
         for( __lock_size_t i = 0; i < mask.size; i++ ) {

libcfa/src/concurrency/monitor.hfa

@@ -23 +23 @@
 
 trait is_monitor(dtype T) {
-        monitor_desc * get_monitor( T & );
+        $monitor * get_monitor( T & );
         void ^?{}( T & mutex );
 };
 
-static inline void ?{}(monitor_desc & this) with( this ) {
+static inline void ?{}($monitor & this) with( this ) {
         lock{};
         entry_queue{};
@@ -39 +39 @@
 }
 
-static inline void ^?{}(monitor_desc & ) {}
+static inline void ^?{}($monitor & ) {}
 
 struct monitor_guard_t {
-        monitor_desc **         m;
+        $monitor **     m;
         __lock_size_t           count;
         __monitor_group_t prev;
 };
 
-void ?{}( monitor_guard_t & this, monitor_desc ** m, __lock_size_t count, void (*func)() );
+void ?{}( monitor_guard_t & this, $monitor ** m, __lock_size_t count, void (*func)() );
 void ^?{}( monitor_guard_t & this );
 
 struct monitor_dtor_guard_t {
-        monitor_desc *    m;
+        $monitor *    m;
         __monitor_group_t prev;
+        bool join;
 };
 
-void ?{}( monitor_dtor_guard_t & this, monitor_desc ** m, void (*func)() );
+void ?{}( monitor_dtor_guard_t & this, $monitor ** m, void (*func)(), bool join );
 void ^?{}( monitor_dtor_guard_t & this );
 
@@ -72 +73 @@
 
         // The monitor this criterion concerns
-        monitor_desc * target;
+        $monitor * target;
 
         // The parent node to which this criterion belongs
@@ -87 +88 @@
 struct __condition_node_t {
         // Thread that needs to be woken when all criteria are met
-        thread_desc * waiting_thread;
+        $thread * waiting_thread;
 
         // Array of criteria (Criterions are contiguous in memory)
@@ -106 +107 @@
 }
 
-void ?{}(__condition_node_t & this, thread_desc * waiting_thread, __lock_size_t count, uintptr_t user_info );
+void ?{}(__condition_node_t & this, $thread * waiting_thread, __lock_size_t count, uintptr_t user_info );
 void ?{}(__condition_criterion_t & this );
-void ?{}(__condition_criterion_t & this, monitor_desc * target, __condition_node_t * owner );
+void ?{}(__condition_criterion_t & this, $monitor * target, __condition_node_t * owner );
 
 struct condition {
@@ -115 +116 @@
 
         // Array of monitor pointers (Monitors are NOT contiguous in memory)
-        monitor_desc ** monitors;
+        $monitor ** monitors;
 
         // Number of monitors in the array
@@ -131 +132 @@
 
               void wait        ( condition & this, uintptr_t user_info = 0 );
+static inline bool is_empty    ( condition & this ) { return this.blocked.head == 1p; }
               bool signal      ( condition & this );
               bool signal_block( condition & this );
-static inline bool is_empty    ( condition & this ) { return !this.blocked.head; }
+static inline bool signal_all  ( condition & this ) { bool ret = false; while(!is_empty(this)) { ret = signal(this) || ret; } return ret; }
          uintptr_t front       ( condition & this );
 

libcfa/src/concurrency/mutex.cfa

@@ -30 +30 @@
         this.lock{};
         this.blocked_threads{};
+        this.is_locked = false;
 }
 
@@ -39 +40 @@
         lock( lock __cfaabi_dbg_ctx2 );
         if( is_locked ) {
-                append( blocked_threads, kernelTLS.this_thread );
-                BlockInternal( &lock );
+                append( blocked_threads, active_thread() );
+                unlock( lock );
+                park();
         }
         else {
@@ -62 +64 @@
         lock( this.lock __cfaabi_dbg_ctx2 );
         this.is_locked = (this.blocked_threads != 0);
-        WakeThread(
+        unpark(
                 pop_head( this.blocked_threads )
         );
@@ -84 +86 @@
         lock( lock __cfaabi_dbg_ctx2 );
         if( owner == 0p ) {
-                owner = kernelTLS.this_thread;
+                owner = active_thread();
                 recursion_count = 1;
                 unlock( lock );
         }
-        else if( owner == kernelTLS.this_thread ) {
+        else if( owner == active_thread() ) {
                 recursion_count++;
                 unlock( lock );
         }
         else {
-                append( blocked_threads, kernelTLS.this_thread );
-                BlockInternal( &lock );
+                append( blocked_threads, active_thread() );
+                unlock( lock );
+                park();
         }
 }
@@ -102 +105 @@
         lock( lock __cfaabi_dbg_ctx2 );
         if( owner == 0p ) {
-                owner = kernelTLS.this_thread;
+                owner = active_thread();
                 recursion_count = 1;
                 ret = true;
         }
-        else if( owner == kernelTLS.this_thread ) {
+        else if( owner == active_thread() ) {
                 recursion_count++;
                 ret = true;
@@ -118 +121 @@
         recursion_count--;
         if( recursion_count == 0 ) {
-                thread_desc * thrd = pop_head( blocked_threads );
+                $thread * thrd = pop_head( blocked_threads );
                 owner = thrd;
                 recursion_count = (thrd ? 1 : 0);
-                WakeThread( thrd );
+                unpark( thrd );
         }
         unlock( lock );
@@ -138 +141 @@
 void notify_one(condition_variable & this) with(this) {
         lock( lock __cfaabi_dbg_ctx2 );
-        WakeThread(
+        unpark(
                 pop_head( this.blocked_threads )
         );
@@ -147 +150 @@
         lock( lock __cfaabi_dbg_ctx2 );
         while(this.blocked_threads) {
-                WakeThread(
+                unpark(
                         pop_head( this.blocked_threads )
                 );
@@ -156 +159 @@
 void wait(condition_variable & this) {
         lock( this.lock __cfaabi_dbg_ctx2 );
-        append( this.blocked_threads, kernelTLS.this_thread );
-        BlockInternal( &this.lock );
+        append( this.blocked_threads, active_thread() );
+        unlock( this.lock );
+        park();
 }
 
@@ -163 +167 @@
 void wait(condition_variable & this, L & l) {
         lock( this.lock __cfaabi_dbg_ctx2 );
-        append( this.blocked_threads, kernelTLS.this_thread );
-        void __unlock(void) {
-                unlock(l);
-                unlock(this.lock);
-        }
-        BlockInternal( __unlock );
+        append( this.blocked_threads, active_thread() );
+        unlock(l);
+        unlock(this.lock);
+        park();
         lock(l);
 }

libcfa/src/concurrency/mutex.hfa

@@ -36 +36 @@
 
         // List of blocked threads
-        __queue_t(struct thread_desc) blocked_threads;
+        __queue_t(struct $thread) blocked_threads;
 
         // Locked flag
@@ -55 +55 @@
 
         // List of blocked threads
-        __queue_t(struct thread_desc) blocked_threads;
+        __queue_t(struct $thread) blocked_threads;
 
         // Current thread owning the lock
-        struct thread_desc * owner;
+        struct $thread * owner;
 
         // Number of recursion level
@@ -83 +83 @@
 
         // List of blocked threads
-        __queue_t(struct thread_desc) blocked_threads;
+        __queue_t(struct $thread) blocked_threads;
 };
 

libcfa/src/concurrency/preemption.cfa

@@ -10 +10 @@
 // Created On       : Mon Jun 5 14:20:42 2017
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Thu Dec  5 16:34:05 2019
-// Update Count     : 43
+// Last Modified On : Fri Nov  6 07:42:13 2020
+// Update Count     : 54
 //
 
@@ -19 +19 @@
 #include <assert.h>
 
-extern "C" {
 #include <errno.h>
 #include <stdio.h>
@@ -25 +24 @@
 #include <unistd.h>
 #include <limits.h>                                                                             // PTHREAD_STACK_MIN
-}
 
 #include "bits/signal.hfa"
+#include "kernel_private.hfa"
 
 #if !defined(__CFA_DEFAULT_PREEMPTION__)
@@ -39 +38 @@
 // FwdDeclarations : timeout handlers
 static void preempt( processor   * this );
-static void timeout( thread_desc * this );
+static void timeout( $thread * this );
 
 // FwdDeclarations : Signal handlers
 static void sigHandler_ctxSwitch( __CFA_SIGPARMS__ );
+static void sigHandler_alarm    ( __CFA_SIGPARMS__ );
 static void sigHandler_segv     ( __CFA_SIGPARMS__ );
 static void sigHandler_ill      ( __CFA_SIGPARMS__ );
@@ -56 +56 @@
 #elif defined( __x86_64 )
 #define CFA_REG_IP gregs[REG_RIP]
-#elif defined( __ARM_ARCH )
+#elif defined( __arm__ )
 #define CFA_REG_IP arm_pc
+#elif defined( __aarch64__ )
+#define CFA_REG_IP pc
 #else
-#error unknown hardware architecture
+#error unsupported hardware architecture
 #endif
 
@@ -83 +85 @@
 // Get next expired node
 static inline alarm_node_t * get_expired( alarm_list_t * alarms, Time currtime ) {
-        if( !alarms->head ) return 0p;                                          // If no alarms return null
-        if( alarms->head->alarm >= currtime ) return 0p;        // If alarms head not expired return null
+        if( ! & (*alarms)`first ) return 0p;                                            // If no alarms return null
+        if( (*alarms)`first.alarm >= currtime ) return 0p;      // If alarms head not expired return null
         return pop(alarms);                                                                     // Otherwise just pop head
 }
 
 // Tick one frame of the Discrete Event Simulation for alarms
-static void tick_preemption() {
+static void tick_preemption(void) {
         alarm_node_t * node = 0p;                                                       // Used in the while loop but cannot be declared in the while condition
         alarm_list_t * alarms = &event_kernel->alarms;          // Local copy for ease of reading
@@ -97 +99 @@
         while( node = get_expired( alarms, currtime ) ) {
                 // __cfaabi_dbg_print_buffer_decl( " KERNEL: preemption tick.\n" );
+                Duration period = node->period;
+                if( period == 0) {
+                        node->set = false;                  // Node is one-shot, just mark it as not pending
+                }
 
                 // Check if this is a kernel
-                if( node->kernel_alarm ) {
+                if( node->type == Kernel ) {
                         preempt( node->proc );
                 }
+                else if( node->type == User ) {
+                        timeout( node->thrd );
+                }
                 else {
-                        timeout( node->thrd );
+                        node->callback(*node);
                 }
 
                 // Check if this is a periodic alarm
-                Duration period = node->period;
                 if( period > 0 ) {
                         // __cfaabi_dbg_print_buffer_local( " KERNEL: alarm period is %lu.\n", period.tv );
@@ -113 +121 @@
                         insert( alarms, node );             // Reinsert the node for the next time it triggers
                 }
-                else {
-                        node->set = false;                  // Node is one-shot, just mark it as not pending
-                }
         }
 
         // If there are still alarms pending, reset the timer
-        if( alarms->head ) {
-                __cfaabi_dbg_print_buffer_decl( " KERNEL: @%ju(%ju) resetting alarm to %ju.\n", currtime.tv, __kernel_get_time().tv, (alarms->head->alarm - currtime).tv);
-                Duration delta = alarms->head->alarm - currtime;
-                Duration caped = max(delta, 50`us);
+        if( & (*alarms)`first ) {
+                __cfadbg_print_buffer_decl(preemption, " KERNEL: @%ju(%ju) resetting alarm to %ju.\n", currtime.tv, __kernel_get_time().tv, (alarms->head->alarm - currtime).tv);
+                Duration delta = (*alarms)`first.alarm - currtime;
+                Duration capped = max(delta, 50`us);
                 // itimerval tim  = { caped };
                 // __cfaabi_dbg_print_buffer_local( "    Values are %lu, %lu, %lu %lu.\n", delta.tv, caped.tv, tim.it_value.tv_sec, tim.it_value.tv_usec);
 
-                __kernel_set_timer( caped );
+                __kernel_set_timer( capped );
         }
 }
@@ -158 +163 @@
 // Kernel Signal Tools
 //=============================================================================================
-
-__cfaabi_dbg_debug_do( static thread_local void * last_interrupt = 0; )
+// In a user-level threading system, there are handful of thread-local variables where this problem occurs on the ARM.
+//
+// For each kernel thread running user-level threads, there is a flag variable to indicate if interrupts are
+// enabled/disabled for that kernel thread. Therefore, this variable is made thread local.
+//
+// For example, this code fragment sets the state of the "interrupt" variable in thread-local memory.
+//
+// _Thread_local volatile int interrupts;
+// int main() {
+//     interrupts = 0; // disable interrupts }
+//
+// which generates the following code on the ARM
+//
+// (gdb) disassemble main
+// Dump of assembler code for function main:
+//    0x0000000000000610 <+0>:  mrs     x1, tpidr_el0
+//    0x0000000000000614 <+4>:  mov     w0, #0x0                        // #0
+//    0x0000000000000618 <+8>:  add     x1, x1, #0x0, lsl #12
+//    0x000000000000061c <+12>: add     x1, x1, #0x10
+//    0x0000000000000620 <+16>: str     wzr, [x1]
+//    0x0000000000000624 <+20>: ret
+//
+// The mrs moves a pointer from coprocessor register tpidr_el0 into register x1.  Register w0 is set to 0. The two adds
+// increase the TLS pointer with the displacement (offset) 0x10, which is the location in the TSL of variable
+// "interrupts".  Finally, 0 is stored into "interrupts" through the pointer in register x1 that points into the
+// TSL. Now once x1 has the pointer to the location of the TSL for kernel thread N, it can be be preempted at a
+// user-level and the user thread is put on the user-level ready-queue. When the preempted thread gets to the front of
+// the user-level ready-queue it is run on kernel thread M. It now stores 0 into "interrupts" back on kernel thread N,
+// turning off interrupt on the wrong kernel thread.
+//
+// On the x86, the following code is generated for the same code fragment.
+//
+// (gdb) disassemble main
+// Dump of assembler code for function main:
+//    0x0000000000400420 <+0>:  movl   $0x0,%fs:0xfffffffffffffffc
+//    0x000000000040042c <+12>: xor    %eax,%eax
+//    0x000000000040042e <+14>: retq
+//
+// and there is base-displacement addressing used to atomically reset variable "interrupts" off of the TSL pointer in
+// register "fs".
+//
+// Hence, the ARM has base-displacement address for the general purpose registers, BUT not to the coprocessor
+// registers. As a result, generating the address for the write into variable "interrupts" is no longer atomic.
+//
+// Note this problem does NOT occur when just using multiple kernel threads because the preemption ALWAYS restarts the
+// thread on the same kernel thread.
+//
+// The obvious question is why does ARM use a coprocessor register to store the TSL pointer given that coprocessor
+// registers are second-class registers with respect to the instruction set. One possible answer is that they did not
+// want to dedicate one of the general registers to hold the TLS pointer and there was a free coprocessor register
+// available.
+
+//-----------------------------------------------------------------------------
+// Some assembly required
+#define __cfaasm_label(label, when) when: asm volatile goto(".global __cfaasm_" #label "_" #when "\n" "__cfaasm_" #label "_" #when ":":::"memory":when)
+
+//----------
+// special case for preemption since used often
+bool __preemption_enabled() {
+        // create a assembler label before
+        // marked as clobber all to avoid movement
+        __cfaasm_label(check, before);
+
+        // access tls as normal
+        bool enabled = __cfaabi_tls.preemption_state.enabled;
+
+        // create a assembler label after
+        // marked as clobber all to avoid movement
+        __cfaasm_label(check, after);
+        return enabled;
+}
+
+struct asm_region {
+        void * before;
+        void * after;
+};
+
+static inline bool __cfaasm_in( void * ip, struct asm_region & region ) {
+        return ip >= region.before && ip <= region.after;
+}
+
+
+//----------
+// Get data from the TLS block
+// struct asm_region __cfaasm_get;
+uintptr_t __cfatls_get( unsigned long int offset ) __attribute__((__noinline__)); //no inline to avoid problems
+uintptr_t __cfatls_get( unsigned long int offset ) {
+        // create a assembler label before
+        // marked as clobber all to avoid movement
+        __cfaasm_label(get, before);
+
+        // access tls as normal (except for pointer arithmetic)
+        uintptr_t val = *(uintptr_t*)((uintptr_t)&__cfaabi_tls + offset);
+
+        // create a assembler label after
+        // marked as clobber all to avoid movement
+        __cfaasm_label(get, after);
+        return val;
+}
 
 extern "C" {
         // Disable interrupts by incrementing the counter
         void disable_interrupts() {
-                with( kernelTLS.preemption_state ) {
+                // create a assembler label before
+                // marked as clobber all to avoid movement
+                __cfaasm_label(dsable, before);
+
+                with( __cfaabi_tls.preemption_state ) {
                         #if GCC_VERSION > 50000
                         static_assert(__atomic_always_lock_free(sizeof(enabled), &enabled), "Must be lock-free");
@@ -181 +287 @@
                         verify( new_val < 65_000u );              // If this triggers someone is disabling interrupts without enabling them
                 }
+
+                // create a assembler label after
+                // marked as clobber all to avoid movement
+                __cfaasm_label(dsable, after);
+
         }
 
         // Enable interrupts by decrementing the counter
-        // If counter reaches 0, execute any pending CtxSwitch
+        // If counter reaches 0, execute any pending __cfactx_switch
         void enable_interrupts( __cfaabi_dbg_ctx_param ) {
-                processor   * proc = kernelTLS.this_processor; // Cache the processor now since interrupts can start happening after the atomic store
-                thread_desc * thrd = kernelTLS.this_thread;       // Cache the thread now since interrupts can start happening after the atomic store
-
-                with( kernelTLS.preemption_state ){
+                // Cache the processor now since interrupts can start happening after the atomic store
+                processor   * proc = __cfaabi_tls.this_processor;
+                /* paranoid */ verify( proc );
+
+                with( __cfaabi_tls.preemption_state ){
                         unsigned short prev = disable_count;
                         disable_count -= 1;
-                        verify( prev != 0u );                     // If this triggers someone is enabled already enabled interruptsverify( prev != 0u );
+
+                        // If this triggers someone is enabled already enabled interruptsverify( prev != 0u );
+                        /* paranoid */ verify( prev != 0u );
 
                         // Check if we need to prempt the thread because an interrupt was missed
                         if( prev == 1 ) {
                                 #if GCC_VERSION > 50000
-                                static_assert(__atomic_always_lock_free(sizeof(enabled), &enabled), "Must be lock-free");
+                                        static_assert(__atomic_always_lock_free(sizeof(enabled), &enabled), "Must be lock-free");
                                 #endif
 
@@ -209 +323 @@
                                 if( proc->pending_preemption ) {
                                         proc->pending_preemption = false;
-                                        BlockInternal( thrd );
+                                        force_yield( __POLL_PREEMPTION );
                                 }
                         }
@@ -219 +333 @@
 
         // Disable interrupts by incrementint the counter
-        // Don't execute any pending CtxSwitch even if counter reaches 0
+        // Don't execute any pending __cfactx_switch even if counter reaches 0
         void enable_interrupts_noPoll() {
-                unsigned short prev = kernelTLS.preemption_state.disable_count;
-                kernelTLS.preemption_state.disable_count -= 1;
-                verifyf( prev != 0u, "Incremented from %u\n", prev );                     // If this triggers someone is enabled already enabled interrupts
+                unsigned short prev = __cfaabi_tls.preemption_state.disable_count;
+                __cfaabi_tls.preemption_state.disable_count -= 1;
+                // If this triggers someone is enabled already enabled interrupts
+                /* paranoid */ verifyf( prev != 0u, "Incremented from %u\n", prev );
                 if( prev == 1 ) {
                         #if GCC_VERSION > 50000
-                        static_assert(__atomic_always_lock_free(sizeof(kernelTLS.preemption_state.enabled), &kernelTLS.preemption_state.enabled), "Must be lock-free");
+                                static_assert(__atomic_always_lock_free(sizeof(__cfaabi_tls.preemption_state.enabled), &__cfaabi_tls.preemption_state.enabled), "Must be lock-free");
                         #endif
                         // Set enabled flag to true
                         // should be atomic to avoid preemption in the middle of the operation.
                         // use memory order RELAXED since there is no inter-thread on this variable requirements
-                        __atomic_store_n(&kernelTLS.preemption_state.enabled, true, __ATOMIC_RELAXED);
+                        __atomic_store_n(&__cfaabi_tls.preemption_state.enabled, true, __ATOMIC_RELAXED);
 
                         // Signal the compiler that a fence is needed but only for signal handlers
@@ -238 +353 @@
         }
 }
+
+//-----------------------------------------------------------------------------
+// Kernel Signal Debug
+void __cfaabi_check_preemption() {
+        bool ready = __preemption_enabled();
+        if(!ready) { abort("Preemption should be ready"); }
+
+        __cfaasm_label(debug, before);
+
+                sigset_t oldset;
+                int ret;
+                ret = pthread_sigmask(0, ( const sigset_t * ) 0p, &oldset);  // workaround trac#208: cast should be unnecessary
+                if(ret != 0) { abort("ERROR sigprocmask returned %d", ret); }
+
+                ret = sigismember(&oldset, SIGUSR1);
+                if(ret <  0) { abort("ERROR sigismember returned %d", ret); }
+                if(ret == 1) { abort("ERROR SIGUSR1 is disabled"); }
+
+                ret = sigismember(&oldset, SIGALRM);
+                if(ret <  0) { abort("ERROR sigismember returned %d", ret); }
+                if(ret == 0) { abort("ERROR SIGALRM is enabled"); }
+
+                ret = sigismember(&oldset, SIGTERM);
+                if(ret <  0) { abort("ERROR sigismember returned %d", ret); }
+                if(ret == 1) { abort("ERROR SIGTERM is disabled"); }
+
+        __cfaasm_label(debug, after);
+}
+
+#ifdef __CFA_WITH_VERIFY__
+bool __cfaabi_dbg_in_kernel() {
+        return !__preemption_enabled();
+}
+#endif
+
+#undef __cfaasm_label
+
+//-----------------------------------------------------------------------------
+// Signal handling
 
 // sigprocmask wrapper : unblock a single signal
@@ -257 +411 @@
 
         if ( pthread_sigmask( SIG_BLOCK, &mask, 0p ) == -1 ) {
-            abort( "internal error, pthread_sigmask" );
+                abort( "internal error, pthread_sigmask" );
         }
 }
@@ -268 +422 @@
 
 // reserved for future use
-static void timeout( thread_desc * this ) {
-        //TODO : implement waking threads
-}
+static void timeout( $thread * this ) {
+        unpark( this );
+}
+
+//-----------------------------------------------------------------------------
+// Some assembly required
+#if defined( __i386 )
+        #ifdef __PIC__
+                #define RELOC_PRELUDE( label ) \
+                        "calll   .Lcfaasm_prelude_" #label "$pb\n\t" \
+                        ".Lcfaasm_prelude_" #label "$pb:\n\t" \
+                        "popl    %%eax\n\t" \
+                        ".Lcfaasm_prelude_" #label "_end:\n\t" \
+                        "addl    $_GLOBAL_OFFSET_TABLE_+(.Lcfaasm_prelude_" #label "_end-.Lcfaasm_prelude_" #label "$pb), %%eax\n\t"
+                #define RELOC_PREFIX ""
+                #define RELOC_SUFFIX "@GOT(%%eax)"
+        #else
+                #define RELOC_PREFIX "$"
+                #define RELOC_SUFFIX ""
+        #endif
+        #define __cfaasm_label( label ) struct asm_region label = \
+                ({ \
+                        struct asm_region region; \
+                        asm( \
+                                RELOC_PRELUDE( label ) \
+                                "movl " RELOC_PREFIX "__cfaasm_" #label "_before" RELOC_SUFFIX ", %[vb]\n\t" \
+                                "movl " RELOC_PREFIX "__cfaasm_" #label "_after"  RELOC_SUFFIX ", %[va]\n\t" \
+                                 : [vb]"=r"(region.before), [va]"=r"(region.after) \
+                        ); \
+                        region; \
+                });
+#elif defined( __x86_64 )
+        #ifdef __PIC__
+                #define RELOC_PREFIX ""
+                #define RELOC_SUFFIX "@GOTPCREL(%%rip)"
+        #else
+                #define RELOC_PREFIX "$"
+                #define RELOC_SUFFIX ""
+        #endif
+        #define __cfaasm_label( label ) struct asm_region label = \
+                ({ \
+                        struct asm_region region; \
+                        asm( \
+                                "movq " RELOC_PREFIX "__cfaasm_" #label "_before" RELOC_SUFFIX ", %[vb]\n\t" \
+                                "movq " RELOC_PREFIX "__cfaasm_" #label "_after"  RELOC_SUFFIX ", %[va]\n\t" \
+                                 : [vb]"=r"(region.before), [va]"=r"(region.after) \
+                        ); \
+                        region; \
+                });
+#elif defined( __aarch64__ )
+        #ifdef __PIC__
+                // Note that this works only for gcc
+                #define __cfaasm_label( label ) struct asm_region label = \
+                ({ \
+                        struct asm_region region; \
+                        asm( \
+                                "adrp %[vb], _GLOBAL_OFFSET_TABLE_"                              "\n\t" \
+                                "ldr  %[vb], [%[vb], #:gotpage_lo15:__cfaasm_" #label "_before]" "\n\t" \
+                                "adrp %[va], _GLOBAL_OFFSET_TABLE_"                              "\n\t" \
+                                "ldr  %[va], [%[va], #:gotpage_lo15:__cfaasm_" #label "_after]"  "\n\t" \
+                                 : [vb]"=r"(region.before), [va]"=r"(region.after) \
+                        ); \
+                        region; \
+                });
+        #else
+                #error this is not the right thing to do
+                /*
+                #define __cfaasm_label( label ) struct asm_region label = \
+                ({ \
+                        struct asm_region region; \
+                        asm( \
+                                "adrp %[vb], __cfaasm_" #label "_before"              "\n\t" \
+                                "add  %[vb], %[vb], :lo12:__cfaasm_" #label "_before" "\n\t" \
+                                "adrp %[va], :got:__cfaasm_" #label "_after"          "\n\t" \
+                                "add  %[va], %[va], :lo12:__cfaasm_" #label "_after"  "\n\t" \
+                                 : [vb]"=r"(region.before), [va]"=r"(region.after) \
+                        ); \
+                        region; \
+                });
+                */
+        #endif
+#else
+        #error unknown hardware architecture
+#endif
 
 // KERNEL ONLY
-// Check if a CtxSwitch signal handler shoud defer
+// Check if a __cfactx_switch signal handler shoud defer
 // If true  : preemption is safe
 // If false : preemption is unsafe and marked as pending
-static inline bool preemption_ready() {
+static inline bool preemption_ready( void * ip ) {
+        // Get all the region for which it is not safe to preempt
+        __cfaasm_label( get    );
+        __cfaasm_label( check  );
+        __cfaasm_label( dsable );
+        __cfaasm_label( debug  );
+
         // Check if preemption is safe
-        bool ready = kernelTLS.preemption_state.enabled && ! kernelTLS.preemption_state.in_progress;
-
+        bool ready = true;
+        if( __cfaasm_in( ip, get    ) ) { ready = false; goto EXIT; };
+        if( __cfaasm_in( ip, check  ) ) { ready = false; goto EXIT; };
+        if( __cfaasm_in( ip, dsable ) ) { ready = false; goto EXIT; };
+        if( __cfaasm_in( ip, debug  ) ) { ready = false; goto EXIT; };
+        if( !__cfaabi_tls.preemption_state.enabled) { ready = false; goto EXIT; };
+        if( __cfaabi_tls.preemption_state.in_progress ) { ready = false; goto EXIT; };
+
+EXIT:
         // Adjust the pending flag accordingly
-        kernelTLS.this_processor->pending_preemption = !ready;
+        __cfaabi_tls.this_processor->pending_preemption = !ready;
         return ready;
 }
@@ -291 +539 @@
 // Startup routine to activate preemption
 // Called from kernel_startup
-void kernel_start_preemption() {
+void __kernel_alarm_startup() {
         __cfaabi_dbg_print_safe( "Kernel : Starting preemption\n" );
 
         // Start with preemption disabled until ready
-        kernelTLS.preemption_state.enabled = false;
-        kernelTLS.preemption_state.disable_count = 1;
+        __cfaabi_tls.preemption_state.enabled = false;
+        __cfaabi_tls.preemption_state.disable_count = 1;
 
         // Initialize the event kernel
@@ -303 +551 @@
 
         // Setup proper signal handlers
-        __cfaabi_sigaction( SIGUSR1, sigHandler_ctxSwitch, SA_SIGINFO | SA_RESTART ); // CtxSwitch handler
+        __cfaabi_sigaction( SIGUSR1, sigHandler_ctxSwitch, SA_SIGINFO | SA_RESTART ); // __cfactx_switch handler
+        __cfaabi_sigaction( SIGALRM, sigHandler_alarm    , SA_SIGINFO | SA_RESTART ); // debug handler
 
         signal_block( SIGALRM );
 
-        alarm_stack = create_pthread( &alarm_thread, alarm_loop, 0p );
+        alarm_stack = __create_pthread( &alarm_thread, alarm_loop, 0p );
 }
 
 // Shutdown routine to deactivate preemption
 // Called from kernel_shutdown
-void kernel_stop_preemption() {
+void __kernel_alarm_shutdown() {
         __cfaabi_dbg_print_safe( "Kernel : Preemption stopping\n" );
 
@@ -326 +575 @@
         // Wait for the preemption thread to finish
 
-        pthread_join( alarm_thread, 0p );
-        free( alarm_stack );
+        __destroy_pthread( alarm_thread, alarm_stack, 0p );
 
         // Preemption is now fully stopped
@@ -353 +601 @@
 // Kernel Signal Handlers
 //=============================================================================================
+__cfaabi_dbg_debug_do( static thread_local void * last_interrupt = 0; )
 
 // Context switch signal handler
 // Receives SIGUSR1 signal and causes the current thread to yield
 static void sigHandler_ctxSwitch( __CFA_SIGPARMS__ ) {
-        __cfaabi_dbg_debug_do( last_interrupt = (void *)(cxt->uc_mcontext.CFA_REG_IP); )
+        void * ip = (void *)(cxt->uc_mcontext.CFA_REG_IP);
+        __cfaabi_dbg_debug_do( last_interrupt = ip; )
 
         // SKULLDUGGERY: if a thread creates a processor and the immediately deletes it,
         // the interrupt that is supposed to force the kernel thread to preempt might arrive
-        // before the kernel thread has even started running. When that happens an iterrupt
-        // we a null 'this_processor' will be caught, just ignore it.
-        if(! kernelTLS.this_processor ) return;
+        // before the kernel thread has even started running. When that happens, an interrupt
+        // with a null 'this_processor' will be caught, just ignore it.
+        if(! __cfaabi_tls.this_processor ) return;
 
         choose(sfp->si_value.sival_int) {
                 case PREEMPT_NORMAL   : ;// Normal case, nothing to do here
-                case PREEMPT_TERMINATE: verify( __atomic_load_n( &kernelTLS.this_processor->do_terminate, __ATOMIC_SEQ_CST ) );
+                case PREEMPT_TERMINATE: verify( __atomic_load_n( &__cfaabi_tls.this_processor->do_terminate, __ATOMIC_SEQ_CST ) );
                 default:
                         abort( "internal error, signal value is %d", sfp->si_value.sival_int );
@@ -373 +623 @@
 
         // Check if it is safe to preempt here
-        if( !preemption_ready() ) { return; }
-
-        __cfaabi_dbg_print_buffer_decl( " KERNEL: preempting core %p (%p @ %p).\n", kernelTLS.this_processor, kernelTLS.this_thread, (void *)(cxt->uc_mcontext.CFA_REG_IP) );
+        if( !preemption_ready( ip ) ) { return; }
+
+        __cfaabi_dbg_print_buffer_decl( " KERNEL: preempting core %p (%p @ %p).\n", __cfaabi_tls.this_processor, __cfaabi_tls.this_thread, (void *)(cxt->uc_mcontext.CFA_REG_IP) );
 
         // Sync flag : prevent recursive calls to the signal handler
-        kernelTLS.preemption_state.in_progress = true;
+        __cfaabi_tls.preemption_state.in_progress = true;
 
         // Clear sighandler mask before context switching.
@@ -388 +638 @@
         }
 
-        // TODO: this should go in finish action
         // Clear the in progress flag
-        kernelTLS.preemption_state.in_progress = false;
+        __cfaabi_tls.preemption_state.in_progress = false;
 
         // Preemption can occur here
 
-        BlockInternal( kernelTLS.this_thread ); // Do the actual CtxSwitch
-}
+        force_yield( __ALARM_PREEMPTION ); // Do the actual __cfactx_switch
+}
+
+static void sigHandler_alarm( __CFA_SIGPARMS__ ) {
+        abort("SIGALRM should never reach the signal handler");
+}
+
+#if !defined(__CFA_NO_STATISTICS__)
+        int __print_alarm_stats = 0;
+#endif
 
 // Main of the alarm thread
 // Waits on SIGALRM and send SIGUSR1 to whom ever needs it
 static void * alarm_loop( __attribute__((unused)) void * args ) {
+        __processor_id_t id;
+        id.full_proc = false;
+        id.id = doregister(&id);
+        __cfaabi_tls.this_proc_id = &id;
+
+        #if !defined(__CFA_NO_STATISTICS__)
+                struct __stats_t local_stats;
+                __cfaabi_tls.this_stats = &local_stats;
+                __init_stats( &local_stats );
+        #endif
+
         // Block sigalrms to control when they arrive
         sigset_t mask;
@@ -457 +725 @@
 EXIT:
         __cfaabi_dbg_print_safe( "Kernel : Preemption thread stopping\n" );
+        unregister(&id);
+
+        #if !defined(__CFA_NO_STATISTICS__)
+                if( 0 != __print_alarm_stats ) {
+                        __print_stats( &local_stats, __print_alarm_stats, "Alarm", "Thread", 0p );
+                }
+        #endif
         return 0p;
 }
-
-//=============================================================================================
-// Kernel Signal Debug
-//=============================================================================================
-
-void __cfaabi_check_preemption() {
-        bool ready = kernelTLS.preemption_state.enabled;
-        if(!ready) { abort("Preemption should be ready"); }
-
-        sigset_t oldset;
-        int ret;
-        ret = pthread_sigmask(0, 0p, &oldset);
-        if(ret != 0) { abort("ERROR sigprocmask returned %d", ret); }
-
-        ret = sigismember(&oldset, SIGUSR1);
-        if(ret <  0) { abort("ERROR sigismember returned %d", ret); }
-        if(ret == 1) { abort("ERROR SIGUSR1 is disabled"); }
-
-        ret = sigismember(&oldset, SIGALRM);
-        if(ret <  0) { abort("ERROR sigismember returned %d", ret); }
-        if(ret == 0) { abort("ERROR SIGALRM is enabled"); }
-
-        ret = sigismember(&oldset, SIGTERM);
-        if(ret <  0) { abort("ERROR sigismember returned %d", ret); }
-        if(ret == 1) { abort("ERROR SIGTERM is disabled"); }
-}
-
-#ifdef __CFA_WITH_VERIFY__
-bool __cfaabi_dbg_in_kernel() {
-        return !kernelTLS.preemption_state.enabled;
-}
-#endif
 
 // Local Variables: //

libcfa/src/concurrency/preemption.hfa

@@ -16 +16 @@
 #pragma once
 
+#include "bits/locks.hfa"
 #include "alarm.hfa"
-#include "kernel_private.hfa"
 
-void kernel_start_preemption();
-void kernel_stop_preemption();
+struct event_kernel_t {
+        alarm_list_t alarms;
+        __spinlock_t lock;
+};
+
+extern event_kernel_t * event_kernel;
+
 void update_preemption( processor * this, Duration duration );
 

libcfa/src/concurrency/thread.cfa

@@ -19 +19 @@
 
 #include "kernel_private.hfa"
+#include "exception.hfa"
 
 #define __CFA_INVOKE_PRIVATE__
 #include "invoke.h"
 
-extern "C" {
-        #include <fenv.h>
-        #include <stddef.h>
-}
-
-//extern volatile thread_local processor * this_processor;
-
 //-----------------------------------------------------------------------------
 // Thread ctors and dtors
-void ?{}(thread_desc & this, const char * const name, cluster & cl, void * storage, size_t storageSize ) with( this ) {
+void ?{}($thread & this, const char * const name, cluster & cl, void * storage, size_t storageSize ) with( this ) {
         context{ 0p, 0p };
         self_cor{ name, storage, storageSize };
+        ticket = TICKET_RUNNING;
         state = Start;
+        preempted = __NO_PREEMPTION;
         curr_cor = &self_cor;
         self_mon.owner = &this;
@@ -41 +37 @@
         self_mon_p = &self_mon;
         curr_cluster = &cl;
-        next = 0p;
+        link.next = 0p;
+        link.prev = 0p;
+        link.preferred = -1;
+        #if defined( __CFA_WITH_VERIFY__ )
+                canary = 0x0D15EA5E0D15EA5Ep;
+        #endif
+
+        seqable.next = 0p;
+        seqable.back = 0p;
 
         node.next = 0p;
@@ -50 +54 @@
 }
 
-void ^?{}(thread_desc& this) with( this ) {
+void ^?{}($thread& this) with( this ) {
+        #if defined( __CFA_WITH_VERIFY__ )
+                canary = 0xDEADDEADDEADDEADp;
+        #endif
         unregister(curr_cluster, this);
         ^self_cor{};
 }
 
+FORALL_DATA_INSTANCE(ThreadCancelled, (dtype thread_t), (thread_t))
+
+forall(dtype T)
+void copy(ThreadCancelled(T) * dst, ThreadCancelled(T) * src) {
+        dst->virtual_table = src->virtual_table;
+        dst->the_thread = src->the_thread;
+        dst->the_exception = src->the_exception;
+}
+
+forall(dtype T)
+const char * msg(ThreadCancelled(T) *) {
+        return "ThreadCancelled";
+}
+
+forall(dtype T)
+static void default_thread_cancel_handler(ThreadCancelled(T) & ) {
+        abort( "Unhandled thread cancellation.\n" );
+}
+
+forall(dtype T | is_thread(T) | IS_EXCEPTION(ThreadCancelled, (T)))
+void ?{}( thread_dtor_guard_t & this,
+                T & thrd, void(*defaultResumptionHandler)(ThreadCancelled(T) &)) {
+        $monitor * m = get_monitor(thrd);
+        $thread * desc = get_thread(thrd);
+
+        // Setup the monitor guard
+        void (*dtor)(T& mutex this) = ^?{};
+        bool join = defaultResumptionHandler != (void(*)(ThreadCancelled(T)&))0;
+        (this.mg){&m, (void(*)())dtor, join};
+
+
+        /* paranoid */ verifyf( Halted == desc->state || Cancelled == desc->state, "Expected thread to be Halted or Cancelled, was %d\n", (int)desc->state );
+
+        // After the guard set-up and any wait, check for cancellation.
+        struct _Unwind_Exception * cancellation = desc->self_cor.cancellation;
+        if ( likely( 0p == cancellation ) ) {
+                return;
+        } else if ( Cancelled == desc->state ) {
+                return;
+        }
+        desc->state = Cancelled;
+        if (!join) {
+                defaultResumptionHandler = default_thread_cancel_handler;
+        }
+
+        ThreadCancelled(T) except;
+        // TODO: Remove explitate vtable set once trac#186 is fixed.
+        except.virtual_table = &get_exception_vtable(&except);
+        except.the_thread = &thrd;
+        except.the_exception = __cfaehm_cancellation_exception( cancellation );
+        throwResume except;
+
+        except.the_exception->virtual_table->free( except.the_exception );
+        free( cancellation );
+        desc->self_cor.cancellation = 0p;
+}
+
+void ^?{}( thread_dtor_guard_t & this ) {
+        ^(this.mg){};
+}
+
+//-----------------------------------------------------------------------------
+// Starting and stopping threads
+forall( dtype T | is_thread(T) )
+void __thrd_start( T & this, void (*main_p)(T &) ) {
+        $thread * this_thrd = get_thread(this);
+
+        disable_interrupts();
+        __cfactx_start(main_p, get_coroutine(this), this, __cfactx_invoke_thread);
+
+        this_thrd->context.[SP, FP] = this_thrd->self_cor.context.[SP, FP];
+        /* paranoid */ verify( this_thrd->context.SP );
+
+        __schedule_thread( this_thrd );
+        enable_interrupts( __cfaabi_dbg_ctx );
+}
+
+//-----------------------------------------------------------------------------
+// Support for threads that don't ues the thread keyword
 forall( dtype T | sized(T) | is_thread(T) | { void ?{}(T&); } )
 void ?{}( scoped(T)& this ) with( this ) {
@@ -73 +159 @@
 
 //-----------------------------------------------------------------------------
-// Starting and stopping threads
-forall( dtype T | is_thread(T) )
-void __thrd_start( T & this, void (*main_p)(T &) ) {
-        thread_desc * this_thrd = get_thread(this);
-        thread_desc * curr_thrd = TL_GET( this_thread );
-
-        disable_interrupts();
-        CtxStart(main_p, get_coroutine(this), this, CtxInvokeThread);
-
-        this_thrd->context.[SP, FP] = this_thrd->self_cor.context.[SP, FP];
-        verify( this_thrd->context.SP );
-        // CtxSwitch( &curr_thrd->context, &this_thrd->context );
-
-        ScheduleThread(this_thrd);
-        enable_interrupts( __cfaabi_dbg_ctx );
+forall(dtype T | is_thread(T) | IS_RESUMPTION_EXCEPTION(ThreadCancelled, (T)))
+T & join( T & this ) {
+        thread_dtor_guard_t guard = { this, defaultResumptionHandler };
+        return this;
 }
 
-void yield( void ) {
-        // Safety note : This could cause some false positives due to preemption
-      verify( TL_GET( preemption_state.enabled ) );
-        BlockInternal( TL_GET( this_thread ) );
-        // Safety note : This could cause some false positives due to preemption
-      verify( TL_GET( preemption_state.enabled ) );
-}
-
-void yield( unsigned times ) {
-        for( unsigned i = 0; i < times; i++ ) {
-                yield();
-        }
+uint64_t thread_rand() {
+        disable_interrupts();
+        uint64_t ret = __tls_rand();
+        enable_interrupts( __cfaabi_dbg_ctx );
+        return ret;
 }
 

libcfa/src/concurrency/thread.hfa

@@ -22 +22 @@
 #include "kernel.hfa"
 #include "monitor.hfa"
+#include "exception.hfa"
 
 //-----------------------------------------------------------------------------
 // thread trait
 trait is_thread(dtype T) {
-      void ^?{}(T& mutex this);
-      void main(T& this);
-      thread_desc* get_thread(T& this);
+        void ^?{}(T& mutex this);
+        void main(T& this);
+        $thread* get_thread(T& this);
 };
 
-#define DECL_THREAD(X) thread_desc* get_thread(X& this) { return &this.__thrd; } void main(X& this)
+FORALL_DATA_EXCEPTION(ThreadCancelled, (dtype thread_t), (thread_t)) (
+        thread_t * the_thread;
+        exception_t * the_exception;
+);
+
+forall(dtype T)
+void copy(ThreadCancelled(T) * dst, ThreadCancelled(T) * src);
+
+forall(dtype T)
+const char * msg(ThreadCancelled(T) *);
+
+// define that satisfies the trait without using the thread keyword
+#define DECL_THREAD(X) $thread* get_thread(X& this) __attribute__((const)) { return &this.__thrd; } void main(X& this)
+
+// Inline getters for threads/coroutines/monitors
+forall( dtype T | is_thread(T) )
+static inline $coroutine* get_coroutine(T & this) __attribute__((const)) { return &get_thread(this)->self_cor; }
 
 forall( dtype T | is_thread(T) )
-static inline coroutine_desc* get_coroutine(T & this) {
-        return &get_thread(this)->self_cor;
-}
+static inline $monitor  * get_monitor  (T & this) __attribute__((const)) { return &get_thread(this)->self_mon; }
 
-forall( dtype T | is_thread(T) )
-static inline monitor_desc* get_monitor(T & this) {
-        return &get_thread(this)->self_mon;
-}
+static inline $coroutine* get_coroutine($thread * this) __attribute__((const)) { return &this->self_cor; }
+static inline $monitor  * get_monitor  ($thread * this) __attribute__((const)) { return &this->self_mon; }
 
-static inline coroutine_desc* get_coroutine(thread_desc * this) {
-        return &this->self_cor;
-}
-
-static inline monitor_desc* get_monitor(thread_desc * this) {
-        return &this->self_mon;
-}
-
+//-----------------------------------------------------------------------------
+// forward declarations needed for threads
 extern struct cluster * mainCluster;
 
@@ -58 +65 @@
 //-----------------------------------------------------------------------------
 // Ctors and dtors
-void ?{}(thread_desc & this, const char * const name, struct cluster & cl, void * storage, size_t storageSize );
-void ^?{}(thread_desc & this);
+void ?{}($thread & this, const char * const name, struct cluster & cl, void * storage, size_t storageSize );
+void ^?{}($thread & this);
 
-static inline void ?{}(thread_desc & this)                                                                  { this{ "Anonymous Thread", *mainCluster, 0p, 65000 }; }
-static inline void ?{}(thread_desc & this, size_t stackSize )                                               { this{ "Anonymous Thread", *mainCluster, 0p, stackSize }; }
-static inline void ?{}(thread_desc & this, void * storage, size_t storageSize )                             { this{ "Anonymous Thread", *mainCluster, storage, storageSize }; }
-static inline void ?{}(thread_desc & this, struct cluster & cl )                                            { this{ "Anonymous Thread", cl, 0p, 65000 }; }
-static inline void ?{}(thread_desc & this, struct cluster & cl, size_t stackSize )                          { this{ "Anonymous Thread", cl, 0p, stackSize }; }
-static inline void ?{}(thread_desc & this, struct cluster & cl, void * storage, size_t storageSize )        { this{ "Anonymous Thread", cl, storage, storageSize }; }
-static inline void ?{}(thread_desc & this, const char * const name)                                         { this{ name, *mainCluster, 0p, 65000 }; }
-static inline void ?{}(thread_desc & this, const char * const name, struct cluster & cl )                   { this{ name, cl, 0p, 65000 }; }
-static inline void ?{}(thread_desc & this, const char * const name, struct cluster & cl, size_t stackSize ) { this{ name, cl, 0p, stackSize }; }
+static inline void ?{}($thread & this)                                                                  { this{ "Anonymous Thread", *mainCluster, 0p, 65000 }; }
+static inline void ?{}($thread & this, size_t stackSize )                                               { this{ "Anonymous Thread", *mainCluster, 0p, stackSize }; }
+static inline void ?{}($thread & this, void * storage, size_t storageSize )                             { this{ "Anonymous Thread", *mainCluster, storage, storageSize }; }
+static inline void ?{}($thread & this, struct cluster & cl )                                            { this{ "Anonymous Thread", cl, 0p, 65000 }; }
+static inline void ?{}($thread & this, struct cluster & cl, size_t stackSize )                          { this{ "Anonymous Thread", cl, 0p, stackSize }; }
+static inline void ?{}($thread & this, struct cluster & cl, void * storage, size_t storageSize )        { this{ "Anonymous Thread", cl, storage, storageSize }; }
+static inline void ?{}($thread & this, const char * const name)                                         { this{ name, *mainCluster, 0p, 65000 }; }
+static inline void ?{}($thread & this, const char * const name, struct cluster & cl )                   { this{ name, cl, 0p, 65000 }; }
+static inline void ?{}($thread & this, const char * const name, struct cluster & cl, size_t stackSize ) { this{ name, cl, 0p, stackSize }; }
+
+struct thread_dtor_guard_t {
+        monitor_dtor_guard_t mg;
+};
+
+forall( dtype T | is_thread(T) | IS_EXCEPTION(ThreadCancelled, (T)) )
+void ?{}( thread_dtor_guard_t & this, T & thrd, void(*)(ThreadCancelled(T) &) );
+void ^?{}( thread_dtor_guard_t & this );
 
 //-----------------------------------------------------------------------------
@@ -88 +103 @@
 void ^?{}( scoped(T)& this );
 
-void yield();
-void yield( unsigned times );
+//-----------------------------------------------------------------------------
+// Scheduler API
 
-static inline struct thread_desc * active_thread () { return TL_GET( this_thread ); }
+//----------
+// Park thread: block until corresponding call to unpark, won't block if unpark is already called
+void park( void );
+
+//----------
+// Unpark a thread, if the thread is already blocked, schedule it
+//                  if the thread is not yet block, signal that it should rerun immediately
+void unpark( $thread * this );
+
+forall( dtype T | is_thread(T) )
+static inline void unpark( T & this ) { if(!&this) return; unpark( get_thread( this ) );}
+
+//----------
+// Yield: force thread to block and be rescheduled
+bool force_yield( enum __Preemption_Reason );
+
+//----------
+// sleep: force thread to block and be rescheduled after Duration duration
+void sleep( Duration duration );
+
+//----------
+// join
+forall( dtype T | is_thread(T) | IS_RESUMPTION_EXCEPTION(ThreadCancelled, (T)) )
+T & join( T & this );
 
 // Local Variables: //

libcfa/src/containers/vector.hfa

@@ -10 +10 @@
 // Created On       : Tue Jul  5 18:00:07 2016
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Sat Jul 22 10:01:18 2017
-// Update Count     : 3
+// Last Modified On : Wed Jun 17 11:02:46 2020
+// Update Count     : 4
 //
 
 #pragma once
 
-extern "C" {
 #include <stdbool.h>
-}
 
 //------------------------------------------------------------------------------

libcfa/src/exception.c

@@ -9 +9 @@
 // Author           : Andrew Beach
 // Created On       : Mon Jun 26 15:13:00 2017
-// Last Modified By : Peter A. Buhr
-// Last Modified On : Thu Feb 22 18:17:34 2018
-// Update Count     : 11
+// Last Modified By : Andrew Beach
+// Last Modified On : Tue Oct 27 16:27:00 2020
+// Update Count     : 35
 //
 
+// Normally we would get this from the CFA prelude.
 #include <stddef.h> // for size_t
 
+#include <unwind.h> // for struct _Unwind_Exception {...};
+
 #include "exception.h"
-
-// Implementation of the secret header.
 
 #include <stdlib.h>
 #include <stdio.h>
-#include <unwind.h>
 #include <bits/debug.hfa>
-
-// FIX ME: temporary hack to keep ARM build working
+#include "concurrency/invoke.h"
+#include "stdhdr/assert.h"
+
+#if defined( __ARM_ARCH )
+#warning FIX ME: temporary hack to keep ARM build working
 #ifndef _URC_FATAL_PHASE1_ERROR
-#define _URC_FATAL_PHASE1_ERROR 2
+#define _URC_FATAL_PHASE1_ERROR 3
 #endif // ! _URC_FATAL_PHASE1_ERROR
 #ifndef _URC_FATAL_PHASE2_ERROR
 #define _URC_FATAL_PHASE2_ERROR 2
 #endif // ! _URC_FATAL_PHASE2_ERROR
+#endif // __ARM_ARCH
 
 #include "lsda.h"
 
+/* The exception class for our exceptions. Because of the vendor component
+ * its value would not be standard.
+ * Vendor: UWPL
+ * Language: CFA\0
+ */
+const _Unwind_Exception_Class __cfaehm_exception_class = 0x4c50575500414643;
 
 // Base exception vtable is abstract, you should not have base exceptions.
-struct __cfaabi_ehm__base_exception_t_vtable
-                ___cfaabi_ehm__base_exception_t_vtable_instance = {
+struct __cfaehm_base_exception_t_vtable
+                ___cfaehm_base_exception_t_vtable_instance = {
         .parent = NULL,
         .size = 0,
@@ -47 +57 @@
 
 
-// Temperary global exception context. Does not work with concurency.
-struct exception_context_t {
-    struct __cfaabi_ehm__try_resume_node * top_resume;
-    struct __cfaabi_ehm__try_resume_node * current_resume;
-
-    exception_t * current_exception;
-    int current_handler_index;
-} shared_stack = {NULL, NULL, 0, 0};
-
 // Get the current exception context.
 // There can be a single global until multithreading occurs, then each stack
-// needs its own. It will have to be updated to handle that.
-struct exception_context_t * this_exception_context() {
+// needs its own. We get this from libcfathreads (no weak attribute).
+__attribute__((weak)) struct exception_context_t * this_exception_context() {
+        static struct exception_context_t shared_stack = {NULL, NULL};
         return &shared_stack;
 }
-//#define SAVE_EXCEPTION_CONTEXT(to_name)
-//struct exception_context_t * to_name = this_exception_context();
-//exception * this_exception() {
-//    return this_exception_context()->current_exception;
-//}
-
-
-// This macro should be the only thing that needs to change across machines.
-// Used in the personality function, way down in termination.
-// struct _Unwind_Context * -> _Unwind_Reason_Code(*)(exception_t *)
-#define MATCHER_FROM_CONTEXT(ptr_to_context) \
-        (*(_Unwind_Reason_Code(**)(exception_t *))(_Unwind_GetCFA(ptr_to_context) + 8))
 
 
 // RESUMPTION ================================================================
 
-void __cfaabi_ehm__throw_resume(exception_t * except) {
-
-        __cfaabi_dbg_print_safe("Throwing resumption exception\n");
-
-        struct __cfaabi_ehm__try_resume_node * original_head = shared_stack.current_resume;
-        struct __cfaabi_ehm__try_resume_node * current =
-                (original_head) ? original_head->next : shared_stack.top_resume;
-
-        for ( ; current ; current = current->next) {
-                shared_stack.current_resume = current;
-                if (current->handler(except)) {
-                        shared_stack.current_resume = original_head;
-                        return;
+static void reset_top_resume(struct __cfaehm_try_resume_node ** store) {
+        this_exception_context()->top_resume = *store;
+}
+
+void __cfaehm_throw_resume(exception_t * except, void (*defaultHandler)(exception_t *)) {
+        struct exception_context_t * context = this_exception_context();
+
+        __cfadbg_print_safe(exception, "Throwing resumption exception\n");
+
+        {
+                __attribute__((cleanup(reset_top_resume)))
+                struct __cfaehm_try_resume_node * original_head = context->top_resume;
+                struct __cfaehm_try_resume_node * current = context->top_resume;
+
+                for ( ; current ; current = current->next) {
+                        context->top_resume = current->next;
+                        if (current->handler(except)) {
+                                return;
+                        }
                 }
-        }
-
-        __cfaabi_dbg_print_safe("Unhandled exception\n");
-        shared_stack.current_resume = original_head;
-
-        // Fall back to termination:
-        __cfaabi_ehm__throw_terminate(except);
-        // TODO: Default handler for resumption.
+        } // End the search and return to the top of the stack.
+
+        // No handler found, fall back to the default operation.
+        __cfadbg_print_safe(exception, "Unhandled exception\n");
+        defaultHandler(except);
 }
 
@@ -106 +99 @@
 // be added after the node is built but before it is made the top node.
 
-void __cfaabi_ehm__try_resume_setup(struct __cfaabi_ehm__try_resume_node * node,
+void __cfaehm_try_resume_setup(struct __cfaehm_try_resume_node * node,
                         _Bool (*handler)(exception_t * except)) {
-        node->next = shared_stack.top_resume;
+        struct exception_context_t * context = this_exception_context();
+        node->next = context->top_resume;
         node->handler = handler;
-        shared_stack.top_resume = node;
-}
-
-void __cfaabi_ehm__try_resume_cleanup(struct __cfaabi_ehm__try_resume_node * node) {
-        shared_stack.top_resume = node->next;
-}
-
-
-// TERMINATION ===============================================================
-
-// MEMORY MANAGEMENT (still for integers)
-// May have to move to cfa for constructors and destructors (references).
-
-struct __cfaabi_ehm__node {
-        struct __cfaabi_ehm__node * next;
-};
+        context->top_resume = node;
+}
+
+void __cfaehm_try_resume_cleanup(struct __cfaehm_try_resume_node * node) {
+        struct exception_context_t * context = this_exception_context();
+        context->top_resume = node->next;
+}
+
+
+// MEMORY MANAGEMENT =========================================================
 
 #define NODE_TO_EXCEPT(node) ((exception_t *)(1 + (node)))
-#define EXCEPT_TO_NODE(except) ((struct __cfaabi_ehm__node *)(except) - 1)
+#define EXCEPT_TO_NODE(except) ((struct __cfaehm_node *)(except) - 1)
+#define UNWIND_TO_NODE(unwind) ((struct __cfaehm_node *)(unwind))
+#define NULL_MAP(map, ptr) ((ptr) ? (map(ptr)) : NULL)
+
+// How to clean up an exception in various situations.
+static void __cfaehm_exception_cleanup(
+                _Unwind_Reason_Code reason,
+                struct _Unwind_Exception * exception) {
+        switch (reason) {
+        case _URC_FOREIGN_EXCEPTION_CAUGHT:
+                // This one we could clean-up to allow cross-language exceptions.
+        case _URC_FATAL_PHASE1_ERROR:
+        case _URC_FATAL_PHASE2_ERROR:
+        default:
+                abort();
+        }
+}
 
 // Creates a copy of the indicated exception and sets current_exception to it.
-static void __cfaabi_ehm__allocate_exception( exception_t * except ) {
+static void __cfaehm_allocate_exception( exception_t * except ) {
         struct exception_context_t * context = this_exception_context();
 
         // Allocate memory for the exception.
-        struct __cfaabi_ehm__node * store = malloc(
-                sizeof( struct __cfaabi_ehm__node ) + except->virtual_table->size );
+        struct __cfaehm_node * store = malloc(
+                sizeof( struct __cfaehm_node ) + except->virtual_table->size );
 
         if ( ! store ) {
@@ -143 +147 @@
         }
 
+        // Initialize the node:
+        exception_t * except_store = NODE_TO_EXCEPT(store);
+        store->unwind_exception.exception_class = __cfaehm_exception_class;
+        store->unwind_exception.exception_cleanup = __cfaehm_exception_cleanup;
+        store->handler_index = 0;
+        except->virtual_table->copy( except_store, except );
+
         // Add the node to the list:
-        store->next = EXCEPT_TO_NODE(context->current_exception);
-        context->current_exception = NODE_TO_EXCEPT(store);
-
-        // Copy the exception to storage.
-        except->virtual_table->copy( context->current_exception, except );
+        store->next = NULL_MAP(EXCEPT_TO_NODE, context->current_exception);
+        context->current_exception = except_store;
 }
 
 // Delete the provided exception, unsetting current_exception if relivant.
-static void __cfaabi_ehm__delete_exception( exception_t * except ) {
-        struct exception_context_t * context = this_exception_context();
-
-        __cfaabi_dbg_print_safe("Deleting Exception\n");
+static void __cfaehm_delete_exception( exception_t * except ) {
+        struct exception_context_t * context = this_exception_context();
+
+        __cfadbg_print_safe(exception, "Deleting Exception\n");
 
         // Remove the exception from the list.
-        struct __cfaabi_ehm__node * to_free = EXCEPT_TO_NODE(except);
-        struct __cfaabi_ehm__node * node;
+        struct __cfaehm_node * to_free = EXCEPT_TO_NODE(except);
+        struct __cfaehm_node * node;
 
         if ( context->current_exception == except ) {
                 node = to_free->next;
-                context->current_exception = (node) ? NODE_TO_EXCEPT(node) : 0;
+                context->current_exception = NULL_MAP(NODE_TO_EXCEPT, node);
         } else {
                 node = EXCEPT_TO_NODE(context->current_exception);
                 // It may always be in the first or second position.
-                while( to_free != node->next ) {
+                while ( to_free != node->next ) {
                         node = node->next;
                 }
@@ -178 +186 @@
 }
 
-// If this isn't a rethrow (*except==0), delete the provided exception.
-void __cfaabi_ehm__cleanup_terminate( void * except ) {
-        if ( *(void**)except ) __cfaabi_ehm__delete_exception( *(exception_t **)except );
-}
-
-
-// We need a piece of storage to raise the exception
-struct _Unwind_Exception this_exception_storage;
+// CANCELLATION ==============================================================
 
 // Function needed by force unwind
@@ -192 +193 @@
                 int version,
                 _Unwind_Action actions,
-                _Unwind_Exception_Class exceptionClass,
+                _Unwind_Exception_Class exception_class,
                 struct _Unwind_Exception * unwind_exception,
-                struct _Unwind_Context * context,
-                void * some_param) {
-        if( actions & _UA_END_OF_STACK  ) exit(1);
-        if( actions & _UA_CLEANUP_PHASE ) return _URC_NO_REASON;
-
-        return _URC_FATAL_PHASE2_ERROR;
+                struct _Unwind_Context * unwind_context,
+                void * stop_param) {
+        // Verify actions follow the rules we expect.
+        verify(actions & _UA_CLEANUP_PHASE);
+        verify(actions & _UA_FORCE_UNWIND);
+        verify(!(actions & _UA_SEARCH_PHASE));
+        verify(!(actions & _UA_HANDLER_FRAME));
+
+        if ( actions & _UA_END_OF_STACK ) {
+                abort();
+        } else {
+                return _URC_NO_REASON;
+        }
+}
+
+__attribute__((weak)) _Unwind_Reason_Code
+__cfaehm_cancellation_unwind( struct _Unwind_Exception * exception ) {
+        return _Unwind_ForcedUnwind( exception, _Stop_Fn, (void*)0x22 );
+}
+
+// Cancel the current stack, prefroming approprate clean-up and messaging.
+void __cfaehm_cancel_stack( exception_t * exception ) {
+        __cfaehm_allocate_exception( exception );
+
+        struct exception_context_t * context = this_exception_context();
+        struct __cfaehm_node * node = EXCEPT_TO_NODE(context->current_exception);
+
+        // Preform clean-up of any extra active exceptions.
+        while ( node->next ) {
+                struct __cfaehm_node * to_free = node->next;
+                node->next = to_free->next;
+                exception_t * except = NODE_TO_EXCEPT( to_free );
+                except->virtual_table->free( except );
+            free( to_free );
+        }
+
+        _Unwind_Reason_Code ret;
+        ret = __cfaehm_cancellation_unwind( &node->unwind_exception );
+        printf("UNWIND ERROR %d after force unwind\n", ret);
+        abort();
+}
+
+
+// TERMINATION ===============================================================
+
+// If this isn't a rethrow (*except==0), delete the provided exception.
+void __cfaehm_cleanup_terminate( void * except ) {
+        if ( *(void**)except ) __cfaehm_delete_exception( *(exception_t **)except );
+}
+
+static void __cfaehm_cleanup_default( exception_t ** except ) {
+        __cfaehm_delete_exception( *except );
+        *except = NULL;
 }
 
 // The exception that is being thrown must already be stored.
-__attribute__((noreturn)) void __cfaabi_ehm__begin_unwind(void) {
-        if ( ! this_exception_context()->current_exception ) {
+static void __cfaehm_begin_unwind(void(*defaultHandler)(exception_t *)) {
+        struct exception_context_t * context = this_exception_context();
+        if ( NULL == context->current_exception ) {
                 printf("UNWIND ERROR missing exception in begin unwind\n");
                 abort();
         }
-
+        struct _Unwind_Exception * storage =
+                &EXCEPT_TO_NODE(context->current_exception)->unwind_exception;
 
         // Call stdlibc to raise the exception
-        _Unwind_Reason_Code ret = _Unwind_RaiseException( &this_exception_storage );
+        __cfadbg_print_safe(exception, "Begin unwinding (storage &p, context %p)\n", storage, context);
+        _Unwind_Reason_Code ret = _Unwind_RaiseException( storage );
 
         // If we reach here it means something happened. For resumption to work we need to find a way
@@ -220 +271 @@
         // the whole stack.
 
-        if( ret == _URC_END_OF_STACK ) {
-                // No proper handler was found. This can be handled in many ways, C++ calls std::terminate.
-                // Here we force unwind the stack, basically raising a cancellation.
-                printf("Uncaught exception %p\n", &this_exception_storage);
-
-                ret = _Unwind_ForcedUnwind( &this_exception_storage, _Stop_Fn, (void*)0x22 );
-                printf("UNWIND ERROR %d after force unwind\n", ret);
+        // We did not simply reach the end of the stack without finding a handler. This is an error.
+        if ( ret != _URC_END_OF_STACK ) {
+                printf("UNWIND ERROR %d after raise exception\n", ret);
                 abort();
         }
 
-        // We did not simply reach the end of the stack without finding a handler. This is an error.
-        printf("UNWIND ERROR %d after raise exception\n", ret);
+        // No handler found, go to the default operation.
+        __cfadbg_print_safe(exception, "Uncaught exception %p\n", storage);
+
+        __attribute__((cleanup(__cfaehm_cleanup_default)))
+        exception_t * exception = context->current_exception;
+        defaultHandler( exception );
+}
+
+void __cfaehm_throw_terminate( exception_t * val, void (*defaultHandler)(exception_t *) ) {
+        __cfadbg_print_safe(exception, "Throwing termination exception\n");
+
+        __cfaehm_allocate_exception( val );
+        __cfaehm_begin_unwind( defaultHandler );
+}
+
+static __attribute__((noreturn)) void __cfaehm_rethrow_adapter( exception_t * except ) {
+        // TODO: Print some error message.
+        (void)except;
         abort();
 }
 
-void __cfaabi_ehm__throw_terminate( exception_t * val ) {
-        __cfaabi_dbg_print_safe("Throwing termination exception\n");
-
-        __cfaabi_ehm__allocate_exception( val );
-        __cfaabi_ehm__begin_unwind();
-}
-
-void __cfaabi_ehm__rethrow_terminate(void) {
-        __cfaabi_dbg_print_safe("Rethrowing termination exception\n");
-
-        __cfaabi_ehm__begin_unwind();
-}
-
-#if defined(PIC)
-#warning Exceptions not yet supported when using Position-Independent Code
-__attribute__((noinline))
-void __cfaabi_ehm__try_terminate(void (*try_block)(),
-                void (*catch_block)(int index, exception_t * except),
-                __attribute__((unused)) int (*match_block)(exception_t * except)) {
+void __cfaehm_rethrow_terminate(void) {
+        __cfadbg_print_safe(exception, "Rethrowing termination exception\n");
+
+        __cfaehm_begin_unwind( __cfaehm_rethrow_adapter );
         abort();
 }
-#else // PIC
+
+#if defined( __x86_64 ) || defined( __i386 )
 // This is our personality routine. For every stack frame annotated with
 // ".cfi_personality 0x3,__gcfa_personality_v0" this function will be called twice when unwinding.
 //  Once in the search phase and once in the cleanup phase.
-_Unwind_Reason_Code __gcfa_personality_v0 (
-                int version, _Unwind_Action actions, unsigned long long exceptionClass,
-                struct _Unwind_Exception* unwind_exception,
-                struct _Unwind_Context* context)
+_Unwind_Reason_Code __gcfa_personality_v0(
+                int version,
+                _Unwind_Action actions,
+                unsigned long long exception_class,
+                struct _Unwind_Exception * unwind_exception,
+                struct _Unwind_Context * unwind_context)
 {
 
-        //__cfaabi_dbg_print_safe("CFA: 0x%lx\n", _Unwind_GetCFA(context));
-        __cfaabi_dbg_print_safe("Personality function (%d, %x, %llu, %p, %p):",
-                        version, actions, exceptionClass, unwind_exception, context);
-
-        // If we've reached the end of the stack then there is nothing much we can do...
-        if( actions & _UA_END_OF_STACK ) return _URC_END_OF_STACK;
-
+        //__cfadbg_print_safe(exception, "CFA: 0x%lx\n", _Unwind_GetCFA(context));
+        __cfadbg_print_safe(exception, "Personality function (%d, %x, %llu, %p, %p):",
+                        version, actions, exception_class, unwind_exception, unwind_context);
+
+        // Verify that actions follow the rules we expect.
+        // This function should never be called at the end of the stack.
+        verify(!(actions & _UA_END_OF_STACK));
+        // Either only the search phase flag is set or...
         if (actions & _UA_SEARCH_PHASE) {
-                __cfaabi_dbg_print_safe(" lookup phase");
-        }
-        else if (actions & _UA_CLEANUP_PHASE) {
-                __cfaabi_dbg_print_safe(" cleanup phase");
-        }
-        // Just in case, probably can't actually happen
-        else {
-                printf(" error\n");
-                return _URC_FATAL_PHASE1_ERROR;
+                verify(actions == _UA_SEARCH_PHASE);
+                __cfadbg_print_safe(exception, " lookup phase");
+        // ... we are in clean-up phase.
+        } else {
+                verify(actions & _UA_CLEANUP_PHASE);
+                __cfadbg_print_safe(exception, " cleanup phase");
+                // We shouldn't be the handler frame during forced unwind.
+                if (actions & _UA_HANDLER_FRAME) {
+                        verify(!(actions & _UA_FORCE_UNWIND));
+                        __cfadbg_print_safe(exception, " (handler frame)");
+                } else if (actions & _UA_FORCE_UNWIND) {
+                        __cfadbg_print_safe(exception, " (force unwind)");
+                }
         }
 
         // Get a pointer to the language specific data from which we will read what we need
-        const unsigned char * lsd = (const unsigned char*) _Unwind_GetLanguageSpecificData( context );
-
-        if( !lsd ) {    //Nothing to do, keep unwinding
+        const unsigned char * lsd = _Unwind_GetLanguageSpecificData( unwind_context );
+
+        if ( !lsd ) {   //Nothing to do, keep unwinding
                 printf(" no LSD");
                 goto UNWIND;
@@ -295 +351 @@
         // Get the instuction pointer and a reading pointer into the exception table
         lsda_header_info lsd_info;
-        const unsigned char * cur_ptr = parse_lsda_header(context, lsd, &lsd_info);
-        _Unwind_Ptr instruction_ptr = _Unwind_GetIP( context );
+        const unsigned char * cur_ptr = parse_lsda_header(unwind_context, lsd, &lsd_info);
+        _Unwind_Ptr instruction_ptr = _Unwind_GetIP(unwind_context);
+
+        struct exception_context_t * context = this_exception_context();
 
         // Linearly search the table for stuff to do
-        while( cur_ptr < lsd_info.action_table ) {
+        while ( cur_ptr < lsd_info.action_table ) {
                 _Unwind_Ptr callsite_start;
                 _Unwind_Ptr callsite_len;
@@ -312 +370 @@
 
                 // Have we reach the correct frame info yet?
-                if( lsd_info.Start + callsite_start + callsite_len < instruction_ptr ) {
+                if ( lsd_info.Start + callsite_start + callsite_len < instruction_ptr ) {
 #ifdef __CFA_DEBUG_PRINT__
                         void * ls = (void*)lsd_info.Start;
@@ -320 +378 @@
                         void * ep = (void*)lsd_info.Start + callsite_start + callsite_len;
                         void * ip = (void*)instruction_ptr;
-                        __cfaabi_dbg_print_safe("\nfound %p - %p (%p, %p, %p), looking for %p\n",
+                        __cfadbg_print_safe(exception, "\nfound %p - %p (%p, %p, %p), looking for %p\n",
                                         bp, ep, ls, cs, cl, ip);
 #endif // __CFA_DEBUG_PRINT__
@@ -327 +385 @@
 
                 // Have we gone too far?
-                if( lsd_info.Start + callsite_start > instruction_ptr ) {
+                if ( lsd_info.Start + callsite_start > instruction_ptr ) {
                         printf(" gone too far");
                         break;
                 }
 
-                // Something to do?
-                if( callsite_landing_pad ) {
-                        // Which phase are we in
-                        if (actions & _UA_SEARCH_PHASE) {
-                                // In search phase, these means we found a potential handler we must check.
-
-                                // We have arbitrarily decided that 0 means nothing to do and 1 means there is
-                                // a potential handler. This doesn't seem to conflict the gcc default behavior.
-                                if (callsite_action != 0) {
-                                        // Now we want to run some code to see if the handler matches
-                                        // This is the tricky part where we want to the power to run arbitrary code
-                                        // However, generating a new exception table entry and try routine every time
-                                        // is way more expansive than we might like
-                                        // The information we have is :
-                                        //  - The GR (Series of registers)
-                                        //    GR1=GP Global Pointer of frame ref by context
-                                        //  - The instruction pointer
-                                        //  - The instruction pointer info (???)
-                                        //  - The CFA (Canonical Frame Address)
-                                        //  - The BSP (Probably the base stack pointer)
-
-
-                                        // The current apprach uses one exception table entry per try block
-                                        _uleb128_t imatcher;
-                                        // Get the relative offset to the {...}?
-                                        cur_ptr = read_uleb128(cur_ptr, &imatcher);
-
-                                        _Unwind_Reason_Code (*matcher)(exception_t *) =
-                                                MATCHER_FROM_CONTEXT(context);
-                                        int index = matcher(shared_stack.current_exception);
-                                        _Unwind_Reason_Code ret = (0 == index)
-                                                ? _URC_CONTINUE_UNWIND : _URC_HANDLER_FOUND;
-                                        shared_stack.current_handler_index = index;
-
-                                        // Based on the return value, check if we matched the exception
-                                        if( ret == _URC_HANDLER_FOUND) {
-                                                __cfaabi_dbg_print_safe(" handler found\n");
-                                        } else {
-                                                __cfaabi_dbg_print_safe(" no handler\n");
-                                        }
-                                        return ret;
+                // Check for what we must do:
+                if ( 0 == callsite_landing_pad ) {
+                        // Nothing to do, move along
+                        __cfadbg_print_safe(exception, " no landing pad");
+                } else if (actions & _UA_SEARCH_PHASE) {
+                        // In search phase, these means we found a potential handler we must check.
+
+                        // We have arbitrarily decided that 0 means nothing to do and 1 means there is
+                        // a potential handler. This doesn't seem to conflict the gcc default behavior.
+                        if (callsite_action != 0) {
+                                // Now we want to run some code to see if the handler matches
+                                // This is the tricky part where we want to the power to run arbitrary code
+                                // However, generating a new exception table entry and try routine every time
+                                // is way more expansive than we might like
+                                // The information we have is :
+                                //  - The GR (Series of registers)
+                                //    GR1=GP Global Pointer of frame ref by context
+                                //  - The instruction pointer
+                                //  - The instruction pointer info (???)
+                                //  - The CFA (Canonical Frame Address)
+                                //  - The BSP (Probably the base stack pointer)
+
+                                // The current apprach uses one exception table entry per try block
+                                _uleb128_t imatcher;
+                                // Get the relative offset to the {...}?
+                                cur_ptr = read_uleb128(cur_ptr, &imatcher);
+
+                                _Unwind_Word match_pos =
+#                               if defined( __x86_64 )
+                                    _Unwind_GetCFA(unwind_context) + 8;
+#                               elif defined( __i386 )
+                                    _Unwind_GetCFA(unwind_context) + 24;
+#                               elif defined( __ARM_ARCH )
+#                                   warning FIX ME: check if anything needed for ARM
+                                    42;
+#                               endif
+                                int (*matcher)(exception_t *) = *(int(**)(exception_t *))match_pos;
+
+                                int index = matcher(context->current_exception);
+                                _Unwind_Reason_Code ret = (0 == index)
+                                        ? _URC_CONTINUE_UNWIND : _URC_HANDLER_FOUND;
+                                UNWIND_TO_NODE(unwind_exception)->handler_index = index;
+
+                                // Based on the return value, check if we matched the exception
+                                if (ret == _URC_HANDLER_FOUND) {
+                                        __cfadbg_print_safe(exception, " handler found\n");
+                                } else {
+                                        // TODO: Continue the search if there is more in the table.
+                                        __cfadbg_print_safe(exception, " no handler\n");
                                 }
-
-                                // This is only a cleanup handler, ignore it
-                                __cfaabi_dbg_print_safe(" no action");
+                                return ret;
                         }
-                        else if (actions & _UA_CLEANUP_PHASE) {
-
-                                if( (callsite_action != 0) && !(actions & _UA_HANDLER_FRAME) ){
-                                        // If this is a potential exception handler
-                                        // but not the one that matched the exception in the seach phase,
-                                        // just ignore it
-                                        goto UNWIND;
-                                }
-
-                                // We need to run some clean-up or a handler
-                                // These statment do the right thing but I don't know any specifics at all
-                                _Unwind_SetGR( context, __builtin_eh_return_data_regno(0), (_Unwind_Ptr) unwind_exception );
-                                _Unwind_SetGR( context, __builtin_eh_return_data_regno(1), 0 );
-
-                                // I assume this sets the instruction pointer to the adress of the landing pad
-                                // It doesn't actually set it, it only state the value that needs to be set once we return _URC_INSTALL_CONTEXT
-                                _Unwind_SetIP( context, ((lsd_info.LPStart) + (callsite_landing_pad)) );
-
-                                __cfaabi_dbg_print_safe(" action\n");
-
-                                // Return have some action to run
-                                return _URC_INSTALL_CONTEXT;
+
+                        // This is only a cleanup handler, ignore it
+                        __cfadbg_print_safe(exception, " no action");
+                } else {
+                        // In clean-up phase, no destructors here but this could be the handler.
+
+                        if ( (callsite_action != 0) && !(actions & _UA_HANDLER_FRAME) ){
+                                // If this is a potential exception handler
+                                // but not the one that matched the exception in the seach phase,
+                                // just ignore it
+                                goto UNWIND;
                         }
+
+                        // We need to run some clean-up or a handler
+                        // These statment do the right thing but I don't know any specifics at all
+                        _Unwind_SetGR( unwind_context, __builtin_eh_return_data_regno(0),
+                                (_Unwind_Ptr)unwind_exception );
+                        _Unwind_SetGR( unwind_context, __builtin_eh_return_data_regno(1), 0 );
+
+                        // I assume this sets the instruction pointer to the adress of the landing pad
+                        // It doesn't actually set it, it only state the value that needs to be set once we
+                        // return _URC_INSTALL_CONTEXT
+                        _Unwind_SetIP( unwind_context, ((lsd_info.LPStart) + (callsite_landing_pad)) );
+
+                        __cfadbg_print_safe(exception, " action\n");
+
+                        // Return have some action to run
+                        return _URC_INSTALL_CONTEXT;
                 }
-
-                // Nothing to do, move along
-                __cfaabi_dbg_print_safe(" no landing pad");
         }
         // No handling found
-        __cfaabi_dbg_print_safe(" table end reached\n");
+        __cfadbg_print_safe(exception, " table end reached");
 
         UNWIND:
-        __cfaabi_dbg_print_safe(" unwind\n");
+        __cfadbg_print_safe(exception, " unwind\n");
 
         // Keep unwinding the stack
         return _URC_CONTINUE_UNWIND;
 }
+
+#pragma GCC push_options
+#pragma GCC optimize(0)
 
 // Try statements are hoisted out see comments for details. While this could probably be unique
 // and simply linked from libcfa but there is one problem left, see the exception table for details
 __attribute__((noinline))
-void __cfaabi_ehm__try_terminate(void (*try_block)(),
+void __cfaehm_try_terminate(void (*try_block)(),
                 void (*catch_block)(int index, exception_t * except),
                 __attribute__((unused)) int (*match_block)(exception_t * except)) {
@@ -425 +494 @@
         //! printf("%p %p %p %p\n", &try_block, &catch_block, &match_block, &xy);
 
-        // Setup statments: These 2 statments won't actually result in any code, they only setup global tables.
-        // However, they clobber gcc cancellation support from gcc.  We can replace the personality routine but
-        // replacing the exception table gcc generates is not really doable, it generates labels based on how the
-        // assembly works.
-
         // Setup the personality routine and exception table.
+        // Unforturnately these clobber gcc cancellation support which means we can't get access to
+        // the attribute cleanup tables at the same time. We would have to inspect the assembly to
+        // create a new set ourselves.
+#ifdef __PIC__
+        asm volatile (".cfi_personality 0x9b,CFA.ref.__gcfa_personality_v0");
+        asm volatile (".cfi_lsda 0x1b, .LLSDACFA2");
+#else
         asm volatile (".cfi_personality 0x3,__gcfa_personality_v0");
         asm volatile (".cfi_lsda 0x3, .LLSDACFA2");
+#endif
 
         // Label which defines the start of the area for which the handler is setup.
@@ -450 +522 @@
         // Label which defines the end of the area for which the handler is setup.
         asm volatile (".TRYEND:");
-        // Label which defines the start of the exception landing pad.  Basically what is called when the exception is
-        // caught.  Note, if multiple handlers are given, the multiplexing should be done by the generated code, not the
-        // exception runtime.
+        // Label which defines the start of the exception landing pad. Basically what is called when
+        // the exception is caught. Note, if multiple handlers are given, the multiplexing should be
+        // done by the generated code, not the exception runtime.
         asm volatile (".CATCH:");
 
         // Exception handler
-        catch_block( shared_stack.current_handler_index,
-                     shared_stack.current_exception );
+        // Note: Saving the exception context on the stack breaks termination exceptions.
+        catch_block( EXCEPT_TO_NODE( this_exception_context()->current_exception )->handler_index,
+                     this_exception_context()->current_exception );
 }
 
@@ -464 +537 @@
 // have a single call to the try routine.
 
-#if defined( __i386 ) || defined( __x86_64 )
+#ifdef __PIC__
+asm (
+        // HEADER
+        ".LFECFA1:\n"
+        "       .globl  __gcfa_personality_v0\n"
+        "       .section        .gcc_except_table,\"a\",@progbits\n"
+        // TABLE HEADER (important field is the BODY length at the end)
+        ".LLSDACFA2:\n"
+        "       .byte   0xff\n"
+        "       .byte   0xff\n"
+        "       .byte   0x1\n"
+        "       .uleb128 .LLSDACSECFA2-.LLSDACSBCFA2\n"
+        // BODY (language specific data)
+        // This uses language specific data and can be modified arbitrarily
+        // We use handled area offset, handled area length,
+        // handler landing pad offset and 1 (action code, gcc seems to use 0).
+        ".LLSDACSBCFA2:\n"
+        "       .uleb128 .TRYSTART-__cfaehm_try_terminate\n"
+        "       .uleb128 .TRYEND-.TRYSTART\n"
+        "       .uleb128 .CATCH-__cfaehm_try_terminate\n"
+        "       .uleb128 1\n"
+        ".LLSDACSECFA2:\n"
+        // TABLE FOOTER
+        "       .text\n"
+        "       .size   __cfaehm_try_terminate, .-__cfaehm_try_terminate\n"
+);
+
+// Somehow this piece of helps with the resolution of debug symbols.
+__attribute__((unused)) static const int dummy = 0;
+
+asm (
+        // Add a hidden symbol which points at the function.
+        "       .hidden CFA.ref.__gcfa_personality_v0\n"
+        "       .weak   CFA.ref.__gcfa_personality_v0\n"
+        // No clue what this does specifically
+        "       .section        .data.rel.local.CFA.ref.__gcfa_personality_v0,\"awG\",@progbits,CFA.ref.__gcfa_personality_v0,comdat\n"
+        "       .align 8\n"
+        "       .type CFA.ref.__gcfa_personality_v0, @object\n"
+        "       .size CFA.ref.__gcfa_personality_v0, 8\n"
+        "CFA.ref.__gcfa_personality_v0:\n"
+#if defined( __x86_64 )
+        "       .quad __gcfa_personality_v0\n"
+#else // then __i386
+        "       .long __gcfa_personality_v0\n"
+#endif
+);
+#else // __PIC__
 asm (
         // HEADER
@@ -479 +598 @@
         ".LLSDACSBCFA2:\n"
         //      Handled area start (relative to start of function)
-        "       .uleb128 .TRYSTART-__cfaabi_ehm__try_terminate\n"
+        "       .uleb128 .TRYSTART-__cfaehm_try_terminate\n"
         //      Handled area length
         "       .uleb128 .TRYEND-.TRYSTART\n"
         //      Handler landing pad address (relative to start of function)
-        "       .uleb128 .CATCH-__cfaabi_ehm__try_terminate\n"
+        "       .uleb128 .CATCH-__cfaehm_try_terminate\n"
         //      Action code, gcc seems to always use 0.
         "       .uleb128 1\n"
@@ -489 +608 @@
         ".LLSDACSECFA2:\n"
         "       .text\n"
-        "       .size   __cfaabi_ehm__try_terminate, .-__cfaabi_ehm__try_terminate\n"
+        "       .size   __cfaehm_try_terminate, .-__cfaehm_try_terminate\n"
         "       .ident  \"GCC: (Ubuntu 6.2.0-3ubuntu11~16.04) 6.2.0 20160901\"\n"
-//      "       .section        .note.GNU-stack,\"x\",@progbits\n"
+        "       .section        .note.GNU-stack,\"x\",@progbits\n"
 );
-#endif // __i386 || __x86_64
-#endif // PIC
+#endif // __PIC__
+
+#pragma GCC pop_options
+
+#elif defined( __ARM_ARCH )
+_Unwind_Reason_Code __gcfa_personality_v0(
+                int version,
+                _Unwind_Action actions,
+                unsigned long long exception_class,
+                struct _Unwind_Exception * unwind_exception,
+                struct _Unwind_Context * unwind_context) {
+        return _URC_CONTINUE_UNWIND;
+}
+
+__attribute__((noinline))
+void __cfaehm_try_terminate(void (*try_block)(),
+                void (*catch_block)(int index, exception_t * except),
+                __attribute__((unused)) int (*match_block)(exception_t * except)) {
+}
+#else
+        #error unsupported hardware architecture
+#endif // __x86_64 || __i386

libcfa/src/exception.h

@@ -5 +5 @@
 // file "LICENCE" distributed with Cforall.
 //
-// exception.h -- Builtins for exception handling.
+// exception.h -- Internal exception handling definitions.
 //
 // Author           : Andrew Beach
 // Created On       : Mon Jun 26 15:11:00 2017
-// Last Modified By : Peter A. Buhr
-// Last Modified On : Thu Feb 22 18:11:15 2018
-// Update Count     : 8
+// Last Modified By : Andrew Beach
+// Last Modified On : Tue Oct 27 14:45:00 2020
+// Update Count     : 11
 //
 
 #pragma once
 
+// This could be considered several headers. All are internal to the exception
+// system but needed to depending on whether they are C/Cforall code and
+// whether or not they are part of the builtins.
 
 #ifdef __cforall
@@ -21 +24 @@
 #endif
 
-struct __cfaabi_ehm__base_exception_t;
-typedef struct __cfaabi_ehm__base_exception_t exception_t;
-struct __cfaabi_ehm__base_exception_t_vtable {
-        const struct __cfaabi_ehm__base_exception_t_vtable * parent;
+// Included in C code or the built-ins.
+#if !defined(__cforall) || defined(__cforall_builtins__)
+
+struct __cfaehm_base_exception_t;
+typedef struct __cfaehm_base_exception_t exception_t;
+struct __cfaehm_base_exception_t_vtable {
+        const struct __cfaehm_base_exception_t_vtable * parent;
         size_t size;
-        void (*copy)(struct __cfaabi_ehm__base_exception_t *this,
-                     struct __cfaabi_ehm__base_exception_t * other);
-        void (*free)(struct __cfaabi_ehm__base_exception_t *this);
-        const char * (*msg)(struct __cfaabi_ehm__base_exception_t *this);
+        void (*copy)(struct __cfaehm_base_exception_t *this,
+                     struct __cfaehm_base_exception_t * other);
+        void (*free)(struct __cfaehm_base_exception_t *this);
+        const char * (*msg)(struct __cfaehm_base_exception_t *this);
 };
-struct __cfaabi_ehm__base_exception_t {
-        struct __cfaabi_ehm__base_exception_t_vtable const * virtual_table;
+struct __cfaehm_base_exception_t {
+        struct __cfaehm_base_exception_t_vtable const * virtual_table;
 };
-extern struct __cfaabi_ehm__base_exception_t_vtable
-        ___cfaabi_ehm__base_exception_t_vtable_instance;
+extern struct __cfaehm_base_exception_t_vtable
+        ___cfaehm_base_exception_t_vtable_instance;
 
 
+void __cfaehm_cancel_stack(exception_t * except) __attribute__((noreturn));
+
 // Used in throw statement translation.
-void __cfaabi_ehm__throw_terminate(exception_t * except) __attribute__((noreturn));
-void __cfaabi_ehm__rethrow_terminate() __attribute__((noreturn));
-void __cfaabi_ehm__throw_resume(exception_t * except);
+void __cfaehm_throw_terminate(exception_t * except, void (*)(exception_t *));
+void __cfaehm_rethrow_terminate() __attribute__((noreturn));
+void __cfaehm_throw_resume(exception_t * except, void (*)(exception_t *));
 
 // Function catches termination exceptions.
-void __cfaabi_ehm__try_terminate(
-    void (*try_block)(),
-    void (*catch_block)(int index, exception_t * except),
-    int (*match_block)(exception_t * except));
+void __cfaehm_try_terminate(
+        void (*try_block)(),
+        void (*catch_block)(int index, exception_t * except),
+        int (*match_block)(exception_t * except));
 
 // Clean-up the exception in catch blocks.
-void __cfaabi_ehm__cleanup_terminate(void * except);
+void __cfaehm_cleanup_terminate(void * except);
 
 // Data structure creates a list of resume handlers.
-struct __cfaabi_ehm__try_resume_node {
-    struct __cfaabi_ehm__try_resume_node * next;
-    _Bool (*handler)(exception_t * except);
+struct __cfaehm_try_resume_node {
+        struct __cfaehm_try_resume_node * next;
+        _Bool (*handler)(exception_t * except);
 };
 
 // These act as constructor and destructor for the resume node.
-void __cfaabi_ehm__try_resume_setup(
-    struct __cfaabi_ehm__try_resume_node * node,
-    _Bool (*handler)(exception_t * except));
-void __cfaabi_ehm__try_resume_cleanup(
-    struct __cfaabi_ehm__try_resume_node * node);
+void __cfaehm_try_resume_setup(
+        struct __cfaehm_try_resume_node * node,
+        _Bool (*handler)(exception_t * except));
+void __cfaehm_try_resume_cleanup(
+        struct __cfaehm_try_resume_node * node);
 
 // Check for a standard way to call fake deconstructors.
-struct __cfaabi_ehm__cleanup_hook {};
+struct __cfaehm_cleanup_hook {};
+
+#endif
+
+// Included in C code and the library.
+#if !defined(__cforall) || !defined(__cforall_builtins__)
+struct __cfaehm_node {
+        struct _Unwind_Exception unwind_exception;
+        struct __cfaehm_node * next;
+        int handler_index;
+};
+
+static inline exception_t * __cfaehm_cancellation_exception(
+                struct _Unwind_Exception * unwind_exception ) {
+        return (exception_t *)(1 + (struct __cfaehm_node *)unwind_exception);
+}
+#endif
 
 #ifdef __cforall
 }
+
+// Built-ins not visible in C.
+#if defined(__cforall_builtins__)
+
+// Not all the built-ins can be expressed in C. These can't be
+// implemented in the .c file either so they all have to be inline.
+
+trait is_exception(dtype exceptT, dtype virtualT) {
+        /* The first field must be a pointer to a virtual table.
+         * That virtual table must be a decendent of the base exception virtual table.
+         */
+        virtualT const & get_exception_vtable(exceptT *);
+        // Always returns the virtual table for this type (associated types hack).
+};
+
+trait is_termination_exception(dtype exceptT, dtype virtualT | is_exception(exceptT, virtualT)) {
+        void defaultTerminationHandler(exceptT &);
+};
+
+trait is_resumption_exception(dtype exceptT, dtype virtualT | is_exception(exceptT, virtualT)) {
+        void defaultResumptionHandler(exceptT &);
+};
+
+forall(dtype exceptT, dtype virtualT | is_termination_exception(exceptT, virtualT))
+static inline void $throw(exceptT & except) {
+        __cfaehm_throw_terminate(
+                (exception_t *)&except,
+                (void(*)(exception_t *))defaultTerminationHandler
+        );
+}
+
+forall(dtype exceptT, dtype virtualT | is_resumption_exception(exceptT, virtualT))
+static inline void $throwResume(exceptT & except) {
+        __cfaehm_throw_resume(
+                (exception_t *)&except,
+                (void(*)(exception_t *))defaultResumptionHandler
+        );
+}
+
+forall(dtype exceptT, dtype virtualT | is_exception(exceptT, virtualT))
+static inline void cancel_stack(exceptT & except) __attribute__((noreturn)) {
+        __cfaehm_cancel_stack( (exception_t *)&except );
+}
+
+forall(dtype exceptT, dtype virtualT | is_exception(exceptT, virtualT))
+static inline void defaultTerminationHandler(exceptT & except) {
+        return cancel_stack( except );
+}
+
+forall(dtype exceptT, dtype virtualT | is_exception(exceptT, virtualT))
+static inline void defaultResumptionHandler(exceptT & except) {
+        throw except;
+}
+
 #endif
+
+#endif

libcfa/src/executor.cfa

@@ -4 +4 @@
 // buffer.
 
-#include <bits/containers.hfa>
 #include <thread.hfa>
-#include <stdio.h>
+#include <containers/list.hfa>
 
-forall( dtype T )
-monitor Buffer {                                        // unbounded buffer
-    __queue_t( T ) queue;                               // unbounded list of work requests
-    condition delay;
-}; // Buffer
-forall( dtype T | is_node(T) ) {
-    void insert( Buffer( T ) & mutex buf, T * elem ) with(buf) {
-        append( queue, elem );                          // insert element into buffer
-        signal( delay );                                // restart
-    } // insert
+forall( dtype T | $dlistable(T, T) ) {
+        monitor Buffer {                                                                        // unbounded buffer
+                dlist( T, T ) queue;                                                    // unbounded list of work requests
+                condition delay;
+        }; // Buffer
 
-    T * remove( Buffer( T ) & mutex buf ) with(buf) {
-        if ( queue.head != 0 ) wait( delay );                   // no request to process ? => wait
-//      return pop_head( queue );
-    } // remove
-} // distribution
+        void insert( Buffer(T) & mutex buf, T * elem ) with(buf) {
+                dlist( T, T ) * qptr = &queue;                                  // workaround https://cforall.uwaterloo.ca/trac/ticket/166
+                insert_last( *qptr, *elem );                                    // insert element into buffer
+                signal( delay );                                                                // restart
+        } // insert
 
-struct WRequest {                                       // client request, no return
-    void (* action)( void );
-    WRequest * next;                                    // intrusive queue field
+        T * remove( Buffer(T) & mutex buf ) with(buf) {
+                dlist( T, T ) * qptr = &queue;                                  // workaround https://cforall.uwaterloo.ca/trac/ticket/166
+                // if ( (*qptr)`is_empty ) wait( delay );                       // no request to process ? => wait
+          if ( (*qptr)`is_empty ) return 0p;                            // no request to process ? => wait
+                return &pop_first( *qptr );
+        } // remove
+} // forall
+
+struct WRequest {                                                                               // client request, no return
+        void (* action)( void );
+        DLISTED_MGD_IMPL_IN(WRequest)
 }; // WRequest
+DLISTED_MGD_IMPL_OUT(WRequest)
 
-WRequest *& get_next( WRequest & this ) { return this.next; }
-void ?{}( WRequest & req ) with(req) { action = 0; next = 0; }
-void ?{}( WRequest & req, void (* action)( void ) ) with(req) { req.action = action; next = 0; }
+void ?{}( WRequest & req ) with(req) { action = 0; }
+void ?{}( WRequest & req, void (* action)( void ) ) with(req) { req.action = action; }
 bool stop( WRequest & req ) { return req.action == 0; }
 void doit( WRequest & req ) { req.action(); }
 
-// Each worker has its own work buffer to reduce contention between client and server. Hence, work requests arrive and
-// are distributed into buffers in a roughly round-robin order.
+// Each worker has its own set (when requests buffers > workers) of work buffers to reduce contention between client
+// and server, where work requests arrive and are distributed into buffers in a roughly round-robin order.
 
 thread Worker {
-    Buffer( WRequest ) * requests;
-    unsigned int start, range;
+        Buffer(WRequest) * requests;
+        WRequest * request;
+        unsigned int start, range;
 }; // Worker
 
 void main( Worker & w ) with(w) {
-    for ( int i = 0;; i = (i + 1) % range ) {
-        WRequest * request = remove( requests[i + start] );
-      if ( ! request ) { yield(); continue; }
-      if ( stop( *request ) ) break;
-        doit( *request );
-        delete( request );
-    } // for
+        for ( int i = 0;; i = (i + 1) % range ) {
+                request = remove( requests[i + start] );
+          if ( ! request ) { yield(); continue; }
+          if ( stop( *request ) ) break;
+                doit( *request );
+                delete( request );
+        } // for
 } // Worker::main
 
-void ?{}( Worker & worker, cluster * wc, Buffer( WRequest ) * requests, unsigned int start, unsigned int range ) {
-    (*get_thread(worker)){ *wc };                       // create on given cluster
-    worker.[requests, start, range] = [requests, start, range];
+void ?{}( Worker & worker, cluster * wc, Buffer(WRequest) * requests, unsigned int start, unsigned int range ) {
+        ((thread &)worker){ *wc };
+        worker.[requests, request, start, range] = [requests, 0p, start, range];
 } // ?{}
 
+WRequest * current_request( Worker & worker ) { return worker.request; }
+
 struct Executor {
-    cluster * cluster;                                  // if workers execute on separate cluster
-    processor ** processors;                            // array of virtual processors adding parallelism for workers
-    Buffer( WRequest ) * requests;                      // list of work requests
-    Worker ** workers;                                  // array of workers executing work requests
-    unsigned int nprocessors, nworkers, nmailboxes;     // number of mailboxes/workers/processor tasks
-    bool sepClus;                                       // use same or separate cluster for executor
+        cluster * cluster;                                                                      // if workers execute on separate cluster
+        processor ** processors;                                                        // array of virtual processors adding parallelism for workers
+        Buffer(WRequest) * requests;                                            // list of work requests
+        Worker ** workers;                                                                      // array of workers executing work requests
+        unsigned int nprocessors, nworkers, nrqueues;           // number of processors/threads/request queues
+        bool sepClus;                                                                           // use same or separate cluster for executor
+        unsigned int next;                                                                      // demultiplexed across worker buffers
 }; // Executor
 
-static thread_local unsigned int next;                  // demultiplexed across worker buffers
 unsigned int tickets( Executor & ex ) with(ex) {
-    //return uFetchAdd( next, 1 ) % nmailboxes;
-    return next++ % nmailboxes;                         // no locking, interference randomizes
+        //return uFetchAdd( next, 1 ) % nrqueues;
+        return next++ % nrqueues;                                                       // no locking, interference randomizes
 } // tickets
 
-void ?{}( Executor & ex, unsigned int np, unsigned int nw, unsigned int nm, bool sc = false ) with(ex) {
-    [nprocessors, nworkers, nmailboxes, sepClus] = [np, nw, nm, sc];
-    assert( nmailboxes >= nworkers );
-    cluster = sepClus ? new( "Executor" ) : active_cluster();
-    processors = (processor **)anew( nprocessors );
-    requests = anew( nmailboxes );
-    workers = (Worker **)anew( nworkers );
+void ?{}( Executor & ex, unsigned int np, unsigned int nw, unsigned int nr, bool sc = false ) with(ex) {
+        [nprocessors, nworkers, nrqueues, sepClus] = [np, nw, nr, sc];
+        assert( nrqueues >= nworkers );
+        cluster = sepClus ? new( "Executor" ) : active_cluster();
+        processors = aalloc( nprocessors );
+        requests = anew( nrqueues );
+        workers = aalloc( nworkers );
 
-    for ( i; nprocessors ) {
-        processors[ i ] = new( *cluster );
-    } // for
+        for ( i; nprocessors ) {
+                processors[i] = new( *cluster );
+        } // for
 
-    unsigned int reqPerWorker = nmailboxes / nworkers, extras = nmailboxes % nworkers;
-    for ( unsigned int i = 0, step = 0; i < nworkers; i += 1, step += reqPerWorker + ( i < extras ? 1 : 0 ) ) {
-        workers[ i ] = new( cluster, requests, step, reqPerWorker + ( i < extras ? 1 : 0 ) );
-    } // for
+        unsigned int reqPerWorker = nrqueues / nworkers, extras = nrqueues % nworkers;
+//      for ( unsigned int i = 0, start = 0, range; i < nworkers; i += 1, start += range ) {
+    for ( i; nworkers : start; 0u ~ @ ~ range : range; ) {
+            range = reqPerWorker + ( i < extras ? 1 : 0 );
+                workers[i] = new( cluster, requests, start, range );
+        } // for
 } // ?{}
 
 void ?{}( Executor & ex, unsigned int nprocessors, unsigned int nworkers, bool sepClus = false ) {
-    ex{ nprocessors, nworkers, nworkers, sepClus };
+        ex{ nprocessors, nworkers, nworkers, sepClus };
 }
 void ?{}( Executor & ex, unsigned int nprocessors, bool sepClus = false ) {
-    ex{ nprocessors, nprocessors, nprocessors, sepClus };
+        ex{ nprocessors, nprocessors, nprocessors, sepClus };
 }
-void ?{}( Executor & ex ) {                             // special for current cluster
-    ex{ 0, active_cluster()->nprocessors, false };
+void ?{}( Executor & ex ) {                                                             // special for current cluster, no processors added
+        ex{ 0, active_cluster()->nprocessors, false };
 }
 void ^?{}( Executor & ex ) with(ex) {
-    // Add one sentinel per worker to stop them. Since in destructor, no new work should be queued.  Cannot combine next
-    // two loops and only have a single sentinel because workers arrive in arbitrary order, so worker1 may take the
-    // single sentinel while waiting for worker 0 to end.
+        // Add one sentinel per worker to stop them. Since in destructor, no new external work should be queued.  Cannot
+        // combine next two loops and only have a single sentinel because workers arrive in arbitrary order, so worker1 may
+        // take the single sentinel while waiting for worker 0 to end.
 
-    WRequest sentinel[nworkers];
-    unsigned int reqPerWorker = nmailboxes / nworkers;
-    for ( unsigned int i = 0, step = 0; i < nworkers; i += 1, step += reqPerWorker ) {
-        insert( requests[step], &sentinel[i] );         // force eventually termination
-    } // for
-    for ( i; nworkers ) {
-        delete( workers[ i ] );
-    } // for
-    for ( i; nprocessors ) {
-        delete( processors[ i ] );
-    } // for
+        WRequest sentinel[nworkers];
+        unsigned int reqPerWorker = nrqueues / nworkers;
+        for ( unsigned int i = 0, step = 0; i < nworkers; i += 1, step += reqPerWorker ) {
+                insert( requests[step], &sentinel[i] );                 // force eventually termination
+        } // for
+        for ( i; nworkers ) {
+                delete( workers[i] );
+        } // for
+        for ( i; nprocessors ) {
+                delete( processors[i] );
+        } // for
 
-    delete( workers );
-    delete( requests );
-    delete( processors );
-    if ( sepClus ) { delete( cluster ); }
+        free( workers );
+//      adelete( nrqueues, requests );
+        for ( i; nrqueues ) ^?{}( requests[i] );                        // FIX ME: problem with resolver
+        free( requests );
+        free( processors );
+        if ( sepClus ) { delete( cluster ); }
 } // ^?{}
 
 void send( Executor & ex, void (* action)( void ) ) {   // asynchronous call, no return value
-    WRequest * node = new( action );
-    insert( ex.requests[tickets( ex )], node );
+        WRequest * node = new( action );
+        insert( ex.requests[tickets( ex )], node );
 } // send
+
 
 int counter = 0;
 
-void workie( void ) {
-    __atomic_add_fetch( &counter, 1, __ATOMIC_SEQ_CST );
-//    fprintf( stderr, "workie\n" );
+void work( void ) {
+        __atomic_add_fetch( &counter, 1, __ATOMIC_SEQ_CST );
+        // fprintf( stderr, "workie\n" );
 }
 
-int main() {
-    {
-        Executor exector;
-        for ( i; 3000 ) {
-            send( exector, workie );
-            if ( i % 100 ) yield();
-        } // for
-    }
-    printf( "%d\n", counter );
+int main( int argc, char * argv[] ) {
+        int times = 1_000_000;
+        if ( argc == 2 ) times = atoi( argv[1] );
+        processor p[7];
+        {
+                Executor exector;
+                for ( i; times ) {
+                        send( exector, work );
+                        if ( i % 100 == 0 ) yield();
+                } // for
+        }
+        printf( "%d\n", counter );
 }
 
 // Local Variables: //
+// tab-width: 4" //
 // compile-command: "cfa executor.cfa" //
 // End: //

libcfa/src/fstream.cfa

@@ -10 +10 @@
 // Created On       : Wed May 27 17:56:53 2015
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Fri Nov 29 06:56:46 2019
-// Update Count     : 355
+// Last Modified On : Fri Jun 19 16:24:54 2020
+// Update Count     : 384
 //
 
@@ -26 +26 @@
 
 
-//*********************************** ofstream ***********************************
+// *********************************** ofstream ***********************************
 
 
@@ -32 +32 @@
 
 void ?{}( ofstream & os, void * file ) {
-        os.file = file;
-        os.sepDefault = true;
-        os.sepOnOff = false;
-        os.nlOnOff = true;
-        os.prt = false;
-        os.sawNL = false;
+        os.$file = file;
+        os.$sepDefault = true;
+        os.$sepOnOff = false;
+        os.$nlOnOff = true;
+        os.$prt = false;
+        os.$sawNL = false;
+        $sepSetCur( os, sepGet( os ) );
         sepSet( os, " " );
-        sepSetCur( os, sepGet( os ) );
         sepSetTuple( os, ", " );
 } // ?{}
 
 // private
-bool sepPrt( ofstream & os ) { setNL( os, false ); return os.sepOnOff; }
-void sepReset( ofstream & os ) { os.sepOnOff = os.sepDefault; }
-void sepReset( ofstream & os, bool reset ) { os.sepDefault = reset; os.sepOnOff = os.sepDefault; }
-const char * sepGetCur( ofstream & os ) { return os.sepCur; }
-void sepSetCur( ofstream & os, const char * sepCur ) { os.sepCur = sepCur; }
-bool getNL( ofstream & os ) { return os.sawNL; }
-void setNL( ofstream & os, bool state ) { os.sawNL = state; }
-bool getANL( ofstream & os ) { return os.nlOnOff; }
-bool getPrt( ofstream & os ) { return os.prt; }
-void setPrt( ofstream & os, bool state ) { os.prt = state; }
+bool $sepPrt( ofstream & os ) { $setNL( os, false ); return os.$sepOnOff; }
+void $sepReset( ofstream & os ) { os.$sepOnOff = os.$sepDefault; }
+void $sepReset( ofstream & os, bool reset ) { os.$sepDefault = reset; os.$sepOnOff = os.$sepDefault; }
+const char * $sepGetCur( ofstream & os ) { return os.$sepCur; }
+void $sepSetCur( ofstream & os, const char sepCur[] ) { os.$sepCur = sepCur; }
+bool $getNL( ofstream & os ) { return os.$sawNL; }
+void $setNL( ofstream & os, bool state ) { os.$sawNL = state; }
+bool $getANL( ofstream & os ) { return os.$nlOnOff; }
+bool $getPrt( ofstream & os ) { return os.$prt; }
+void $setPrt( ofstream & os, bool state ) { os.$prt = state; }
 
 // public
-void ?{}( ofstream & os ) { os.file = 0; }
-
-void ?{}( ofstream & os, const char * name, const char * mode ) {
+void ?{}( ofstream & os ) { os.$file = 0p; }
+
+void ?{}( ofstream & os, const char name[], const char mode[] ) {
         open( os, name, mode );
 } // ?{}
 
-void ?{}( ofstream & os, const char * name ) {
+void ?{}( ofstream & os, const char name[] ) {
         open( os, name, "w" );
 } // ?{}
@@ -70 +70 @@
 } // ^?{}
 
-void sepOn( ofstream & os ) { os.sepOnOff = ! getNL( os ); }
-void sepOff( ofstream & os ) { os.sepOnOff = false; }
+void sepOn( ofstream & os ) { os.$sepOnOff = ! $getNL( os ); }
+void sepOff( ofstream & os ) { os.$sepOnOff = false; }
 
 bool sepDisable( ofstream & os ) {
-        bool temp = os.sepDefault;
-        os.sepDefault = false;
-        sepReset( os );
+        bool temp = os.$sepDefault;
+        os.$sepDefault = false;
+        $sepReset( os );
         return temp;
 } // sepDisable
 
 bool sepEnable( ofstream & os ) {
-        bool temp = os.sepDefault;
-        os.sepDefault = true;
-        if ( os.sepOnOff ) sepReset( os );                                      // start of line ?
+        bool temp = os.$sepDefault;
+        os.$sepDefault = true;
+        if ( os.$sepOnOff ) $sepReset( os );                            // start of line ?
         return temp;
 } // sepEnable
 
-void nlOn( ofstream & os ) { os.nlOnOff = true; }
-void nlOff( ofstream & os ) { os.nlOnOff = false; }
-
-const char * sepGet( ofstream & os ) { return os.separator; }
-void sepSet( ofstream & os, const char * s ) {
+void nlOn( ofstream & os ) { os.$nlOnOff = true; }
+void nlOff( ofstream & os ) { os.$nlOnOff = false; }
+
+const char * sepGet( ofstream & os ) { return os.$separator; }
+void sepSet( ofstream & os, const char s[] ) {
         assert( s );
-        strncpy( os.separator, s, sepSize - 1 );
-        os.separator[sepSize - 1] = '\0';
+        strncpy( os.$separator, s, sepSize - 1 );
+        os.$separator[sepSize - 1] = '\0';
 } // sepSet
 
-const char * sepGetTuple( ofstream & os ) { return os.tupleSeparator; }
-void sepSetTuple( ofstream & os, const char * s ) {
+const char * sepGetTuple( ofstream & os ) { return os.$tupleSeparator; }
+void sepSetTuple( ofstream & os, const char s[] ) {
         assert( s );
-        strncpy( os.tupleSeparator, s, sepSize - 1 );
-        os.tupleSeparator[sepSize - 1] = '\0';
+        strncpy( os.$tupleSeparator, s, sepSize - 1 );
+        os.$tupleSeparator[sepSize - 1] = '\0';
 } // sepSet
 
 void ends( ofstream & os ) {
-        if ( getANL( os ) ) nl( os );
-        else setPrt( os, false );                                                       // turn off
+        if ( $getANL( os ) ) nl( os );
+        else $setPrt( os, false );                                                      // turn off
         if ( &os == &exit ) exit( EXIT_FAILURE );
         if ( &os == &abort ) abort();
@@ -112 +112 @@
 
 int fail( ofstream & os ) {
-        return os.file == 0 || ferror( (FILE *)(os.file) );
+        return os.$file == 0 || ferror( (FILE *)(os.$file) );
 } // fail
 
 int flush( ofstream & os ) {
-        return fflush( (FILE *)(os.file) );
+        return fflush( (FILE *)(os.$file) );
 } // flush
 
-void open( ofstream & os, const char * name, const char * mode ) {
+void open( ofstream & os, const char name[], const char mode[] ) {
         FILE * file = fopen( name, mode );
         #ifdef __CFA_DEBUG__
-        if ( file == 0 ) {
-                abort | IO_MSG "open output file \"" | name | "\"" | nl | strerror( errno );
+        if ( file == 0p ) {
+                throw (Open_Failure){ os };
+                // abort | IO_MSG "open output file \"" | name | "\"" | nl | strerror( errno );
         } // if
         #endif // __CFA_DEBUG__
@@ -129 +130 @@
 } // open
 
-void open( ofstream & os, const char * name ) {
+void open( ofstream & os, const char name[] ) {
         open( os, name, "w" );
 } // open
 
 void close( ofstream & os ) {
-        if ( (FILE *)(os.file) == stdout || (FILE *)(os.file) == stderr ) return;
-
-        if ( fclose( (FILE *)(os.file) ) == EOF ) {
+  if ( (FILE *)(os.$file) == 0p ) return;
+  if ( (FILE *)(os.$file) == (FILE *)stdout || (FILE *)(os.$file) == (FILE *)stderr ) return;
+
+        if ( fclose( (FILE *)(os.$file) ) == EOF ) {
                 abort | IO_MSG "close output" | nl | strerror( errno );
         } // if
+        os.$file = 0p;
 } // close
 
-ofstream & write( ofstream & os, const char * data, size_t size ) {
+ofstream & write( ofstream & os, const char data[], size_t size ) {
         if ( fail( os ) ) {
                 abort | IO_MSG "attempt write I/O on failed stream";
         } // if
 
-        if ( fwrite( data, 1, size, (FILE *)(os.file) ) != size ) {
+        if ( fwrite( data, 1, size, (FILE *)(os.$file) ) != size ) {
                 abort | IO_MSG "write" | nl | strerror( errno );
         } // if
@@ -155 +158 @@
         va_list args;
         va_start( args, format );
-        int len = vfprintf( (FILE *)(os.file), format, args );
+        int len = vfprintf( (FILE *)(os.$file), format, args );
         if ( len == EOF ) {
-                if ( ferror( (FILE *)(os.file) ) ) {
+                if ( ferror( (FILE *)(os.$file) ) ) {
                         abort | IO_MSG "invalid write";
                 } // if
@@ -163 +166 @@
         va_end( args );
 
-        setPrt( os, true );                                                                     // called in output cascade
-        sepReset( os );                                                                         // reset separator
+        $setPrt( os, true );                                                            // called in output cascade
+        $sepReset( os );                                                                        // reset separator
         return len;
 } // fmt
@@ -179 +182 @@
 
 
-//*********************************** ifstream ***********************************
+// *********************************** ifstream ***********************************
 
 
 // private
 void ?{}( ifstream & is, void * file ) {
-        is.file = file;
-        is.nlOnOff = false;
+        is.$file = file;
+        is.$nlOnOff = false;
 } // ?{}
 
 // public
-void ?{}( ifstream & is ) {     is.file = 0; }
-
-void ?{}( ifstream & is, const char * name, const char * mode ) {
+void ?{}( ifstream & is ) { is.$file = 0p; }
+
+void ?{}( ifstream & is, const char name[], const char mode[] ) {
         open( is, name, mode );
 } // ?{}
 
-void ?{}( ifstream & is, const char * name ) {
+void ?{}( ifstream & is, const char name[] ) {
         open( is, name, "r" );
 } // ?{}
@@ -203 +206 @@
 } // ^?{}
 
-void nlOn( ifstream & os ) { os.nlOnOff = true; }
-void nlOff( ifstream & os ) { os.nlOnOff = false; }
-bool getANL( ifstream & os ) { return os.nlOnOff; }
+void nlOn( ifstream & os ) { os.$nlOnOff = true; }
+void nlOff( ifstream & os ) { os.$nlOnOff = false; }
+bool getANL( ifstream & os ) { return os.$nlOnOff; }
 
 int fail( ifstream & is ) {
-        return is.file == 0 || ferror( (FILE *)(is.file) );
+        return is.$file == 0p || ferror( (FILE *)(is.$file) );
 } // fail
 
 int eof( ifstream & is ) {
-        return feof( (FILE *)(is.file) );
+        return feof( (FILE *)(is.$file) );
 } // eof
 
-void open( ifstream & is, const char * name, const char * mode ) {
+void open( ifstream & is, const char name[], const char mode[] ) {
         FILE * file = fopen( name, mode );
         #ifdef __CFA_DEBUG__
-        if ( file == 0 ) {
-                abort | IO_MSG "open input file \"" | name | "\"" | nl | strerror( errno );
+        if ( file == 0p ) {
+                throw (Open_Failure){ is };
+                // abort | IO_MSG "open input file \"" | name | "\"" | nl | strerror( errno );
         } // if
         #endif // __CFA_DEBUG__
-        is.file = file;
+        is.$file = file;
 } // open
 
-void open( ifstream & is, const char * name ) {
+void open( ifstream & is, const char name[] ) {
         open( is, name, "r" );
 } // open
 
 void close( ifstream & is ) {
-        if ( (FILE *)(is.file) == stdin ) return;
-
-        if ( fclose( (FILE *)(is.file) ) == EOF ) {
+  if ( (FILE *)(is.$file) == 0p ) return;
+  if ( (FILE *)(is.$file) == (FILE *)stdin ) return;
+
+        if ( fclose( (FILE *)(is.$file) ) == EOF ) {
                 abort | IO_MSG "close input" | nl | strerror( errno );
         } // if
+        is.$file = 0p;
 } // close
 
@@ -242 +248 @@
         } // if
 
-        if ( fread( data, size, 1, (FILE *)(is.file) ) == 0 ) {
+        if ( fread( data, size, 1, (FILE *)(is.$file) ) == 0 ) {
                 abort | IO_MSG "read" | nl | strerror( errno );
         } // if
@@ -253 +259 @@
         } // if
 
-        if ( ungetc( c, (FILE *)(is.file) ) == EOF ) {
+        if ( ungetc( c, (FILE *)(is.$file) ) == EOF ) {
                 abort | IO_MSG "ungetc" | nl | strerror( errno );
         } // if
@@ -263 +269 @@
 
         va_start( args, format );
-        int len = vfscanf( (FILE *)(is.file), format, args );
+        int len = vfscanf( (FILE *)(is.$file), format, args );
         if ( len == EOF ) {
-                if ( ferror( (FILE *)(is.file) ) ) {
+                if ( ferror( (FILE *)(is.$file) ) ) {
                         abort | IO_MSG "invalid read";
                 } // if
@@ -276 +282 @@
 ifstream & sin = sinFile, & stdin = sinFile;
 
+
+// *********************************** exceptions ***********************************
+
+
+void ?{}( Open_Failure & this, ofstream & ostream ) {
+        VTABLE_INIT(this, Open_Failure);
+        this.ostream = &ostream;
+        this.tag = 1;
+}
+void ?{}( Open_Failure & this, ifstream & istream ) {
+        VTABLE_INIT(this, Open_Failure);
+        this.istream = &istream;
+        this.tag = 0;
+}
+const char * Open_Failure_msg(Open_Failure * this) {
+        return "Open_Failure";
+}
+VTABLE_INSTANCE(Open_Failure)(Open_Failure_msg);
+void throwOpen_Failure( ofstream & ostream ) {
+        Open_Failure exc = { ostream };
+}
+void throwOpen_Failure( ifstream & istream ) {
+        Open_Failure exc = { istream };
+}
+
 // Local Variables: //
 // tab-width: 4 //

libcfa/src/fstream.hfa

@@ -10 +10 @@
 // Created On       : Wed May 27 17:56:53 2015
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Fri Nov 29 06:56:02 2019
-// Update Count     : 168
+// Last Modified On : Fri Jun 19 16:29:17 2020
+// Update Count     : 189
 //
 
@@ -17 +17 @@
 
 #include "iostream.hfa"
+#include <exception.hfa>
 
 
-//*********************************** ofstream ***********************************
+// *********************************** ofstream ***********************************
 
 
 enum { sepSize = 16 };
 struct ofstream {
-        void * file;
-        bool sepDefault;
-        bool sepOnOff;
-        bool nlOnOff;
-        bool prt;                                                                                       // print text
-        bool sawNL;
-        const char * sepCur;
-        char separator[sepSize];
-        char tupleSeparator[sepSize];
+        void * $file;
+        bool $sepDefault;
+        bool $sepOnOff;
+        bool $nlOnOff;
+        bool $prt;                                                                                      // print text
+        bool $sawNL;
+        const char * $sepCur;
+        char $separator[sepSize];
+        char $tupleSeparator[sepSize];
 }; // ofstream
 
 // private
-bool sepPrt( ofstream & );
-void sepReset( ofstream & );
-void sepReset( ofstream &, bool );
-const char * sepGetCur( ofstream & );
-void sepSetCur( ofstream &, const char * );
-bool getNL( ofstream & );
-void setNL( ofstream &, bool );
-bool getANL( ofstream & );
-bool getPrt( ofstream & );
-void setPrt( ofstream &, bool );
+bool $sepPrt( ofstream & );
+void $sepReset( ofstream & );
+void $sepReset( ofstream &, bool );
+const char * $sepGetCur( ofstream & );
+void $sepSetCur( ofstream &, const char [] );
+bool $getNL( ofstream & );
+void $setNL( ofstream &, bool );
+bool $getANL( ofstream & );
+bool $getPrt( ofstream & );
+void $setPrt( ofstream &, bool );
 
 // public
@@ -56 +57 @@
 
 const char * sepGet( ofstream & );
-void sepSet( ofstream &, const char * );
+void sepSet( ofstream &, const char [] );
 const char * sepGetTuple( ofstream & );
-void sepSetTuple( ofstream &, const char * );
+void sepSetTuple( ofstream &, const char [] );
 
 void ends( ofstream & os );
 int fail( ofstream & );
 int flush( ofstream & );
-void open( ofstream &, const char * name, const char * mode );
-void open( ofstream &, const char * name );
+void open( ofstream &, const char name[], const char mode[] );
+void open( ofstream &, const char name[] );
 void close( ofstream & );
-ofstream & write( ofstream &, const char * data, size_t size );
-int fmt( ofstream &, const char format[], ... );
+ofstream & write( ofstream &, const char data[], size_t size );
+int fmt( ofstream &, const char format[], ... ) __attribute__(( format(printf, 2, 3) ));
 
 void ?{}( ofstream & os );
-void ?{}( ofstream & os, const char * name, const char * mode );
-void ?{}( ofstream & os, const char * name );
+void ?{}( ofstream & os, const char name[], const char mode[] );
+void ?{}( ofstream & os, const char name[] );
 void ^?{}( ofstream & os );
 
@@ -78 +79 @@
 
 
-//*********************************** ifstream ***********************************
+// *********************************** ifstream ***********************************
 
 
 struct ifstream {
-        void * file;
-        bool nlOnOff;
+        void * $file;
+        bool $nlOnOff;
 }; // ifstream
 
@@ -92 +93 @@
 int fail( ifstream & is );
 int eof( ifstream & is );
-void open( ifstream & is, const char * name, const char * mode );
-void open( ifstream & is, const char * name );
+void open( ifstream & is, const char name[], const char mode[] );
+void open( ifstream & is, const char name[] );
 void close( ifstream & is );
 ifstream & read( ifstream & is, char * data, size_t size );
 ifstream & ungetc( ifstream & is, char c );
-int fmt( ifstream &, const char format[], ... );
+int fmt( ifstream &, const char format[], ... ) __attribute__(( format(scanf, 2, 3) ));
 
 void ?{}( ifstream & is );
-void ?{}( ifstream & is, const char * name, const char * mode );
-void ?{}( ifstream & is, const char * name );
+void ?{}( ifstream & is, const char name[], const char mode[] );
+void ?{}( ifstream & is, const char name[] );
 void ^?{}( ifstream & is );
 
 extern ifstream & sin, & stdin;                                                 // aliases
+
+
+// *********************************** exceptions ***********************************
+
+
+DATA_EXCEPTION(Open_Failure)(
+        union {
+                ofstream * ostream;
+                ifstream * istream;
+        };
+        // TEMPORARY: need polymorphic exceptions
+        int tag;                                                                                        // 1 => ostream; 0 => istream
+);
+
+void ?{}( Open_Failure & this, ofstream & ostream );
+void ?{}( Open_Failure & this, ifstream & istream );
 
 // Local Variables: //

libcfa/src/gmp.hfa

@@ -10 +10 @@
 // Created On       : Tue Apr 19 08:43:43 2016
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Sat Jul 13 15:25:05 2019
-// Update Count     : 27
+// Last Modified On : Sun Feb  9 09:56:54 2020
+// Update Count     : 31
 //
 
@@ -24 +24 @@
 
 static inline {
-        // constructor
+        // constructor, zero_t/one_t are unnecessary because of relationship with signed/unsigned int
         void ?{}( Int & this ) { mpz_init( this.mpz ); }
         void ?{}( Int & this, Int init ) { mpz_init_set( this.mpz, init.mpz ); }
-        void ?{}( Int & this, zero_t ) { mpz_init_set_si( this.mpz, 0 ); }
-        void ?{}( Int & this, one_t ) { mpz_init_set_si( this.mpz, 1 ); }
         void ?{}( Int & this, signed long int init ) { mpz_init_set_si( this.mpz, init ); }
         void ?{}( Int & this, unsigned long int init ) { mpz_init_set_ui( this.mpz, init ); }
-        void ?{}( Int & this, const char * val ) { if ( mpz_init_set_str( this.mpz, val, 0 ) ) abort(); }
+        void ?{}( Int & this, const char val[] ) { if ( mpz_init_set_str( this.mpz, val, 0 ) ) abort(); }
         void ^?{}( Int & this ) { mpz_clear( this.mpz ); }
 
@@ -37 +35 @@
         Int ?`mp( signed long int init ) { return (Int){ init }; }
         Int ?`mp( unsigned long int init ) { return (Int){ init }; }
-        Int ?`mp( const char * init ) { return (Int){ init }; }
+        Int ?`mp( const char init[] ) { return (Int){ init }; }
 
         // assignment
@@ -43 +41 @@
         Int ?=?( Int & lhs, long int rhs ) { mpz_set_si( lhs.mpz, rhs ); return lhs; }
         Int ?=?( Int & lhs, unsigned long int rhs ) { mpz_set_ui( lhs.mpz, rhs ); return lhs; }
-        Int ?=?( Int & lhs, const char * rhs ) { if ( mpz_set_str( lhs.mpz, rhs, 0 ) ) { abort | "invalid string conversion"; } return lhs; }
+        Int ?=?( Int & lhs, const char rhs[] ) { if ( mpz_set_str( lhs.mpz, rhs, 0 ) ) { abort | "invalid string conversion"; } return lhs; }
 
         char ?=?( char & lhs, Int rhs ) { char val = mpz_get_si( rhs.mpz ); lhs = val; return lhs; }
@@ -265 +263 @@
         forall( dtype ostype | ostream( ostype ) ) {
                 ostype & ?|?( ostype & os, Int mp ) {
-                        if ( sepPrt( os ) ) fmt( os, "%s", sepGetCur( os ) );
+                        if ( $sepPrt( os ) ) fmt( os, "%s", $sepGetCur( os ) );
                         gmp_printf( "%Zd", mp.mpz );
                         sepOn( os );

libcfa/src/heap.cfa

@@ -5 +5 @@
 // file "LICENCE" distributed with Cforall.
 //
-// heap.c --
+// heap.cfa --
 //
 // Author           : Peter A. Buhr
 // Created On       : Tue Dec 19 21:58:35 2017
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Sun Dec  8 21:01:31 2019
-// Update Count     : 647
+// Last Modified On : Wed Dec 16 12:28:25 2020
+// Update Count     : 1023
 //
 
 #include <unistd.h>                                                                             // sbrk, sysconf
+#include <stdlib.h>                                                                             // EXIT_FAILURE
 #include <stdbool.h>                                                                    // true, false
 #include <stdio.h>                                                                              // snprintf, fileno
 #include <errno.h>                                                                              // errno
 #include <string.h>                                                                             // memset, memcpy
-extern "C" {
+#include <limits.h>                                                                             // ULONG_MAX
+#include <malloc.h>                                                                             // memalign, malloc_usable_size
 #include <sys/mman.h>                                                                   // mmap, munmap
-} // extern "C"
-
-// #comment TD : Many of these should be merged into math I believe
-#include "bits/align.hfa"                                                               // libPow2
+
+#include "bits/align.hfa"                                                               // libAlign
 #include "bits/defs.hfa"                                                                // likely, unlikely
 #include "bits/locks.hfa"                                                               // __spinlock_t
 #include "startup.hfa"                                                                  // STARTUP_PRIORITY_MEMORY
-//#include "stdlib.hfa"                                                                 // bsearchl
-#include "malloc.h"
-
-#define MIN(x, y) (y > x ? x : y)
+#include "math.hfa"                                                                             // ceiling
+#include "bitmanip.hfa"                                                                 // is_pow2, ceiling2
 
 static bool traceHeap = false;
@@ -74 +72 @@
         // Define the default extension heap amount in units of bytes. When the uC++ supplied heap reaches the brk address,
         // the brk address is extended by the extension amount.
-        __CFA_DEFAULT_HEAP_EXPANSION__ = (1 * 1024 * 1024),
+        __CFA_DEFAULT_HEAP_EXPANSION__ = (10 * 1024 * 1024),
 
         // Define the mmap crossover point during allocation. Allocations less than this amount are allocated from buckets;
@@ -91 +89 @@
 
 #ifdef __CFA_DEBUG__
-static unsigned int allocFree;                                                  // running total of allocations minus frees
+static size_t allocUnfreed;                                                             // running total of allocations minus frees
 
 static void prtUnfreed() {
-        if ( allocFree != 0 ) {
+        if ( allocUnfreed != 0 ) {
                 // DO NOT USE STREAMS AS THEY MAY BE UNAVAILABLE AT THIS POINT.
                 char helpText[512];
-                int len = snprintf( helpText, sizeof(helpText), "CFA warning (UNIX pid:%ld) : program terminating with %u(0x%x) bytes of storage allocated but not freed.\n"
+                int len = snprintf( helpText, sizeof(helpText), "CFA warning (UNIX pid:%ld) : program terminating with %zu(0x%zx) bytes of storage allocated but not freed.\n"
                                                         "Possible cause is unfreed storage allocated by the program or system/library routines called from the program.\n",
-                                                        (long int)getpid(), allocFree, allocFree ); // always print the UNIX pid
+                                                        (long int)getpid(), allocUnfreed, allocUnfreed ); // always print the UNIX pid
                 __cfaabi_bits_write( STDERR_FILENO, helpText, len ); // print debug/nodebug
         } // if
@@ -106 +104 @@
 extern "C" {
         void heapAppStart() {                                                           // called by __cfaabi_appready_startup
-                allocFree = 0;
+                allocUnfreed = 0;
         } // heapAppStart
 
@@ -118 +116 @@
 
 // statically allocated variables => zero filled.
-static size_t pageSize;                                                                 // architecture pagesize
+size_t __page_size;                                                                             // architecture pagesize
+int __map_prot;                                                                                 // common mmap/mprotect protection
 static size_t heapExpand;                                                               // sbrk advance
 static size_t mmapStart;                                                                // cross over point for mmap
@@ -127 +126 @@
 #define LOCKFREE 1
 #define BUCKETLOCK SPINLOCK
-#if BUCKETLOCK == LOCKFREE
-#include <uStackLF.h>
+#if BUCKETLOCK == SPINLOCK
+#elif BUCKETLOCK == LOCKFREE
+#include <stackLockFree.hfa>
+#else
+        #error undefined lock type for bucket lock
 #endif // LOCKFREE
 
@@ -136 +138 @@
 
 struct HeapManager {
-//      struct FreeHeader;                                                                      // forward declaration
-
         struct Storage {
                 struct Header {                                                                 // header
@@ -145 +145 @@
                                                 struct {                                                // 4-byte word => 8-byte header, 8-byte word => 16-byte header
                                                         #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ && __SIZEOF_POINTER__ == 4
-                                                        uint32_t padding;                       // unused, force home/blocksize to overlay alignment in fake header
+                                                        uint64_t padding;                       // unused, force home/blocksize to overlay alignment in fake header
                                                         #endif // __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ && __SIZEOF_POINTER__ == 4
 
                                                         union {
-//                                                              FreeHeader * home;              // allocated block points back to home locations (must overlay alignment)
+                                                                // FreeHeader * home;           // allocated block points back to home locations (must overlay alignment)
+                                                                // 2nd low-order bit => zero filled
                                                                 void * home;                    // allocated block points back to home locations (must overlay alignment)
                                                                 size_t blockSize;               // size for munmap (must overlay alignment)
-                                                                #if BUCKLOCK == SPINLOCK
+                                                                #if BUCKETLOCK == SPINLOCK
                                                                 Storage * next;                 // freed block points next freed block of same size
                                                                 #endif // SPINLOCK
                                                         };
+                                                        size_t size;                            // allocation size in bytes
 
                                                         #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ && __SIZEOF_POINTER__ == 4
-                                                        uint32_t padding;                       // unused, force home/blocksize to overlay alignment in fake header
+                                                        uint64_t padding;                       // unused, force home/blocksize to overlay alignment in fake header
                                                         #endif // __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ && __SIZEOF_POINTER__ == 4
                                                 };
-                                                // future code
-                                                #if BUCKLOCK == LOCKFREE
-                                                Stack<Storage>::Link next;              // freed block points next freed block of same size (double-wide)
+                                                #if BUCKETLOCK == LOCKFREE
+                                                Link(Storage) next;                             // freed block points next freed block of same size (double-wide)
                                                 #endif // LOCKFREE
                                         };
                                 } real; // RealHeader
+
                                 struct FakeHeader {
                                         #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
-                                        uint32_t alignment;                                     // low-order bits of home/blockSize used for tricks
+                                        uint32_t alignment;                                     // 1st low-order bit => fake header & alignment
                                         #endif // __ORDER_LITTLE_ENDIAN__
 
@@ -187 +189 @@
 
         struct FreeHeader {
-                #if BUCKLOCK == SPINLOCK
+                #if BUCKETLOCK == SPINLOCK
                 __spinlock_t lock;                                                              // must be first field for alignment
                 Storage * freeList;
-                #elif BUCKLOCK == LOCKFREE
-                // future code
-                StackLF<Storage> freeList;
                 #else
-                        #error undefined lock type for bucket lock
-                #endif // SPINLOCK
+                StackLF(Storage) freeList;
+                #endif // BUCKETLOCK
                 size_t blockSize;                                                               // size of allocations on this list
         }; // FreeHeader
@@ -208 +207 @@
 }; // HeapManager
 
+#if BUCKETLOCK == LOCKFREE
+static inline {
+        Link(HeapManager.Storage) * ?`next( HeapManager.Storage * this ) { return &this->header.kind.real.next; }
+        void ?{}( HeapManager.FreeHeader & ) {}
+        void ^?{}( HeapManager.FreeHeader & ) {}
+} // distribution
+#endif // LOCKFREE
+
 static inline size_t getKey( const HeapManager.FreeHeader & freeheader ) { return freeheader.blockSize; }
 
@@ -214 +221 @@
 #define __STATISTICS__
 
-// Bucket size must be multiple of 16.
-// Powers of 2 are common allocation sizes, so make powers of 2 generate the minimum required size.
+// Size of array must harmonize with NoBucketSizes and individual bucket sizes must be multiple of 16.
+// Smaller multiples of 16 and powers of 2 are common allocation sizes, so make them generate the minimum required bucket size.
+// malloc(0) returns 0p, so no bucket is necessary for 0 bytes returning an address that can be freed.
 static const unsigned int bucketSizes[] @= {                    // different bucket sizes
-        16, 32, 48, 64 + sizeof(HeapManager.Storage), // 4
-        96, 112, 128 + sizeof(HeapManager.Storage), // 3
+        16 + sizeof(HeapManager.Storage), 32 + sizeof(HeapManager.Storage), 48 + sizeof(HeapManager.Storage), 64 + sizeof(HeapManager.Storage), // 4
+        96 + sizeof(HeapManager.Storage), 112 + sizeof(HeapManager.Storage), 128 + sizeof(HeapManager.Storage), // 3
         160, 192, 224, 256 + sizeof(HeapManager.Storage), // 4
         320, 384, 448, 512 + sizeof(HeapManager.Storage), // 4
@@ -236 +244 @@
 };
 
-static_assert( NoBucketSizes == sizeof(bucketSizes) / sizeof(bucketSizes[0]), "size of bucket array wrong" );
+static_assert( NoBucketSizes == sizeof(bucketSizes) / sizeof(bucketSizes[0] ), "size of bucket array wrong" );
 
 #ifdef FASTLOOKUP
@@ -243 +251 @@
 #endif // FASTLOOKUP
 
-static int mmapFd = -1;                                                                 // fake or actual fd for anonymous file
+static const off_t mmapFd = -1;                                                 // fake or actual fd for anonymous file
 #ifdef __CFA_DEBUG__
 static bool heapBoot = 0;                                                               // detect recursion during boot
 #endif // __CFA_DEBUG__
+
+// The constructor for heapManager is called explicitly in memory_startup.
 static HeapManager heapManager __attribute__(( aligned (128) )) @= {}; // size of cache line to prevent false sharing
 
@@ -252 +262 @@
 #ifdef __STATISTICS__
 // Heap statistics counters.
+static unsigned int malloc_calls;
+static unsigned long long int malloc_storage;
+static unsigned int aalloc_calls;
+static unsigned long long int aalloc_storage;
+static unsigned int calloc_calls;
+static unsigned long long int calloc_storage;
+static unsigned int memalign_calls;
+static unsigned long long int memalign_storage;
+static unsigned int amemalign_calls;
+static unsigned long long int amemalign_storage;
+static unsigned int cmemalign_calls;
+static unsigned long long int cmemalign_storage;
+static unsigned int resize_calls;
+static unsigned long long int resize_storage;
+static unsigned int realloc_calls;
+static unsigned long long int realloc_storage;
+static unsigned int free_calls;
+static unsigned long long int free_storage;
+static unsigned int mmap_calls;
 static unsigned long long int mmap_storage;
-static unsigned int mmap_calls;
+static unsigned int munmap_calls;
 static unsigned long long int munmap_storage;
-static unsigned int munmap_calls;
+static unsigned int sbrk_calls;
 static unsigned long long int sbrk_storage;
-static unsigned int sbrk_calls;
-static unsigned long long int malloc_storage;
-static unsigned int malloc_calls;
-static unsigned long long int free_storage;
-static unsigned int free_calls;
-static unsigned long long int calloc_storage;
-static unsigned int calloc_calls;
-static unsigned long long int memalign_storage;
-static unsigned int memalign_calls;
-static unsigned long long int cmemalign_storage;
-static unsigned int cmemalign_calls;
-static unsigned long long int realloc_storage;
-static unsigned int realloc_calls;
 // Statistics file descriptor (changed by malloc_stats_fd).
-static int statfd = STDERR_FILENO;                                              // default stderr
+static int stat_fd = STDERR_FILENO;                                             // default stderr
 
 // Use "write" because streams may be shutdown when calls are made.
 static void printStats() {
-        char helpText[512];
+        char helpText[1024];
         __cfaabi_bits_print_buffer( STDERR_FILENO, helpText, sizeof(helpText),
                                                                         "\nHeap statistics:\n"
                                                                         "  malloc: calls %u / storage %llu\n"
+                                                                        "  aalloc: calls %u / storage %llu\n"
                                                                         "  calloc: calls %u / storage %llu\n"
                                                                         "  memalign: calls %u / storage %llu\n"
+                                                                        "  amemalign: calls %u / storage %llu\n"
                                                                         "  cmemalign: calls %u / storage %llu\n"
+                                                                        "  resize: calls %u / storage %llu\n"
                                                                         "  realloc: calls %u / storage %llu\n"
                                                                         "  free: calls %u / storage %llu\n"
@@ -288 +307 @@
                                                                         "  sbrk: calls %u / storage %llu\n",
                                                                         malloc_calls, malloc_storage,
+                                                                        aalloc_calls, aalloc_storage,
                                                                         calloc_calls, calloc_storage,
                                                                         memalign_calls, memalign_storage,
+                                                                        amemalign_calls, amemalign_storage,
                                                                         cmemalign_calls, cmemalign_storage,
+                                                                        resize_calls, resize_storage,
                                                                         realloc_calls, realloc_storage,
                                                                         free_calls, free_storage,
@@ -300 +322 @@
 
 static int printStatsXML( FILE * stream ) {                             // see malloc_info
-        char helpText[512];
+        char helpText[1024];
         int len = snprintf( helpText, sizeof(helpText),
                                                 "<malloc version=\"1\">\n"
@@ -307 +329 @@
                                                 "</sizes>\n"
                                                 "<total type=\"malloc\" count=\"%u\" size=\"%llu\"/>\n"
+                                                "<total type=\"aalloc\" count=\"%u\" size=\"%llu\"/>\n"
                                                 "<total type=\"calloc\" count=\"%u\" size=\"%llu\"/>\n"
                                                 "<total type=\"memalign\" count=\"%u\" size=\"%llu\"/>\n"
+                                                "<total type=\"amemalign\" count=\"%u\" size=\"%llu\"/>\n"
                                                 "<total type=\"cmemalign\" count=\"%u\" size=\"%llu\"/>\n"
+                                                "<total type=\"resize\" count=\"%u\" size=\"%llu\"/>\n"
                                                 "<total type=\"realloc\" count=\"%u\" size=\"%llu\"/>\n"
                                                 "<total type=\"free\" count=\"%u\" size=\"%llu\"/>\n"
@@ -317 +342 @@
                                                 "</malloc>",
                                                 malloc_calls, malloc_storage,
+                                                aalloc_calls, aalloc_storage,
                                                 calloc_calls, calloc_storage,
                                                 memalign_calls, memalign_storage,
+                                                amemalign_calls, amemalign_storage,
                                                 cmemalign_calls, cmemalign_storage,
+                                                resize_calls, resize_storage,
                                                 realloc_calls, realloc_storage,
                                                 free_calls, free_storage,
@@ -332 +360 @@
 
 
-// static inline void noMemory() {
-//      abort( "Heap memory exhausted at %zu bytes.\n"
-//                 "Possible cause is very large memory allocation and/or large amount of unfreed storage allocated by the program or system/library routines.",
-//                 ((char *)(sbrk( 0 )) - (char *)(heapManager.heapBegin)) );
-// } // noMemory
-
-
-static inline void checkAlign( size_t alignment ) {
-        if ( alignment < libAlign() || ! libPow2( alignment ) ) {
-                abort( "Alignment %zu for memory allocation is less than %d and/or not a power of 2.", alignment, libAlign() );
-        } // if
-} // checkAlign
-
-
-static inline bool setHeapExpand( size_t value ) {
-  if ( heapExpand < pageSize ) return true;
-        heapExpand = value;
-        return false;
-} // setHeapExpand
-
-
 // thunk problem
 size_t Bsearchl( unsigned int key, const unsigned int * vals, size_t dim ) {
@@ -369 +376 @@
 
 static inline bool setMmapStart( size_t value ) {               // true => mmapped, false => sbrk
-  if ( value < pageSize || bucketSizes[NoBucketSizes - 1] < value ) return true;
+  if ( value < __page_size || bucketSizes[NoBucketSizes - 1] < value ) return false;
         mmapStart = value;                                                                      // set global
 
@@ -376 +383 @@
         assert( maxBucketsUsed < NoBucketSizes );                       // subscript failure ?
         assert( mmapStart <= bucketSizes[maxBucketsUsed] ); // search failure ?
-        return false;
+        return true;
 } // setMmapStart
 
 
-static inline void checkHeader( bool check, const char * name, void * addr ) {
+// <-------+----------------------------------------------------> bsize (bucket size)
+// |header |addr
+//==================================================================================
+//                   align/offset |
+// <-----------------<------------+-----------------------------> bsize (bucket size)
+//                   |fake-header | addr
+#define headerAddr( addr ) ((HeapManager.Storage.Header *)( (char *)addr - sizeof(HeapManager.Storage) ))
+#define realHeader( header ) ((HeapManager.Storage.Header *)((char *)header - header->kind.fake.offset))
+
+// <-------<<--------------------- dsize ---------------------->> bsize (bucket size)
+// |header |addr
+//==================================================================================
+//                   align/offset |
+// <------------------------------<<---------- dsize --------->>> bsize (bucket size)
+//                   |fake-header |addr
+#define dataStorage( bsize, addr, header ) (bsize - ( (char *)addr - (char *)header ))
+
+
+static inline void checkAlign( size_t alignment ) {
+        if ( alignment < libAlign() || ! is_pow2( alignment ) ) {
+                abort( "Alignment %zu for memory allocation is less than %d and/or not a power of 2.", alignment, libAlign() );
+        } // if
+} // checkAlign
+
+
+static inline void checkHeader( bool check, const char name[], void * addr ) {
         if ( unlikely( check ) ) {                                                      // bad address ?
                 abort( "Attempt to %s storage %p with address outside the heap.\n"
@@ -391 +423 @@
 static inline void fakeHeader( HeapManager.Storage.Header *& header, size_t & alignment ) {
         if ( unlikely( (header->kind.fake.alignment & 1) == 1 ) ) { // fake header ?
-                size_t offset = header->kind.fake.offset;
                 alignment = header->kind.fake.alignment & -2;   // remove flag from value
                 #ifdef __CFA_DEBUG__
                 checkAlign( alignment );                                                // check alignment
                 #endif // __CFA_DEBUG__
-                header = (HeapManager.Storage.Header *)((char *)header - offset);
+                header = realHeader( header );                                  // backup from fake to real header
+        } else {
+                alignment = libAlign();                                                 // => no fake header
         } // if
 } // fakeHeader
 
 
-// <-------+----------------------------------------------------> bsize (bucket size)
-// |header |addr
-//==================================================================================
-//                                | alignment
-// <-----------------<------------+-----------------------------> bsize (bucket size)
-//                   |fake-header | addr
-#define headerAddr( addr ) ((HeapManager.Storage.Header *)( (char *)addr - sizeof(HeapManager.Storage) ))
-
-// <-------<<--------------------- dsize ---------------------->> bsize (bucket size)
-// |header |addr
-//==================================================================================
-//                                | alignment
-// <------------------------------<<---------- dsize --------->>> bsize (bucket size)
-//                   |fake-header |addr
-#define dataStorage( bsize, addr, header ) (bsize - ( (char *)addr - (char *)header ))
-
-
-static inline bool headers( const char * name __attribute__(( unused )), void * addr, HeapManager.Storage.Header *& header, HeapManager.FreeHeader *& freeElem, size_t & size, size_t & alignment ) with ( heapManager ) {
+static inline bool headers( const char name[] __attribute__(( unused )), void * addr, HeapManager.Storage.Header *& header, HeapManager.FreeHeader *& freeElem,
+                                                        size_t & size, size_t & alignment ) with( heapManager ) {
         header = headerAddr( addr );
 
-        if ( unlikely( heapEnd < addr ) ) {                                     // mmapped ?
+  if ( unlikely( addr < heapBegin || heapEnd < addr ) ) { // mmapped ?
                 fakeHeader( header, alignment );
                 size = header->kind.real.blockSize & -3;                // mmap size
@@ -428 +445 @@
 
         #ifdef __CFA_DEBUG__
-        checkHeader( addr < heapBegin || header < (HeapManager.Storage.Header *)heapBegin, name, addr ); // bad low address ?
+        checkHeader( header < (HeapManager.Storage.Header *)heapBegin, name, addr ); // bad low address ?
         #endif // __CFA_DEBUG__
 
@@ -449 +466 @@
 } // headers
 
-
-static inline void * extend( size_t size ) with ( heapManager ) {
+#ifdef __CFA_DEBUG__
+#if __SIZEOF_POINTER__ == 4
+#define MASK 0xdeadbeef
+#else
+#define MASK 0xdeadbeefdeadbeef
+#endif
+#define STRIDE size_t
+
+static void * Memset( void * addr, STRIDE size ) {              // debug only
+        if ( size % sizeof(STRIDE) != 0 ) abort( "Memset() : internal error, size %zd not multiple of %zd.", size, sizeof(STRIDE) );
+        if ( (STRIDE)addr % sizeof(STRIDE) != 0 ) abort( "Memset() : internal error, addr %p not multiple of %zd.", addr, sizeof(STRIDE) );
+
+        STRIDE * end = (STRIDE *)addr + size / sizeof(STRIDE);
+        for ( STRIDE * p = (STRIDE *)addr; p < end; p += 1 ) *p = MASK;
+        return addr;
+} // Memset
+#endif // __CFA_DEBUG__
+
+
+#define NO_MEMORY_MSG "insufficient heap memory available for allocating %zd new bytes."
+
+static inline void * extend( size_t size ) with( heapManager ) {
         lock( extlock __cfaabi_dbg_ctx2 );
         ptrdiff_t rem = heapRemaining - size;
@@ -456 +493 @@
                 // If the size requested is bigger than the current remaining storage, increase the size of the heap.
 
-                size_t increase = libCeiling( size > heapExpand ? size : heapExpand, libAlign() );
-                if ( sbrk( increase ) == (void *)-1 ) {
+                size_t increase = ceiling2( size > heapExpand ? size : heapExpand, __page_size );
+                // Do not call abort or strerror( errno ) as they may call malloc.
+                if ( sbrk( increase ) == (void *)-1 ) {                 // failed, no memory ?
                         unlock( extlock );
-                        errno = ENOMEM;
-                        return 0p;
+                        __cfaabi_bits_print_nolock( STDERR_FILENO, NO_MEMORY_MSG, size );
+                        _exit( EXIT_FAILURE );
+                } // if
+                if ( mprotect( (char *)heapEnd + heapRemaining, increase, __map_prot ) ) {
+                        unlock( extlock );
+                        __cfaabi_bits_print_nolock( STDERR_FILENO, "extend() : internal error, mprotect failure, heapEnd:%p size:%zd, errno:%d.\n", heapEnd, increase, errno );
+                        _exit( EXIT_FAILURE );
                 } // if
                 #ifdef __STATISTICS__
@@ -468 +511 @@
                 #ifdef __CFA_DEBUG__
                 // Set new memory to garbage so subsequent uninitialized usages might fail.
-                memset( (char *)heapEnd + heapRemaining, '\377', increase );
+                memset( (char *)heapEnd + heapRemaining, '\xde', increase );
+                //Memset( (char *)heapEnd + heapRemaining, increase );
                 #endif // __CFA_DEBUG__
                 rem = heapRemaining + increase - size;
@@ -481 +525 @@
 
 
-static inline void * doMalloc( size_t size ) with ( heapManager ) {
+static inline void * doMalloc( size_t size ) with( heapManager ) {
         HeapManager.Storage * block;                                            // pointer to new block of storage
 
@@ -487 +531 @@
         // along with the block and is a multiple of the alignment size.
 
-  if ( unlikely( size > ~0ul - sizeof(HeapManager.Storage) ) ) return 0p;
+  if ( unlikely( size > ULONG_MAX - sizeof(HeapManager.Storage) ) ) return 0p;
         size_t tsize = size + sizeof(HeapManager.Storage);
         if ( likely( tsize < mmapStart ) ) {                            // small size => sbrk
@@ -497 +541 @@
                         posn = Bsearchl( (unsigned int)tsize, bucketSizes, (size_t)maxBucketsUsed );
                 HeapManager.FreeHeader * freeElem = &freeLists[posn];
-                // #ifdef FASTLOOKUP
-                // if ( tsize < LookupSizes )
-                //      freeElem = &freeLists[lookup[tsize]];
-                // else
-                // #endif // FASTLOOKUP
-                //      freeElem = bsearchl( tsize, freeLists, (size_t)maxBucketsUsed ); // binary search
-                // HeapManager.FreeHeader * freeElem =
-                //      #ifdef FASTLOOKUP
-                //      tsize < LookupSizes ? &freeLists[lookup[tsize]] :
-                //      #endif // FASTLOOKUP
-                //      bsearchl( tsize, freeLists, (size_t)maxBucketsUsed ); // binary search
-                assert( freeElem <= &freeLists[maxBucketsUsed] ); // subscripting error ?
-                assert( tsize <= freeElem->blockSize );                 // search failure ?
+                verify( freeElem <= &freeLists[maxBucketsUsed] ); // subscripting error ?
+                verify( tsize <= freeElem->blockSize );                 // search failure ?
                 tsize = freeElem->blockSize;                                    // total space needed for request
 
                 // Spin until the lock is acquired for this particular size of block.
 
-                #if defined( SPINLOCK )
+                #if BUCKETLOCK == SPINLOCK
                 lock( freeElem->lock __cfaabi_dbg_ctx2 );
                 block = freeElem->freeList;                                             // remove node from stack
                 #else
-                block = freeElem->freeList.pop();
-                #endif // SPINLOCK
+                block = pop( freeElem->freeList );
+                #endif // BUCKETLOCK
                 if ( unlikely( block == 0p ) ) {                                // no free block ?
-                        #if defined( SPINLOCK )
+                        #if BUCKETLOCK == SPINLOCK
                         unlock( freeElem->lock );
-                        #endif // SPINLOCK
+                        #endif // BUCKETLOCK
 
                         // Freelist for that size was empty, so carve it out of the heap if there's enough left, or get some more
@@ -529 +562 @@
 
                         block = (HeapManager.Storage *)extend( tsize ); // mutual exclusion on call
-  if ( unlikely( block == 0p ) ) return 0p;
-                #if defined( SPINLOCK )
+                #if BUCKETLOCK == SPINLOCK
                 } else {
                         freeElem->freeList = block->header.kind.real.next;
                         unlock( freeElem->lock );
-                #endif // SPINLOCK
+                #endif // BUCKETLOCK
                 } // if
 
                 block->header.kind.real.home = freeElem;                // pointer back to free list of apropriate size
         } else {                                                                                        // large size => mmap
-  if ( unlikely( size > ~0ul - pageSize ) ) return 0p;
-                tsize = libCeiling( tsize, pageSize );                  // must be multiple of page size
+  if ( unlikely( size > ULONG_MAX - __page_size ) ) return 0p;
+                tsize = ceiling2( tsize, __page_size );                 // must be multiple of page size
                 #ifdef __STATISTICS__
                 __atomic_add_fetch( &mmap_calls, 1, __ATOMIC_SEQ_CST );
                 __atomic_add_fetch( &mmap_storage, tsize, __ATOMIC_SEQ_CST );
                 #endif // __STATISTICS__
-                block = (HeapManager.Storage *)mmap( 0, tsize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, mmapFd, 0 );
-                if ( block == (HeapManager.Storage *)MAP_FAILED ) {
+
+                block = (HeapManager.Storage *)mmap( 0, tsize, __map_prot, MAP_PRIVATE | MAP_ANONYMOUS, mmapFd, 0 );
+                if ( block == (HeapManager.Storage *)MAP_FAILED ) { // failed ?
+                        if ( errno == ENOMEM ) abort( NO_MEMORY_MSG, tsize ); // no memory
                         // Do not call strerror( errno ) as it may call malloc.
-                        abort( "(HeapManager &)0x%p.doMalloc() : internal error, mmap failure, size:%zu error:%d.", &heapManager, tsize, errno );
-                } // if
+                        abort( "(HeapManager &)0x%p.doMalloc() : internal error, mmap failure, size:%zu errno:%d.", &heapManager, tsize, errno );
+                } //if
                 #ifdef __CFA_DEBUG__
                 // Set new memory to garbage so subsequent uninitialized usages might fail.
-                memset( block, '\377', tsize );
+                memset( block, '\xde', tsize );
+                //Memset( block, tsize );
                 #endif // __CFA_DEBUG__
                 block->header.kind.real.blockSize = tsize;              // storage size for munmap
         } // if
 
+        block->header.kind.real.size = size;                            // store allocation size
         void * addr = &(block->data);                                           // adjust off header to user bytes
+        verify( ((uintptr_t)addr & (libAlign() - 1)) == 0 ); // minimum alignment ?
 
         #ifdef __CFA_DEBUG__
-        assert( ((uintptr_t)addr & (libAlign() - 1)) == 0 ); // minimum alignment ?
-        __atomic_add_fetch( &allocFree, tsize, __ATOMIC_SEQ_CST );
+        __atomic_add_fetch( &allocUnfreed, tsize, __ATOMIC_SEQ_CST );
         if ( traceHeap() ) {
                 enum { BufferSize = 64 };
                 char helpText[BufferSize];
                 int len = snprintf( helpText, BufferSize, "%p = Malloc( %zu ) (allocated %zu)\n", addr, size, tsize );
-                // int len = snprintf( helpText, BufferSize, "Malloc %p %zu\n", addr, size );
                 __cfaabi_bits_write( STDERR_FILENO, helpText, len ); // print debug/nodebug
         } // if
@@ -575 +610 @@
 
 
-static inline void doFree( void * addr ) with ( heapManager ) {
+static inline void doFree( void * addr ) with( heapManager ) {
         #ifdef __CFA_DEBUG__
         if ( unlikely( heapManager.heapBegin == 0p ) ) {
@@ -592 +627 @@
                 #endif // __STATISTICS__
                 if ( munmap( header, size ) == -1 ) {
-                        #ifdef __CFA_DEBUG__
                         abort( "Attempt to deallocate storage %p not allocated or with corrupt header.\n"
                                    "Possible cause is invalid pointer.",
                                    addr );
-                        #endif // __CFA_DEBUG__
                 } // if
         } else {
                 #ifdef __CFA_DEBUG__
                 // Set free memory to garbage so subsequent usages might fail.
-                memset( ((HeapManager.Storage *)header)->data, '\377', freeElem->blockSize - sizeof( HeapManager.Storage ) );
+                memset( ((HeapManager.Storage *)header)->data, '\xde', freeElem->blockSize - sizeof( HeapManager.Storage ) );
+                //Memset( ((HeapManager.Storage *)header)->data, freeElem->blockSize - sizeof( HeapManager.Storage ) );
                 #endif // __CFA_DEBUG__
 
@@ -607 +641 @@
                 free_storage += size;
                 #endif // __STATISTICS__
-                #if defined( SPINLOCK )
+                #if BUCKETLOCK == SPINLOCK
                 lock( freeElem->lock __cfaabi_dbg_ctx2 );               // acquire spin lock
                 header->kind.real.next = freeElem->freeList;    // push on stack
@@ -613 +647 @@
                 unlock( freeElem->lock );                                               // release spin lock
                 #else
-                freeElem->freeList.push( *(HeapManager.Storage *)header );
-                #endif // SPINLOCK
+                push( freeElem->freeList, *(HeapManager.Storage *)header );
+                #endif // BUCKETLOCK
         } // if
 
         #ifdef __CFA_DEBUG__
-        __atomic_add_fetch( &allocFree, -size, __ATOMIC_SEQ_CST );
+        __atomic_add_fetch( &allocUnfreed, -size, __ATOMIC_SEQ_CST );
         if ( traceHeap() ) {
-                enum { BufferSize = 64 };
-                char helpText[BufferSize];
+                char helpText[64];
                 int len = snprintf( helpText, sizeof(helpText), "Free( %p ) size:%zu\n", addr, size );
                 __cfaabi_bits_write( STDERR_FILENO, helpText, len ); // print debug/nodebug
@@ -629 +662 @@
 
 
-size_t prtFree( HeapManager & manager ) with ( manager ) {
+size_t prtFree( HeapManager & manager ) with( manager ) {
         size_t total = 0;
         #ifdef __STATISTICS__
@@ -641 +674 @@
                 #endif // __STATISTICS__
 
-                #if defined( SPINLOCK )
+                #if BUCKETLOCK == SPINLOCK
                 for ( HeapManager.Storage * p = freeLists[i].freeList; p != 0p; p = p->header.kind.real.next ) {
                 #else
-                for ( HeapManager.Storage * p = freeLists[i].freeList.top(); p != 0p; p = p->header.kind.real.next.top ) {
-                #endif // SPINLOCK
+                        for(;;) {
+//              for ( HeapManager.Storage * p = top( freeLists[i].freeList ); p != 0p; p = (p)`next->top ) {
+//              for ( HeapManager.Storage * p = top( freeLists[i].freeList ); p != 0p; /* p = getNext( p )->top */) {
+//                      HeapManager.Storage * temp = p->header.kind.real.next.top; // FIX ME: direct assignent fails, initialization works`
+//                      typeof(p) temp = (( p )`next)->top;                     // FIX ME: direct assignent fails, initialization works`
+//                      p = temp;
+                #endif // BUCKETLOCK
                         total += size;
                         #ifdef __STATISTICS__
@@ -665 +703 @@
 
 
-static void ?{}( HeapManager & manager ) with ( manager ) {
-        pageSize = sysconf( _SC_PAGESIZE );
+static void ?{}( HeapManager & manager ) with( manager ) {
+        __page_size = sysconf( _SC_PAGESIZE );
+        __map_prot = PROT_READ | PROT_WRITE | PROT_EXEC;
 
         for ( unsigned int i = 0; i < NoBucketSizes; i += 1 ) { // initialize the free lists
@@ -680 +719 @@
         #endif // FASTLOOKUP
 
-        if ( setMmapStart( default_mmap_start() ) ) {
+        if ( ! setMmapStart( default_mmap_start() ) ) {
                 abort( "HeapManager : internal error, mmap start initialization failure." );
         } // if
@@ -686 +725 @@
 
         char * end = (char *)sbrk( 0 );
-        sbrk( (char *)libCeiling( (long unsigned int)end, libAlign() ) - end ); // move start of heap to multiple of alignment
-        heapBegin = heapEnd = sbrk( 0 );                                        // get new start point
+        heapBegin = heapEnd = sbrk( (char *)ceiling2( (long unsigned int)end, __page_size ) - end ); // move start of heap to multiple of alignment
 } // HeapManager
 
@@ -695 +733 @@
         if ( traceHeapTerm() ) {
                 printStats();
-                // if ( prtfree() ) prtFree( heapManager, true );
+                // prtUnfreed() called in heapAppStop()
         } // if
         #endif // __STATISTICS__
@@ -704 +742 @@
 void memory_startup( void ) {
         #ifdef __CFA_DEBUG__
-        if ( unlikely( heapBoot ) ) {                                           // check for recursion during system boot
-                // DO NOT USE STREAMS AS THEY MAY BE UNAVAILABLE AT THIS POINT.
+        if ( heapBoot ) {                                                                       // check for recursion during system boot
                 abort( "boot() : internal error, recursively invoked during system boot." );
         } // if
@@ -711 +748 @@
         #endif // __CFA_DEBUG__
 
-        //assert( heapManager.heapBegin != 0 );
+        //verify( heapManager.heapBegin != 0 );
         //heapManager{};
-        if ( heapManager.heapBegin == 0p ) heapManager{};
+        if ( heapManager.heapBegin == 0p ) heapManager{};       // sanity check
 } // memory_startup
 
@@ -723 +760 @@
 
 static inline void * mallocNoStats( size_t size ) {             // necessary for malloc statistics
-        //assert( heapManager.heapBegin != 0 );
-        if ( unlikely( heapManager.heapBegin == 0p ) ) heapManager{}; // called before memory_startup ?
-        void * addr = doMalloc( size );
-        if ( unlikely( addr == 0p ) ) errno = ENOMEM;           // POSIX
-        return addr;
+        verify( heapManager.heapBegin != 0p );                          // called before memory_startup ?
+  if ( unlikely( size ) == 0 ) return 0p;                               // 0 BYTE ALLOCATION RETURNS NULL POINTER
+
+#if __SIZEOF_POINTER__ == 8
+        verify( size < ((typeof(size_t))1 << 48) );
+#endif // __SIZEOF_POINTER__ == 8
+        return doMalloc( size );
 } // mallocNoStats
 
 
-static inline void * callocNoStats( size_t noOfElems, size_t elemSize ) {
-        size_t size = noOfElems * elemSize;
+static inline void * callocNoStats( size_t dim, size_t elemSize ) {
+        size_t size = dim * elemSize;
+  if ( unlikely( size ) == 0 ) return 0p;                               // 0 BYTE ALLOCATION RETURNS NULL POINTER
         char * addr = (char *)mallocNoStats( size );
-  if ( unlikely( addr == 0p ) ) return 0p;
 
         HeapManager.Storage.Header * header;
         HeapManager.FreeHeader * freeElem;
         size_t bsize, alignment;
-        bool mapped __attribute__(( unused )) = headers( "calloc", addr, header, freeElem, bsize, alignment );
         #ifndef __CFA_DEBUG__
+        bool mapped =
+        #endif // __CFA_DEBUG__
+                headers( "calloc", addr, header, freeElem, bsize, alignment );
+        #ifndef __CFA_DEBUG__
+
         // Mapped storage is zero filled, but in debug mode mapped memory is scrubbed in doMalloc, so it has to be reset to zero.
         if ( ! mapped )
         #endif // __CFA_DEBUG__
-                // Zero entire data space even when > than size => realloc without a new allocation and zero fill works.
-                // <-------00000000000000000000000000000000000000000000000000000> bsize (bucket size)
+                // <-------0000000000000000000000000000UUUUUUUUUUUUUUUUUUUUUUUUU> bsize (bucket size) U => undefined
                 // `-header`-addr                      `-size
-                memset( addr, '\0', bsize - sizeof(HeapManager.Storage) ); // set to zeros
+                memset( addr, '\0', size );                                             // set to zeros
 
         header->kind.real.blockSize |= 2;                                       // mark as zero filled
@@ -754 +796 @@
 
 
-static inline void * memalignNoStats( size_t alignment, size_t size ) { // necessary for malloc statistics
+static inline void * memalignNoStats( size_t alignment, size_t size ) {
+  if ( unlikely( size ) == 0 ) return 0p;                               // 0 BYTE ALLOCATION RETURNS NULL POINTER
+
         #ifdef __CFA_DEBUG__
         checkAlign( alignment );                                                        // check alignment
@@ -772 +816 @@
         // add sizeof(Storage) for fake header
         char * addr = (char *)mallocNoStats( size + alignment - libAlign() + sizeof(HeapManager.Storage) );
-  if ( unlikely( addr == 0p ) ) return addr;
 
         // address in the block of the "next" alignment address
-        char * user = (char *)libCeiling( (uintptr_t)(addr + sizeof(HeapManager.Storage)), alignment );
+        char * user = (char *)ceiling2( (uintptr_t)(addr + sizeof(HeapManager.Storage)), alignment );
 
         // address of header from malloc
         HeapManager.Storage.Header * realHeader = headerAddr( addr );
+        realHeader->kind.real.size = size;                                      // correct size to eliminate above alignment offset
         // address of fake header * before* the alignment location
         HeapManager.Storage.Header * fakeHeader = headerAddr( user );
@@ -790 +834 @@
 
 
-static inline void * cmemalignNoStats( size_t alignment, size_t noOfElems, size_t elemSize ) {
-        size_t size = noOfElems * elemSize;
+static inline void * cmemalignNoStats( size_t alignment, size_t dim, size_t elemSize ) {
+        size_t size = dim * elemSize;
+  if ( unlikely( size ) == 0 ) return 0p;                               // 0 BYTE ALLOCATION RETURNS NULL POINTER
         char * addr = (char *)memalignNoStats( alignment, size );
-  if ( unlikely( addr == 0p ) ) return 0p;
+
         HeapManager.Storage.Header * header;
         HeapManager.FreeHeader * freeElem;
         size_t bsize;
-        bool mapped __attribute__(( unused )) = headers( "cmemalign", addr, header, freeElem, bsize, alignment );
         #ifndef __CFA_DEBUG__
+        bool mapped =
+        #endif // __CFA_DEBUG__
+                headers( "cmemalign", addr, header, freeElem, bsize, alignment );
+
         // Mapped storage is zero filled, but in debug mode mapped memory is scrubbed in doMalloc, so it has to be reset to zero.
+        #ifndef __CFA_DEBUG__
         if ( ! mapped )
         #endif // __CFA_DEBUG__
-                memset( addr, '\0', dataStorage( bsize, addr, header ) ); // set to zeros
-        header->kind.real.blockSize |= 2;                               // mark as zero filled
-
+                // <-------0000000000000000000000000000UUUUUUUUUUUUUUUUUUUUUUUUU> bsize (bucket size) U => undefined
+                // `-header`-addr                      `-size
+                memset( addr, '\0', size );                                             // set to zeros
+
+        header->kind.real.blockSize |= 2;                                       // mark as zero filled
         return addr;
 } // cmemalignNoStats
 
 
-// supported mallopt options
-#ifndef M_MMAP_THRESHOLD
-#define M_MMAP_THRESHOLD (-1)
-#endif // M_TOP_PAD
-#ifndef M_TOP_PAD
-#define M_TOP_PAD (-2)
-#endif // M_TOP_PAD
-
-
 extern "C" {
-        // The malloc() function allocates size bytes and returns a pointer to the allocated memory. The memory is not
-        // initialized. If size is 0, then malloc() returns either 0p, or a unique pointer value that can later be
-        // successfully passed to free().
+        // Allocates size bytes and returns a pointer to the allocated memory.  The contents are undefined. If size is 0,
+        // then malloc() returns a unique pointer value that can later be successfully passed to free().
         void * malloc( size_t size ) {
                 #ifdef __STATISTICS__
@@ -831 +872 @@
         } // malloc
 
-        // The calloc() function allocates memory for an array of nmemb elements of size bytes each and returns a pointer to
-        // the allocated memory. The memory is set to zero. If nmemb or size is 0, then calloc() returns either 0p, or a
-        // unique pointer value that can later be successfully passed to free().
-        void * calloc( size_t noOfElems, size_t elemSize ) {
+
+        // Same as malloc() except size bytes is an array of dim elements each of elemSize bytes.
+        void * aalloc( size_t dim, size_t elemSize ) {
+                size_t size = dim * elemSize;
+                #ifdef __STATISTICS__
+                __atomic_add_fetch( &aalloc_calls, 1, __ATOMIC_SEQ_CST );
+                __atomic_add_fetch( &aalloc_storage, size, __ATOMIC_SEQ_CST );
+                #endif // __STATISTICS__
+
+                return mallocNoStats( size );
+        } // aalloc
+
+
+        // Same as aalloc() with memory set to zero.
+        void * calloc( size_t dim, size_t elemSize ) {
                 #ifdef __STATISTICS__
                 __atomic_add_fetch( &calloc_calls, 1, __ATOMIC_SEQ_CST );
-                __atomic_add_fetch( &calloc_storage, noOfElems * elemSize, __ATOMIC_SEQ_CST );
-                #endif // __STATISTICS__
-
-                return callocNoStats( noOfElems, elemSize );
+                __atomic_add_fetch( &calloc_storage, dim * elemSize, __ATOMIC_SEQ_CST );
+                #endif // __STATISTICS__
+
+                return callocNoStats( dim, elemSize );
         } // calloc
 
-        // The realloc() function changes the size of the memory block pointed to by ptr to size bytes. The contents will be
-        // unchanged in the range from the start of the region up to the minimum of the old and new sizes. If the new size
-        // is larger than the old size, the added memory will not be initialized.  If ptr is 0p, then the call is
-        // equivalent to malloc(size), for all values of size; if size is equal to zero, and ptr is not 0p, then the call
-        // is equivalent to free(ptr). Unless ptr is 0p, it must have been returned by an earlier call to malloc(),
-        // calloc() or realloc(). If the area pointed to was moved, a free(ptr) is done.
-        void * realloc( void * oaddr, size_t size ) {
-                #ifdef __STATISTICS__
-                __atomic_add_fetch( &realloc_calls, 1, __ATOMIC_SEQ_CST );
+
+        // Change the size of the memory block pointed to by oaddr to size bytes. The contents are undefined.  If oaddr is
+        // 0p, then the call is equivalent to malloc(size), for all values of size; if size is equal to zero, and oaddr is
+        // not 0p, then the call is equivalent to free(oaddr). Unless oaddr is 0p, it must have been returned by an earlier
+        // call to malloc(), alloc(), calloc() or realloc(). If the area pointed to was moved, a free(oaddr) is done.
+        void * resize( void * oaddr, size_t size ) {
+                #ifdef __STATISTICS__
+                __atomic_add_fetch( &resize_calls, 1, __ATOMIC_SEQ_CST );
                 #endif // __STATISTICS__
 
                 // If size is equal to 0, either NULL or a pointer suitable to be passed to free() is returned.
-          if ( unlikely( size == 0 ) ) { free( oaddr ); return mallocNoStats( size ); } // special cases
-          if ( unlikely( oaddr == 0p ) ) return mallocNoStats( size );
+          if ( unlikely( size == 0 ) ) { free( oaddr ); return 0p; } // special cases
+          if ( unlikely( oaddr == 0p ) ) {
+                        #ifdef __STATISTICS__
+                        __atomic_add_fetch( &resize_storage, size, __ATOMIC_SEQ_CST );
+                        #endif // __STATISTICS__
+                        return mallocNoStats( size );
+                } // if
 
                 HeapManager.Storage.Header * header;
                 HeapManager.FreeHeader * freeElem;
-                size_t bsize, oalign = 0;
+                size_t bsize, oalign;
+                headers( "resize", oaddr, header, freeElem, bsize, oalign );
+                size_t odsize = dataStorage( bsize, oaddr, header ); // data storage available in bucket
+
+                // same size, DO NOT preserve STICKY PROPERTIES.
+                if ( oalign == libAlign() && size <= odsize && odsize <= size * 2 ) { // allow 50% wasted storage for smaller size
+                        header->kind.real.blockSize &= -2;                      // no alignment and turn off 0 fill
+                        header->kind.real.size = size;                          // reset allocation size
+                        return oaddr;
+                } // if
+
+                #ifdef __STATISTICS__
+                __atomic_add_fetch( &resize_storage, size, __ATOMIC_SEQ_CST );
+                #endif // __STATISTICS__
+
+                // change size, DO NOT preserve STICKY PROPERTIES.
+                free( oaddr );
+                return mallocNoStats( size );                                   // create new area
+        } // resize
+
+
+        // Same as resize() but the contents are unchanged in the range from the start of the region up to the minimum of
+        // the old and new sizes.
+        void * realloc( void * oaddr, size_t size ) {
+                #ifdef __STATISTICS__
+                __atomic_add_fetch( &realloc_calls, 1, __ATOMIC_SEQ_CST );
+                #endif // __STATISTICS__
+
+                // If size is equal to 0, either NULL or a pointer suitable to be passed to free() is returned.
+          if ( unlikely( size == 0 ) ) { free( oaddr ); return 0p; } // special cases
+          if ( unlikely( oaddr == 0p ) ) {
+                        #ifdef __STATISTICS__
+                        __atomic_add_fetch( &realloc_storage, size, __ATOMIC_SEQ_CST );
+                        #endif // __STATISTICS__
+                        return mallocNoStats( size );
+                } // if
+
+                HeapManager.Storage.Header * header;
+                HeapManager.FreeHeader * freeElem;
+                size_t bsize, oalign;
                 headers( "realloc", oaddr, header, freeElem, bsize, oalign );
 
                 size_t odsize = dataStorage( bsize, oaddr, header ); // data storage available in bucket
-          if ( size <= odsize && odsize <= size * 2 ) { // allow up to 50% wasted storage in smaller size
-                        // Do not know size of original allocation => cannot do 0 fill for any additional space because do not know
-                        // where to start filling, i.e., do not overwrite existing values in space.
-                        //
-                        // This case does not result in a new profiler entry because the previous one still exists and it must match with
-                        // the free for this memory.  Hence, this realloc does not appear in the profiler output.
+                size_t osize = header->kind.real.size;                  // old allocation size
+                bool ozfill = (header->kind.real.blockSize & 2); // old allocation zero filled
+          if ( unlikely( size <= odsize ) && odsize <= size * 2 ) { // allow up to 50% wasted storage
+                        header->kind.real.size = size;                          // reset allocation size
+                        if ( unlikely( ozfill ) && size > osize ) {     // previous request zero fill and larger ?
+                                memset( (char *)oaddr + osize, '\0', size - osize ); // initialize added storage
+                        } // if
                         return oaddr;
                 } // if
 
                 #ifdef __STATISTICS__
-                __atomic_add_fetch( &realloc_storage, size, __ATOMIC_SEQ_CST );
+                __atomic_add_fetch( &realloc_storage, size, __ATOMIC_SEQ_CST );
                 #endif // __STATISTICS__
 
@@ -880 +976 @@
 
                 void * naddr;
-                if ( unlikely( oalign != 0 ) ) {                                // previous request memalign?
-                        if ( unlikely( header->kind.real.blockSize & 2 ) ) { // previous request zero fill
-                                naddr = cmemalignNoStats( oalign, 1, size ); // create new aligned area
-                        } else {
-                                naddr = memalignNoStats( oalign, size ); // create new aligned area
+                if ( likely( oalign == libAlign() ) ) {                 // previous request not aligned ?
+                        naddr = mallocNoStats( size );                          // create new area
+                } else {
+                        naddr = memalignNoStats( oalign, size );        // create new aligned area
+                } // if
+
+                headers( "realloc", naddr, header, freeElem, bsize, oalign );
+                memcpy( naddr, oaddr, min( osize, size ) );             // copy bytes
+                free( oaddr );
+
+                if ( unlikely( ozfill ) ) {                                             // previous request zero fill ?
+                        header->kind.real.blockSize |= 2;                       // mark new request as zero filled
+                        if ( size > osize ) {                                           // previous request larger ?
+                                memset( (char *)naddr + osize, '\0', size - osize ); // initialize added storage
                         } // if
-                } else {
-                        if ( unlikely( header->kind.real.blockSize & 2 ) ) { // previous request zero fill
-                                naddr = callocNoStats( 1, size );               // create new area
-                        } else {
-                                naddr = mallocNoStats( size );                  // create new area
-                        } // if
-                } // if
-          if ( unlikely( naddr == 0p ) ) return 0p;
-
-                headers( "realloc", naddr, header, freeElem, bsize, oalign );
-                size_t ndsize = dataStorage( bsize, naddr, header ); // data storage avilable in bucket
-                // To preserve prior fill, the entire bucket must be copied versus the size.
-                memcpy( naddr, oaddr, MIN( odsize, ndsize ) );  // copy bytes
-                free( oaddr );
+                } // if
                 return naddr;
         } // realloc
 
-        // The obsolete function memalign() allocates size bytes and returns a pointer to the allocated memory. The memory
-        // address will be a multiple of alignment, which must be a power of two.
+
+        // Same as malloc() except the memory address is a multiple of alignment, which must be a power of two. (obsolete)
         void * memalign( size_t alignment, size_t size ) {
                 #ifdef __STATISTICS__
@@ -915 +1007 @@
 
 
-        // The cmemalign() function is the same as calloc() with memory alignment.
-        void * cmemalign( size_t alignment, size_t noOfElems, size_t elemSize ) {
+        // Same as aalloc() with memory alignment.
+        void * amemalign( size_t alignment, size_t dim, size_t elemSize ) {
+                size_t size = dim * elemSize;
                 #ifdef __STATISTICS__
                 __atomic_add_fetch( &cmemalign_calls, 1, __ATOMIC_SEQ_CST );
-                __atomic_add_fetch( &cmemalign_storage, noOfElems * elemSize, __ATOMIC_SEQ_CST );
-                #endif // __STATISTICS__
-
-                return cmemalignNoStats( alignment, noOfElems, elemSize );
+                __atomic_add_fetch( &cmemalign_storage, size, __ATOMIC_SEQ_CST );
+                #endif // __STATISTICS__
+
+                return memalignNoStats( alignment, size );
+        } // amemalign
+
+
+        // Same as calloc() with memory alignment.
+        void * cmemalign( size_t alignment, size_t dim, size_t elemSize ) {
+                #ifdef __STATISTICS__
+                __atomic_add_fetch( &cmemalign_calls, 1, __ATOMIC_SEQ_CST );
+                __atomic_add_fetch( &cmemalign_storage, dim * elemSize, __ATOMIC_SEQ_CST );
+                #endif // __STATISTICS__
+
+                return cmemalignNoStats( alignment, dim, elemSize );
         } // cmemalign
 
-        // The function aligned_alloc() is the same as memalign(), except for the added restriction that size should be a
-        // multiple of alignment.
+
+        // Same as memalign(), but ISO/IEC 2011 C11 Section 7.22.2 states: the value of size shall be an integral multiple
+    // of alignment. This requirement is universally ignored.
         void * aligned_alloc( size_t alignment, size_t size ) {
                 return memalign( alignment, size );
@@ -932 +1037 @@
 
 
-        // The function posix_memalign() allocates size bytes and places the address of the allocated memory in *memptr. The
-        // address of the allocated memory will be a multiple of alignment, which must be a power of two and a multiple of
-        // sizeof(void *). If size is 0, then posix_memalign() returns either 0p, or a unique pointer value that can later
-        // be successfully passed to free(3).
+        // Allocates size bytes and places the address of the allocated memory in *memptr. The address of the allocated
+        // memory shall be a multiple of alignment, which must be a power of two and a multiple of sizeof(void *). If size
+        // is 0, then posix_memalign() returns either 0p, or a unique pointer value that can later be successfully passed to
+        // free(3).
         int posix_memalign( void ** memptr, size_t alignment, size_t size ) {
-          if ( alignment < sizeof(void *) || ! libPow2( alignment ) ) return EINVAL; // check alignment
+          if ( alignment < libAlign() || ! is_pow2( alignment ) ) return EINVAL; // check alignment
                 * memptr = memalign( alignment, size );
-          if ( unlikely( * memptr == 0p ) ) return ENOMEM;
                 return 0;
         } // posix_memalign
 
-        // The obsolete function valloc() allocates size bytes and returns a pointer to the allocated memory. The memory
-        // address will be a multiple of the page size.  It is equivalent to memalign(sysconf(_SC_PAGESIZE),size).
+
+        // Allocates size bytes and returns a pointer to the allocated memory. The memory address shall be a multiple of the
+        // page size.  It is equivalent to memalign(sysconf(_SC_PAGESIZE),size).
         void * valloc( size_t size ) {
-                return memalign( pageSize, size );
+                return memalign( __page_size, size );
         } // valloc
 
 
-        // The free() function frees the memory space pointed to by ptr, which must have been returned by a previous call to
-        // malloc(), calloc() or realloc().  Otherwise, or if free(ptr) has already been called before, undefined behavior
-        // occurs. If ptr is 0p, no operation is performed.
+        // Same as valloc but rounds size to multiple of page size.
+        void * pvalloc( size_t size ) {
+                return memalign( __page_size, ceiling2( size, __page_size ) );
+        } // pvalloc
+
+
+        // Frees the memory space pointed to by ptr, which must have been returned by a previous call to malloc(), calloc()
+        // or realloc().  Otherwise, or if free(ptr) has already been called before, undefined behaviour occurs. If ptr is
+        // 0p, no operation is performed.
         void free( void * addr ) {
                 #ifdef __STATISTICS__
@@ -973 +1084 @@
 
 
-        // The malloc_alignment() function returns the alignment of the allocation.
+        // Returns the alignment of an allocation.
         size_t malloc_alignment( void * addr ) {
           if ( unlikely( addr == 0p ) ) return libAlign();      // minimum alignment
@@ -980 +1091 @@
                         return header->kind.fake.alignment & -2;        // remove flag from value
                 } else {
-                        return libAlign ();                                                     // minimum alignment
+                        return libAlign();                                                      // minimum alignment
                 } // if
         } // malloc_alignment
 
 
-        // The malloc_zero_fill() function returns true if the allocation is zero filled, i.e., initially allocated by calloc().
+        // Set the alignment for an the allocation and return previous alignment or 0 if no alignment.
+        size_t $malloc_alignment_set( void * addr, size_t alignment ) {
+          if ( unlikely( addr == 0p ) ) return libAlign();      // minimum alignment
+                size_t ret;
+                HeapManager.Storage.Header * header = headerAddr( addr );
+                if ( (header->kind.fake.alignment & 1) == 1 ) { // fake header ?
+                        ret = header->kind.fake.alignment & -2;         // remove flag from old value
+                        header->kind.fake.alignment = alignment | 1; // add flag to new value
+                } else {
+                        ret = 0;                                                                        // => no alignment to change
+                } // if
+                return ret;
+        } // $malloc_alignment_set
+
+
+        // Returns true if the allocation is zero filled, e.g., allocated by calloc().
         bool malloc_zero_fill( void * addr ) {
           if ( unlikely( addr == 0p ) ) return false;           // null allocation is not zero fill
                 HeapManager.Storage.Header * header = headerAddr( addr );
                 if ( (header->kind.fake.alignment & 1) == 1 ) { // fake header ?
-                        header = (HeapManager.Storage.Header *)((char *)header - header->kind.fake.offset);
-                } // if
-                return (header->kind.real.blockSize & 2) != 0;  // zero filled (calloc/cmemalign) ?
+                        header = realHeader( header );                          // backup from fake to real header
+                } // if
+                return (header->kind.real.blockSize & 2) != 0;  // zero filled ?
         } // malloc_zero_fill
 
-
-        // The malloc_usable_size() function returns the number of usable bytes in the block pointed to by ptr, a pointer to
-        // a block of memory allocated by malloc(3) or a related function.
+        // Set allocation is zero filled and return previous zero filled.
+        bool $malloc_zero_fill_set( void * addr ) {
+          if ( unlikely( addr == 0p ) ) return false;           // null allocation is not zero fill
+                HeapManager.Storage.Header * header = headerAddr( addr );
+                if ( (header->kind.fake.alignment & 1) == 1 ) { // fake header ?
+                        header = realHeader( header );                          // backup from fake to real header
+                } // if
+                bool ret = (header->kind.real.blockSize & 2) != 0; // zero filled ?
+                header->kind.real.blockSize |= 2;                               // mark as zero filled
+                return ret;
+        } // $malloc_zero_fill_set
+
+
+        // Returns original total allocation size (not bucket size) => array size is dimension * sizeif(T).
+        size_t malloc_size( void * addr ) {
+          if ( unlikely( addr == 0p ) ) return 0;                       // null allocation has zero size
+                HeapManager.Storage.Header * header = headerAddr( addr );
+                if ( (header->kind.fake.alignment & 1) == 1 ) { // fake header ?
+                        header = realHeader( header );                          // backup from fake to real header
+                } // if
+                return header->kind.real.size;
+        } // malloc_size
+
+        // Set allocation size and return previous size.
+        size_t $malloc_size_set( void * addr, size_t size ) {
+          if ( unlikely( addr == 0p ) ) return 0;                       // null allocation has 0 size
+                HeapManager.Storage.Header * header = headerAddr( addr );
+                if ( (header->kind.fake.alignment & 1) == 1 ) { // fake header ?
+                        header = realHeader( header );                          // backup from fake to real header
+                } // if
+                size_t ret = header->kind.real.size;
+                header->kind.real.size = size;
+                return ret;
+        } // $malloc_size_set
+
+
+        // Returns the number of usable bytes in the block pointed to by ptr, a pointer to a block of memory allocated by
+        // malloc or a related function.
         size_t malloc_usable_size( void * addr ) {
           if ( unlikely( addr == 0p ) ) return 0;                       // null allocation has 0 size
@@ -1005 +1166 @@
 
                 headers( "malloc_usable_size", addr, header, freeElem, bsize, alignment );
-                return dataStorage( bsize, addr, header );      // data storage in bucket
+                return dataStorage( bsize, addr, header );              // data storage in bucket
         } // malloc_usable_size
 
 
-        // The malloc_stats() function prints (on default standard error) statistics about memory allocated by malloc(3) and
-        // related functions.
+        // Prints (on default standard error) statistics about memory allocated by malloc and related functions.
         void malloc_stats( void ) {
                 #ifdef __STATISTICS__
@@ -1018 +1178 @@
         } // malloc_stats
 
-        // The malloc_stats_fd() function changes the file descripter where malloc_stats() writes the statistics.
+
+        // Changes the file descripter where malloc_stats() writes statistics.
         int malloc_stats_fd( int fd __attribute__(( unused )) ) {
                 #ifdef __STATISTICS__
-                int temp = statfd;
-                statfd = fd;
+                int temp = stat_fd;
+                stat_fd = fd;
                 return temp;
                 #else
@@ -1030 +1191 @@
 
 
-        // The mallopt() function adjusts parameters that control the behavior of the memory-allocation functions (see
-        // malloc(3)). The param argument specifies the parameter to be modified, and value specifies the new value for that
-        // parameter.
+        // Adjusts parameters that control the behaviour of the memory-allocation functions (see malloc). The param argument
+        // specifies the parameter to be modified, and value specifies the new value for that parameter.
         int mallopt( int option, int value ) {
                 choose( option ) {
                   case M_TOP_PAD:
-                        if ( setHeapExpand( value ) ) return 1;
+                        heapExpand = ceiling2( value, __page_size ); return 1;
                   case M_MMAP_THRESHOLD:
                         if ( setMmapStart( value ) ) return 1;
+                        break;
                 } // switch
                 return 0;                                                                               // error, unsupported
         } // mallopt
 
-        // The malloc_trim() function attempts to release free memory at the top of the heap (by calling sbrk(2) with a
-        // suitable argument).
+
+        // Attempt to release free memory at the top of the heap (by calling sbrk with a suitable argument).
         int malloc_trim( size_t ) {
                 return 0;                                                                               // => impossible to release memory
@@ -1050 +1211 @@
 
 
-        // The malloc_info() function exports an XML string that describes the current state of the memory-allocation
-        // implementation in the caller.  The string is printed on the file stream stream.  The exported string includes
-        // information about all arenas (see malloc(3)).
+        // Exports an XML string that describes the current state of the memory-allocation implementation in the caller.
+        // The string is printed on the file stream stream.  The exported string includes information about all arenas (see
+        // malloc).
         int malloc_info( int options, FILE * stream ) {
-                if ( options != 0 ) { errno = EINVAL; return -1; }
+          if ( options != 0 ) { errno = EINVAL; return -1; }
+                #ifdef __STATISTICS__
                 return printStatsXML( stream );
+                #else
+                return 0;                                                                               // unsupported
+                #endif // __STATISTICS__
         } // malloc_info
 
 
-        // The malloc_get_state() function records the current state of all malloc(3) internal bookkeeping variables (but
-        // not the actual contents of the heap or the state of malloc_hook(3) functions pointers).  The state is recorded in
-        // a system-dependent opaque data structure dynamically allocated via malloc(3), and a pointer to that data
-        // structure is returned as the function result.  (It is the caller's responsibility to free(3) this memory.)
+        // Records the current state of all malloc internal bookkeeping variables (but not the actual contents of the heap
+        // or the state of malloc_hook functions pointers).  The state is recorded in a system-dependent opaque data
+        // structure dynamically allocated via malloc, and a pointer to that data structure is returned as the function
+        // result.  (The caller must free this memory.)
         void * malloc_get_state( void ) {
                 return 0p;                                                                              // unsupported
@@ -1068 +1233 @@
 
 
-        // The malloc_set_state() function restores the state of all malloc(3) internal bookkeeping variables to the values
-        // recorded in the opaque data structure pointed to by state.
-        int malloc_set_state( void * ptr ) {
+        // Restores the state of all malloc internal bookkeeping variables to the values recorded in the opaque data
+        // structure pointed to by state.
+        int malloc_set_state( void * ) {
                 return 0;                                                                               // unsupported
         } // malloc_set_state
@@ -1077 +1242 @@
 
 // Must have CFA linkage to overload with C linkage realloc.
-void * realloc( void * oaddr, size_t nalign, size_t size ) {
+void * resize( void * oaddr, size_t nalign, size_t size ) {
         #ifdef __STATISTICS__
-        __atomic_add_fetch( &realloc_calls, 1, __ATOMIC_SEQ_CST );
+        __atomic_add_fetch( &resize_calls, 1, __ATOMIC_SEQ_CST );
         #endif // __STATISTICS__
 
-        // If size is equal to 0, either NULL or a pointer suitable to be passed to free() is returned.
-  if ( unlikely( size == 0 ) ) { free( oaddr ); return mallocNoStats( size ); } // special cases
-  if ( unlikely( oaddr == 0p ) ) return mallocNoStats( size );
-
-        if ( unlikely( nalign == 0 ) ) nalign = libAlign();     // reset alignment to minimum
+        if ( unlikely( nalign < libAlign() ) ) nalign = libAlign(); // reset alignment to minimum
         #ifdef __CFA_DEBUG__
         else
@@ -1092 +1253 @@
         #endif // __CFA_DEBUG__
 
-        HeapManager.Storage.Header * header;
-        HeapManager.FreeHeader * freeElem;
-        size_t bsize, oalign = 0;
-        headers( "realloc", oaddr, header, freeElem, bsize, oalign );
-        size_t odsize = dataStorage( bsize, oaddr, header ); // data storage available in bucket
-
-  if ( oalign != 0 && (uintptr_t)oaddr % nalign == 0 ) { // has alignment and just happens to work out
-                headerAddr( oaddr )->kind.fake.alignment = nalign | 1; // update alignment (could be the same)
-                return realloc( oaddr, size );
+        // If size is equal to 0, either NULL or a pointer suitable to be passed to free() is returned.
+  if ( unlikely( size == 0 ) ) { free( oaddr ); return 0p; } // special cases
+  if ( unlikely( oaddr == 0p ) ) {
+                #ifdef __STATISTICS__
+                __atomic_add_fetch( &resize_storage, size, __ATOMIC_SEQ_CST );
+                #endif // __STATISTICS__
+                return memalignNoStats( nalign, size );
+        } // if
+
+        // Attempt to reuse existing alignment.
+        HeapManager.Storage.Header * header = headerAddr( oaddr );
+        bool isFakeHeader = header->kind.fake.alignment & 1; // old fake header ?
+        size_t oalign;
+        if ( isFakeHeader ) {
+                oalign = header->kind.fake.alignment & -2;              // old alignment
+                if ( (uintptr_t)oaddr % nalign == 0                             // lucky match ?
+                         && ( oalign <= nalign                                          // going down
+                                  || (oalign >= nalign && oalign <= 256) ) // little alignment storage wasted ?
+                        ) {
+                        headerAddr( oaddr )->kind.fake.alignment = nalign | 1; // update alignment (could be the same)
+                        HeapManager.FreeHeader * freeElem;
+                        size_t bsize, oalign;
+                        headers( "resize", oaddr, header, freeElem, bsize, oalign );
+                        size_t odsize = dataStorage( bsize, oaddr, header ); // data storage available in bucket
+
+                        if ( size <= odsize && odsize <= size * 2 ) { // allow 50% wasted data storage
+                                headerAddr( oaddr )->kind.fake.alignment = nalign | 1; // update alignment (could be the same)
+
+                                header->kind.real.blockSize &= -2;              // turn off 0 fill
+                                header->kind.real.size = size;                  // reset allocation size
+                                return oaddr;
+                        } // if
+                } // if
+        } else if ( ! isFakeHeader                                                      // old real header (aligned on libAlign) ?
+                                && nalign == libAlign() ) {                             // new alignment also on libAlign => no fake header needed
+                return resize( oaddr, size );                                   // duplicate special case checks
         } // if
 
         #ifdef __STATISTICS__
+        __atomic_add_fetch( &resize_storage, size, __ATOMIC_SEQ_CST );
+        #endif // __STATISTICS__
+
+        // change size, DO NOT preserve STICKY PROPERTIES.
+        free( oaddr );
+        return memalignNoStats( nalign, size );                         // create new aligned area
+} // resize
+
+
+void * realloc( void * oaddr, size_t nalign, size_t size ) {
+        if ( unlikely( nalign < libAlign() ) ) nalign = libAlign(); // reset alignment to minimum
+        #ifdef __CFA_DEBUG__
+        else
+                checkAlign( nalign );                                                   // check alignment
+        #endif // __CFA_DEBUG__
+
+        // If size is equal to 0, either NULL or a pointer suitable to be passed to free() is returned.
+  if ( unlikely( size == 0 ) ) { free( oaddr ); return 0p; } // special cases
+  if ( unlikely( oaddr == 0p ) ) {
+                #ifdef __STATISTICS__
+                __atomic_add_fetch( &realloc_calls, 1, __ATOMIC_SEQ_CST );
+                __atomic_add_fetch( &realloc_storage, size, __ATOMIC_SEQ_CST );
+                #endif // __STATISTICS__
+                return memalignNoStats( nalign, size );
+        } // if
+
+        // Attempt to reuse existing alignment.
+        HeapManager.Storage.Header * header = headerAddr( oaddr );
+        bool isFakeHeader = header->kind.fake.alignment & 1; // old fake header ?
+        size_t oalign;
+        if ( isFakeHeader ) {
+                oalign = header->kind.fake.alignment & -2;              // old alignment
+                if ( (uintptr_t)oaddr % nalign == 0                             // lucky match ?
+                         && ( oalign <= nalign                                          // going down
+                                  || (oalign >= nalign && oalign <= 256) ) // little alignment storage wasted ?
+                        ) {
+                        headerAddr( oaddr )->kind.fake.alignment = nalign | 1; // update alignment (could be the same)
+                        return realloc( oaddr, size );                          // duplicate alignment and special case checks
+                } // if
+        } else if ( ! isFakeHeader                                                      // old real header (aligned on libAlign) ?
+                                && nalign == libAlign() )                               // new alignment also on libAlign => no fake header needed
+                return realloc( oaddr, size );                                  // duplicate alignment and special case checks
+
+        #ifdef __STATISTICS__
+        __atomic_add_fetch( &realloc_calls, 1, __ATOMIC_SEQ_CST );
         __atomic_add_fetch( &realloc_storage, size, __ATOMIC_SEQ_CST );
         #endif // __STATISTICS__
 
+        HeapManager.FreeHeader * freeElem;
+        size_t bsize;
+        headers( "realloc", oaddr, header, freeElem, bsize, oalign );
+
         // change size and copy old content to new storage
 
-        void * naddr;
-        if ( unlikely( header->kind.real.blockSize & 2 ) ) { // previous request zero fill
-                naddr = cmemalignNoStats( nalign, 1, size );    // create new aligned area
-        } else {
-                naddr = memalignNoStats( nalign, size );                // create new aligned area
-        } // if
+        size_t osize = header->kind.real.size;                          // old allocation size
+        bool ozfill = (header->kind.real.blockSize & 2);        // old allocation zero filled
+
+        void * naddr = memalignNoStats( nalign, size );         // create new aligned area
 
         headers( "realloc", naddr, header, freeElem, bsize, oalign );
-        size_t ndsize = dataStorage( bsize, naddr, header ); // data storage avilable in bucket
-        // To preserve prior fill, the entire bucket must be copied versus the size.
-        memcpy( naddr, oaddr, MIN( odsize, ndsize ) );          // copy bytes
+        memcpy( naddr, oaddr, min( osize, size ) );                     // copy bytes
         free( oaddr );
+
+        if ( unlikely( ozfill ) ) {                                                     // previous request zero fill ?
+                header->kind.real.blockSize |= 2;                               // mark new request as zero filled
+                if ( size > osize ) {                                                   // previous request larger ?
+                        memset( (char *)naddr + osize, '\0', size - osize ); // initialize added storage
+                } // if
+        } // if
         return naddr;
 } // realloc

libcfa/src/interpose.cfa

@@ -10 +10 @@
 // Created On       : Wed Mar 29 16:10:31 2017
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Thu Jan 30 17:47:32 2020
-// Update Count     : 156
+// Last Modified On : Fri Mar 13 17:35:37 2020
+// Update Count     : 178
 //
 
 #include <stdarg.h>                                                                             // va_start, va_end
+#include <stdio.h>
 #include <string.h>                                                                             // strlen
 #include <unistd.h>                                                                             // _exit, getpid
@@ -29 +30 @@
 #include "bits/signal.hfa"                                                              // sigHandler_?
 #include "startup.hfa"                                                                  // STARTUP_PRIORITY_CORE
+#include <assert.h>
 
 //=============================================================================================
@@ -40 +42 @@
 
 typedef void (* generic_fptr_t)(void);
-generic_fptr_t interpose_symbol( const char * symbol, const char * version ) {
+generic_fptr_t interpose_symbol( const char symbol[], const char version[] ) {
         const char * error;
 
@@ -142 +144 @@
 void abort( const char fmt[], ... ) __attribute__(( format(printf, 1, 2), __nothrow__, __leaf__, __noreturn__ ));
 void abort( bool signalAbort, const char fmt[], ... ) __attribute__(( format(printf, 2, 3), __nothrow__, __leaf__, __noreturn__ ));
+void __abort( bool signalAbort, const char fmt[], va_list args ) __attribute__(( __nothrow__, __leaf__, __noreturn__ ));
 
 extern "C" {
         void abort( void ) __attribute__(( __nothrow__, __leaf__, __noreturn__ )) {
-                abort( false, NULL ); // FIX ME: 0p does not work
+                abort( false, "%s", "" );
         }
 
@@ -151 +154 @@
                 va_list argp;
                 va_start( argp, fmt );
-                abort( false, fmt, argp );
+                __abort( false, fmt, argp );
                 va_end( argp );
         }
@@ -161 +164 @@
 
 void * kernel_abort( void ) __attribute__(( __nothrow__, __leaf__, __weak__ )) { return 0p; }
-void kernel_abort_msg( void * data, char * buffer, int size ) __attribute__(( __nothrow__, __leaf__, __weak__ )) {}
+void kernel_abort_msg( void * data, char buffer[], int size ) __attribute__(( __nothrow__, __leaf__, __weak__ )) {}
 // See concurrency/kernel.cfa for strong definition used in multi-processor mode.
 int kernel_abort_lastframe( void ) __attribute__(( __nothrow__, __leaf__, __weak__ )) { return 4; }
@@ -169 +172 @@
 
 static void __cfaabi_backtrace( int start ) {
-        enum {
-                Frames = 50,                                                                    // maximum number of stack frames
-        };
+        enum { Frames = 50, };                                                          // maximum number of stack frames
         int last = kernel_abort_lastframe();                            // skip last N stack frames
 
         void * array[Frames];
         size_t size = backtrace( array, Frames );
-        char ** messages = backtrace_symbols( array, size );
+        char ** messages = backtrace_symbols( array, size ); // does not demangle names
 
         *index( messages[0], '(' ) = '\0';                                      // find executable name
@@ -184 +185 @@
                 char * name = 0p, * offset_begin = 0p, * offset_end = 0p;
 
-                for ( char * p = messages[i]; *p; ++p ) {               // find parantheses and +offset
+                for ( char * p = messages[i]; *p; p += 1 ) {    // find parantheses and +offset
                         //__cfaabi_bits_print_nolock( "X %s\n", p);
                         if ( *p == '(' ) {
@@ -219 +220 @@
 }
 
-void abort( bool signalAbort, const char fmt[], ... ) {
-        void * kernel_data = kernel_abort();                            // must be done here to lock down kernel
-        int len;
-
-        signal( SIGABRT, SIG_DFL );                                                     // prevent final "real" abort from recursing to handler
-
-        len = snprintf( abort_text, abort_text_size, "Cforall Runtime error (UNIX pid:%ld) ", (long int)getpid() ); // use UNIX pid (versus getPid)
-        __cfaabi_bits_write( STDERR_FILENO, abort_text, len );
-
-        if ( fmt ) {
-                va_list args;
-                va_start( args, fmt );
-
+static volatile int __abort_stage = 0;
+
+// Cannot forward va_list.
+void __abort( bool signalAbort, const char fmt[], va_list args ) {
+        int stage = __atomic_add_fetch( &__abort_stage, 1, __ATOMIC_SEQ_CST );
+
+        // First stage: stop the cforall kernel and print
+        if(stage == 1) {
+                // increment stage
+                stage = __atomic_add_fetch( &__abort_stage, 1, __ATOMIC_SEQ_CST );
+
+                // must be done here to lock down kernel
+                void * kernel_data = kernel_abort();
+                int len;
+
+                signal( SIGABRT, SIG_DFL );                                                     // prevent final "real" abort from recursing to handler
+
+                len = snprintf( abort_text, abort_text_size, "Cforall Runtime error (UNIX pid:%ld) ", (long int)getpid() ); // use UNIX pid (versus getPid)
+                __cfaabi_bits_write( STDERR_FILENO, abort_text, len );
+
+                assert( fmt );
                 len = vsnprintf( abort_text, abort_text_size, fmt, args );
-                va_end( args );
                 __cfaabi_bits_write( STDERR_FILENO, abort_text, len );
 
-                if ( fmt[strlen( fmt ) - 1] != '\n' ) {                 // add optional newline if missing at the end of the format text
-                        __cfaabi_dbg_write( "\n", 1 );
-                }
-        }
-
-        kernel_abort_msg( kernel_data, abort_text, abort_text_size );
-        __cfaabi_backtrace( signalAbort ? 4 : 3 );
-
-        __cabi_libc.abort();                                                            // print stack trace in handler
+                // add optional newline if missing at the end of the format text
+                if ( fmt[strlen( fmt ) - 1] != '\n' ) {
+                        __cfaabi_bits_write( STDERR_FILENO, "\n", 1 );
+                } // if
+                kernel_abort_msg( kernel_data, abort_text, abort_text_size );
+        }
+
+        // Second stage: print the backtrace
+        if(stage == 2) {
+                // increment stage
+                stage = __atomic_add_fetch( &__abort_stage, 1, __ATOMIC_SEQ_CST );
+
+                // print stack trace in handler
+                __cfaabi_backtrace( signalAbort ? 4 : 2 );
+        }
+
+        do {
+                // Finally call abort
+                __cabi_libc.abort();
+
+                // Loop so that we never return
+        } while(true);
 }
 
@@ -250 +271 @@
         va_list args;
         va_start( args, fmt );
-        abort( false, fmt, args );
+        __abort( false, fmt, args );
+    // CONTROL NEVER REACHES HERE!
         va_end( args );
+}
+
+void abort( bool signalAbort, const char fmt[], ... ) {
+    va_list args;
+    va_start( args, fmt );
+    __abort( signalAbort, fmt, args );
+    // CONTROL NEVER REACHES HERE!
+    va_end( args );
 }
 

libcfa/src/iostream.cfa

@@ -10 +10 @@
 // Created On       : Wed May 27 17:56:53 2015
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Sat Jul 13 08:07:59 2019
-// Update Count     : 821
+// Last Modified On : Mon Aug 24 08:31:35 2020
+// Update Count     : 1130
 //
 
 #include "iostream.hfa"
 
-extern "C" {
 #include <stdio.h>
 #include <stdbool.h>                                                                    // true/false
-//#include <string.h>                                                                   // strlen, strcmp
+#include <stdint.h>                                                                             // UINT64_MAX
+#include <float.h>                                                                              // DBL_DIG, LDBL_DIG
+#include <complex.h>                                                                    // creal, cimag
+//#include <string.h>                                                                   // strlen, strcmp, memcpy
+extern "C" {
 extern size_t strlen (const char *__s) __attribute__ ((__nothrow__ , __leaf__)) __attribute__ ((__pure__)) __attribute__ ((__nonnull__ (1)));
 extern int strcmp (const char *__s1, const char *__s2) __attribute__ ((__nothrow__ , __leaf__)) __attribute__ ((__pure__)) __attribute__ ((__nonnull__ (1, 2)));
 extern char *strcpy (char *__restrict __dest, const char *__restrict __src) __attribute__ ((__nothrow__ , __leaf__)) __attribute__ ((__nonnull__ (1, 2)));
 extern void *memcpy (void *__restrict __dest, const void *__restrict __src, size_t __n) __attribute__ ((__nothrow__ , __leaf__)) __attribute__ ((__nonnull__ (1, 2)));
-#include <float.h>                                                                              // DBL_DIG, LDBL_DIG
-#include <math.h>                                                                               // isfinite
-#include <complex.h>                                                                    // creal, cimag
 } // extern "C"
 
-
-//*********************************** ostream ***********************************
+#include "math.hfa"                                                                             // isfinite, floor, ceiling_div
+#include "bitmanip.hfa"                                                                 // high1
+
+
+// *********************************** ostream ***********************************
 
 
 forall( dtype ostype | ostream( ostype ) ) {
-        ostype & ?|?( ostype & os, zero_t ) {
-                if ( sepPrt( os ) ) fmt( os, "%s", sepGetCur( os ) );
-                fmt( os, "%d", 0n );
-                return os;
-        } // ?|?
-        void ?|?( ostype & os, zero_t z ) {
-                (ostype &)(os | z); ends( os );
-        } // ?|?
-
-        ostype & ?|?( ostype & os, one_t ) {
-                if ( sepPrt( os ) ) fmt( os, "%s", sepGetCur( os ) );
-                fmt( os, "%d", 1n );
-                return os;
-        } // ?|?
-        void ?|?( ostype & os, one_t o ) {
-                (ostype &)(os | o); ends( os );
-        } // ?|?
-
         ostype & ?|?( ostype & os, bool b ) {
-                if ( sepPrt( os ) ) fmt( os, "%s", sepGetCur( os ) );
+                if ( $sepPrt( os ) ) fmt( os, "%s", $sepGetCur( os ) );
                 fmt( os, "%s", b ? "true" : "false" );
                 return os;
@@ -63 +48 @@
         ostype & ?|?( ostype & os, char c ) {
                 fmt( os, "%c", c );
-                if ( c == '\n' ) setNL( os, true );
+                if ( c == '\n' ) $setNL( os, true );
                 return sepOff( os );
         } // ?|?
@@ -71 +56 @@
 
         ostype & ?|?( ostype & os, signed char sc ) {
-                if ( sepPrt( os ) ) fmt( os, "%s", sepGetCur( os ) );
+                if ( $sepPrt( os ) ) fmt( os, "%s", $sepGetCur( os ) );
                 fmt( os, "%hhd", sc );
                 return os;
@@ -80 +65 @@
 
         ostype & ?|?( ostype & os, unsigned char usc ) {
-                if ( sepPrt( os ) ) fmt( os, "%s", sepGetCur( os ) );
+                if ( $sepPrt( os ) ) fmt( os, "%s", $sepGetCur( os ) );
                 fmt( os, "%hhu", usc );
                 return os;
@@ -89 +74 @@
 
         ostype & ?|?( ostype & os, short int si ) {
-                if ( sepPrt( os ) ) fmt( os, "%s", sepGetCur( os ) );
+                if ( $sepPrt( os ) ) fmt( os, "%s", $sepGetCur( os ) );
                 fmt( os, "%hd", si );
                 return os;
@@ -98 +83 @@
 
         ostype & ?|?( ostype & os, unsigned short int usi ) {
-                if ( sepPrt( os ) ) fmt( os, "%s", sepGetCur( os ) );
+                if ( $sepPrt( os ) ) fmt( os, "%s", $sepGetCur( os ) );
                 fmt( os, "%hu", usi );
                 return os;
@@ -107 +92 @@
 
         ostype & ?|?( ostype & os, int i ) {
-                if ( sepPrt( os ) ) fmt( os, "%s", sepGetCur( os ) );
+                if ( $sepPrt( os ) ) fmt( os, "%s", $sepGetCur( os ) );
                 fmt( os, "%d", i );
                 return os;
@@ -116 +101 @@
 
         ostype & ?|?( ostype & os, unsigned int ui ) {
-                if ( sepPrt( os ) ) fmt( os, "%s", sepGetCur( os ) );
+                if ( $sepPrt( os ) ) fmt( os, "%s", $sepGetCur( os ) );
                 fmt( os, "%u", ui );
                 return os;
@@ -125 +110 @@
 
         ostype & ?|?( ostype & os, long int li ) {
-                if ( sepPrt( os ) ) fmt( os, "%s", sepGetCur( os ) );
+                if ( $sepPrt( os ) ) fmt( os, "%s", $sepGetCur( os ) );
                 fmt( os, "%ld", li );
                 return os;
@@ -134 +119 @@
 
         ostype & ?|?( ostype & os, unsigned long int uli ) {
-                if ( sepPrt( os ) ) fmt( os, "%s", sepGetCur( os ) );
+                if ( $sepPrt( os ) ) fmt( os, "%s", $sepGetCur( os ) );
                 fmt( os, "%lu", uli );
                 return os;
@@ -143 +128 @@
 
         ostype & ?|?( ostype & os, long long int lli ) {
-                if ( sepPrt( os ) ) fmt( os, "%s", sepGetCur( os ) );
+                if ( $sepPrt( os ) ) fmt( os, "%s", $sepGetCur( os ) );
                 fmt( os, "%lld", lli );
                 return os;
@@ -152 +137 @@
 
         ostype & ?|?( ostype & os, unsigned long long int ulli ) {
-                if ( sepPrt( os ) ) fmt( os, "%s", sepGetCur( os ) );
+                if ( $sepPrt( os ) ) fmt( os, "%s", $sepGetCur( os ) );
                 fmt( os, "%llu", ulli );
                 return os;
@@ -159 +144 @@
                 (ostype &)(os | ulli); ends( os );
         } // ?|?
+
+#if defined( __SIZEOF_INT128__ )
+        //      UINT64_MAX 18_446_744_073_709_551_615_ULL
+        #define P10_UINT64 10_000_000_000_000_000_000_ULL       // 19 zeroes
+
+        static inline void base10_128( ostype & os, unsigned int128 val ) {
+#if defined(__GNUC__) && __GNUC_PREREQ(7,0)                             // gcc version >= 7
+                if ( val > P10_UINT64 ) {
+#else
+                if ( (uint64_t)(val >> 64) != 0 || (uint64_t)val > P10_UINT64 ) { // patch gcc 5 & 6 -O3 bug
+#endif // __GNUC_PREREQ(7,0)
+                        base10_128( os, val / P10_UINT64 );                     // recursive
+                        fmt( os, "%.19lu", (uint64_t)(val % P10_UINT64) );
+                } else {
+                        fmt( os, "%lu", (uint64_t)val );
+                } // if
+        } // base10_128
+
+        static inline void base10_128( ostype & os, int128 val ) {
+                if ( val < 0 ) {
+                        fmt( os, "-" );                                                         // leading negative sign
+                        val = -val;
+                } // if
+                base10_128( os, (unsigned int128)val );                 // print zero/positive value
+        } // base10_128
+
+        ostype & ?|?( ostype & os, int128 llli ) {
+                if ( $sepPrt( os ) ) fmt( os, "%s", $sepGetCur( os ) );
+                base10_128( os, llli );
+                return os;
+        } // ?|?
+        void & ?|?( ostype & os, int128 llli ) {
+                (ostype &)(os | llli); ends( os );
+        } // ?|?
+
+        ostype & ?|?( ostype & os, unsigned int128 ullli ) {
+                if ( $sepPrt( os ) ) fmt( os, "%s", $sepGetCur( os ) );
+                base10_128( os, ullli );
+                return os;
+        } // ?|?
+        void & ?|?( ostype & os, unsigned int128 ullli ) {
+                (ostype &)(os | ullli); ends( os );
+        } // ?|?
+#endif // __SIZEOF_INT128__
 
         #define PrintWithDP( os, format, val, ... ) \
@@ -175 +204 @@
 
         ostype & ?|?( ostype & os, float f ) {
-                if ( sepPrt( os ) ) fmt( os, "%s", sepGetCur( os ) );
+                if ( $sepPrt( os ) ) fmt( os, "%s", $sepGetCur( os ) );
                 PrintWithDP( os, "%g", f );
                 return os;
@@ -184 +213 @@
 
         ostype & ?|?( ostype & os, double d ) {
-                if ( sepPrt( os ) ) fmt( os, "%s", sepGetCur( os ) );
+                if ( $sepPrt( os ) ) fmt( os, "%s", $sepGetCur( os ) );
                 PrintWithDP( os, "%.*lg", d, DBL_DIG );
                 return os;
@@ -193 +222 @@
 
         ostype & ?|?( ostype & os, long double ld ) {
-                if ( sepPrt( os ) ) fmt( os, "%s", sepGetCur( os ) );
+                if ( $sepPrt( os ) ) fmt( os, "%s", $sepGetCur( os ) );
                 PrintWithDP( os, "%.*Lg", ld, LDBL_DIG );
                 return os;
@@ -202 +231 @@
 
         ostype & ?|?( ostype & os, float _Complex fc ) {
-                if ( sepPrt( os ) ) fmt( os, "%s", sepGetCur( os ) );
+                if ( $sepPrt( os ) ) fmt( os, "%s", $sepGetCur( os ) );
 //              os | crealf( fc ) | nonl;
                 PrintWithDP( os, "%g", crealf( fc ) );
@@ -214 +243 @@
 
         ostype & ?|?( ostype & os, double _Complex dc ) {
-                if ( sepPrt( os ) ) fmt( os, "%s", sepGetCur( os ) );
+                if ( $sepPrt( os ) ) fmt( os, "%s", $sepGetCur( os ) );
 //              os | creal( dc ) | nonl;
                 PrintWithDP( os, "%.*lg", creal( dc ), DBL_DIG );
@@ -226 +255 @@
 
         ostype & ?|?( ostype & os, long double _Complex ldc ) {
-                if ( sepPrt( os ) ) fmt( os, "%s", sepGetCur( os ) );
+                if ( $sepPrt( os ) ) fmt( os, "%s", $sepGetCur( os ) );
 //              os | creall( ldc ) || nonl;
                 PrintWithDP( os, "%.*Lg", creall( ldc ), LDBL_DIG );
@@ -237 +266 @@
         } // ?|?
 
-        ostype & ?|?( ostype & os, const char * str ) {
+        ostype & ?|?( ostype & os, const char str[] ) {
                 enum { Open = 1, Close, OpenClose };
                 static const unsigned char mask[256] @= {
@@ -257 +286 @@
                 // first character IS NOT spacing or closing punctuation => add left separator
                 unsigned char ch = str[0];                                              // must make unsigned
-                if ( sepPrt( os ) && mask[ ch ] != Close && mask[ ch ] != OpenClose ) {
-                        fmt( os, "%s", sepGetCur( os ) );
+                if ( $sepPrt( os ) && mask[ ch ] != Close && mask[ ch ] != OpenClose ) {
+                        fmt( os, "%s", $sepGetCur( os ) );
                 } // if
 
                 // if string starts line, must reset to determine open state because separator is off
-                sepReset( os );                                                                 // reset separator
+                $sepReset( os );                                                                // reset separator
 
                 // last character IS spacing or opening punctuation => turn off separator for next item
                 size_t len = strlen( str );
                 ch = str[len - 1];                                                              // must make unsigned
-                if ( sepPrt( os ) && mask[ ch ] != Open && mask[ ch ] != OpenClose ) {
+                if ( $sepPrt( os ) && mask[ ch ] != Open && mask[ ch ] != OpenClose ) {
                         sepOn( os );
                 } else {
                         sepOff( os );
                 } // if
-                if ( ch == '\n' ) setNL( os, true );                    // check *AFTER* sepPrt call above as it resets NL flag
+                if ( ch == '\n' ) $setNL( os, true );                   // check *AFTER* $sepPrt call above as it resets NL flag
                 return write( os, str, len );
         } // ?|?
-        void ?|?( ostype & os, const char * str ) {
+
+        void ?|?( ostype & os, const char str[] ) {
                 (ostype &)(os | str); ends( os );
         } // ?|?
 
 //      ostype & ?|?( ostype & os, const char16_t * str ) {
-//              if ( sepPrt( os ) ) fmt( os, "%s", sepGetCur( os ) );
+//              if ( $sepPrt( os ) ) fmt( os, "%s", $sepGetCur( os ) );
 //              fmt( os, "%ls", str );
 //              return os;
@@ -287 +317 @@
 // #if ! ( __ARM_ARCH_ISA_ARM == 1 && __ARM_32BIT_STATE == 1 ) // char32_t == wchar_t => ambiguous
 //      ostype & ?|?( ostype & os, const char32_t * str ) {
-//              if ( sepPrt( os ) ) fmt( os, "%s", sepGetCur( os ) );
+//              if ( $sepPrt( os ) ) fmt( os, "%s", $sepGetCur( os ) );
 //              fmt( os, "%ls", str );
 //              return os;
@@ -294 +324 @@
 
 //      ostype & ?|?( ostype & os, const wchar_t * str ) {
-//              if ( sepPrt( os ) ) fmt( os, "%s", sepGetCur( os ) );
+//              if ( $sepPrt( os ) ) fmt( os, "%s", $sepGetCur( os ) );
 //              fmt( os, "%ls", str );
 //              return os;
@@ -300 +330 @@
 
         ostype & ?|?( ostype & os, const void * p ) {
-                if ( sepPrt( os ) ) fmt( os, "%s", sepGetCur( os ) );
+                if ( $sepPrt( os ) ) fmt( os, "%s", $sepGetCur( os ) );
                 fmt( os, "%p", p );
                 return os;
@@ -315 +345 @@
         void ?|?( ostype & os, ostype & (* manip)( ostype & ) ) {
                 (ostype &)(manip( os ));
-                if ( getPrt( os ) ) ends( os );                                 // something printed ?
-                setPrt( os, false );                                                    // turn off
+                if ( $getPrt( os ) ) ends( os );                                // something printed ?
+                $setPrt( os, false );                                                   // turn off
         } // ?|?
 
@@ -329 +359 @@
         ostype & nl( ostype & os ) {
                 (ostype &)(os | '\n');
-                setPrt( os, false );                                                    // turn off
-                setNL( os, true );
+                $setPrt( os, false );                                                   // turn off
+                $setNL( os, true );
                 flush( os );
                 return sepOff( os );                                                    // prepare for next line
@@ -336 +366 @@
 
         ostype & nonl( ostype & os ) {
-                setPrt( os, false );                                                    // turn off
+                $setPrt( os, false );                                                   // turn off
                 return os;
         } // nonl
@@ -375 +405 @@
         ostype & ?|?( ostype & os, T arg, Params rest ) {
                 (ostype &)(os | arg);                                                   // print first argument
-                sepSetCur( os, sepGetTuple( os ) );                             // switch to tuple separator
+                $sepSetCur( os, sepGetTuple( os ) );                    // switch to tuple separator
                 (ostype &)(os | rest);                                                  // print remaining arguments
-                sepSetCur( os, sepGet( os ) );                                  // switch to regular separator
+                $sepSetCur( os, sepGet( os ) );                                 // switch to regular separator
                 return os;
         } // ?|?
@@ -383 +413 @@
                 // (ostype &)(?|?( os, arg, rest )); ends( os );
                 (ostype &)(os | arg);                                                   // print first argument
-                sepSetCur( os, sepGetTuple( os ) );                             // switch to tuple separator
+                $sepSetCur( os, sepGetTuple( os ) );                    // switch to tuple separator
                 (ostype &)(os | rest);                                                  // print remaining arguments
-                sepSetCur( os, sepGet( os ) );                                  // switch to regular separator
+                $sepSetCur( os, sepGet( os ) );                                 // switch to regular separator
                 ends( os );
         } // ?|?
@@ -403 +433 @@
 } // distribution
 
-//*********************************** manipulators ***********************************
-
-//*********************************** integral ***********************************
+// *********************************** manipulators ***********************************
+
+// *********************************** integral ***********************************
 
 static const char * shortbin[] = { "0", "1", "10", "11", "100", "101", "110", "111", "1000", "1001", "1010", "1011", "1100", "1101", "1110", "1111" };
@@ -411 +441 @@
 
 // Default prefix for non-decimal prints is 0b, 0, 0x.
-#define IntegralFMTImpl( T, CODE, IFMTNP, IFMTP ) \
+#define IntegralFMTImpl( T, IFMTNP, IFMTP ) \
 forall( dtype ostype | ostream( ostype ) ) { \
         ostype & ?|?( ostype & os, _Ostream_Manip(T) f ) { \
-                if ( sepPrt( os ) ) fmt( os, "%s", sepGetCur( os ) ); \
+                if ( $sepPrt( os ) ) fmt( os, "%s", $sepGetCur( os ) ); \
 \
                 if ( f.base == 'b' || f.base == 'B' ) {                 /* bespoke binary format */ \
-                        int bits;                                                                                                       \
-                        if ( f.val == (T){0} ) bits = 1;                        /* force at least one bit to print */ \
-                        else bits = sizeof(long long int) * 8 - __builtin_clzll( f.val ); /* position of most significant bit */ \
-                        bits = bits > sizeof(f.val) * 8 ? sizeof(f.val) * 8 : bits; \
-                        int spaces = f.wd - bits;                                       /* can be negative */ \
-                        if ( ! f.flags.nobsdp ) { spaces -= 2; }        /* base prefix takes space */ \
-                        /* printf( "%d %d\n", bits, spaces ); */ \
+                        int bits = high1( f.val );                                      /* position of most significant bit */ \
+                        if ( bits == 0 ) bits = 1;                                      /* 0 value => force one bit to print */ \
+                        int spaces; \
                         if ( ! f.flags.left ) {                                         /* right justified ? */ \
                                 /* Note, base prefix then zero padding or spacing then prefix. */ \
-                                if ( f.flags.pad0 || f.flags.pc ) { \
+                                if ( f.flags.pc ) { \
+                                        spaces = f.wd - f.pc; \
+                                        if ( ! f.flags.nobsdp ) { spaces -= 2; } /* base prefix takes space */ \
+                                        if ( spaces > 0 ) fmt( os, "%*s", spaces, " " ); /* space pad */ \
                                         if ( ! f.flags.nobsdp ) { fmt( os, "0%c", f.base ); } \
-                                        if ( f.flags.pc ) spaces = f.pc - bits; \
+                                        spaces = f.pc - bits; \
                                         if ( spaces > 0 ) fmt( os, "%0*d", spaces, 0 ); /* zero pad */ \
                                 } else { \
-                                        if ( spaces > 0 ) fmt( os, "%*s", spaces, " " ); /* space pad */ \
-                                        if ( ! f.flags.nobsdp ) { fmt( os, "0%c", f.base ); } \
+                                        spaces = f.wd - bits; \
+                                        if ( ! f.flags.nobsdp ) { spaces -= 2; } /* base prefix takes space */ \
+                                        if ( f.flags.pad0 ) { \
+                                                if ( ! f.flags.nobsdp ) { fmt( os, "0%c", f.base ); } \
+                                                if ( spaces > 0 ) fmt( os, "%0*d", spaces, 0 ); /* zero pad */ \
+                                        } else { \
+                                                if ( spaces > 0 ) fmt( os, "%*s", spaces, " " ); /* space pad */ \
+                                                if ( ! f.flags.nobsdp ) { fmt( os, "0%c", f.base ); } \
+                                        } /* if */ \
                                 } /* if */ \
-                        } else if ( ! f.flags.nobsdp ) { \
-                                fmt( os, "0%c", f.base ); \
+                        } else { \
+                                if ( ! f.flags.nobsdp ) fmt( os, "0%c", f.base ); \
+                                if ( f.flags.pc ) { \
+                                        spaces = f.pc - bits; \
+                                        if ( spaces > 0 ) fmt( os, "%0*d", spaces, 0 ); /* zero pad */ \
+                                        spaces = f.wd - f.pc; \
+                                } else { /* pad0 flag ignored with left flag */ \
+                                        spaces = f.wd - bits; \
+                                } /* if */ \
+                                if ( ! f.flags.nobsdp ) { spaces -= 2; } /* base prefix takes space */ \
                         } /* if */ \
-                        int shift = (bits - 1) / 4 * 4; /* floor( bits - 1, 4 ) */ \
+                        int shift = floor( bits - 1, 4 ); \
                         typeof( f.val ) temp = f.val; \
                         fmt( os, "%s", shortbin[(temp >> shift) & 0xf] ); \
@@ -448 +492 @@
                         if ( f.flags.left && spaces > 0 ) fmt( os, "%*s", spaces, " " ); \
                         return os; \
-                } /* if  */ \
+                } /* if */ \
 \
                 char fmtstr[sizeof(IFMTP)];                                             /* sizeof includes '\0' */ \
@@ -458 +502 @@
                 if ( ! f.flags.nobsdp ) { fmtstr[star] = '#'; star -= 1; } \
                 if ( f.flags.left ) { fmtstr[star] = '-'; star -= 1; } \
-                if ( f.flags.sign && f.base == CODE ) { fmtstr[star] = '+'; star -= 1; } \
+                if ( f.flags.sign ) { fmtstr[star] = '+'; star -= 1; } \
                 if ( f.flags.pad0 && ! f.flags.pc ) { fmtstr[star] = '0'; star -= 1; } \
                 fmtstr[star] = '%'; \
 \
                 if ( ! f.flags.pc ) {                                                   /* no precision */ \
-                        /* printf( "%s\n", &fmtstr[star] ); */ \
                         fmtstr[sizeof(IFMTNP)-2] = f.base;                      /* sizeof includes '\0' */ \
+                        /* printf( "%s %c\n", &fmtstr[star], f.base ); */ \
                         fmt( os, &fmtstr[star], f.wd, f.val ); \
                 } else {                                                                                /* precision */ \
                         fmtstr[sizeof(IFMTP)-2] = f.base;                       /* sizeof includes '\0' */ \
-                        /* printf( "%s\n", &fmtstr[star] ); */ \
+                        /* printf( "%s %c\n", &fmtstr[star], f.base ); */ \
                         fmt( os, &fmtstr[star], f.wd, f.pc, f.val ); \
                 } /* if */ \
@@ -476 +520 @@
 } // distribution
 
-IntegralFMTImpl( signed char, 'd', "%    *hh ", "%    *.*hh " )
-IntegralFMTImpl( unsigned char, 'u', "%    *hh ", "%    *.*hh " )
-IntegralFMTImpl( signed short int, 'd', "%    *h ", "%    *.*h " )
-IntegralFMTImpl( unsigned short int, 'u', "%    *h ", "%    *.*h " )
-IntegralFMTImpl( signed int, 'd', "%    * ", "%    *.* " )
-IntegralFMTImpl( unsigned int, 'u', "%    * ", "%    *.* " )
-IntegralFMTImpl( signed long int, 'd', "%    *l ", "%    *.*l " )
-IntegralFMTImpl( unsigned long int, 'u', "%    *l ", "%    *.*l " )
-IntegralFMTImpl( signed long long int, 'd', "%    *ll ", "%    *.*ll " )
-IntegralFMTImpl( unsigned long long int, 'u', "%    *ll ", "%    *.*ll " )
-
-//*********************************** floating point ***********************************
+IntegralFMTImpl( signed char, "%    *hh ", "%    *.*hh " )
+IntegralFMTImpl( unsigned char, "%    *hh ", "%    *.*hh " )
+IntegralFMTImpl( signed short int, "%    *h ", "%    *.*h " )
+IntegralFMTImpl( unsigned short int, "%    *h ", "%    *.*h " )
+IntegralFMTImpl( signed int, "%    * ", "%    *.* " )
+IntegralFMTImpl( unsigned int, "%    * ", "%    *.* " )
+IntegralFMTImpl( signed long int, "%    *l ", "%    *.*l " )
+IntegralFMTImpl( unsigned long int, "%    *l ", "%    *.*l " )
+IntegralFMTImpl( signed long long int, "%    *ll ", "%    *.*ll " )
+IntegralFMTImpl( unsigned long long int, "%    *ll ", "%    *.*ll " )
+
+#if 0
+#if defined( __SIZEOF_INT128__ )
+// Default prefix for non-decimal prints is 0b, 0, 0x.
+#define IntegralFMTImpl128( T, SIGNED, CODE, IFMTNP, IFMTP ) \
+forall( dtype ostype | ostream( ostype ) ) \
+static void base10_128( ostype & os, _Ostream_Manip(T) f ) { \
+        if ( f.val > UINT64_MAX ) { \
+                unsigned long long int lsig = f.val % P10_UINT64; \
+                f.val /= P10_UINT64; /* msig */ \
+                base10_128( os, f ); /* recursion */ \
+                _Ostream_Manip(unsigned long long int) fmt @= { lsig, 0, 19, 'u', { .all : 0 } }; \
+                fmt.flags.nobsdp = true; \
+                /* printf( "fmt1 %c %lld %d\n", fmt.base, fmt.val, fmt.all ); */ \
+                sepOff( os ); \
+                (ostype &)(os | fmt); \
+        } else { \
+                /* printf( "fmt2 %c %lld %d\n", f.base, (unsigned long long int)f.val, f.all ); */ \
+                _Ostream_Manip(SIGNED long long int) fmt @= { (SIGNED long long int)f.val, f.wd, f.pc, f.base, { .all : f.all } }; \
+                (ostype &)(os | fmt); \
+        } /* if */ \
+} /* base10_128 */ \
+forall( dtype ostype | ostream( ostype ) ) { \
+        ostype & ?|?( ostype & os, _Ostream_Manip(T) f ) { \
+                if ( $sepPrt( os ) ) fmt( os, "%s", $sepGetCur( os ) ); \
+\
+                if ( f.base == 'b' | f.base == 'B' | f.base == 'o' | f.base == 'x' | f.base == 'X' ) { \
+                        unsigned long long int msig = (unsigned long long int)(f.val >> 64); \
+                        unsigned long long int lsig = (unsigned long long int)(f.val); \
+                        _Ostream_Manip(SIGNED long long int) fmt @= { msig, f.wd, f.pc, f.base, { .all : f.all } }; \
+                        _Ostream_Manip(unsigned long long int) fmt2 @= { lsig, 0, 0, f.base, { .all : 0 } }; \
+                        if ( msig == 0 ) { \
+                                fmt.val = lsig; \
+                                (ostype &)(os | fmt); \
+                        } else { \
+                                fmt2.flags.pad0 = fmt2.flags.nobsdp = true;     \
+                                if ( f.base == 'b' | f.base == 'B' ) { \
+                                        if ( fmt.flags.pc && fmt.pc > 64 ) fmt.pc -= 64; else { fmt.flags.pc = false; fmt.pc = 0; } \
+                                        if ( fmt.flags.left ) { \
+                                                fmt.flags.left = false; \
+                                                fmt.wd = 0; \
+                                                /* printf( "L %llo %llo %llo %d %d '%c' %x\n", msig, lsig, fmt.val, fmt.wd, fmt.pc, fmt.base, fmt.all ); */ \
+                                                fmt2.flags.left = true; \
+                                                int msigd = high1( msig ); \
+                                                fmt2.wd = f.wd - (fmt.pc > msigd ? fmt.pc : msigd); \
+                                                if ( ! fmt.flags.nobsdp ) fmt2.wd -= 2; /* compensate for 0b base specifier */ \
+                                                if ( (int)fmt2.wd < 64 ) fmt2.wd = 64; /* cast deals with negative value */ \
+                                                fmt2.flags.pc = true; fmt2.pc = 64; \
+                                        } else { \
+                                                if ( fmt.wd > 64 ) fmt.wd -= 64; \
+                                                else fmt.wd = 1; \
+                                                /* printf( "R %llo %llo %llo %d %d '%c' %x\n", msig, lsig, fmt.val, fmt.wd, fmt.pc, fmt.base, fmt.all ); */ \
+                                                fmt2.wd = 64; \
+                                        } /* if */ \
+                                        /* printf( "C %llo %d %d '%c' %x\n", fmt2.val, fmt2.wd, fmt2.pc, fmt2.base, fmt2.all ); */ \
+                                        (ostype &)(os | fmt | "" | fmt2); \
+                                } else if ( f.base == 'o' ) { \
+                                        if ( fmt.flags.pc && fmt.pc > 22 ) fmt.pc -= 22; else { fmt.flags.pc = false; fmt.pc = 0; } \
+                                        fmt.val = (unsigned long long int)fmt.val >> 2; \
+                                        fmt2.val = ((msig & 0x3) << 1) + ((lsig & 0x8000000000000000U) != 0); \
+                                        if ( fmt.flags.left ) { \
+                                                fmt.flags.left = false; \
+                                                fmt.wd = 0; \
+                                                /* printf( "L %llo %llo %llo %d %d '%c' %x %llo %d %d '%c' %x\n", msig, lsig, fmt.val, fmt.wd, fmt.pc, fmt.base, fmt.all, fmt2.val, fmt2.wd, fmt2.pc, fmt2.base, fmt2.all ); */ \
+                                                (ostype &)(os | fmt | "" | fmt2); \
+                                                sepOff( os ); \
+                                                fmt2.flags.left = true; \
+                                                int msigd = ceiling_div( high1( fmt.val ), 3 ); \
+                                                fmt2.wd = f.wd - (fmt.pc > msigd ? fmt.pc : msigd); \
+                                                if ( ! fmt.flags.nobsdp ) fmt2.wd -= 1; /* compensate for 0 base specifier */ \
+                                                if ( (int)fmt2.wd < 21 ) fmt2.wd = 21; /* cast deals with negative value */ \
+                                                fmt2.flags.pc = true; fmt2.pc = 21; \
+                                        } else { \
+                                                if ( fmt.wd > 22 ) fmt.wd -= 22; \
+                                                else fmt.wd = 1; \
+                                                /* printf( "R %llo %llo %llo %d %d '%c' %x %llo %d %d '%c' %x\n", msig, lsig, fmt.val, fmt.wd, fmt.pc, fmt.base, fmt.all, fmt2.val, fmt2.wd, fmt2.pc, fmt2.base, fmt2.all ); */ \
+                                                (ostype &)(os | fmt | "" | fmt2); \
+                                                sepOff( os ); \
+                                                fmt2.wd = 21; \
+                                        } /* if */ \
+                                        fmt2.val = lsig & 0x7fffffffffffffffU; \
+                                        /* printf( "\nC %llo %d %d '%c' %x\n", fmt2.val, fmt2.wd, fmt2.pc, fmt2.base, fmt2.all ); */ \
+                                        (ostype &)(os | fmt2); \
+                                } else { /* f.base == 'x'  | f.base == 'X' */ \
+                                        if ( fmt.flags.pc && fmt.pc > 16 ) fmt.pc -= 16; else { fmt.flags.pc = false; fmt.pc = 0; } \
+                                        if ( fmt.flags.left ) { \
+                                                fmt.flags.left = false; \
+                                                fmt.wd = 0; \
+                                                /* printf( "L %llo %llo %llo %d %d '%c' %x\n", msig, lsig, fmt.val, fmt.wd, fmt.pc, fmt.base, fmt.all ); */ \
+                                                fmt2.flags.left = true; \
+                                                int msigd = high1( msig ); \
+                                                fmt2.wd = f.wd - (fmt.pc > msigd ? fmt.pc : msigd); \
+                                                if ( ! fmt.flags.nobsdp ) fmt2.wd -= 2; /* compensate for 0x base specifier */ \
+                                                if ( (int)fmt2.wd < 16 ) fmt2.wd = 16; /* cast deals with negative value */ \
+                                                fmt2.flags.pc = true; fmt2.pc = 16; \
+                                        } else { \
+                                                if ( fmt.wd > 16 ) fmt.wd -= 16; \
+                                                else fmt.wd = 1; \
+                                                /* printf( "R %llo %llo %llo %d %d '%c' %x\n", msig, lsig, fmt.val, fmt.wd, fmt.pc, fmt.base, fmt.all ); */ \
+                                                fmt2.wd = 16; \
+                                        } /* if */ \
+                                        /* printf( "C %llo %d %d '%c' %x\n", fmt2.val, fmt2.wd, fmt2.pc, fmt2.base, fmt2.all ); */ \
+                                        (ostype &)(os | fmt | "" | fmt2); \
+                                } /* if */ \
+                        } /* if */ \
+                } else { \
+                        if ( CODE == 'd' ) { \
+                                if ( f.val < 0 )  { fmt( os, "-" ); sepOff( os ); f.val = -f.val; f.flags.sign = false; } \
+                        } /* if */ \
+                        base10_128( os, f ); \
+                } /* if */ \
+                return os; \
+        } /* ?|? */ \
+        void ?|?( ostype & os, _Ostream_Manip(T) f ) { (ostype &)(os | f); ends( os ); } \
+} // distribution
+
+IntegralFMTImpl128( int128, signed, 'd', "%    *ll ", "%    *.*ll " )
+IntegralFMTImpl128( unsigned int128, unsigned, 'u', "%    *ll ", "%    *.*ll " )
+#endif // __SIZEOF_INT128__
+#endif // 0
+
+#if 1
+#if defined( __SIZEOF_INT128__ )
+// Default prefix for non-decimal prints is 0b, 0, 0x.
+forall( dtype ostype | ostream( ostype ) )
+static inline void base_128( ostype & os, unsigned int128 val, unsigned int128 power, _Ostream_Manip(uint64_t) & f, unsigned int maxdig, unsigned int bits, unsigned int cnt = 0 ) {
+        int wd = 1;                                                                                     // f.wd is never 0 because 0 implies left-pad
+        if ( val > power ) {                                                            // subdivide value into printable 64-bit values
+                base_128( os, val / power, power, f, maxdig, bits, cnt + 1 ); // recursive
+                f.val = val % power;
+                if ( cnt == 1 && f.flags.left ) { wd = f.wd; f.wd = maxdig; } // copy f.wd and reset for printing middle chunk
+                // printf( "R val:%#lx(%lu) wd:%u pc:%u base:%c neg:%d pc:%d left:%d nobsdp:%d sign:%d pad0:%d\n",
+                //              f.val, f.val, f.wd, f.pc, f.base, f.flags.neg, f.flags.pc, f.flags.left, f.flags.nobsdp, f.flags.sign, f.flags.pad0 ); 
+                (ostype &)(os | f);
+                if ( cnt == 1 ) {
+                        if ( f.flags.left ) { wd -= maxdig; f.wd = wd < 0 ? 1 : wd; } // update and restore f.wd for printing end chunk
+                        sepOff( os );                                                           // no seperator between chunks
+                } // if
+        } else {                                                                                        // print start chunk
+                f.val = val;
+                // f.pc is unsigned => use wd
+                if ( f.flags.pc && f.pc > maxdig * cnt ) { wd = f.pc - maxdig * cnt; f.pc = wd < 0 ? 0 : wd; }
+                else { f.flags.pc = false; f.pc = 0; }
+
+                if ( ! f.flags.left ) {                                                 // right justify
+                        wd = f.wd - maxdig * cnt;
+                        f.wd = wd < 0 ? 1 : wd;
+                        wd = maxdig;
+                } else {                                                                                // left justify
+                        if ( cnt != 0 ) {                                                       // value >= 2^64 ?
+                                unsigned int dig, bs = 0;
+                                // compute size of prefix digits and base
+                                if ( f.base == 'd' || f.base == 'u' ) { // no base prefix
+                                        dig = ceil( log10( f.val ) );           // use floating-point
+                                        if ( f.base == 'd' && (f.flags.neg || f.flags.sign) ) bs = 1; // sign ?
+                                } else {
+                                        dig = ceiling_div( high1( f.val ), bits );
+                                        if ( ! f.flags.nobsdp ) {                       // base prefix ?
+                                                if ( f.base == 'o' ) {
+                                                        // 0 prefix for octal is not added for precision with leading zero
+                                                        if ( f.pc <= dig ) bs = 1;      // 1 character prefix
+                                                } else bs = 2;                                  // 2 character prefix
+                                        } // if
+                                } // if
+                                wd = f.wd - (f.pc > dig ? f.pc : dig) - bs; // precision > leading digits ?
+                                if ( wd < 0 ) wd = 1;
+                                f.wd = 1;
+                        } // if
+                        // all manipulators handled implicitly for value < 2^64
+                } // if
+                // prior checks ensure wd not negative
+
+                if ( f.flags.neg ) f.val = -f.val;
+                // printf( "L val:%#lx(%lu) wd:%u pc:%u base:%c neg:%d pc:%d left:%d nobsdp:%d sign:%d pad0:%d\n",
+                //              f.val, f.val, f.wd, f.pc, f.base, f.flags.neg, f.flags.pc, f.flags.left, f.flags.nobsdp, f.flags.sign, f.flags.pad0 ); 
+                (ostype &)(os | f);
+
+                // remaining middle and end chunks are padded with 0s on the left
+                if ( ! f.flags.left ) { f.flags.pad0 = true; f.flags.pc = false; } // left pad with 0s
+                else { f.pc = maxdig; f.flags.pc = true; }              // left pad with precision
+
+                if ( cnt != 0 ) sepOff( os );                                   // no seperator between chunks
+                f.wd = wd;                                                                              // reset f.wd for next chunk
+                f.flags.sign = false;                                                   // no leading +/- sign
+                f.flags.nobsdp = true;                                                  // no leading base prefix
+        } // if
+} // base_128
+
+#define IntegralFMTImpl128( T ) \
+forall( dtype ostype | ostream( ostype ) ) { \
+        ostype & ?|?( ostype & os, _Ostream_Manip(T) f ) { \
+                _Ostream_Manip(uint64_t) fmt; \
+                fmt.[wd, pc, base, all] = f.[wd, pc, base, all]; \
+                if ( f.base == 'b' | f.base == 'B' ) { \
+                        base_128( os, f.val, (unsigned int128)1 << 64, fmt, 64, 1 ); \
+                } else if ( f.base == 'o' ) { \
+                        base_128( os, f.val, (unsigned int128)1 << 63, fmt, 21, 3 ); \
+                } else if ( f.base == 'd' || f.base == 'u' ) { \
+                        if ( f.base == 'd' && f.val < 0 ) { f.val = -f.val; fmt.flags.neg = true; } \
+                        base_128( os, f.val, (unsigned int128)10_000_000_000_000_000_000UL, fmt, 19, 0 ); \
+                } else { \
+                        base_128( os, f.val, (unsigned int128)1 << 64, fmt, 16, 4 ); \
+                } /* if */ \
+                return os; \
+        } /* ?|? */ \
+        void ?|?( ostype & os, _Ostream_Manip(T) f ) { (ostype &)(os | f); ends( os ); } \
+} // distribution
+
+IntegralFMTImpl128( int128 )
+IntegralFMTImpl128( unsigned int128 )
+#endif // __SIZEOF_INT128__
+#endif // 0
+
+// *********************************** floating point ***********************************
 
 #define PrintWithDP2( os, format, val, ... ) \
@@ -513 +769 @@
 forall( dtype ostype | ostream( ostype ) ) { \
         ostype & ?|?( ostype & os, _Ostream_Manip(T) f ) { \
-                if ( sepPrt( os ) ) fmt( os, "%s", sepGetCur( os ) ); \
+                if ( $sepPrt( os ) ) fmt( os, "%s", $sepGetCur( os ) ); \
                 char fmtstr[sizeof(DFMTP)];                                             /* sizeof includes '\0' */ \
                 if ( ! f.flags.pc ) memcpy( &fmtstr, DFMTNP, sizeof(DFMTNP) ); \
@@ -536 +792 @@
                 return os; \
         } /* ?|? */ \
+\
         void ?|?( ostype & os, _Ostream_Manip(T) f ) { (ostype &)(os | f); ends( os ); } \
 } // distribution
@@ -542 +799 @@
 FloatingPointFMTImpl( long double, "%    *L ", "%    *.*L " )
 
-//*********************************** character ***********************************
+// *********************************** character ***********************************
 
 forall( dtype ostype | ostream( ostype ) ) {
@@ -555 +812 @@
                 } // if
 
-                if ( sepPrt( os ) ) fmt( os, "%s", sepGetCur( os ) );
+                if ( $sepPrt( os ) ) fmt( os, "%s", $sepGetCur( os ) );
 
                 #define CFMTNP "% * "
@@ -571 +828 @@
                 return os;
         } // ?|?
+
         void ?|?( ostype & os, _Ostream_Manip(char) f ) { (ostype &)(os | f); ends( os ); }
 } // distribution
 
-//*********************************** C string ***********************************
+// *********************************** C string ***********************************
 
 forall( dtype ostype | ostream( ostype ) ) {
@@ -592 +850 @@
                 } // if
 
-                if ( sepPrt( os ) ) fmt( os, "%s", sepGetCur( os ) );
+                if ( $sepPrt( os ) ) fmt( os, "%s", $sepGetCur( os ) );
 
                 #define SFMTNP "% * "
@@ -616 +874 @@
                 return os;
         } // ?|?
+
         void ?|?( ostype & os, _Ostream_Manip(const char *) f ) { (ostype &)(os | f); ends( os ); }
 } // distribution
 
 
-//*********************************** istream ***********************************
+// *********************************** istream ***********************************
 
 
@@ -697 +956 @@
         } // ?|?
 
+#if defined( __SIZEOF_INT128__ )
+        istype & ?|?( istype & is, int128 & i128 ) {
+                return (istype &)(is | (unsigned int128 &)i128);
+        } // ?|?
+
+        istype & ?|?( istype & is, unsigned int128 & ui128 ) {
+                char s[40];
+                bool sign = false;
+
+                if ( fmt( is, " %[-]", s ) == 1 ) sign = true;  // skip whitespace, negative sign ?
+                // If the input is too large, the value returned is undefined. If there is no input, no value is returned
+                if ( fmt( is, "%39[0-9]%*[0-9]", s ) == 1 ) {   // take first 39 characters, ignore remaining
+                        ui128 = 0;
+                        for ( unsigned int i = 0; s[i] != '\0'; i += 1 ) {
+                                ui128 = ui128 * 10 + s[i] - '0';
+                        } // for
+                        if ( sign ) ui128 = -ui128;
+                } else if ( sign ) ungetc( is, '-' );                   // return minus when no digits
+                return is;
+        } // ?|?
+#endif // __SIZEOF_INT128__
 
         istype & ?|?( istype & is, float & f ) {
@@ -735 +1015 @@
         } // ?|?
 
-        // istype & ?|?( istype & is, const char * fmt ) {
+        // istype & ?|?( istype & is, const char fmt[] ) {
         //      fmt( is, fmt, "" );
         //      return is;
@@ -766 +1046 @@
 } // distribution
 
-//*********************************** manipulators ***********************************
+// *********************************** manipulators ***********************************
 
 forall( dtype istype | istream( istype ) )

libcfa/src/iostream.hfa

@@ -10 +10 @@
 // Created On       : Wed May 27 17:56:53 2015
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Fri Jul 12 12:08:38 2019
-// Update Count     : 334
+// Last Modified On : Tue Aug 11 22:16:14 2020
+// Update Count     : 350
 //
 
@@ -19 +19 @@
 
 
-//*********************************** ostream ***********************************
+// *********************************** ostream ***********************************
 
 
 trait ostream( dtype ostype ) {
         // private
-        bool sepPrt( ostype & );                                                        // get separator state (on/off)
-        void sepReset( ostype & );                                                      // set separator state to default state
-        void sepReset( ostype &, bool );                                        // set separator and default state
-        const char * sepGetCur( ostype & );                                     // get current separator string
-        void sepSetCur( ostype &, const char * );                       // set current separator string
-        bool getNL( ostype & );                                                         // check newline
-        void setNL( ostype &, bool );                                           // saw newline
-        bool getANL( ostype & );                                                        // get auto newline (on/off)
-        bool getPrt( ostype & );                                                        // get fmt called in output cascade
-        void setPrt( ostype &, bool );                                          // set fmt called in output cascade
+        bool $sepPrt( ostype & );                                                       // get separator state (on/off)
+        void $sepReset( ostype & );                                                     // set separator state to default state
+        void $sepReset( ostype &, bool );                                       // set separator and default state
+        const char * $sepGetCur( ostype & );                            // get current separator string
+        void $sepSetCur( ostype &, const char [] );                     // set current separator string
+        bool $getNL( ostype & );                                                        // check newline
+        void $setNL( ostype &, bool );                                          // saw newline
+        bool $getANL( ostype & );                                                       // get auto newline (on/off)
+        bool $getPrt( ostype & );                                                       // get fmt called in output cascade
+        void $setPrt( ostype &, bool );                                         // set fmt called in output cascade
         // public
         void sepOn( ostype & );                                                         // turn separator state on
@@ -43 +43 @@
 
         const char * sepGet( ostype & );                                        // get separator string
-        void sepSet( ostype &, const char * );                          // set separator to string (15 character maximum)
+        void sepSet( ostype &, const char [] );                         // set separator to string (15 character maximum)
         const char * sepGetTuple( ostype & );                           // get tuple separator string
-        void sepSetTuple( ostype &, const char * );                     // set tuple separator to string (15 character maximum)
+        void sepSetTuple( ostype &, const char [] );            // set tuple separator to string (15 character maximum)
 
         void ends( ostype & os );                                                       // end of output statement
         int fail( ostype & );
         int flush( ostype & );
-        void open( ostype & os, const char * name, const char * mode );
+        void open( ostype & os, const char name[], const char mode[] );
         void close( ostype & os );
-        ostype & write( ostype &, const char *, size_t );
+        ostype & write( ostype &, const char [], size_t );
         int fmt( ostype &, const char format[], ... ) __attribute__(( format(printf, 2, 3) ));
 }; // ostream
@@ -67 +67 @@
 
 forall( dtype ostype | ostream( ostype ) ) {
-        ostype & ?|?( ostype &, zero_t );
-        void ?|?( ostype &, zero_t );
-        ostype & ?|?( ostype &, one_t );
-        void ?|?( ostype &, one_t );
-
         ostype & ?|?( ostype &, bool );
         void ?|?( ostype &, bool );
@@ -98 +93 @@
         ostype & ?|?( ostype &, unsigned long long int );
         void ?|?( ostype &, unsigned long long int );
+#if defined( __SIZEOF_INT128__ )
+        ostype & ?|?( ostype &, int128 );
+        void ?|?( ostype &, int128 );
+        ostype & ?|?( ostype &, unsigned int128 );
+        void ?|?( ostype &, unsigned int128 );
+#endif // __SIZEOF_INT128__
 
         ostype & ?|?( ostype &, float );
@@ -113 +114 @@
         void ?|?( ostype &, long double _Complex );
 
-        ostype & ?|?( ostype &, const char * );
-        void ?|?( ostype &, const char * );
+        ostype & ?|?( ostype &, const char [] );
+        void ?|?( ostype &, const char [] );
         // ostype & ?|?( ostype &, const char16_t * );
 #if ! ( __ARM_ARCH_ISA_ARM == 1 && __ARM_32BIT_STATE == 1 ) // char32_t == wchar_t => ambiguous
@@ -150 +151 @@
 } // distribution
 
-//*********************************** manipulators ***********************************
+// *********************************** manipulators ***********************************
 
 forall( otype T )
@@ -160 +161 @@
                 unsigned char all;
                 struct {
+                        unsigned char neg:1;                                            // val is negative
                         unsigned char pc:1;                                                     // precision specified
                         unsigned char left:1;                                           // left justify
@@ -169 +171 @@
 }; // _Ostream_Manip
 
-//*********************************** integral ***********************************
+// *********************************** integral ***********************************
 
 // See 6.7.9. 19) The initialization shall occur in initializer list order, each initializer provided for a particular
@@ -206 +208 @@
 IntegralFMTDecl( signed long long int, 'd' )
 IntegralFMTDecl( unsigned long long int, 'u' )
-
-//*********************************** floating point ***********************************
+#if defined( __SIZEOF_INT128__ )
+IntegralFMTDecl( int128, 'd' )
+IntegralFMTDecl( unsigned int128, 'u' )
+#endif // __SIZEOF_INT128__
+
+// *********************************** floating point ***********************************
 
 // Default suffix for values with no fraction is "."
@@ -236 +242 @@
 FloatingPointFMTDecl( long double )
 
-//*********************************** character ***********************************
+// *********************************** character ***********************************
 
 static inline {
@@ -253 +259 @@
 } // ?|?
 
-//*********************************** C string ***********************************
+// *********************************** C string ***********************************
 
 static inline {
-        _Ostream_Manip(const char *) bin( const char * s ) { return (_Ostream_Manip(const char *))@{ s, 1, 0, 'b', { .all : 0 } }; }
-        _Ostream_Manip(const char *) oct( const char * s ) { return (_Ostream_Manip(const char *))@{ s, 1, 0, 'o', { .all : 0 } }; }
-        _Ostream_Manip(const char *) hex( const char * s ) { return (_Ostream_Manip(const char *))@{ s, 1, 0, 'x', { .all : 0 } }; }
-        _Ostream_Manip(const char *) wd( unsigned int w, const char * s ) { return (_Ostream_Manip(const char *))@{ s, w, 0, 's', { .all : 0 } }; }
-        _Ostream_Manip(const char *) wd( unsigned int w, unsigned char pc, const char * s ) { return (_Ostream_Manip(const char *))@{ s, w, pc, 's', { .flags.pc : true } }; }
+        _Ostream_Manip(const char *) bin( const char s[] ) { return (_Ostream_Manip(const char *))@{ s, 1, 0, 'b', { .all : 0 } }; }
+        _Ostream_Manip(const char *) oct( const char s[] ) { return (_Ostream_Manip(const char *))@{ s, 1, 0, 'o', { .all : 0 } }; }
+        _Ostream_Manip(const char *) hex( const char s[] ) { return (_Ostream_Manip(const char *))@{ s, 1, 0, 'x', { .all : 0 } }; }
+        _Ostream_Manip(const char *) wd( unsigned int w, const char s[] ) { return (_Ostream_Manip(const char *))@{ s, w, 0, 's', { .all : 0 } }; }
+        _Ostream_Manip(const char *) wd( unsigned int w, unsigned char pc, const char s[] ) { return (_Ostream_Manip(const char *))@{ s, w, pc, 's', { .flags.pc : true } }; }
         _Ostream_Manip(const char *) & wd( unsigned int w, _Ostream_Manip(const char *) & fmt ) { fmt.wd = w; return fmt; }
         _Ostream_Manip(const char *) & wd( unsigned int w, unsigned char pc, _Ostream_Manip(const char *) & fmt ) { fmt.wd = w; fmt.pc = pc; fmt.flags.pc = true; return fmt; }
@@ -272 +278 @@
 
 
-//*********************************** istream ***********************************
+// *********************************** istream ***********************************
 
 
@@ -281 +287 @@
         int fail( istype & );
         int eof( istype & );
-        void open( istype & is, const char * name );
+        void open( istype & is, const char name[] );
         void close( istype & is );
         istype & read( istype &, char *, size_t );
@@ -304 +310 @@
         istype & ?|?( istype &, unsigned int & );
         istype & ?|?( istype &, long int & );
+        istype & ?|?( istype &, unsigned long int & );
         istype & ?|?( istype &, long long int & );
-        istype & ?|?( istype &, unsigned long int & );
         istype & ?|?( istype &, unsigned long long int & );
+#if defined( __SIZEOF_INT128__ )
+        istype & ?|?( istype &, int128 & );
+        istype & ?|?( istype &, unsigned int128 & );
+#endif // __SIZEOF_INT128__
 
         istype & ?|?( istype &, float & );
@@ -316 +326 @@
         istype & ?|?( istype &, long double _Complex & );
 
-//      istype & ?|?( istype &, const char * );
+//      istype & ?|?( istype &, const char [] );
         istype & ?|?( istype &, char * );
 
@@ -326 +336 @@
 } // distribution
 
-//*********************************** manipulators ***********************************
+// *********************************** manipulators ***********************************
 
 struct _Istream_Cstr {
@@ -343 +353 @@
 static inline {
         _Istream_Cstr skip( unsigned int n ) { return (_Istream_Cstr){ 0p, 0p, n, { .all : 0 } }; }
-        _Istream_Cstr skip( const char * scanset ) { return (_Istream_Cstr){ 0p, scanset, -1, { .all : 0 } }; }
-        _Istream_Cstr incl( const char * scanset, char * s ) { return (_Istream_Cstr){ s, scanset, -1, { .flags.inex : false } }; }
-        _Istream_Cstr & incl( const char * scanset, _Istream_Cstr & fmt ) { fmt.scanset = scanset; fmt.flags.inex = false; return fmt; }
-        _Istream_Cstr excl( const char * scanset, char * s ) { return (_Istream_Cstr){ s, scanset, -1, { .flags.inex : true } }; }
-        _Istream_Cstr & excl( const char * scanset, _Istream_Cstr & fmt ) { fmt.scanset = scanset; fmt.flags.inex = true; return fmt; }
-        _Istream_Cstr ignore( const char * s ) { return (_Istream_Cstr)@{ s, 0p, -1, { .flags.ignore : true } }; }
+        _Istream_Cstr skip( const char scanset[] ) { return (_Istream_Cstr){ 0p, scanset, -1, { .all : 0 } }; }
+        _Istream_Cstr incl( const char scanset[], char * s ) { return (_Istream_Cstr){ s, scanset, -1, { .flags.inex : false } }; }
+        _Istream_Cstr & incl( const char scanset[], _Istream_Cstr & fmt ) { fmt.scanset = scanset; fmt.flags.inex = false; return fmt; }
+        _Istream_Cstr excl( const char scanset[], char * s ) { return (_Istream_Cstr){ s, scanset, -1, { .flags.inex : true } }; }
+        _Istream_Cstr & excl( const char scanset[], _Istream_Cstr & fmt ) { fmt.scanset = scanset; fmt.flags.inex = true; return fmt; }
+        _Istream_Cstr ignore( char s[] ) { return (_Istream_Cstr)@{ s, 0p, -1, { .flags.ignore : true } }; }
         _Istream_Cstr & ignore( _Istream_Cstr & fmt ) { fmt.flags.ignore = true; return fmt; }
-        _Istream_Cstr wdi( unsigned int w, char * s ) { return (_Istream_Cstr)@{ s, 0p, w, { .all : 0 } }; }
+        _Istream_Cstr wdi( unsigned int w, char s[] ) { return (_Istream_Cstr)@{ s, 0p, w, { .all : 0 } }; }
         _Istream_Cstr & wdi( unsigned int w, _Istream_Cstr & fmt ) { fmt.wd = w; return fmt; }
 } // distribution
@@ -360 +370 @@
 
 static inline {
-        _Istream_Char ignore( const char c ) { return (_Istream_Char)@{ true }; }
+        _Istream_Char ignore( const char ) { return (_Istream_Char)@{ true }; }
         _Istream_Char & ignore( _Istream_Char & fmt ) { fmt.ignore = true; return fmt; }
 } // distribution
 forall( dtype istype | istream( istype ) ) istype & ?|?( istype & is, _Istream_Char f );
 
-forall( otype T )
+forall( dtype T | sized( T ) )
 struct _Istream_Manip {
         T & val;                                                                                        // polymorphic base-type
@@ -403 +413 @@
 
 
-//*********************************** time ***********************************
+// *********************************** time ***********************************
 
 

libcfa/src/math.hfa

@@ -10 +10 @@
 // Created On       : Mon Apr 18 23:37:04 2016
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Fri Jul 13 11:02:15 2018
-// Update Count     : 116
+// Last Modified On : Mon Aug 24 08:56:20 2020
+// Update Count     : 126
 //
 
@@ -19 +19 @@
 #include <complex.h>
 
+//---------------------------------------
+
+#include "common.hfa"
+
 //---------------------- General ----------------------
 
-static inline float ?%?( float x, float y ) { return fmodf( x, y ); }
-static inline float fmod( float x, float y ) { return fmodf( x, y ); }
-static inline double ?%?( double x, double y ) { return fmod( x, y ); }
-// extern "C" { double fmod( double, double ); }
-static inline long double ?%?( long double x, long double y ) { return fmodl( x, y ); }
-static inline long double fmod( long double x, long double y ) { return fmodl( x, y ); }
-
-static inline float remainder( float x, float y ) { return remainderf( x, y ); }
-// extern "C" { double remainder( double, double ); }
-static inline long double remainder( long double x, long double y ) { return remainderl( x, y ); }
-
-static inline float remquo( float x, float y, int * quo ) { return remquof( x, y, quo ); }
-// extern "C" { double remquo( double x, double y, int * quo ); }
-static inline long double remquo( long double x, long double y, int * quo ) { return remquol( x, y, quo ); }
-static inline [ int, float ] remquo( float x, float y ) { int quo; x = remquof( x, y, &quo ); return [ quo, x ]; }
-static inline [ int, double ] remquo( double x, double y ) { int quo; x = remquo( x, y, &quo ); return [ quo, x ]; }
-static inline [ int, long double ] remquo( long double x, long double y ) { int quo; x = remquol( x, y, &quo ); return [ quo, x ]; }
-
-static inline [ float, float ] div( float x, float y ) { y = modff( x / y, &x ); return [ x, y ]; }
-static inline [ double, double ] div( double x, double y ) { y = modf( x / y, &x ); return [ x, y ]; }
-static inline [ long double, long double ] div( long double x, long double y ) { y = modfl( x / y, &x ); return [ x, y ]; }
-
-static inline float fma( float x, float y, float z ) { return fmaf( x, y, z ); }
-// extern "C" { double fma( double, double, double ); }
-static inline long double fma( long double x, long double y, long double z ) { return fmal( x, y, z ); }
-
-static inline float fdim( float x, float y ) { return fdimf( x, y ); }
-// extern "C" { double fdim( double, double ); }
-static inline long double fdim( long double x, long double y ) { return fdiml( x, y ); }
-
-static inline float nan( const char * tag ) { return nanf( tag ); }
-// extern "C" { double nan( const char * ); }
-static inline long double nan( const char * tag ) { return nanl( tag ); }
+static inline {
+        float ?%?( float x, float y ) { return fmodf( x, y ); }
+        float fmod( float x, float y ) { return fmodf( x, y ); }
+        double ?%?( double x, double y ) { return fmod( x, y ); }
+        // extern "C" { double fmod( double, double ); }
+        long double ?%?( long double x, long double y ) { return fmodl( x, y ); }
+        long double fmod( long double x, long double y ) { return fmodl( x, y ); }
+
+        float remainder( float x, float y ) { return remainderf( x, y ); }
+        // extern "C" { double remainder( double, double ); }
+        long double remainder( long double x, long double y ) { return remainderl( x, y ); }
+
+        float remquo( float x, float y, int * quo ) { return remquof( x, y, quo ); }
+        // extern "C" { double remquo( double x, double y, int * quo ); }
+        long double remquo( long double x, long double y, int * quo ) { return remquol( x, y, quo ); }
+        [ int, float ] remquo( float x, float y ) { int quo; x = remquof( x, y, &quo ); return [ quo, x ]; }
+        [ int, double ] remquo( double x, double y ) { int quo; x = remquo( x, y, &quo ); return [ quo, x ]; }
+        [ int, long double ] remquo( long double x, long double y ) { int quo; x = remquol( x, y, &quo ); return [ quo, x ]; }
+
+        [ float, float ] div( float x, float y ) { y = modff( x / y, &x ); return [ x, y ]; }
+        [ double, double ] div( double x, double y ) { y = modf( x / y, &x ); return [ x, y ]; }
+        [ long double, long double ] div( long double x, long double y ) { y = modfl( x / y, &x ); return [ x, y ]; }
+
+        float fma( float x, float y, float z ) { return fmaf( x, y, z ); }
+        // extern "C" { double fma( double, double, double ); }
+        long double fma( long double x, long double y, long double z ) { return fmal( x, y, z ); }
+
+        float fdim( float x, float y ) { return fdimf( x, y ); }
+        // extern "C" { double fdim( double, double ); }
+        long double fdim( long double x, long double y ) { return fdiml( x, y ); }
+
+        float nan( const char tag[] ) { return nanf( tag ); }
+        // extern "C" { double nan( const char [] ); }
+        long double nan( const char tag[] ) { return nanl( tag ); }
+} // distribution
 
 //---------------------- Exponential ----------------------
 
-static inline float exp( float x ) { return expf( x ); }
-// extern "C" { double exp( double ); }
-static inline long double exp( long double x ) { return expl( x ); }
-static inline float _Complex exp( float _Complex x ) { return cexpf( x ); }
-static inline double _Complex exp( double _Complex x ) { return cexp( x ); }
-static inline long double _Complex exp( long double _Complex x ) { return cexpl( x ); }
-
-static inline float exp2( float x ) { return exp2f( x ); }
-// extern "C" { double exp2( double ); }
-static inline long double exp2( long double x ) { return exp2l( x ); }
-//static inline float _Complex exp2( float _Complex x ) { return cexp2f( x ); }
-//static inline double _Complex exp2( double _Complex x ) { return cexp2( x ); }
-//static inline long double _Complex exp2( long double _Complex x ) { return cexp2l( x ); }
-
-static inline float expm1( float x ) { return expm1f( x ); }
-// extern "C" { double expm1( double ); }
-static inline long double expm1( long double x ) { return expm1l( x ); }
-
-static inline float pow( float x, float y ) { return powf( x, y ); }
-// extern "C" { double pow( double, double ); }
-static inline long double pow( long double x, long double y ) { return powl( x, y ); }
-static inline float _Complex pow( float _Complex x, float _Complex y ) { return cpowf( x, y ); }
-static inline double _Complex pow( double _Complex x, double _Complex y ) { return cpow( x, y ); }
-static inline long double _Complex pow( long double _Complex x, long double _Complex y ) { return cpowl( x, y ); }
+static inline {
+        float exp( float x ) { return expf( x ); }
+        // extern "C" { double exp( double ); }
+        long double exp( long double x ) { return expl( x ); }
+        float _Complex exp( float _Complex x ) { return cexpf( x ); }
+        double _Complex exp( double _Complex x ) { return cexp( x ); }
+        long double _Complex exp( long double _Complex x ) { return cexpl( x ); }
+
+        float exp2( float x ) { return exp2f( x ); }
+        // extern "C" { double exp2( double ); }
+        long double exp2( long double x ) { return exp2l( x ); }
+        //float _Complex exp2( float _Complex x ) { return cexp2f( x ); }
+        //double _Complex exp2( double _Complex x ) { return cexp2( x ); }
+        //long double _Complex exp2( long double _Complex x ) { return cexp2l( x ); }
+
+        float expm1( float x ) { return expm1f( x ); }
+        // extern "C" { double expm1( double ); }
+        long double expm1( long double x ) { return expm1l( x ); }
+
+        float pow( float x, float y ) { return powf( x, y ); }
+        // extern "C" { double pow( double, double ); }
+        long double pow( long double x, long double y ) { return powl( x, y ); }
+        float _Complex pow( float _Complex x, float _Complex y ) { return cpowf( x, y ); }
+        double _Complex pow( double _Complex x, double _Complex y ) { return cpow( x, y ); }
+        long double _Complex pow( long double _Complex x, long double _Complex y ) { return cpowl( x, y ); }
+} // distribution
 
 //---------------------- Logarithm ----------------------
 
-static inline float log( float x ) { return logf( x ); }
-// extern "C" { double log( double ); }
-static inline long double log( long double x ) { return logl( x ); }
-static inline float _Complex log( float _Complex x ) { return clogf( x ); }
-static inline double _Complex log( double _Complex x ) { return clog( x ); }
-static inline long double _Complex log( long double _Complex x ) { return clogl( x ); }
-
-static inline float log2( float x ) { return log2f( x ); }
-// extern "C" { double log2( double ); }
-static inline long double log2( long double x ) { return log2l( x ); }
-// static inline float _Complex log2( float _Complex x ) { return clog2f( x ); }
-// static inline double _Complex log2( double _Complex x ) { return clog2( x ); }
-// static inline long double _Complex log2( long double _Complex x ) { return clog2l( x ); }
-
-static inline float log10( float x ) { return log10f( x ); }
-// extern "C" { double log10( double ); }
-static inline long double log10( long double x ) { return log10l( x ); }
-// static inline float _Complex log10( float _Complex x ) { return clog10f( x ); }
-// static inline double _Complex log10( double _Complex x ) { return clog10( x ); }
-// static inline long double _Complex log10( long double _Complex x ) { return clog10l( x ); }
-
-static inline float log1p( float x ) { return log1pf( x ); }
-// extern "C" { double log1p( double ); }
-static inline long double log1p( long double x ) { return log1pl( x ); }
-
-static inline int ilogb( float x ) { return ilogbf( x ); }
-// extern "C" { int ilogb( double ); }
-static inline int ilogb( long double x ) { return ilogbl( x ); }
-
-static inline float logb( float x ) { return logbf( x ); }
-// extern "C" { double logb( double ); }
-static inline long double logb( long double x ) { return logbl( x ); }
-
-static inline float sqrt( float x ) { return sqrtf( x ); }
-// extern "C" { double sqrt( double ); }
-static inline long double sqrt( long double x ) { return sqrtl( x ); }
-static inline float _Complex sqrt( float _Complex x ) { return csqrtf( x ); }
-static inline double _Complex sqrt( double _Complex x ) { return csqrt( x ); }
-static inline long double _Complex sqrt( long double _Complex x ) { return csqrtl( x ); }
-
-static inline float cbrt( float x ) { return cbrtf( x ); }
-// extern "C" { double cbrt( double ); }
-static inline long double cbrt( long double x ) { return cbrtl( x ); }
-
-static inline float hypot( float x, float y ) { return hypotf( x, y ); }
-// extern "C" { double hypot( double, double ); }
-static inline long double hypot( long double x, long double y ) { return hypotl( x, y ); }
+static inline {
+        float log( float x ) { return logf( x ); }
+        // extern "C" { double log( double ); }
+        long double log( long double x ) { return logl( x ); }
+        float _Complex log( float _Complex x ) { return clogf( x ); }
+        double _Complex log( double _Complex x ) { return clog( x ); }
+        long double _Complex log( long double _Complex x ) { return clogl( x ); }
+
+        float log2( float x ) { return log2f( x ); }
+        // extern "C" { double log2( double ); }
+        long double log2( long double x ) { return log2l( x ); }
+        // float _Complex log2( float _Complex x ) { return clog2f( x ); }
+        // double _Complex log2( double _Complex x ) { return clog2( x ); }
+        // long double _Complex log2( long double _Complex x ) { return clog2l( x ); }
+
+        float log10( float x ) { return log10f( x ); }
+        // extern "C" { double log10( double ); }
+        long double log10( long double x ) { return log10l( x ); }
+        // float _Complex log10( float _Complex x ) { return clog10f( x ); }
+        // double _Complex log10( double _Complex x ) { return clog10( x ); }
+        // long double _Complex log10( long double _Complex x ) { return clog10l( x ); }
+
+        float log1p( float x ) { return log1pf( x ); }
+        // extern "C" { double log1p( double ); }
+        long double log1p( long double x ) { return log1pl( x ); }
+
+        int ilogb( float x ) { return ilogbf( x ); }
+        // extern "C" { int ilogb( double ); }
+        int ilogb( long double x ) { return ilogbl( x ); }
+
+        float logb( float x ) { return logbf( x ); }
+        // extern "C" { double logb( double ); }
+        long double logb( long double x ) { return logbl( x ); }
+
+        float sqrt( float x ) { return sqrtf( x ); }
+        // extern "C" { double sqrt( double ); }
+        long double sqrt( long double x ) { return sqrtl( x ); }
+        float _Complex sqrt( float _Complex x ) { return csqrtf( x ); }
+        double _Complex sqrt( double _Complex x ) { return csqrt( x ); }
+        long double _Complex sqrt( long double _Complex x ) { return csqrtl( x ); }
+
+        float cbrt( float x ) { return cbrtf( x ); }
+        // extern "C" { double cbrt( double ); }
+        long double cbrt( long double x ) { return cbrtl( x ); }
+
+        float hypot( float x, float y ) { return hypotf( x, y ); }
+        // extern "C" { double hypot( double, double ); }
+        long double hypot( long double x, long double y ) { return hypotl( x, y ); }
+} // distribution
 
 //---------------------- Trigonometric ----------------------
 
-static inline float sin( float x ) { return sinf( x ); }
-// extern "C" { double sin( double ); }
-static inline long double sin( long double x ) { return sinl( x ); }
-static inline float _Complex sin( float _Complex x ) { return csinf( x ); }
-static inline double _Complex sin( double _Complex x ) { return csin( x ); }
-static inline long double _Complex sin( long double _Complex x ) { return csinl( x ); }
-
-static inline float cos( float x ) { return cosf( x ); }
-// extern "C" { double cos( double ); }
-static inline long double cos( long double x ) { return cosl( x ); }
-static inline float _Complex cos( float _Complex x ) { return ccosf( x ); }
-static inline double _Complex cos( double _Complex x ) { return ccos( x ); }
-static inline long double _Complex cos( long double _Complex x ) { return ccosl( x ); }
-
-static inline float tan( float x ) { return tanf( x ); }
-// extern "C" { double tan( double ); }
-static inline long double tan( long double x ) { return tanl( x ); }
-static inline float _Complex tan( float _Complex x ) { return ctanf( x ); }
-static inline double _Complex tan( double _Complex x ) { return ctan( x ); }
-static inline long double _Complex tan( long double _Complex x ) { return ctanl( x ); }
-
-static inline float asin( float x ) { return asinf( x ); }
-// extern "C" { double asin( double ); }
-static inline long double asin( long double x ) { return asinl( x ); }
-static inline float _Complex asin( float _Complex x ) { return casinf( x ); }
-static inline double _Complex asin( double _Complex x ) { return casin( x ); }
-static inline long double _Complex asin( long double _Complex x ) { return casinl( x ); }
-
-static inline float acos( float x ) { return acosf( x ); }
-// extern "C" { double acos( double ); }
-static inline long double acos( long double x ) { return acosl( x ); }
-static inline float _Complex acos( float _Complex x ) { return cacosf( x ); }
-static inline double _Complex acos( double _Complex x ) { return cacos( x ); }
-static inline long double _Complex acos( long double _Complex x ) { return cacosl( x ); }
-
-static inline float atan( float x ) { return atanf( x ); }
-// extern "C" { double atan( double ); }
-static inline long double atan( long double x ) { return atanl( x ); }
-static inline float _Complex atan( float _Complex x ) { return catanf( x ); }
-static inline double _Complex atan( double _Complex x ) { return catan( x ); }
-static inline long double _Complex atan( long double _Complex x ) { return catanl( x ); }
-
-static inline float atan2( float x, float y ) { return atan2f( x, y ); }
-// extern "C" { double atan2( double, double ); }
-static inline long double atan2( long double x, long double y ) { return atan2l( x, y ); }
-
-// alternative name for atan2
-static inline float atan( float x, float y ) { return atan2f( x, y ); }
-static inline double atan( double x, double y ) { return atan2( x, y ); }
-static inline long double atan( long double x, long double y ) { return atan2l( x, y ); }
+static inline {
+        float sin( float x ) { return sinf( x ); }
+        // extern "C" { double sin( double ); }
+        long double sin( long double x ) { return sinl( x ); }
+        float _Complex sin( float _Complex x ) { return csinf( x ); }
+        double _Complex sin( double _Complex x ) { return csin( x ); }
+        long double _Complex sin( long double _Complex x ) { return csinl( x ); }
+
+        float cos( float x ) { return cosf( x ); }
+        // extern "C" { double cos( double ); }
+        long double cos( long double x ) { return cosl( x ); }
+        float _Complex cos( float _Complex x ) { return ccosf( x ); }
+        double _Complex cos( double _Complex x ) { return ccos( x ); }
+        long double _Complex cos( long double _Complex x ) { return ccosl( x ); }
+
+        float tan( float x ) { return tanf( x ); }
+        // extern "C" { double tan( double ); }
+        long double tan( long double x ) { return tanl( x ); }
+        float _Complex tan( float _Complex x ) { return ctanf( x ); }
+        double _Complex tan( double _Complex x ) { return ctan( x ); }
+        long double _Complex tan( long double _Complex x ) { return ctanl( x ); }
+
+        float asin( float x ) { return asinf( x ); }
+        // extern "C" { double asin( double ); }
+        long double asin( long double x ) { return asinl( x ); }
+        float _Complex asin( float _Complex x ) { return casinf( x ); }
+        double _Complex asin( double _Complex x ) { return casin( x ); }
+        long double _Complex asin( long double _Complex x ) { return casinl( x ); }
+
+        float acos( float x ) { return acosf( x ); }
+        // extern "C" { double acos( double ); }
+        long double acos( long double x ) { return acosl( x ); }
+        float _Complex acos( float _Complex x ) { return cacosf( x ); }
+        double _Complex acos( double _Complex x ) { return cacos( x ); }
+        long double _Complex acos( long double _Complex x ) { return cacosl( x ); }
+
+        float atan( float x ) { return atanf( x ); }
+        // extern "C" { double atan( double ); }
+        long double atan( long double x ) { return atanl( x ); }
+        float _Complex atan( float _Complex x ) { return catanf( x ); }
+        double _Complex atan( double _Complex x ) { return catan( x ); }
+        long double _Complex atan( long double _Complex x ) { return catanl( x ); }
+
+        float atan2( float x, float y ) { return atan2f( x, y ); }
+        // extern "C" { double atan2( double, double ); }
+        long double atan2( long double x, long double y ) { return atan2l( x, y ); }
+
+        // alternative name for atan2
+        float atan( float x, float y ) { return atan2f( x, y ); }
+        double atan( double x, double y ) { return atan2( x, y ); }
+        long double atan( long double x, long double y ) { return atan2l( x, y ); }
+} // distribution
 
 //---------------------- Hyperbolic ----------------------
 
-static inline float sinh( float x ) { return sinhf( x ); }
-// extern "C" { double sinh( double ); }
-static inline long double sinh( long double x ) { return sinhl( x ); }
-static inline float _Complex sinh( float _Complex x ) { return csinhf( x ); }
-static inline double _Complex sinh( double _Complex x ) { return csinh( x ); }
-static inline long double _Complex sinh( long double _Complex x ) { return csinhl( x ); }
-
-static inline float cosh( float x ) { return coshf( x ); }
-// extern "C" { double cosh( double ); }
-static inline long double cosh( long double x ) { return coshl( x ); }
-static inline float _Complex cosh( float _Complex x ) { return ccoshf( x ); }
-static inline double _Complex cosh( double _Complex x ) { return ccosh( x ); }
-static inline long double _Complex cosh( long double _Complex x ) { return ccoshl( x ); }
-
-static inline float tanh( float x ) { return tanhf( x ); }
-// extern "C" { double tanh( double ); }
-static inline long double tanh( long double x ) { return tanhl( x ); }
-static inline float _Complex tanh( float _Complex x ) { return ctanhf( x ); }
-static inline double _Complex tanh( double _Complex x ) { return ctanh( x ); }
-static inline long double _Complex tanh( long double _Complex x ) { return ctanhl( x ); }
-
-static inline float asinh( float x ) { return asinhf( x ); }
-// extern "C" { double asinh( double ); }
-static inline long double asinh( long double x ) { return asinhl( x ); }
-static inline float _Complex asinh( float _Complex x ) { return casinhf( x ); }
-static inline double _Complex asinh( double _Complex x ) { return casinh( x ); }
-static inline long double _Complex asinh( long double _Complex x ) { return casinhl( x ); }
-
-static inline float acosh( float x ) { return acoshf( x ); }
-// extern "C" { double acosh( double ); }
-static inline long double acosh( long double x ) { return acoshl( x ); }
-static inline float _Complex acosh( float _Complex x ) { return cacoshf( x ); }
-static inline double _Complex acosh( double _Complex x ) { return cacosh( x ); }
-static inline long double _Complex acosh( long double _Complex x ) { return cacoshl( x ); }
-
-static inline float atanh( float x ) { return atanhf( x ); }
-// extern "C" { double atanh( double ); }
-static inline long double atanh( long double x ) { return atanhl( x ); }
-static inline float _Complex atanh( float _Complex x ) { return catanhf( x ); }
-static inline double _Complex atanh( double _Complex x ) { return catanh( x ); }
-static inline long double _Complex atanh( long double _Complex x ) { return catanhl( x ); }
+static inline {
+        float sinh( float x ) { return sinhf( x ); }
+        // extern "C" { double sinh( double ); }
+        long double sinh( long double x ) { return sinhl( x ); }
+        float _Complex sinh( float _Complex x ) { return csinhf( x ); }
+        double _Complex sinh( double _Complex x ) { return csinh( x ); }
+        long double _Complex sinh( long double _Complex x ) { return csinhl( x ); }
+
+        float cosh( float x ) { return coshf( x ); }
+        // extern "C" { double cosh( double ); }
+        long double cosh( long double x ) { return coshl( x ); }
+        float _Complex cosh( float _Complex x ) { return ccoshf( x ); }
+        double _Complex cosh( double _Complex x ) { return ccosh( x ); }
+        long double _Complex cosh( long double _Complex x ) { return ccoshl( x ); }
+
+        float tanh( float x ) { return tanhf( x ); }
+        // extern "C" { double tanh( double ); }
+        long double tanh( long double x ) { return tanhl( x ); }
+        float _Complex tanh( float _Complex x ) { return ctanhf( x ); }
+        double _Complex tanh( double _Complex x ) { return ctanh( x ); }
+        long double _Complex tanh( long double _Complex x ) { return ctanhl( x ); }
+
+        float asinh( float x ) { return asinhf( x ); }
+        // extern "C" { double asinh( double ); }
+        long double asinh( long double x ) { return asinhl( x ); }
+        float _Complex asinh( float _Complex x ) { return casinhf( x ); }
+        double _Complex asinh( double _Complex x ) { return casinh( x ); }
+        long double _Complex asinh( long double _Complex x ) { return casinhl( x ); }
+
+        float acosh( float x ) { return acoshf( x ); }
+        // extern "C" { double acosh( double ); }
+        long double acosh( long double x ) { return acoshl( x ); }
+        float _Complex acosh( float _Complex x ) { return cacoshf( x ); }
+        double _Complex acosh( double _Complex x ) { return cacosh( x ); }
+        long double _Complex acosh( long double _Complex x ) { return cacoshl( x ); }
+
+        float atanh( float x ) { return atanhf( x ); }
+        // extern "C" { double atanh( double ); }
+        long double atanh( long double x ) { return atanhl( x ); }
+        float _Complex atanh( float _Complex x ) { return catanhf( x ); }
+        double _Complex atanh( double _Complex x ) { return catanh( x ); }
+        long double _Complex atanh( long double _Complex x ) { return catanhl( x ); }
+} // distribution
 
 //---------------------- Error / Gamma ----------------------
 
-static inline float erf( float x ) { return erff( x ); }
-// extern "C" { double erf( double ); }
-static inline long double erf( long double x ) { return erfl( x ); }
-// float _Complex erf( float _Complex );
-// double _Complex erf( double _Complex );
-// long double _Complex erf( long double _Complex );
-
-static inline float erfc( float x ) { return erfcf( x ); }
-// extern "C" { double erfc( double ); }
-static inline long double erfc( long double x ) { return erfcl( x ); }
-// float _Complex erfc( float _Complex );
-// double _Complex erfc( double _Complex );
-// long double _Complex erfc( long double _Complex );
-
-static inline float lgamma( float x ) { return lgammaf( x ); }
-// extern "C" { double lgamma( double ); }
-static inline long double lgamma( long double x ) { return lgammal( x ); }
-static inline float lgamma( float x, int * sign ) { return lgammaf_r( x, sign ); }
-static inline double lgamma( double x, int * sign ) { return lgamma_r( x, sign ); }
-static inline long double lgamma( long double x, int * sign ) { return lgammal_r( x, sign ); }
-
-static inline float tgamma( float x ) { return tgammaf( x ); }
-// extern "C" { double tgamma( double ); }
-static inline long double tgamma( long double x ) { return tgammal( x ); }
+static inline {
+        float erf( float x ) { return erff( x ); }
+        // extern "C" { double erf( double ); }
+        long double erf( long double x ) { return erfl( x ); }
+        // float _Complex erf( float _Complex );
+        // double _Complex erf( double _Complex );
+        // long double _Complex erf( long double _Complex );
+
+        float erfc( float x ) { return erfcf( x ); }
+        // extern "C" { double erfc( double ); }
+        long double erfc( long double x ) { return erfcl( x ); }
+        // float _Complex erfc( float _Complex );
+        // double _Complex erfc( double _Complex );
+        // long double _Complex erfc( long double _Complex );
+
+        float lgamma( float x ) { return lgammaf( x ); }
+        // extern "C" { double lgamma( double ); }
+        long double lgamma( long double x ) { return lgammal( x ); }
+        float lgamma( float x, int * sign ) { return lgammaf_r( x, sign ); }
+        double lgamma( double x, int * sign ) { return lgamma_r( x, sign ); }
+        long double lgamma( long double x, int * sign ) { return lgammal_r( x, sign ); }
+
+        float tgamma( float x ) { return tgammaf( x ); }
+        // extern "C" { double tgamma( double ); }
+        long double tgamma( long double x ) { return tgammal( x ); }
+} // distribution
 
 //---------------------- Nearest Integer ----------------------
 
-static inline float floor( float x ) { return floorf( x ); }
-// extern "C" { double floor( double ); }
-static inline long double floor( long double x ) { return floorl( x ); }
-
-static inline float ceil( float x ) { return ceilf( x ); }
-// extern "C" { double ceil( double ); }
-static inline long double ceil( long double x ) { return ceill( x ); }
-
-static inline float trunc( float x ) { return truncf( x ); }
-// extern "C" { double trunc( double ); }
-static inline long double trunc( long double x ) { return truncl( x ); }
-
-static inline float rint( float x ) { return rintf( x ); }
-// extern "C" { double rint( double x ); }
-static inline long double rint( long double x ) { return rintl( x ); }
-static inline long int rint( float x ) { return lrintf( x ); }
-static inline long int rint( double x ) { return lrint( x ); }
-static inline long int rint( long double x ) { return lrintl( x ); }
-static inline long long int rint( float x ) { return llrintf( x ); }
-static inline long long int rint( double x ) { return llrint( x ); }
-static inline long long int rint( long double x ) { return llrintl( x ); }
-
-static inline long int lrint( float x ) { return lrintf( x ); }
-// extern "C" { long int lrint( double ); }
-static inline long int lrint( long double x ) { return lrintl( x ); }
-static inline long long int llrint( float x ) { return llrintf( x ); }
-// extern "C" { long long int llrint( double ); }
-static inline long long int llrint( long double x ) { return llrintl( x ); }
-
-static inline float nearbyint( float x ) { return nearbyintf( x ); }
-// extern "C" { double nearbyint( double ); }
-static inline long double nearbyint( long double x ) { return nearbyintl( x ); }
-
-static inline float round( float x ) { return roundf( x ); }
-// extern "C" { double round( double x ); }
-static inline long double round( long double x ) { return roundl( x ); }
-static inline long int round( float x ) { return lroundf( x ); }
-static inline long int round( double x ) { return lround( x ); }
-static inline long int round( long double x ) { return lroundl( x ); }
-static inline long long int round( float x ) { return llroundf( x ); }
-static inline long long int round( double x ) { return llround( x ); }
-static inline long long int round( long double x ) { return llroundl( x ); }
-
-static inline long int lround( float x ) { return lroundf( x ); }
-// extern "C" { long int lround( double ); }
-static inline long int lround( long double x ) { return lroundl( x ); }
-static inline long long int llround( float x ) { return llroundf( x ); }
-// extern "C" { long long int llround( double ); }
-static inline long long int llround( long double x ) { return llroundl( x ); }
+static inline {
+        signed char floor( signed char n, signed char align ) { return n / align * align; }
+        unsigned char floor( unsigned char n, unsigned char align ) { return n / align * align; }
+        short int floor( short int n, short int align ) { return n / align * align; }
+        unsigned short int floor( unsigned short int n, unsigned short int align ) { return n / align * align; }
+        int floor( int n, int align ) { return n / align * align; }
+        unsigned int floor( unsigned int n, unsigned int align ) { return n / align * align; }
+        long int floor( long int n, long int align ) { return n / align * align; }
+        unsigned long int floor( unsigned long int n, unsigned long int align ) { return n / align * align; }
+        long long int floor( long long int n, long long int align ) { return n / align * align; }
+        unsigned long long int floor( unsigned long long int n, unsigned long long int align ) { return n / align * align; }
+
+        // forall( otype T | { T ?/?( T, T ); T ?*?( T, T ); } )
+        // T floor( T n, T align ) { return n / align * align; }
+
+        signed char ceiling_div( signed char n, char align ) { return (n + (align - 1)) / align; }
+        unsigned char ceiling_div( unsigned char n, unsigned char align ) { return (n + (align - 1)) / align; }
+        short int ceiling_div( short int n, short int align ) { return (n + (align - 1)) / align; }
+        unsigned short int ceiling_div( unsigned short int n, unsigned short int align ) { return (n + (align - 1)) / align; }
+        int ceiling_div( int n, int align ) { return (n + (align - 1)) / align; }
+        unsigned int ceiling_div( unsigned int n, unsigned int align ) { return (n + (align - 1)) / align; }
+        long int ceiling_div( long int n, long int align ) { return (n + (align - 1)) / align; }
+        unsigned long int ceiling_div( unsigned long int n, unsigned long int align ) { return (n + (align - 1)) / align; }
+        long long int ceiling_div( long long int n, long long int align ) { return (n + (align - 1)) / align; }
+        unsigned long long int ceiling_div( unsigned long long int n, unsigned long long int align ) { return (n + (align - 1)) / align; }
+
+        // forall( otype T | { T ?+?( T, T ); T ?-?( T, T ); T ?%?( T, T ); } )
+        // T ceiling_div( T n, T align ) { verify( is_pow2( align ) );return (n + (align - 1)) / align; }
+        
+        // gcc notices the div/mod pair and saves both so only one div.
+        signed char ceiling( signed char n, signed char align ) { return floor( n + (n % align != 0 ? align - 1 : 0), align ); }
+        unsigned char ceiling( unsigned char n, unsigned char align ) { return floor( n + (n % align != 0 ? align - 1 : 0), align ); }
+        short int ceiling( short int n, short int align ) { return floor( n + (n % align != 0 ? align - 1 : 0), align ); }
+        unsigned short int ceiling( unsigned short int n, unsigned short int align ) { return floor( n + (n % align != 0 ? align - 1 : 0), align ); }
+        int ceiling( int n, int align ) { return floor( n + (n % align != 0 ? align - 1 : 0), align ); }
+        unsigned int ceiling( unsigned int n, unsigned int align ) { return floor( n + (n % align != 0 ? align - 1 : 0), align ); }
+        long int ceiling( long int n, long int align ) { return floor( n + (n % align != 0 ? align - 1 : 0), align ); }
+        unsigned long int ceiling( unsigned long int n, unsigned long int align ) { return floor( n + (n % align != 0 ? align - 1 : 0) , align); }
+        long long int ceiling( long long int n, long long int align ) { return floor( n + (n % align != 0 ? align - 1 : 0), align ); }
+        unsigned long long int ceiling( unsigned long long int n, unsigned long long int align ) { return floor( n + (n % align != 0 ? align - 1 : 0), align ); }
+
+        // forall( otype T | { void ?{}( T &, one_t ); T ?+?( T, T ); T ?-?( T, T ); T ?/?( T, T ); } )
+        // T ceiling( T n, T align ) { return return floor( n + (n % align != 0 ? align - 1 : 0), align ); *}
+
+        float floor( float x ) { return floorf( x ); }
+        // extern "C" { double floor( double ); }
+        long double floor( long double x ) { return floorl( x ); }
+
+        float ceil( float x ) { return ceilf( x ); }
+        // extern "C" { double ceil( double ); }
+        long double ceil( long double x ) { return ceill( x ); }
+
+        float trunc( float x ) { return truncf( x ); }
+        // extern "C" { double trunc( double ); }
+        long double trunc( long double x ) { return truncl( x ); }
+
+        float rint( float x ) { return rintf( x ); }
+        // extern "C" { double rint( double x ); }
+        long double rint( long double x ) { return rintl( x ); }
+        long int rint( float x ) { return lrintf( x ); }
+        long int rint( double x ) { return lrint( x ); }
+        long int rint( long double x ) { return lrintl( x ); }
+        long long int rint( float x ) { return llrintf( x ); }
+        long long int rint( double x ) { return llrint( x ); }
+        long long int rint( long double x ) { return llrintl( x ); }
+
+        long int lrint( float x ) { return lrintf( x ); }
+        // extern "C" { long int lrint( double ); }
+        long int lrint( long double x ) { return lrintl( x ); }
+        long long int llrint( float x ) { return llrintf( x ); }
+        // extern "C" { long long int llrint( double ); }
+        long long int llrint( long double x ) { return llrintl( x ); }
+
+        float nearbyint( float x ) { return nearbyintf( x ); }
+        // extern "C" { double nearbyint( double ); }
+        long double nearbyint( long double x ) { return nearbyintl( x ); }
+
+        float round( float x ) { return roundf( x ); }
+        // extern "C" { double round( double x ); }
+        long double round( long double x ) { return roundl( x ); }
+        long int round( float x ) { return lroundf( x ); }
+        long int round( double x ) { return lround( x ); }
+        long int round( long double x ) { return lroundl( x ); }
+        long long int round( float x ) { return llroundf( x ); }
+        long long int round( double x ) { return llround( x ); }
+        long long int round( long double x ) { return llroundl( x ); }
+
+        long int lround( float x ) { return lroundf( x ); }
+        // extern "C" { long int lround( double ); }
+        long int lround( long double x ) { return lroundl( x ); }
+        long long int llround( float x ) { return llroundf( x ); }
+        // extern "C" { long long int llround( double ); }
+        long long int llround( long double x ) { return llroundl( x ); }
+} // distribution
 
 //---------------------- Manipulation ----------------------
 
-static inline float copysign( float x, float y ) { return copysignf( x, y ); }
-// extern "C" { double copysign( double, double ); }
-static inline long double copysign( long double x, long double y ) { return copysignl( x, y ); }
-
-static inline float frexp( float x, int * ip ) { return frexpf( x, ip ); }
-// extern "C" { double frexp( double, int * ); }
-static inline long double frexp( long double x, int * ip ) { return frexpl( x, ip ); }
-
-static inline float ldexp( float x, int exp2 ) { return ldexpf( x, exp2 ); }
-// extern "C" { double ldexp( double, int ); }
-static inline long double ldexp( long double x, int exp2 ) { return ldexpl( x, exp2 ); }
-
-static inline [ float, float ] modf( float x ) { float i; x = modff( x, &i ); return [ i, x ]; }
-static inline float modf( float x, float * i ) { return modff( x, i ); }
-static inline [ double, double ] modf( double x ) { double i; x = modf( x, &i ); return [ i, x ]; }
-// extern "C" { double modf( double, double * ); }
-static inline [ long double, long double ] modf( long double x ) { long double i; x = modfl( x, &i ); return [ i, x ]; }
-static inline long double modf( long double x, long double * i ) { return modfl( x, i ); }
-
-static inline float nextafter( float x, float y ) { return nextafterf( x, y ); }
-// extern "C" { double nextafter( double, double ); }
-static inline long double nextafter( long double x, long double y ) { return nextafterl( x, y ); }
-
-static inline float nexttoward( float x, long double y ) { return nexttowardf( x, y ); }
-// extern "C" { double nexttoward( double, long double ); }
-static inline long double nexttoward( long double x, long double y ) { return nexttowardl( x, y ); }
-
-static inline float scalbn( float x, int exp ) { return scalbnf( x, exp ); }
-// extern "C" { double scalbn( double, int ); }
-static inline long double scalbn( long double x, int exp ) { return scalbnl( x, exp ); }
-static inline float scalbn( float x, long int exp ) { return scalblnf( x, exp ); }
-static inline double scalbn( double x, long int exp ) { return scalbln( x, exp ); }
-static inline long double scalbn( long double x, long int exp ) { return scalblnl( x, exp ); }
-
-static inline float scalbln( float x, long int exp ) { return scalblnf( x, exp ); }
-// extern "C" { double scalbln( double, long int ); }
-static inline long double scalbln( long double x, long int exp ) { return scalblnl( x, exp ); }
+static inline {
+        float copysign( float x, float y ) { return copysignf( x, y ); }
+        // extern "C" { double copysign( double, double ); }
+        long double copysign( long double x, long double y ) { return copysignl( x, y ); }
+
+        float frexp( float x, int * ip ) { return frexpf( x, ip ); }
+        // extern "C" { double frexp( double, int * ); }
+        long double frexp( long double x, int * ip ) { return frexpl( x, ip ); }
+
+        float ldexp( float x, int exp2 ) { return ldexpf( x, exp2 ); }
+        // extern "C" { double ldexp( double, int ); }
+        long double ldexp( long double x, int exp2 ) { return ldexpl( x, exp2 ); }
+
+        [ float, float ] modf( float x ) { float i; x = modff( x, &i ); return [ i, x ]; }
+        float modf( float x, float * i ) { return modff( x, i ); }
+        [ double, double ] modf( double x ) { double i; x = modf( x, &i ); return [ i, x ]; }
+        // extern "C" { double modf( double, double * ); }
+        [ long double, long double ] modf( long double x ) { long double i; x = modfl( x, &i ); return [ i, x ]; }
+        long double modf( long double x, long double * i ) { return modfl( x, i ); }
+
+        float nextafter( float x, float y ) { return nextafterf( x, y ); }
+        // extern "C" { double nextafter( double, double ); }
+        long double nextafter( long double x, long double y ) { return nextafterl( x, y ); }
+
+        float nexttoward( float x, long double y ) { return nexttowardf( x, y ); }
+        // extern "C" { double nexttoward( double, long double ); }
+        long double nexttoward( long double x, long double y ) { return nexttowardl( x, y ); }
+
+        float scalbn( float x, int exp ) { return scalbnf( x, exp ); }
+        // extern "C" { double scalbn( double, int ); }
+        long double scalbn( long double x, int exp ) { return scalbnl( x, exp ); }
+        float scalbn( float x, long int exp ) { return scalblnf( x, exp ); }
+        double scalbn( double x, long int exp ) { return scalbln( x, exp ); }
+        long double scalbn( long double x, long int exp ) { return scalblnl( x, exp ); }
+
+        float scalbln( float x, long int exp ) { return scalblnf( x, exp ); }
+        // extern "C" { double scalbln( double, long int ); }
+        long double scalbln( long double x, long int exp ) { return scalblnl( x, exp ); }
+} // distribution
 
 //---------------------------------------
 
-#include "common.hfa"
-
-//---------------------------------------
-
-forall( otype T | { void ?{}( T &, one_t ); T ?+?( T, T ); T ?-?( T, T );T ?*?( T, T ); } )
-T lerp( T x, T y, T a ) { return x * ((T){1} - a) + y * a; }
-
-forall( otype T | { void ?{}( T &, zero_t ); void ?{}( T &, one_t ); int ?<?( T, T ); } )
-T step( T edge, T x ) { return x < edge ? (T){0} : (T){1}; }
-
-forall( otype T | { void ?{}( T &, int ); T clamp( T, T, T ); T ?-?( T, T ); T ?*?( T, T ); T ?/?( T, T ); } )
-T smoothstep( T edge0, T edge1, T x ) { T t = clamp( (x - edge0) / (edge1 - edge0), (T){0}, (T){1} ); return t * t * ((T){3} - (T){2} * t); }
+static inline {
+        forall( otype T | { void ?{}( T &, one_t ); T ?+?( T, T ); T ?-?( T, T );T ?*?( T, T ); } )
+        T lerp( T x, T y, T a ) { return x * ((T){1} - a) + y * a; }
+
+        forall( otype T | { void ?{}( T &, zero_t ); void ?{}( T &, one_t ); int ?<?( T, T ); } )
+        T step( T edge, T x ) { return x < edge ? (T){0} : (T){1}; }
+
+        forall( otype T | { void ?{}( T &, int ); T clamp( T, T, T ); T ?-?( T, T ); T ?*?( T, T ); T ?/?( T, T ); } )
+        T smoothstep( T edge0, T edge1, T x ) { T t = clamp( (x - edge0) / (edge1 - edge0), (T){0}, (T){1} ); return t * t * ((T){3} - (T){2} * t); }
+} // distribution
 
 // Local Variables: //

libcfa/src/rational.cfa

@@ -10 +10 @@
 // Created On       : Wed Apr  6 17:54:28 2016
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Fri Jul 12 18:12:08 2019
-// Update Count     : 184
+// Last Modified On : Sat Feb  8 17:56:36 2020
+// Update Count     : 187
 //
 
@@ -56 +56 @@
         } // rational
 
+        void ?{}( Rational(RationalImpl) & r, zero_t ) {
+                r{ (RationalImpl){0}, (RationalImpl){1} };
+        } // rational
+
+        void ?{}( Rational(RationalImpl) & r, one_t ) {
+                r{ (RationalImpl){1}, (RationalImpl){1} };
+        } // rational
 
         // getter for numerator/denominator

libcfa/src/startup.cfa

@@ -10 +10 @@
 // Created On       : Tue Jul 24 16:21:57 2018
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Fri Dec 13 13:16:45 2019
-// Update Count     : 29
+// Last Modified On : Tue Feb  4 13:03:18 2020
+// Update Count     : 30
 //
 
-#include <time.h>                                                                               // tzset
+#include <time.h>                // tzset
+#include <locale.h>        // setlocale
 #include "startup.hfa"
 
@@ -21 +22 @@
     void __cfaabi_appready_startup( void ) {
                 tzset();                                                                                // initialize time global variables
+                setlocale(LC_NUMERIC, "");
                 #ifdef __CFA_DEBUG__
                 extern void heapAppStart();
@@ -41 +43 @@
 struct __spinlock_t;
 extern "C" {
-        void __cfaabi_dbg_record(struct __spinlock_t & this, const char * prev_name) __attribute__(( weak )) {}
+        void __cfaabi_dbg_record_lock(struct __spinlock_t & this, const char prev_name[]) __attribute__(( weak )) {}
 }
 

libcfa/src/stdhdr/assert.h

@@ -10 +10 @@
 // Created On       : Mon Jul  4 23:25:26 2016
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Mon Jul 31 23:09:32 2017
-// Update Count     : 13
+// Last Modified On : Tue Feb  4 12:58:49 2020
+// Update Count     : 15
 //
 
@@ -27 +27 @@
         #define assertf( expr, fmt, ... ) ((expr) ? ((void)0) : __assert_fail_f(__VSTRINGIFY__(expr), __FILE__, __LINE__, __PRETTY_FUNCTION__, fmt, ## __VA_ARGS__ ))
 
-        void __assert_fail_f( const char *assertion, const char *file, unsigned int line, const char *function, const char *fmt, ... ) __attribute__((noreturn, format( printf, 5, 6) ));
+        void __assert_fail_f( const char assertion[], const char file[], unsigned int line, const char function[], const char fmt[], ... ) __attribute__((noreturn, format( printf, 5, 6) ));
 #endif
 
@@ -33 +33 @@
         #define verify(x) assert(x)
         #define verifyf(x, ...) assertf(x, __VA_ARGS__)
+        #define verifyfail(...)
         #define __CFA_WITH_VERIFY__
 #else
         #define verify(x)
         #define verifyf(x, ...)
+        #define verifyfail(...)
 #endif
 

libcfa/src/stdhdr/bfdlink.h

@@ -10 +10 @@
 // Created On       : Tue Jul 18 07:26:04 2017
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Sat Feb  1 07:15:29 2020
-// Update Count     : 5
+// Last Modified On : Fri Feb  7 19:05:08 2020
+// Update Count     : 6
 // 
 
 // include file uses the CFA keyword "with".
 #if ! defined( with )                                                                   // nesting ?
-#define with ``with``                                                                   // make keyword an identifier
+#define with ``with                                                                             // make keyword an identifier
 #define __CFA_BFDLINK_H__
 #endif

libcfa/src/stdhdr/hwloc.h

@@ -10 +10 @@
 // Created On       : Tue Jul 18 07:45:00 2017
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Sat Feb  1 07:15:39 2020
-// Update Count     : 5
+// Last Modified On : Fri Feb  7 19:05:18 2020
+// Update Count     : 6
 // 
 
 // include file uses the CFA keyword "thread".
 #if ! defined( thread )                                                                 // nesting ?
-#define thread ``thread``                                                               // make keyword an identifier
+#define thread ``thread                                                                 // make keyword an identifier
 #define __CFA_HWLOC_H__
 #endif

libcfa/src/stdhdr/krb5.h

@@ -10 +10 @@
 // Created On       : Tue Jul 18 07:55:44 2017
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Sat Feb  1 07:15:47 2020
-// Update Count     : 5
+// Last Modified On : Fri Feb  7 19:05:35 2020
+// Update Count     : 6
 // 
 
 // include file uses the CFA keyword "enable".
 #if ! defined( enable )                                                                 // nesting ?
-#define enable ``enable``                                                               // make keyword an identifier
+#define enable ``enable                                                                 // make keyword an identifier
 #define __CFA_KRB5_H__
 #endif

libcfa/src/stdhdr/malloc.h

@@ -10 +10 @@
 // Created On       : Thu Jul 20 15:58:16 2017
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Sat Aug 11 09:06:31 2018
-// Update Count     : 10
+// Last Modified On : Wed May 27 14:13:14 2020
+// Update Count     : 18
 // 
-
-
-size_t default_mmap_start();                                                    // CFA extras
-size_t default_heap_expansion();
-
-bool traceHeap();
-bool traceHeapOn();
-bool traceHeapOff();
-
-bool traceHeapTerm();
-bool traceHeapTermOn();
-bool traceHeapTermOff();
-
-bool checkFree();
-bool checkFreeOn();
-bool checkFreeOff();
-
-extern "C" {
-size_t malloc_alignment( void * );
-bool malloc_zero_fill( void * );
-int malloc_stats_fd( int fd );
-void * cmemalign( size_t alignment, size_t noOfElems, size_t elemSize );
-} // extern "C"
 
 extern "C" {
 #include_next <malloc.h>                                                                // has internal check for multiple expansion
 } // extern "C"
+
+#include <heap.hfa>
 
 // Local Variables: //

libcfa/src/stdhdr/math.h

@@ -10 +10 @@
 // Created On       : Mon Jul  4 23:25:26 2016
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Sat Feb  1 07:15:58 2020
-// Update Count     : 14
+// Last Modified On : Fri Feb  7 19:05:27 2020
+// Update Count     : 15
 // 
 
 extern "C" {
 #if ! defined( exception )                                                              // nesting ?
-#define exception ``exception``                                                 // make keyword an identifier
+#define exception ``exception                                                   // make keyword an identifier
 #define __CFA_MATH_H__
 #endif

libcfa/src/stdlib.cfa

@@ -10 +10 @@
 // Created On       : Thu Jan 28 17:10:29 2016
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Wed Nov 20 17:22:47 2019
-// Update Count     : 485
+// Last Modified On : Thu Nov 12 07:46:09 2020
+// Update Count     : 503
 //
 
@@ -20 +20 @@
 #define _XOPEN_SOURCE 600                                                               // posix_memalign, *rand48
 #include <string.h>                                                                             // memcpy, memset
-#include <malloc.h>                                                                             // malloc_usable_size
 //#include <math.h>                                                                             // fabsf, fabs, fabsl
 #include <complex.h>                                                                    // _Complex_I
@@ -27 +26 @@
 //---------------------------------------
 
-forall( dtype T | sized(T) ) {
-        T * alloc_set( T ptr[], size_t dim, char fill ) {       // realloc array with fill
-                size_t olen = malloc_usable_size( ptr );                // current allocation
-                void * nptr = (void *)realloc( (void *)ptr, dim * sizeof(T) ); // C realloc
-                size_t nlen = malloc_usable_size( nptr );               // new allocation
-                if ( nlen > olen ) {                                                    // larger ?
-                        memset( (char *)nptr + olen, (int)fill, nlen - olen ); // initialize added storage
-                } // if
-                return (T *)nptr;
-        } // alloc_set
-
-        T * alloc_align_set( T ptr[], size_t align, char fill ) { // aligned realloc with fill
-                size_t olen = malloc_usable_size( ptr );                // current allocation
-                void * nptr = (void *)realloc( (void *)ptr, align, sizeof(T) ); // CFA realloc
-                // char * nptr = alloc_align( ptr, align );
-                size_t nlen = malloc_usable_size( nptr );               // new allocation
-                if ( nlen > olen ) {                                                    // larger ?
-                        memset( (char *)nptr + olen, (int)fill, nlen - olen ); // initialize added storage
-                } // if
-                return (T *)nptr;
-        } // alloc_align_set
-} // distribution
-
-// allocation/deallocation and constructor/destructor, non-array types
-forall( dtype T | sized(T), ttype Params | { void ?{}( T &, Params ); } )
-T * new( Params p ) {
-        return &(*malloc()){ p };                                                       // run constructor
-} // new
-
-forall( dtype T | sized(T) | { void ^?{}( T & ); } )
-void delete( T * ptr ) {
-        if ( ptr ) {                                                                            // ignore null
-                ^(*ptr){};                                                                              // run destructor
-                free( ptr );
-        } // if
-} // delete
-
-forall( dtype T, ttype Params | sized(T) | { void ^?{}( T & ); void delete( Params ); } )
-void delete( T * ptr, Params rest ) {
-        if ( ptr ) {                                                                            // ignore null
-                ^(*ptr){};                                                                              // run destructor
-                free( ptr );
-        } // if
-        delete( rest );
-} // delete
-
-
-// allocation/deallocation and constructor/destructor, array types
-forall( dtype T | sized(T), ttype Params | { void ?{}( T &, Params ); } )
-T * anew( size_t dim, Params p ) {
+// Cforall allocation/deallocation and constructor/destructor, array types
+
+forall( dtype T | sized(T), ttype TT | { void ?{}( T &, TT ); } )
+T * anew( size_t dim, TT p ) {
         T * arr = alloc( dim );
         for ( unsigned int i = 0; i < dim; i += 1 ) {
@@ -85 +38 @@
 
 forall( dtype T | sized(T) | { void ^?{}( T & ); } )
-void adelete( size_t dim, T arr[] ) {
+void adelete( T arr[] ) {
         if ( arr ) {                                                                            // ignore null
+                size_t dim = malloc_size( arr ) / sizeof( T );
                 for ( int i = dim - 1; i >= 0; i -= 1 ) {               // reverse allocation order, must be unsigned
                         ^(arr[i]){};                                                            // run destructor
@@ -94 +48 @@
 } // adelete
 
-forall( dtype T | sized(T) | { void ^?{}( T & ); }, ttype Params | { void adelete( Params ); } )
-void adelete( size_t dim, T arr[], Params rest ) {
+forall( dtype T | sized(T) | { void ^?{}( T & ); }, ttype TT | { void adelete( TT ); } )
+void adelete( T arr[], TT rest ) {
         if ( arr ) {                                                                            // ignore null
+                size_t dim = malloc_size( arr ) / sizeof( T );
                 for ( int i = dim - 1; i >= 0; i -= 1 ) {               // reverse allocation order, must be unsigned
                         ^(arr[i]){};                                                            // run destructor
@@ -107 +62 @@
 //---------------------------------------
 
-float _Complex strto( const char * sptr, char ** eptr ) {
+float _Complex strto( const char sptr[], char ** eptr ) {
         float re, im;
         char * eeptr;
@@ -118 +73 @@
 } // strto
 
-double _Complex strto( const char * sptr, char ** eptr ) {
+double _Complex strto( const char sptr[], char ** eptr ) {
         double re, im;
         char * eeptr;
@@ -129 +84 @@
 } // strto
 
-long double _Complex strto( const char * sptr, char ** eptr ) {
+long double _Complex strto( const char sptr[], char ** eptr ) {
         long double re, im;
         char * eeptr;
@@ -255 +210 @@
 extern "C" {                                                                                    // override C version
         void srandom( unsigned int seed ) { srand48( (long int)seed ); }
-        long int random( void ) { return mrand48(); }
+        long int random( void ) { return mrand48(); }           // GENERATES POSITIVE AND NEGATIVE VALUES
 } // extern "C"
 

libcfa/src/stdlib.hfa

@@ -10 +10 @@
 // Created On       : Thu Jan 28 17:12:35 2016
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Fri Nov 29 23:08:02 2019
-// Update Count     : 400
+// Last Modified On : Sat Dec 12 13:52:34 2020
+// Update Count     : 536
 //
 
 #pragma once
 
-#include "bits/defs.hfa"
-#include "bits/align.hfa"
+#include "bits/defs.hfa"                                                                // OPTIONAL_THREAD
+#include "bits/align.hfa"                                                               // libAlign
 
 #include <stdlib.h>                                                                             // *alloc, strto*, ato*
-
+#include <heap.hfa>
+
+// Reduce includes by explicitly defining these routines.
 extern "C" {
-        void * memalign( size_t align, size_t size );           // malloc.h
+        void * memalign( size_t alignment, size_t size );       // malloc.h
+        void * pvalloc( size_t size );                                          // malloc.h
         void * memset( void * dest, int fill, size_t size ); // string.h
         void * memcpy( void * dest, const void * src, size_t size ); // string.h
-    void * cmemalign( size_t alignment, size_t noOfElems, size_t elemSize ); // CFA heap
 } // extern "C"
-
-void * realloc( void * oaddr, size_t nalign, size_t size ); // CFA heap
 
 //---------------------------------------
@@ -39 +39 @@
 //---------------------------------------
 
+#include "common.hfa"
+
+//---------------------------------------
+
+// Macro because of returns
+#define $ARRAY_ALLOC( allocation, alignment, dim ) \
+        if ( _Alignof(T) <= libAlign() ) return (T *)(void *)allocation( dim, (size_t)sizeof(T) ); /* C allocation */ \
+        else return (T *)alignment( _Alignof(T), dim, sizeof(T) )
+
 static inline forall( dtype T | sized(T) ) {
-        // C dynamic allocation
+        // CFA safe equivalents, i.e., implicit size specification
 
         T * malloc( void ) {
-                if ( _Alignof(T) <= libAlign() ) return (T *)(void *)malloc( (size_t)sizeof(T) ); // C malloc
+                if ( _Alignof(T) <= libAlign() ) return (T *)(void *)malloc( (size_t)sizeof(T) ); // C allocation
                 else return (T *)memalign( _Alignof(T), sizeof(T) );
         } // malloc
 
+        T * aalloc( size_t dim ) {
+                $ARRAY_ALLOC( aalloc, amemalign, dim );
+        } // aalloc
+
         T * calloc( size_t dim ) {
-                if ( _Alignof(T) <= libAlign() )return (T *)(void *)calloc( dim, sizeof(T) ); // C calloc
-                else return (T *)cmemalign( _Alignof(T), dim, sizeof(T) );
+                $ARRAY_ALLOC( calloc, cmemalign, dim );
         } // calloc
 
+        T * resize( T * ptr, size_t size ) {                            // CFA resize, eliminate return-type cast
+                if ( _Alignof(T) <= libAlign() ) return (T *)(void *)resize( (void *)ptr, size ); // CFA resize
+                else return (T *)(void *)resize( (void *)ptr, _Alignof(T), size ); // CFA resize
+        } // resize
+
         T * realloc( T * ptr, size_t size ) {                           // CFA realloc, eliminate return-type cast
-                return (T *)(void *)realloc( (void *)ptr, size ); // C realloc
+                if ( _Alignof(T) <= libAlign() ) return (T *)(void *)realloc( (void *)ptr, size ); // C realloc
+                else return (T *)(void *)realloc( (void *)ptr, _Alignof(T), size ); // CFA realloc
         } // realloc
 
@@ -60 +78 @@
         } // memalign
 
+        T * amemalign( size_t align, size_t dim ) {
+                return (T *)amemalign( align, dim, sizeof(T) ); // CFA amemalign
+        } // amemalign
+
         T * cmemalign( size_t align, size_t dim  ) {
                 return (T *)cmemalign( align, dim, sizeof(T) ); // CFA cmemalign
@@ -72 +94 @@
         } // posix_memalign
 
-        // Cforall dynamic allocation
-
-        T * alloc( void ) {
-                return malloc();
-        } // alloc
-
-        T * alloc( size_t dim ) {
-                if ( _Alignof(T) <= libAlign() ) return (T *)(void *)malloc( dim * (size_t)sizeof(T) );
-                else return (T *)memalign( _Alignof(T), dim * sizeof(T) );
-        } // alloc
-
-        T * alloc( T ptr[], size_t dim ) {                                      // realloc
-                return (T *)(void *)realloc( (void *)ptr, dim * sizeof(T) ); // C realloc
-        } // alloc
-
-        T * alloc_set( char fill ) {
-                return (T *)memset( (T *)alloc(), (int)fill, sizeof(T) ); // initialize with fill value
-        } // alloc
-
-        T * alloc_set( T fill ) {
-                return (T *)memcpy( (T *)alloc(), &fill, sizeof(T) ); // initialize with fill value
-        } // alloc
-
-        T * alloc_set( size_t dim, char fill ) {
-                return (T *)memset( (T *)alloc( dim ), (int)fill, dim * sizeof(T) ); // initialize with fill value
-        } // alloc
-
-        T * alloc_set( size_t dim, T fill ) {
-                T * r = (T *)alloc( dim );
-                for ( i; dim ) { memcpy( &r[i], &fill, sizeof(T) ); } // initialize with fill value
-                return r;
-        } // alloc
-
-        T * alloc_set( size_t dim, const T fill[] ) {
-                return (T *)memcpy( (T *)alloc( dim ), fill, dim * sizeof(T) ); // initialize with fill value
-        } // alloc
-} // distribution
-
-forall( dtype T | sized(T) ) {
-        T * alloc_set( T ptr[], size_t dim, char fill );        // realloc array with fill
-} // distribution
+        T * valloc( void ) {
+                return (T *)valloc( sizeof(T) );                                // C valloc
+        } // valloc
+
+        T * pvalloc( void ) {
+                return (T *)pvalloc( sizeof(T) );                               // C pvalloc
+        } // pvalloc
+} // distribution
+
+/*
+        FIX ME : fix alloc interface after Ticker Number 214 is resolved, define and add union to S_fill. Then, modify postfix-fill functions to support T * with nmemb, char, and T object of any size. Finally, change alloc_internal.
+        Or, just follow the instructions below for that.
+
+        1. Replace the current forall-block that contains defintions of S_fill and S_realloc with following:
+                forall( dtype T | sized(T) ) {
+                        union  U_fill           { char c; T * a; T t; };
+                        struct S_fill           { char tag; U_fill(T) fill; };
+                        struct S_realloc        { inline T *; };
+                }
+
+        2. Replace all current postfix-fill functions with following for updated S_fill:
+                S_fill(T) ?`fill( char a )                                      { S_fill(T) ret = {'c'}; ret.fill.c = a; return ret; }
+                S_fill(T) ?`fill( T    a )                                      { S_fill(T) ret = {'t'}; memcpy(&ret.fill.t, &a, sizeof(T)); return ret; }
+                S_fill(T) ?`fill( T    a[], size_t nmemb )      { S_fill(T) ret = {'a', nmemb}; ret.fill.a = a; return ret; }
+
+        3. Replace the $alloc_internal function which is outside ttype forall-block with following function:
+                T * $alloc_internal( void * Resize, T * Realloc, size_t Align, size_t Dim, S_fill(T) Fill) {
+                        T * ptr = NULL;
+                        size_t size = sizeof(T);
+                        size_t copy_end = 0;
+
+                        if(Resize) {
+                                ptr = (T*) (void *) resize( (int *)Resize, Align, Dim * size );
+                        } else if (Realloc) {
+                                if (Fill.tag != '0') copy_end = min(malloc_size( Realloc ), Dim * size);
+                                ptr = (T*) (void *) realloc( (int *)Realloc, Align, Dim * size );
+                        } else {
+                                ptr = (T*) (void *) memalign( Align, Dim * size );
+                        }
+
+                        if(Fill.tag == 'c') {
+                                memset( (char *)ptr + copy_end, (int)Fill.fill.c, Dim * size - copy_end );
+                        } else if(Fill.tag == 't') {
+                                for ( int i = copy_end; i <= Dim * size - size ; i += size ) {
+                                        memcpy( (char *)ptr + i, &Fill.fill.t, size );
+                                }
+                        } else if(Fill.tag == 'a') {
+                                memcpy( (char *)ptr + copy_end, Fill.fill.a, min(Dim * size - copy_end, size * Fill.nmemb) );
+                        }
+
+                        return ptr;
+                } // $alloc_internal
+*/
+
+typedef struct S_align                  { inline size_t;  } T_align;
+typedef struct S_resize                 { inline void *;  }     T_resize;
+
+forall( dtype T ) {
+        struct S_fill           { char tag; char c; size_t size; T * at; char t[50]; };
+        struct S_realloc        { inline T *; };
+}
+
+static inline T_align   ?`align   ( size_t a )  { return (T_align){a}; }
+static inline T_resize  ?`resize  ( void * a )  { return (T_resize){a}; }
 
 static inline forall( dtype T | sized(T) ) {
-        T * alloc_align( size_t align ) {
-                return (T *)memalign( align, sizeof(T) );
-        } // alloc_align
-
-        T * alloc_align( size_t align, size_t dim ) {
-                return (T *)memalign( align, dim * sizeof(T) );
-        } // alloc_align
-
-        T * alloc_align( T ptr[], size_t align ) {                      // aligned realloc array
-                return (T *)(void *)realloc( (void *)ptr, align, sizeof(T) ); // CFA realloc
-        } // alloc_align
-
-        T * alloc_align( T ptr[], size_t align, size_t dim ) { // aligned realloc array
-                return (T *)(void *)realloc( (void *)ptr, align, dim * sizeof(T) ); // CFA realloc
-        } // alloc_align
-
-        T * alloc_align_set( size_t align, char fill ) {
-                return (T *)memset( (T *)alloc_align( align ), (int)fill, sizeof(T) ); // initialize with fill value
-        } // alloc_align
-
-        T * alloc_align_set( size_t align, T fill ) {
-                return (T *)memcpy( (T *)alloc_align( align ), &fill, sizeof(T) ); // initialize with fill value
-        } // alloc_align
-
-        T * alloc_align_set( size_t align, size_t dim, char fill ) {
-                return (T *)memset( (T *)alloc_align( align, dim ), (int)fill, dim * sizeof(T) ); // initialize with fill value
-        } // alloc_align
-
-        T * alloc_align_set( size_t align, size_t dim, T fill ) {
-                T * r = (T *)alloc_align( align, dim );
-                for ( i; dim ) { memcpy( &r[i], &fill, sizeof(T) ); } // initialize with fill value
-                return r;
-        } // alloc_align
-
-        T * alloc_align_set( size_t align, size_t dim, const T fill[] ) {
-                return (T *)memcpy( (T *)alloc_align( align, dim ), fill, dim * sizeof(T) );
-        } // alloc_align
-} // distribution
-
-forall( dtype T | sized(T) ) {
-        T * alloc_align_set( T ptr[], size_t align, size_t dim, char fill ); // aligned realloc array with fill
-} // distribution
+        S_fill(T) ?`fill ( T t ) {
+                S_fill(T) ret = { 't' };
+                size_t size = sizeof(T);
+                if(size > sizeof(ret.t)) { printf("ERROR: const object of size greater than 50 bytes given for dynamic memory fill\n"); exit(1); }
+                memcpy( &ret.t, &t, size );
+                return ret;
+        }
+        S_fill(T)               ?`fill ( char c )                               { return (S_fill(T)){ 'c', c }; }
+        S_fill(T)               ?`fill ( T * a )                                { return (S_fill(T)){ 'T', '0', 0, a }; }
+        S_fill(T)               ?`fill ( T a[], size_t nmemb )  { return (S_fill(T)){ 'a', '0', nmemb * sizeof(T), a }; }
+
+        S_realloc(T)    ?`realloc ( T * a )                             { return (S_realloc(T)){a}; }
+
+        T * $alloc_internal( void * Resize, T * Realloc, size_t Align, size_t Dim, S_fill(T) Fill) {
+                T * ptr = NULL;
+                size_t size = sizeof(T);
+                size_t copy_end = 0;
+
+                if ( Resize ) {
+                        ptr = (T*) (void *) resize( (void *)Resize, Align, Dim * size );
+                } else if ( Realloc ) {
+                        if (Fill.tag != '0') copy_end = min(malloc_size( Realloc ), Dim * size);
+                        ptr = (T*) (void *) realloc( (void *)Realloc, Align, Dim * size );
+                } else {
+                        ptr = (T*) (void *) memalign( Align, Dim * size );
+                }
+
+                if(Fill.tag == 'c') {
+                        memset( (char *)ptr + copy_end, (int)Fill.c, Dim * size - copy_end );
+                } else if(Fill.tag == 't') {
+                        for ( int i = copy_end; i < Dim * size; i += size ) {
+                                memcpy( (char *)ptr + i, &Fill.t, size );
+                        }
+                } else if(Fill.tag == 'a') {
+                        memcpy( (char *)ptr + copy_end, Fill.at, min(Dim * size - copy_end, Fill.size) );
+                } else if(Fill.tag == 'T') {
+                        for ( int i = copy_end; i < Dim * size; i += size ) {
+                                memcpy( (char *)ptr + i, Fill.at, size );
+                        }
+                }
+
+                return ptr;
+        } // $alloc_internal
+
+        forall( ttype TT | { T * $alloc_internal( void *, T *, size_t, size_t, S_fill(T), TT ); } ) {
+
+                T * $alloc_internal( void *       , T * Realloc, size_t Align, size_t Dim, S_fill(T) Fill, T_resize Resize, TT rest) {
+                return $alloc_internal( Resize, (T*)0p, Align, Dim, Fill, rest);
+                }
+
+                T * $alloc_internal( void * Resize, T *        , size_t Align, size_t Dim, S_fill(T) Fill, S_realloc(T) Realloc, TT rest) {
+                return $alloc_internal( (void*)0p, Realloc, Align, Dim, Fill, rest);
+                }
+
+                T * $alloc_internal( void * Resize, T * Realloc, size_t      , size_t Dim, S_fill(T) Fill, T_align Align, TT rest) {
+                return $alloc_internal( Resize, Realloc, Align, Dim, Fill, rest);
+                }
+
+                T * $alloc_internal( void * Resize, T * Realloc, size_t Align, size_t Dim, S_fill(T)     , S_fill(T) Fill, TT rest) {
+                return $alloc_internal( Resize, Realloc, Align, Dim, Fill, rest);
+                }
+
+            T * alloc( TT all ) {
+                return $alloc_internal( (void*)0p, (T*)0p, (_Alignof(T) > libAlign() ? _Alignof(T) : libAlign()), (size_t)1, (S_fill(T)){'0'}, all);
+            }
+
+            T * alloc( size_t dim, TT all ) {
+                return $alloc_internal( (void*)0p, (T*)0p, (_Alignof(T) > libAlign() ? _Alignof(T) : libAlign()), dim, (S_fill(T)){'0'}, all);
+            }
+
+        } // distribution TT
+} // distribution T
 
 static inline forall( dtype T | sized(T) ) {
-        // data, non-array types
+        // CFA safe initialization/copy, i.e., implicit size specification, non-array types
         T * memset( T * dest, char fill ) {
                 return (T *)memset( dest, fill, sizeof(T) );
@@ -167 +242 @@
                 return (T *)memcpy( dest, src, sizeof(T) );
         } // memcpy
-} // distribution
-
-static inline forall( dtype T | sized(T) ) {
-        // data, array types
+
+        // CFA safe initialization/copy, i.e., implicit size specification, array types
         T * amemset( T dest[], char fill, size_t dim ) {
                 return (T *)(void *)memset( dest, fill, dim * sizeof(T) ); // C memset
@@ -180 +253 @@
 } // distribution
 
-// allocation/deallocation and constructor/destructor, non-array types
-forall( dtype T | sized(T), ttype Params | { void ?{}( T &, Params ); } ) T * new( Params p );
-forall( dtype T | sized(T) | { void ^?{}( T & ); } ) void delete( T * ptr );
-forall( dtype T, ttype Params | sized(T) | { void ^?{}( T & ); void delete( Params ); } ) void delete( T * ptr, Params rest );
-
-// allocation/deallocation and constructor/destructor, array types
-forall( dtype T | sized(T), ttype Params | { void ?{}( T &, Params ); } ) T * anew( size_t dim, Params p );
-forall( dtype T | sized(T) | { void ^?{}( T & ); } ) void adelete( size_t dim, T arr[] );
-forall( dtype T | sized(T) | { void ^?{}( T & ); }, ttype Params | { void adelete( Params ); } ) void adelete( size_t dim, T arr[], Params rest );
+// CFA deallocation for multiple objects
+static inline forall( dtype T )                                                 // FIX ME, problems with 0p in list
+void free( T * ptr ) {
+        free( (void *)ptr );                                                            // C free
+} // free
+static inline forall( dtype T, ttype TT | { void free( TT ); } )
+void free( T * ptr, TT rest ) {
+        free( ptr );
+        free( rest );
+} // free
+
+// CFA allocation/deallocation and constructor/destructor, non-array types
+static inline forall( dtype T | sized(T), ttype TT | { void ?{}( T &, TT ); } )
+T * new( TT p ) {
+        return &(*(T *)malloc()){ p };                                                  // run constructor
+} // new
+
+static inline forall( dtype T | { void ^?{}( T & ); } )
+void delete( T * ptr ) {
+        // special case for 0-sized object => always call destructor
+        if ( ptr || sizeof(ptr) == 0 ) {                                        // ignore null but not 0-sized objects
+                ^(*ptr){};                                                                              // run destructor
+        } // if
+        free( ptr );                                                                            // always call free
+} // delete
+static inline forall( dtype T, ttype TT | { void ^?{}( T & ); void delete( TT ); } )
+void delete( T * ptr, TT rest ) {
+        delete( ptr );
+        delete( rest );
+} // delete
+
+// CFA allocation/deallocation and constructor/destructor, array types
+forall( dtype T | sized(T), ttype TT | { void ?{}( T &, TT ); } ) T * anew( size_t dim, TT p );
+forall( dtype T | sized(T) | { void ^?{}( T & ); } ) void adelete( T arr[] );
+forall( dtype T | sized(T) | { void ^?{}( T & ); }, ttype TT | { void adelete( TT ); } ) void adelete( T arr[], TT rest );
 
 //---------------------------------------
 
 static inline {
-        int strto( const char * sptr, char ** eptr, int base ) { return (int)strtol( sptr, eptr, base ); }
-        unsigned int strto( const char * sptr, char ** eptr, int base ) { return (unsigned int)strtoul( sptr, eptr, base ); }
-        long int strto( const char * sptr, char ** eptr, int base ) { return strtol( sptr, eptr, base ); }
-        unsigned long int strto( const char * sptr, char ** eptr, int base ) { return strtoul( sptr, eptr, base ); }
-        long long int strto( const char * sptr, char ** eptr, int base ) { return strtoll( sptr, eptr, base ); }
-        unsigned long long int strto( const char * sptr, char ** eptr, int base ) { return strtoull( sptr, eptr, base ); }
-
-        float strto( const char * sptr, char ** eptr ) { return strtof( sptr, eptr ); }
-        double strto( const char * sptr, char ** eptr ) { return strtod( sptr, eptr ); }
-        long double strto( const char * sptr, char ** eptr ) { return strtold( sptr, eptr ); }
-} // distribution
-
-float _Complex strto( const char * sptr, char ** eptr );
-double _Complex strto( const char * sptr, char ** eptr );
-long double _Complex strto( const char * sptr, char ** eptr );
+        int strto( const char sptr[], char ** eptr, int base ) { return (int)strtol( sptr, eptr, base ); }
+        unsigned int strto( const char sptr[], char ** eptr, int base ) { return (unsigned int)strtoul( sptr, eptr, base ); }
+        long int strto( const char sptr[], char ** eptr, int base ) { return strtol( sptr, eptr, base ); }
+        unsigned long int strto( const char sptr[], char ** eptr, int base ) { return strtoul( sptr, eptr, base ); }
+        long long int strto( const char sptr[], char ** eptr, int base ) { return strtoll( sptr, eptr, base ); }
+        unsigned long long int strto( const char sptr[], char ** eptr, int base ) { return strtoull( sptr, eptr, base ); }
+
+        float strto( const char sptr[], char ** eptr ) { return strtof( sptr, eptr ); }
+        double strto( const char sptr[], char ** eptr ) { return strtod( sptr, eptr ); }
+        long double strto( const char sptr[], char ** eptr ) { return strtold( sptr, eptr ); }
+} // distribution
+
+float _Complex strto( const char sptr[], char ** eptr );
+double _Complex strto( const char sptr[], char ** eptr );
+long double _Complex strto( const char sptr[], char ** eptr );
 
 static inline {
-        int ato( const char * sptr ) { return (int)strtol( sptr, 0p, 10 ); }
-        unsigned int ato( const char * sptr ) { return (unsigned int)strtoul( sptr, 0p, 10 ); }
-        long int ato( const char * sptr ) { return strtol( sptr, 0p, 10 ); }
-        unsigned long int ato( const char * sptr ) { return strtoul( sptr, 0p, 10 ); }
-        long long int ato( const char * sptr ) { return strtoll( sptr, 0p, 10 ); }
-        unsigned long long int ato( const char * sptr ) { return strtoull( sptr, 0p, 10 ); }
-
-        float ato( const char * sptr ) { return strtof( sptr, 0p ); }
-        double ato( const char * sptr ) { return strtod( sptr, 0p ); }
-        long double ato( const char * sptr ) { return strtold( sptr, 0p ); }
-
-        float _Complex ato( const char * sptr ) { return strto( sptr, 0p ); }
-        double _Complex ato( const char * sptr ) { return strto( sptr, 0p ); }
-        long double _Complex ato( const char * sptr ) { return strto( sptr, 0p ); }
+        int ato( const char sptr[] ) { return (int)strtol( sptr, 0p, 10 ); }
+        unsigned int ato( const char sptr[] ) { return (unsigned int)strtoul( sptr, 0p, 10 ); }
+        long int ato( const char sptr[] ) { return strtol( sptr, 0p, 10 ); }
+        unsigned long int ato( const char sptr[] ) { return strtoul( sptr, 0p, 10 ); }
+        long long int ato( const char sptr[] ) { return strtoll( sptr, 0p, 10 ); }
+        unsigned long long int ato( const char sptr[] ) { return strtoull( sptr, 0p, 10 ); }
+
+        float ato( const char sptr[] ) { return strtof( sptr, 0p ); }
+        double ato( const char sptr[] ) { return strtod( sptr, 0p ); }
+        long double ato( const char sptr[] ) { return strtold( sptr, 0p ); }
+
+        float _Complex ato( const char sptr[] ) { return strto( sptr, 0p ); }
+        double _Complex ato( const char sptr[] ) { return strto( sptr, 0p ); }
+        long double _Complex ato( const char sptr[] ) { return strto( sptr, 0p ); }
 } // distribution
 
@@ -254 +353 @@
 extern "C" {                                                                                    // override C version
         void srandom( unsigned int seed );
-        long int random( void );
+        long int random( void );                                                        // GENERATES POSITIVE AND NEGATIVE VALUES
+        // For positive values, use unsigned int, e.g., unsigned int r = random() % 100U;
 } // extern "C"
 
@@ -261 +361 @@
         long int random( long int u ) { if ( u < 0 ) return random( u, 0 ); else return random( 0, u ); } // [0,u)
         unsigned long int random( void ) { return lrand48(); }
+        unsigned long int random( unsigned long int u ) { return lrand48() % u; } // [0,u)
         unsigned long int random( unsigned long int l, unsigned long int u ) { if ( u < l ) [u, l] = [l, u]; return lrand48() % (u - l) + l; } // [l,u)
-        unsigned long int random( unsigned long int u ) { return lrand48() % u; } // [0,u)
 
         char random( void ) { return (unsigned long int)random(); }
@@ -283 +383 @@
 //---------------------------------------
 
-#include "common.hfa"
-
-//---------------------------------------
-
-extern bool threading_enabled(void) OPTIONAL_THREAD;
+extern bool threading_enabled( void ) OPTIONAL_THREAD;
 
 // Local Variables: //

libcfa/src/time.cfa

@@ -10 +10 @@
 // Created On       : Tue Mar 27 13:33:14 2018
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Sun Jan  5 17:27:40 2020
-// Update Count     : 69
+// Last Modified On : Tue Feb  4 08:24:18 2020
+// Update Count     : 70
 //
 
@@ -129 +129 @@
 } // dd_mm_yy
 
-size_t strftime( char * buf, size_t size, const char * fmt, Time time ) with( time ) {
+size_t strftime( char buf[], size_t size, const char fmt[], Time time ) with( time ) {
         time_t s = tn / TIMEGRAN;
         tm tm;

libcfa/src/time.hfa

@@ -10 +10 @@
 // Created On       : Wed Mar 14 23:18:57 2018
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Mon Jan  6 12:50:16 2020
-// Update Count     : 653
+// Last Modified On : Wed Jun 17 16:13:00 2020
+// Update Count     : 663
 //
 
@@ -20 +20 @@
 
 #include <time.h>                                                                               // timespec
-extern "C" {
 #include <sys/time.h>                                                                   // timeval
-}
 #include <time_t.hfa>                                                                   // Duration/Time types
 
@@ -91 +89 @@
         int64_t ?`w( Duration dur ) { return dur.tn / (7LL * 24LL * 60LL * 60LL * TIMEGRAN); }
 
+        double ?`dns( Duration dur ) { return dur.tn; }
+        double ?`dus( Duration dur ) { return dur.tn / ((double)TIMEGRAN / 1_000_000.); }
+        double ?`dms( Duration dur ) { return dur.tn / ((double)TIMEGRAN / 1_000.); }
+        double ?`ds( Duration dur ) { return dur.tn / (double)TIMEGRAN; }
+        double ?`dm( Duration dur ) { return dur.tn / (60. * TIMEGRAN); }
+        double ?`dh( Duration dur ) { return dur.tn / (60. * 60. * (double)TIMEGRAN); }
+        double ?`dd( Duration dur ) { return dur.tn / (24. * 60. * 60. * (double)TIMEGRAN); }
+        double ?`dw( Duration dur ) { return dur.tn / (7. * 24. * 60. * 60. * (double)TIMEGRAN); }
+
         Duration max( Duration lhs, Duration rhs ) { return  (lhs.tn < rhs.tn) ? rhs : lhs;}
         Duration min( Duration lhs, Duration rhs ) { return !(rhs.tn < lhs.tn) ? lhs : rhs;}
@@ -191 +198 @@
 } // dmy
 
-size_t strftime( char * buf, size_t size, const char * fmt, Time time );
+size_t strftime( char buf[], size_t size, const char fmt[], Time time );
 
 //------------------------- timeval (cont) -------------------------

libcfa/src/vec/vec.hfa

@@ +1 @@
+//
+// Cforall Version 1.0.0 Copyright (C) 2021 University of Waterloo
+//
+// The contents of this file are covered under the licence agreement in the
+// file "LICENCE" distributed with Cforall.
+//
+// io/types.hfa --
+//
+// Author           : Dimitry Kobets
+// Created On       :
+// Last Modified By :
+// Last Modified On :
+// Update Count     :
+//
+
 #pragma once
 

libcfa/src/vec/vec2.hfa

@@ +1 @@
+//
+// Cforall Version 1.0.0 Copyright (C) 2021 University of Waterloo
+//
+// The contents of this file are covered under the licence agreement in the
+// file "LICENCE" distributed with Cforall.
+//
+// io/types.hfa --
+//
+// Author           : Dimitry Kobets
+// Created On       :
+// Last Modified By :
+// Last Modified On :
+// Update Count     :
+//
+
 #pragma once
 

libcfa/src/vec/vec3.hfa

@@ +1 @@
+//
+// Cforall Version 1.0.0 Copyright (C) 2021 University of Waterloo
+//
+// The contents of this file are covered under the licence agreement in the
+// file "LICENCE" distributed with Cforall.
+//
+// io/types.hfa --
+//
+// Author           : Dimitry Kobets
+// Created On       :
+// Last Modified By :
+// Last Modified On :
+// Update Count     :
+//
+
 #pragma once
 

libcfa/src/vec/vec4.hfa

@@ +1 @@
+//
+// Cforall Version 1.0.0 Copyright (C) 2021 University of Waterloo
+//
+// The contents of this file are covered under the licence agreement in the
+// file "LICENCE" distributed with Cforall.
+//
+// io/types.hfa --
+//
+// Author           : Dimitry Kobets
+// Created On       :
+// Last Modified By :
+// Last Modified On :
+// Update Count     :
+//
+
 #pragma once