Changeset df6cc9d for libcfa

Timestamp:

Oct 19, 2022, 4:43:26 PM (3 years ago)

Author:

Thierry Delisle <tdelisle@…>

Branches:

ADT, ast-experimental, master

Children:

1a45263

Parents:

9cd5bd2 (diff), 135143ba (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the (diff) links above to see all the changes relative to each parent.

Message:

Merge branch 'master' into pthread-emulation

Location:

libcfa/src

Files:

• assert.cfa (modified) (2 diffs)
• bitmanip.hfa (modified) (2 diffs)
• bits/locks.hfa (modified) (2 diffs)
• clock.hfa (modified) (1 diff)
• common.hfa (modified) (7 diffs)
• concurrency/alarm.cfa (modified) (5 diffs)
• concurrency/alarm.hfa (modified) (1 diff)
• concurrency/io.cfa (modified) (11 diffs)
• concurrency/io/types.hfa (modified) (2 diffs)
• concurrency/kernel.hfa (modified) (3 diffs)
• concurrency/kernel/cluster.cfa (modified) (3 diffs)
• concurrency/kernel/cluster.hfa (modified) (3 diffs)
• concurrency/kernel/fwd.hfa (modified) (1 diff)
• concurrency/kernel/private.hfa (modified) (2 diffs)
• concurrency/kernel/startup.cfa (modified) (3 diffs)
• concurrency/preemption.cfa (modified) (7 diffs)
• concurrency/ready_queue.cfa (modified) (4 diffs)
• containers/array.hfa (modified) (6 diffs)
• device/cpu.cfa (modified) (1 diff)
• heap.cfa (modified) (55 diffs)
• heap.hfa (modified) (3 diffs)
• math.hfa (modified) (11 diffs)
• parseargs.cfa (modified) (9 diffs)
• parseargs.hfa (modified) (3 diffs)
• startup.cfa (modified) (2 diffs)
• stdhdr/assert.h (modified) (2 diffs)
• time.hfa (modified) (9 diffs)

libcfa/src/assert.cfa

@@ -25 +25 @@
 
         #define CFA_ASSERT_FMT "Cforall Assertion error \"%s\" from program \"%s\" in \"%s\" at line %d in file \"%s\""
+        #define CFA_WARNING_FMT "Cforall Assertion warning \"%s\" from program \"%s\" in \"%s\" at line %d in file \"%s\""
 
         // called by macro assert in assert.h
@@ -48 +49 @@
                 abort();
         }
+
+        // called by macro warnf
+        // would be cool to remove libcfa_public but it's needed for libcfathread
+        void __assert_warn_f( const char assertion[], const char file[], unsigned int line, const char function[], const char fmt[], ... ) libcfa_public {
+                __cfaabi_bits_acquire();
+                __cfaabi_bits_print_nolock( STDERR_FILENO, CFA_WARNING_FMT ": ", assertion, __progname, function, line, file );
+
+                va_list args;
+                va_start( args, fmt );
+                __cfaabi_bits_print_vararg( STDERR_FILENO, fmt, args );
+                va_end( args );
+
+                __cfaabi_bits_print_nolock( STDERR_FILENO, "\n" );
+                __cfaabi_bits_release();
+        }
 }
 

libcfa/src/bitmanip.hfa

@@ -11 +11 @@
 // Created On       : Sat Mar 14 18:12:27 2020
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Sun Aug 23 21:39:28 2020
-// Update Count     : 140
+// Last Modified On : Sat Oct  8 08:28:15 2022
+// Update Count     : 142
 // 
 
@@ -21 +21 @@
 // Bits are numbered 1-N.
 
-#include <assert.h>
-
 #define __bitsizeof( n ) (sizeof(n) * __CHAR_BIT__)
 
-static inline {
+static inline __attribute__((always_inline)) {
         // Count leading 0 bits.
         unsigned int leading0s( unsigned char n ) { return n != 0 ? __builtin_clz( n ) - (__bitsizeof(unsigned int) - __bitsizeof(n)) : __bitsizeof(n); }

libcfa/src/bits/locks.hfa

@@ -13 +13 @@
 // Created On       : Tue Oct 31 15:14:38 2017
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Mon Sep 19 18:51:53 2022
-// Update Count     : 17
+// Last Modified On : Tue Sep 20 22:09:50 2022
+// Update Count     : 18
 //
 
@@ -64 +64 @@
                         #ifndef NOEXPBACK
                                 // exponential spin
-                        for ( volatile unsigned int s; 0 ~ spin ) Pause();
+                                for ( volatile unsigned int s; 0 ~ spin ) Pause();
 
                                 // slowly increase by powers of 2

libcfa/src/clock.hfa

@@ -13 +13 @@
 // Update Count     : 28
 //
+
+#pragma once
 
 #include <time.hfa>

libcfa/src/common.hfa

@@ -10 +10 @@
 // Created On       : Wed Jul 11 17:54:36 2018
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Wed May  5 14:02:04 2021
-// Update Count     : 18
+// Last Modified On : Sat Oct  8 08:32:57 2022
+// Update Count     : 23
 //
 
@@ -31 +31 @@
         long long int llabs( long long int );
 } // extern "C"
-
-static inline {
+static inline __attribute__((always_inline)) {
         unsigned char abs( signed char v ) { return abs( (int)v ); }
         // use default C routine for int
@@ -44 +43 @@
         long double fabsl( long double );
 } // extern "C"
-static inline {
+static inline __attribute__((always_inline)) {
         float abs( float x ) { return fabsf( x ); }
         double abs( double x ) { return fabs( x ); }
@@ -55 +54 @@
         long double cabsl( long double _Complex );
 } // extern "C"
-static inline {
+static inline __attribute__((always_inline)) {
         float abs( float _Complex x ) { return cabsf( x ); }
         double abs( double _Complex x ) { return cabs( x ); }
@@ -66 +65 @@
 //---------------------------------------
 
-static inline {
+static inline __attribute__((always_inline)) {
         char min( char v1, char v2 ) { return v1 < v2 ? v1 : v2; } // optimization
         int min( int v1, int v2 ) { return v1 < v2 ? v1 : v2; }
@@ -74 +73 @@
         long long int min( long long int v1, long long int v2 ) { return v1 < v2 ? v1 : v2; }
         unsigned long long int min( unsigned long long int v1, unsigned int v2 ) { return v1 < v2 ? v1 : v2; }
-        forall( T | { int ?<?( T, T ); } )
+        forall( T | { int ?<?( T, T ); } )                                      // generic
         T min( T v1, T v2 ) { return v1 < v2 ? v1 : v2; }
 
@@ -84 +83 @@
         long long int max( long long int v1, long long int v2 ) { return v1 > v2 ? v1 : v2; }
         unsigned long long int max( unsigned long long int v1, unsigned long long int v2 ) { return v1 > v2 ? v1 : v2; }
-        forall( T | { int ?>?( T, T ); } )
+        forall( T | { int ?>?( T, T ); } )                                      // generic
         T max( T v1, T v2 ) { return v1 > v2 ? v1 : v2; }
 

libcfa/src/concurrency/alarm.cfa

@@ -55 +55 @@
         this.period  = period;
         this.thrd = thrd;
-        this.timeval = __kernel_get_time() + alarm;
+        this.deadline = __kernel_get_time() + alarm;
         set = false;
         type = User;
@@ -64 +64 @@
         this.period  = period;
         this.proc = proc;
-        this.timeval = __kernel_get_time() + alarm;
+        this.deadline = __kernel_get_time() + alarm;
         set = false;
         type = Kernel;
@@ -72 +72 @@
         this.initial = alarm;
         this.period  = period;
-        this.timeval = __kernel_get_time() + alarm;
+        this.deadline = __kernel_get_time() + alarm;
         set = false;
         type = Callback;
@@ -85 +85 @@
 void insert( alarm_list_t * this, alarm_node_t * n ) {
         alarm_node_t * it = & (*this)`first;
-        while( it && (n->timeval > it->timeval) ) {
+        while( it && (n->deadline > it->deadline) ) {
                 it = & (*it)`next;
         }
@@ -116 +116 @@
 
                 Time curr = __kernel_get_time();
-                __cfadbg_print_safe( preemption, " KERNEL: alarm inserting %p (%lu -> %lu).\n", this, curr.tn, this->timeval.tn );
+                __cfadbg_print_safe( preemption, " KERNEL: alarm inserting %p (%lu -> %lu).\n", this, curr.tn, this->deadline.tn );
                 insert( &alarms, this );
-                __kernel_set_timer( this->timeval - curr);
+                __kernel_set_timer( this->deadline - curr);
                 this->set = true;
         }

libcfa/src/concurrency/alarm.hfa

@@ -57 +57 @@
         };
 
-        Time timeval;           // actual time at which the alarm goes off
+        Time deadline;          // actual time at which the alarm goes off
         enum alarm_type type;   // true if this is not a user defined alarm
         bool set                :1;     // whether or not the alarm has be registered

libcfa/src/concurrency/io.cfa

@@ -201 +201 @@
                 __atomic_unlock(&ctx->cq.lock);
 
-                touch_tsc( cltr->sched.io.tscs, ctx->cq.id, ts_prev, ts_next );
+                touch_tsc( cltr->sched.io.tscs, ctx->cq.id, ts_prev, ts_next, false );
 
                 return true;
         }
 
-        bool __cfa_io_drain( processor * proc ) {
+        bool __cfa_io_drain( struct processor * proc ) {
                 bool local = false;
                 bool remote = false;
@@ -243 +243 @@
                                 /* paranoid */ verify( io.tscs[target].t.tv != ULLONG_MAX );
                                 HELP: if(target < ctxs_count) {
-                                        const unsigned long long cutoff = calc_cutoff(ctsc, ctx->cq.id, ctxs_count, io.data, io.tscs, __shard_factor.io);
-                                        const unsigned long long age = moving_average(ctsc, io.tscs[target].t.tv, io.tscs[target].t.ma);
+                                        const __readyQ_avg_t cutoff = calc_cutoff(ctsc, ctx->cq.id, ctxs_count, io.data, io.tscs, __shard_factor.io, false);
+                                        const __readyQ_avg_t age = moving_average(ctsc, io.tscs[target].t.tv, io.tscs[target].t.ma, false);
                                         __cfadbg_print_safe(io, "Kernel I/O: Help attempt on %u from %u, age %'llu vs cutoff %'llu, %s\n", target, ctx->cq.id, age, cutoff, age > cutoff ? "yes" : "no");
                                         if(age <= cutoff) break HELP;
@@ -273 +273 @@
         }
 
-        bool __cfa_io_flush( processor * proc ) {
+        bool __cfa_io_flush( struct processor * proc ) {
                 /* paranoid */ verify( ! __preemption_enabled() );
                 /* paranoid */ verify( proc );
@@ -353 +353 @@
 
                 disable_interrupts();
-                processor * proc = __cfaabi_tls.this_processor;
+                struct processor * proc = __cfaabi_tls.this_processor;
                 io_context$ * ctx = proc->io.ctx;
                 /* paranoid */ verify( __cfaabi_tls.this_processor );
@@ -433 +433 @@
                 disable_interrupts();
                 __STATS__( true, if(!lazy) io.submit.eagr += 1; )
-                processor * proc = __cfaabi_tls.this_processor;
+                struct processor * proc = __cfaabi_tls.this_processor;
                 io_context$ * ctx = proc->io.ctx;
                 /* paranoid */ verify( __cfaabi_tls.this_processor );
@@ -551 +551 @@
                 enqueue(this.pending, (__outstanding_io&)pa);
 
-                wait( pa.sem );
+                wait( pa.waitctx );
 
                 return pa.ctx;
@@ -578 +578 @@
                                 pa.ctx = ctx;
 
-                                post( pa.sem );
+                                post( pa.waitctx );
                         }
 
@@ -613 +613 @@
                 }
 
-                wait( ei.sem );
+                wait( ei.waitctx );
 
                 __cfadbg_print_safe(io, "Kernel I/O : %u submitted from arbiter\n", have);
@@ -631 +631 @@
                                         __submit_only(&ctx, ei.idxs, ei.have);
 
-                                        post( ei.sem );
+                                        post( ei.waitctx );
                                 }
 
@@ -641 +641 @@
 
         #if defined(CFA_WITH_IO_URING_IDLE)
-                bool __kernel_read(processor * proc, io_future_t & future, iovec & iov, int fd) {
+                bool __kernel_read(struct processor * proc, io_future_t & future, iovec & iov, int fd) {
                         io_context$ * ctx = proc->io.ctx;
                         /* paranoid */ verify( ! __preemption_enabled() );
@@ -692 +692 @@
                 }
 
-                void __cfa_io_idle( processor * proc ) {
+                void __cfa_io_idle( struct processor * proc ) {
                         iovec iov;
                         __atomic_acquire( &proc->io.ctx->cq.lock );

libcfa/src/concurrency/io/types.hfa

@@ -107 +107 @@
         struct __outstanding_io {
                 inline Colable;
-                single_sem sem;
+                oneshot waitctx;
         };
         static inline __outstanding_io *& Next( __outstanding_io * n ) { return (__outstanding_io *)Next( (Colable *)n ); }
@@ -127 +127 @@
         struct __attribute__((aligned(64))) io_context$ {
                 io_arbiter$ * arbiter;
-                processor * proc;
+                struct processor * proc;
 
                 __outstanding_io_queue ext_sq;

libcfa/src/concurrency/kernel.hfa

@@ -136 +136 @@
 
         // Link lists fields
-        inline dlink(processor);
+        dlink(processor) link;
 
         // special init fields
@@ -158 +158 @@
 #endif
 };
-P9_EMBEDDED( processor, dlink(processor) )
+// P9_EMBEDDED( processor, dlink(processor) )
+static inline tytagref( dlink(processor), dlink(processor) ) ?`inner( processor & this ) {
+    dlink(processor) & b = this.link;
+    tytagref( dlink(processor), dlink(processor) ) result = { b };
+    return result;
+}
 
 void  ?{}(processor & this, const char name[], struct cluster & cltr);
@@ -176 +181 @@
 
 // Aligned timestamps which are used by the ready queue and io subsystem
-union __attribute__((aligned(64))) __timestamp_t {
-        struct {
-                volatile unsigned long long tv;
-                volatile unsigned long long ma;
-        } t;
-        char __padding[192];
-};
-
-static inline void  ?{}(__timestamp_t & this) { this.t.tv = 0; this.t.ma = 0; }
-static inline void ^?{}(__timestamp_t &) {}
+union __attribute__((aligned(64))) __timestamp_t;
+
+void  ?{}(__timestamp_t & this);
+void ^?{}(__timestamp_t &);
 
 

libcfa/src/concurrency/kernel/cluster.cfa

@@ -221 +221 @@
 static const unsigned __readyq_single_shard = 2;
 
+void  ?{}(__timestamp_t & this) { this.t.tv = 0; this.t.ma = 0; }
+void ^?{}(__timestamp_t &) {}
+
 //-----------------------------------------------------------------------
 // Check that all the intrusive queues in the data structure are still consistent
@@ -254 +257 @@
 }
 
-static void assign_list(unsigned & valrq, unsigned & valio, dlist(processor) & list, unsigned count) {
-        processor * it = &list`first;
+static void assign_list(unsigned & valrq, unsigned & valio, dlist(struct processor) & list, unsigned count) {
+        struct processor * it = &list`first;
         for(unsigned i = 0; i < count; i++) {
                 /* paranoid */ verifyf( it, "Unexpected null iterator, at index %u of %u\n", i, count);
@@ -278 +281 @@
 
 #if defined(CFA_HAVE_LINUX_IO_URING_H)
-        static void assign_io(io_context$ ** data, size_t count, dlist(processor) & list) {
-                processor * it = &list`first;
+        static void assign_io(io_context$ ** data, size_t count, dlist(struct processor) & list) {
+                struct processor * it = &list`first;
                 while(it) {
                         /* paranoid */ verifyf( it, "Unexpected null iterator\n");

libcfa/src/concurrency/kernel/cluster.hfa

@@ -18 +18 @@
 #include "device/cpu.hfa"
 #include "kernel/private.hfa"
+#include "math.hfa"
 
 #include <limits.h>
+#include <inttypes.h>
+
+#include "clock.hfa"
+
+#if   defined(READYQ_USE_LINEAR_AVG)
+
+// no conversion needed in this case
+static inline __readyQ_avg_t __to_readyQ_avg(unsigned long long intsc) { return intsc; }
+
+// warn normally all ints
+#define warn_large_before warnf( !strict || old_avg < 33_000_000_000, "Suspiciously large previous average: %'llu (%llx), %'" PRId64 "ms \n", old_avg, old_avg, program()`ms )
+#define warn_large_after warnf( !strict || ret < 33_000_000_000, "Suspiciously large new average after %'" PRId64 "ms cputime: %'llu (%llx) from %'llu-%'llu (%'llu, %'llu) and %'llu\n", program()`ms, ret, ret, currtsc, intsc, new_val, new_val / 1000000, old_avg )
+
+// 8X linear factor is just 8 * x
+#define AVG_FACTOR( x ) (8 * (x))
+
+#elif defined(READYQ_USE_LOGDBL_AVG)
+
+// convert to log2 scale but using double
+static inline __readyQ_avg_t __to_readyQ_avg(unsigned long long intsc) { if(unlikely(0 == intsc)) return 0.0; else return log2(intsc); }
+
+#define warn_large_before warnf( !strict || old_avg < 35.0, "Suspiciously large previous average: %'lf, %'" PRId64 "ms \n", old_avg, program()`ms )
+#define warn_large_after warnf( !strict || ret < 35.3, "Suspiciously large new average after %'" PRId64 "ms cputime: %'lf from %'llu-%'llu (%'llu, %'llu) and %'lf\n", program()`ms, ret, currtsc, intsc, new_val, new_val / 1000000, old_avg ); \
+verify(ret >= 0)
+
+// 8X factor in logscale is log2(8X) = log2(8) + log2(X) = 3 + log2(X)
+#define AVG_FACTOR( x ) (3.0 + (x))
+
+// we need to overload the __atomic_load_n because they don't support double
+static inline double __atomic_load_n(volatile double * ptr, int mem) {
+        volatile uint64_t * uptr = (volatile uint64_t *)ptr;
+        _Static_assert(sizeof(*uptr) == sizeof(*ptr));
+        uint64_t ret = 0;
+        ret = __atomic_load_n(uptr, mem);
+        uint64_t *rp = &ret;
+        double ret = *(volatile double *)rp;
+        /* paranoid */ verify( ret == 0 || ret > 3e-100 );
+        return ret;
+}
+
+// we need to overload the __atomic_store_n because they don't support double
+static inline void __atomic_store_n(volatile double * ptr, double val, int mem) {
+        /* paranoid */ verify( val == 0 || val > 3e-100 );
+        volatile uint64_t * uptr = (volatile uint64_t *)ptr;
+        _Static_assert(sizeof(*uptr) == sizeof(*ptr));
+        uint64_t * valp = (uint64_t *)&val;
+        __atomic_store_n(uptr, *valp, mem);
+}
+
+#elif defined(READYQ_USE_LOGDBL_AVG)
+
+//convert to log2 scale but with fix point u32.32 values
+static inline __readyQ_avg_t __to_readyQ_avg(unsigned long long intsc) { return ulog2_32_32(tsc); }
+
+// 8X factor, +3 in logscale (see above) is + 0x3.00000000
+#define AVG_FACTOR( x ) (0x3_00000000ull + (x))
+
+#else
+#error must pick a scheme for averaging
+#endif
 
 //-----------------------------------------------------------------------
 // Calc moving average based on existing average, before and current time.
-static inline unsigned long long moving_average(unsigned long long currtsc, unsigned long long instsc, unsigned long long old_avg) {
-        /* paranoid */ verifyf( old_avg < 15000000000000, "Suspiciously large previous average: %'llu (%llx)\n", old_avg, old_avg );
+static inline __readyQ_avg_t moving_average(unsigned long long currtsc, unsigned long long intsc, __readyQ_avg_t old_avg, bool strict) {
+        (void)strict; // disable the warning around the fact this is unused in release.
+        /* paranoid */ warn_large_before;
 
-        const unsigned long long new_val = currtsc > instsc ? currtsc - instsc : 0;
-        const unsigned long long total_weight = 16;
-        const unsigned long long new_weight   = 4;
-        const unsigned long long old_weight = total_weight - new_weight;
-        const unsigned long long ret = ((new_weight * new_val) + (old_weight * old_avg)) / total_weight;
+        const unsigned long long new_val = currtsc > intsc ? currtsc - intsc : 0;
+        const __readyQ_avg_t total_weight = 16;
+        const __readyQ_avg_t new_weight   = 12;
+        const __readyQ_avg_t old_weight = total_weight - new_weight;
+        const __readyQ_avg_t ret = ((new_weight * __to_readyQ_avg(new_val)) + (old_weight * old_avg)) / total_weight;
+
+        /* paranoid */ warn_large_after;
         return ret;
 }
 
-static inline void touch_tsc(__timestamp_t * tscs, size_t idx, unsigned long long ts_prev, unsigned long long ts_next) {
+static inline void touch_tsc(__timestamp_t * tscs, size_t idx, unsigned long long ts_prev, unsigned long long ts_next, bool strict) {
         if (ts_next == ULLONG_MAX) return;
         unsigned long long now = rdtscl();
-        unsigned long long pma = __atomic_load_n(&tscs[ idx ].t.ma, __ATOMIC_RELAXED);
+        __readyQ_avg_t pma = __atomic_load_n(&tscs[ idx ].t.ma, __ATOMIC_RELAXED);
         __atomic_store_n(&tscs[ idx ].t.tv, ts_next, __ATOMIC_RELAXED);
-        __atomic_store_n(&tscs[ idx ].t.ma, moving_average(now, ts_prev, pma), __ATOMIC_RELAXED);
+        __atomic_store_n(&tscs[ idx ].t.ma, moving_average(now, ts_prev, pma, strict), __ATOMIC_RELAXED);
 }
 
@@ -45 +109 @@
 // Calc age a timestamp should be before needing help.
 forall(Data_t * | { unsigned long long ts(Data_t & this); })
-static inline unsigned long long calc_cutoff(
+static inline __readyQ_avg_t calc_cutoff(
         const unsigned long long ctsc,
         unsigned procid,
@@ -51 +115 @@
         Data_t * data,
         __timestamp_t * tscs,
-        const unsigned shard_factor
+        const unsigned shard_factor,
+        bool strict
 ) {
         unsigned start = procid;
-        unsigned long long max = 0;
+        __readyQ_avg_t max = 0;
         for(i; shard_factor) {
                 unsigned long long ptsc = ts(data[start + i]);
                 if(ptsc != ULLONG_MAX) {
                         /* paranoid */ verify( start + i < count );
-                        unsigned long long tsc = moving_average(ctsc, ptsc, tscs[start + i].t.ma);
-                        if(tsc > max) max = tsc;
+                        __readyQ_avg_t avg = moving_average(ctsc, ptsc, tscs[start + i].t.ma, strict);
+                        if(avg > max) max = avg;
                 }
         }
-        return 8 * max;
+        return AVG_FACTOR( max );
 }
 

libcfa/src/concurrency/kernel/fwd.hfa

@@ -276 +276 @@
                         // intented to be use by wait, wait_any, waitfor, etc. rather than used directly
                         bool retract( future_t & this, oneshot & wait_ctx ) {
-                                struct oneshot * expected = this.ptr;
+                                struct oneshot * expected = &wait_ctx;
 
                                 // attempt to remove the context so it doesn't get consumed.

libcfa/src/concurrency/kernel/private.hfa

@@ -50 +50 @@
         #endif
 #endif
+// #define READYQ_USE_LINEAR_AVG
+#define READYQ_USE_LOGDBL_AVG
+// #define READYQ_USE_LOGINT_AVG
+
+#if   defined(READYQ_USE_LINEAR_AVG)
+typedef unsigned long long __readyQ_avg_t;
+#elif defined(READYQ_USE_LOGDBL_AVG)
+typedef double __readyQ_avg_t;
+#elif defined(READYQ_USE_LOGDBL_AVG)
+typedef unsigned long long __readyQ_avg_t;
+#else
+#error must pick a scheme for averaging
+#endif
 
 extern "C" {
@@ -65 +78 @@
 //-----------------------------------------------------------------------------
 // Scheduler
+union __attribute__((aligned(64))) __timestamp_t {
+        struct {
+                volatile unsigned long long tv;
+                volatile __readyQ_avg_t ma;
+        } t;
+        char __padding[192];
+};
+
 extern "C" {
         void disable_interrupts() OPTIONAL_THREAD;

libcfa/src/concurrency/kernel/startup.cfa

@@ -184 +184 @@
 
 
+extern void heapManagerCtor();
+extern void heapManagerDtor();
+
 //=============================================================================================
 // Kernel Setup logic
@@ -374 +377 @@
         proc->local_data = &__cfaabi_tls;
 
+        heapManagerCtor();                                                                      // initialize heap
+
         __cfa_io_start( proc );
         register_tls( proc );
@@ -425 +430 @@
         unregister_tls( proc );
         __cfa_io_stop( proc );
+
+        heapManagerDtor();                                                                      // de-initialize heap
 
         return 0p;

libcfa/src/concurrency/preemption.cfa

@@ -104 +104 @@
 static inline alarm_node_t * get_expired( alarm_list_t * alarms, Time currtime ) {
         if( ! & (*alarms)`first ) return 0p;                                            // If no alarms return null
-        if( (*alarms)`first.timeval >= currtime ) return 0p;    // If alarms head not expired return null
+        if( (*alarms)`first.deadline >= currtime ) return 0p;   // If alarms head not expired return null
         return pop(alarms);                                                                     // Otherwise just pop head
 }
@@ -140 +140 @@
                 if( period > 0 ) {
                         __cfadbg_print_buffer_local( preemption, " KERNEL: alarm period is %lu.\n", period`ns );
-                        node->timeval = currtime + period;  // Alarm is periodic, add currtime to it (used cached current time)
+                        node->deadline = currtime + period;  // Alarm is periodic, add currtime to it (used cached current time)
                         insert( alarms, node );             // Reinsert the node for the next time it triggers
                 }
@@ -147 +147 @@
         // If there are still alarms pending, reset the timer
         if( & (*alarms)`first ) {
-                Duration delta = (*alarms)`first.timeval - currtime;
+                Duration delta = (*alarms)`first.deadline - currtime;
                 __kernel_set_timer( delta );
         }
@@ -232 +232 @@
 // available.
 
-//-----------------------------------------------------------------------------
-// Some assembly required
-#define __cfaasm_label(label, when) when: asm volatile goto(".global __cfaasm_" #label "_" #when "\n" "__cfaasm_" #label "_" #when ":":::"memory":when)
-
 //----------
 // special case for preemption since used often
-__attribute__((optimize("no-reorder-blocks"))) bool __preemption_enabled() libcfa_nopreempt libcfa_public {
-        // create a assembler label before
-        // marked as clobber all to avoid movement
-        __cfaasm_label(check, before);
-
+bool __preemption_enabled() libcfa_nopreempt libcfa_public {
         // access tls as normal
-        bool enabled = __cfaabi_tls.preemption_state.enabled;
-
-        // Check if there is a pending preemption
-        processor   * proc = __cfaabi_tls.this_processor;
-        bool pending = proc ? proc->pending_preemption : false;
-        if( enabled && pending ) proc->pending_preemption = false;
-
-        // create a assembler label after
-        // marked as clobber all to avoid movement
-        __cfaasm_label(check, after);
-
-        // If we can preempt and there is a pending one
-        // this is a good time to yield
-        if( enabled && pending ) {
-                force_yield( __POLL_PREEMPTION );
-        }
-        return enabled;
-}
-
-struct asm_region {
-        void * before;
-        void * after;
-};
-
-static inline bool __cfaasm_in( void * ip, struct asm_region & region ) {
-        return ip >= region.before && ip <= region.after;
+        return __cfaabi_tls.preemption_state.enabled;
 }
 
@@ -293 +260 @@
 uintptr_t __cfatls_get( unsigned long int offset ) libcfa_nopreempt libcfa_public; //no inline to avoid problems
 uintptr_t __cfatls_get( unsigned long int offset ) {
-        // create a assembler label before
-        // marked as clobber all to avoid movement
-        __cfaasm_label(get, before);
-
         // access tls as normal (except for pointer arithmetic)
         uintptr_t val = *(uintptr_t*)((uintptr_t)&__cfaabi_tls + offset);
 
-        // create a assembler label after
-        // marked as clobber all to avoid movement
-        __cfaasm_label(get, after);
-
         // This is used everywhere, to avoid cost, we DO NOT poll pending preemption
         return val;
@@ -310 +269 @@
 extern "C" {
         // Disable interrupts by incrementing the counter
-        void disable_interrupts() libcfa_nopreempt libcfa_public {
-                // create a assembler label before
-                // marked as clobber all to avoid movement
-                __cfaasm_label(dsable, before);
-
-                with( __cfaabi_tls.preemption_state ) {
-                        #if GCC_VERSION > 50000
-                        static_assert(__atomic_always_lock_free(sizeof(enabled), &enabled), "Must be lock-free");
-                        #endif
-
-                        // Set enabled flag to false
-                        // should be atomic to avoid preemption in the middle of the operation.
-                        // use memory order RELAXED since there is no inter-thread on this variable requirements
-                        __atomic_store_n(&enabled, false, __ATOMIC_RELAXED);
-
-                        // Signal the compiler that a fence is needed but only for signal handlers
-                        __atomic_signal_fence(__ATOMIC_ACQUIRE);
-
-                        __attribute__((unused)) unsigned short new_val = disable_count + 1;
-                        disable_count = new_val;
-                        verify( new_val < 65_000u );              // If this triggers someone is disabling interrupts without enabling them
-                }
-
-                // create a assembler label after
-                // marked as clobber all to avoid movement
-                __cfaasm_label(dsable, after);
-
+        void disable_interrupts() libcfa_nopreempt libcfa_public with( __cfaabi_tls.preemption_state ) {
+                #if GCC_VERSION > 50000
+                static_assert(__atomic_always_lock_free(sizeof(enabled), &enabled), "Must be lock-free");
+                #endif
+
+                // Set enabled flag to false
+                // should be atomic to avoid preemption in the middle of the operation.
+                // use memory order RELAXED since there is no inter-thread on this variable requirements
+                __atomic_store_n(&enabled, false, __ATOMIC_RELAXED);
+
+                // Signal the compiler that a fence is needed but only for signal handlers
+                __atomic_signal_fence(__ATOMIC_ACQUIRE);
+
+                __attribute__((unused)) unsigned short new_val = disable_count + 1;
+                disable_count = new_val;
+                verify( new_val < 65_000u );              // If this triggers someone is disabling interrupts without enabling them
         }
 
@@ -379 +327 @@
         // i.e. on a real processor and not in the kernel
         // (can return true even if no preemption was pending)
-        bool poll_interrupts() libcfa_public {
+        bool poll_interrupts() libcfa_nopreempt libcfa_public {
                 // Cache the processor now since interrupts can start happening after the atomic store
-                processor   * proc = publicTLS_get( this_processor );
+                processor   * proc =  __cfaabi_tls.this_processor;
                 if ( ! proc ) return false;
-                if ( ! __preemption_enabled() ) return false;
-
-                with( __cfaabi_tls.preemption_state ){
-                        // Signal the compiler that a fence is needed but only for signal handlers
-                        __atomic_signal_fence(__ATOMIC_RELEASE);
-                        if( proc->pending_preemption ) {
-                                proc->pending_preemption = false;
-                                force_yield( __POLL_PREEMPTION );
-                        }
+                if ( ! __cfaabi_tls.preemption_state.enabled ) return false;
+
+                // Signal the compiler that a fence is needed but only for signal handlers
+                __atomic_signal_fence(__ATOMIC_RELEASE);
+                if( unlikely( proc->pending_preemption ) ) {
+                        proc->pending_preemption = false;
+                        force_yield( __POLL_PREEMPTION );
                 }
 

libcfa/src/concurrency/ready_queue.cfa

@@ -62 +62 @@
 //-----------------------------------------------------------------------
 __attribute__((hot)) void push(struct cluster * cltr, struct thread$ * thrd, unpark_hint hint) with (cltr->sched) {
-        processor * const proc = kernelTLS().this_processor;
+        struct processor * const proc = kernelTLS().this_processor;
         const bool external = (!proc) || (cltr != proc->cltr);
         const bool remote   = hint == UNPARK_REMOTE;
@@ -116 +116 @@
         /* paranoid */ verify( kernelTLS().this_processor->rdq.id < lanes_count );
 
-        processor * const proc = kernelTLS().this_processor;
+        struct processor * const proc = kernelTLS().this_processor;
         unsigned this = proc->rdq.id;
         /* paranoid */ verify( this < lanes_count );
@@ -139 +139 @@
                 /* paranoid */ verify( readyQ.tscs[target].t.tv != ULLONG_MAX );
                 if(target < lanes_count) {
-                        const unsigned long long cutoff = calc_cutoff(ctsc, proc->rdq.id, lanes_count, cltr->sched.readyQ.data, cltr->sched.readyQ.tscs, __shard_factor.readyq);
-                        const unsigned long long age = moving_average(ctsc, readyQ.tscs[target].t.tv, readyQ.tscs[target].t.ma);
+                        const __readyQ_avg_t cutoff = calc_cutoff(ctsc, proc->rdq.id, lanes_count, cltr->sched.readyQ.data, cltr->sched.readyQ.tscs, __shard_factor.readyq, true);
+                        const __readyQ_avg_t age = moving_average(ctsc, readyQ.tscs[target].t.tv, readyQ.tscs[target].t.ma, false);
                         __cfadbg_print_safe(ready_queue, "Kernel : Help attempt on %u from %u, age %'llu vs cutoff %'llu, %s\n", target, this, age, cutoff, age > cutoff ? "yes" : "no");
                         if(age > cutoff) {
@@ -214 +214 @@
         __STATS( stats.success++; )
 
-        touch_tsc(readyQ.tscs, w, ts_prev, ts_next);
+        touch_tsc(readyQ.tscs, w, ts_prev, ts_next, true);
 
         thrd->preferred = w / __shard_factor.readyq;

libcfa/src/containers/array.hfa

@@ -27 +27 @@
     // -  Given bug of Trac #247, CFA gives sizeof expressions type unsigned long int, when it
     //    should give them type size_t.
-    //    
+    //
     //                          gcc -m32         cfa -m32 given bug         gcc -m64
     // ptrdiff_t                int              int                        long int
@@ -39 +39 @@
     }
 
+    static inline const Timmed & ?[?]( const arpk(N, S, Timmed, Tbase) & a, int i ) {
+        assert( i < N );
+        return (Timmed &) a.strides[i];
+    }
+
     static inline Timmed & ?[?]( arpk(N, S, Timmed, Tbase) & a, unsigned int i ) {
+        assert( i < N );
+        return (Timmed &) a.strides[i];
+    }
+
+    static inline const Timmed & ?[?]( const arpk(N, S, Timmed, Tbase) & a, unsigned int i ) {
         assert( i < N );
         return (Timmed &) a.strides[i];
@@ -49 +59 @@
     }
 
+    static inline const Timmed & ?[?]( const arpk(N, S, Timmed, Tbase) & a, long int i ) {
+        assert( i < N );
+        return (Timmed &) a.strides[i];
+    }
+
     static inline Timmed & ?[?]( arpk(N, S, Timmed, Tbase) & a, unsigned long int i ) {
+        assert( i < N );
+        return (Timmed &) a.strides[i];
+    }
+
+    static inline const Timmed & ?[?]( const arpk(N, S, Timmed, Tbase) & a, unsigned long int i ) {
         assert( i < N );
         return (Timmed &) a.strides[i];
@@ -83 +103 @@
     // Make a FOREACH macro
     #define FE_0(WHAT)
-    #define FE_1(WHAT, X) WHAT(X) 
+    #define FE_1(WHAT, X) WHAT(X)
     #define FE_2(WHAT, X, ...) WHAT(X)FE_1(WHAT, __VA_ARGS__)
     #define FE_3(WHAT, X, ...) WHAT(X)FE_2(WHAT, __VA_ARGS__)
@@ -90 +110 @@
     //... repeat as needed
 
-    #define GET_MACRO(_0,_1,_2,_3,_4,_5,NAME,...) NAME 
+    #define GET_MACRO(_0,_1,_2,_3,_4,_5,NAME,...) NAME
     #define FOR_EACH(action,...) \
     GET_MACRO(_0,__VA_ARGS__,FE_5,FE_4,FE_3,FE_2,FE_1,FE_0)(action,__VA_ARGS__)
@@ -115 +135 @@
 }
 
-#else 
+#else
 
 // Workaround form.  Listing all possibilities up to 4 dims.

libcfa/src/device/cpu.cfa

@@ -359 +359 @@
                 int idxs = count_cache_indexes();
 
+                // Do we actually have a cache?
+                if(idxs == 0) {
+                        // if not just fake the data structure, it makes things easier.
+                        cpu_info.hthrd_count = cpus_c;
+                        cpu_info.llc_count = 0;
+                        struct cpu_map_entry_t * entries = alloc(cpu_info.hthrd_count);
+                        for(i; cpu_info.hthrd_count) {
+                                entries[i].self  = i;
+                                entries[i].start = 0;
+                                entries[i].count = cpu_info.hthrd_count;
+                                entries[i].cache = 0;
+                        }
+                        cpu_info.llc_map = entries;
+                        return;
+                }
+
                 // Count actual cache levels
                 unsigned cache_levels = 0;
                 unsigned llc = 0;
-                if (idxs != 0) {
-                        unsigned char prev = -1u;
-                        void first(unsigned idx, unsigned char level, const char * map, size_t len) {
-                                /* paranoid */ verifyf(level < prev, "Index %u of cpu 0 has cache levels out of order: %u then %u", idx, (unsigned)prev, (unsigned)level);
-                                llc = max(llc, level);
-                                prev = level;
-                                cache_levels++;
-                        }
-                        foreach_cacheidx(0, idxs, first);
-                }
+
+                unsigned char prev = -1u;
+                void first(unsigned idx, unsigned char level, const char * map, size_t len) {
+                        /* paranoid */ verifyf(level < prev, "Index %u of cpu 0 has cache levels out of order: %u then %u", idx, (unsigned)prev, (unsigned)level);
+                        llc = max(llc, level);
+                        prev = level;
+                        cache_levels++;
+                }
+                foreach_cacheidx(0, idxs, first);
 
                 // Read in raw data

libcfa/src/heap.cfa

@@ -10 +10 @@
 // Created On       : Tue Dec 19 21:58:35 2017
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Fri Apr 29 19:05:03 2022
-// Update Count     : 1167
+// Last Modified On : Thu Oct 13 22:21:52 2022
+// Update Count     : 1557
 //
 
+#include <stdio.h>
 #include <string.h>                                                                             // memset, memcpy
 #include <limits.h>                                                                             // ULONG_MAX
@@ -21 +22 @@
 #include <malloc.h>                                                                             // memalign, malloc_usable_size
 #include <sys/mman.h>                                                                   // mmap, munmap
+extern "C" {
 #include <sys/sysinfo.h>                                                                // get_nprocs
+} // extern "C"
 
 #include "bits/align.hfa"                                                               // libAlign
 #include "bits/defs.hfa"                                                                // likely, unlikely
-#include "bits/locks.hfa"                                                               // __spinlock_t
+#include "concurrency/kernel/fwd.hfa"                                   // __POLL_PREEMPTION
 #include "startup.hfa"                                                                  // STARTUP_PRIORITY_MEMORY
-#include "math.hfa"                                                                             // min
+#include "math.hfa"                                                                             // ceiling, min
 #include "bitmanip.hfa"                                                                 // is_pow2, ceiling2
 
-#define FASTLOOKUP
-#define __STATISTICS__
+// supported mallopt options
+#ifndef M_MMAP_THRESHOLD
+#define M_MMAP_THRESHOLD (-1)
+#endif // M_MMAP_THRESHOLD
+
+#ifndef M_TOP_PAD
+#define M_TOP_PAD (-2)
+#endif // M_TOP_PAD
+
+#define FASTLOOKUP                                                                              // use O(1) table lookup from allocation size to bucket size
+#define RETURNSPIN                                                                              // toggle spinlock / lockfree stack
+#define OWNERSHIP                                                                               // return freed memory to owner thread
+
+#define CACHE_ALIGN 64
+#define CALIGN __attribute__(( aligned(CACHE_ALIGN) ))
+
+#define TLSMODEL __attribute__(( tls_model("initial-exec") ))
+
+//#define __STATISTICS__
+
+enum {
+        // The default extension heap amount in units of bytes. When the current heap reaches the brk address, the brk
+        // address is extended by the extension amount.
+        __CFA_DEFAULT_HEAP_EXPANSION__ = 10 * 1024 * 1024,
+
+        // The mmap crossover point during allocation. Allocations less than this amount are allocated from buckets; values
+        // greater than or equal to this value are mmap from the operating system.
+        __CFA_DEFAULT_MMAP_START__ = 512 * 1024 + 1,
+
+        // The default unfreed storage amount in units of bytes. When the uC++ program ends it subtracts this amount from
+        // the malloc/free counter to adjust for storage the program does not free.
+        __CFA_DEFAULT_HEAP_UNFREED__ = 0
+}; // enum
+
+
+//####################### Heap Trace/Print ####################
 
 
@@ -55 +92 @@
 static bool prtFree = false;
 
-static bool prtFree() {
+bool prtFree() {
         return prtFree;
 } // prtFree
 
-static bool prtFreeOn() {
+bool prtFreeOn() {
         bool temp = prtFree;
         prtFree = true;
@@ -65 +102 @@
 } // prtFreeOn
 
-static bool prtFreeOff() {
+bool prtFreeOff() {
         bool temp = prtFree;
         prtFree = false;
@@ -72 +109 @@
 
 
-enum {
-        // The default extension heap amount in units of bytes. When the current heap reaches the brk address, the brk
-        // address is extended by the extension amount.
-        __CFA_DEFAULT_HEAP_EXPANSION__ = 10 * 1024 * 1024,
-
-        // The mmap crossover point during allocation. Allocations less than this amount are allocated from buckets; values
-        // greater than or equal to this value are mmap from the operating system.
-        __CFA_DEFAULT_MMAP_START__ = 512 * 1024 + 1,
-
-        // The default unfreed storage amount in units of bytes. When the uC++ program ends it subtracts this amount from
-        // the malloc/free counter to adjust for storage the program does not free.
-        __CFA_DEFAULT_HEAP_UNFREED__ = 0
-}; // enum
+//######################### Spin Lock #########################
+
+
+// pause to prevent excess processor bus usage
+#if defined( __i386 ) || defined( __x86_64 )
+        #define Pause() __asm__ __volatile__ ( "pause" : : : )
+#elif defined(__ARM_ARCH)
+        #define Pause() __asm__ __volatile__ ( "YIELD" : : : )
+#else
+        #error unsupported architecture
+#endif
+
+typedef volatile uintptr_t SpinLock_t CALIGN;                   // aligned addressable word-size
+
+static inline __attribute__((always_inline)) void lock( volatile SpinLock_t & slock ) {
+        enum { SPIN_START = 4, SPIN_END = 64 * 1024, };
+        unsigned int spin = SPIN_START;
+
+        for ( unsigned int i = 1;; i += 1 ) {
+          if ( slock == 0 && __atomic_test_and_set( &slock, __ATOMIC_SEQ_CST ) == 0 ) break; // Fence
+                for ( volatile unsigned int s = 0; s < spin; s += 1 ) Pause(); // exponential spin
+                spin += spin;                                                                   // powers of 2
+                //if ( i % 64 == 0 ) spin += spin;                              // slowly increase by powers of 2
+                if ( spin > SPIN_END ) spin = SPIN_END;                 // cap spinning
+        } // for
+} // spin_lock
+
+static inline __attribute__((always_inline)) void unlock( volatile SpinLock_t & slock ) {
+        __atomic_clear( &slock, __ATOMIC_SEQ_CST );                     // Fence
+} // spin_unlock
 
 
@@ -120 +174 @@
                 unsigned int free_calls, free_null_calls;
                 unsigned long long int free_storage_request, free_storage_alloc;
-                unsigned int away_pulls, away_pushes;
-                unsigned long long int away_storage_request, away_storage_alloc;
+                unsigned int return_pulls, return_pushes;
+                unsigned long long int return_storage_request, return_storage_alloc;
                 unsigned int mmap_calls, mmap_0_calls;                  // no zero calls
                 unsigned long long int mmap_storage_request, mmap_storage_alloc;
@@ -131 +185 @@
 
 static_assert( sizeof(HeapStatistics) == CntTriples * sizeof(StatsOverlay),
-                           "Heap statistics counter-triplets does not match with array size" );
+                           "Heap statistics counter-triplets does not match with array size" );
 
 static void HeapStatisticsCtor( HeapStatistics & stats ) {
@@ -203 +257 @@
         static_assert( libAlign() >= sizeof( Storage ), "minimum alignment < sizeof( Storage )" );
 
-        struct FreeHeader {
-                size_t blockSize __attribute__(( aligned (8) )); // size of allocations on this list
+        struct __attribute__(( aligned (8) )) FreeHeader {
+                size_t blockSize __attribute__(( aligned(8) )); // size of allocations on this list
                 #if BUCKETLOCK == SPINLOCK
-                __spinlock_t lock;
-                Storage * freeList;
+                #ifdef OWNERSHIP
+                #ifdef RETURNSPIN
+                SpinLock_t returnLock;
+                #endif // RETURNSPIN
+                Storage * returnList;                                                   // other thread return list
+                #endif // OWNERSHIP
+                Storage * freeList;                                                             // thread free list
                 #else
                 StackLF(Storage) freeList;
                 #endif // BUCKETLOCK
-        } __attribute__(( aligned (8) )); // FreeHeader
+                Heap * homeManager;                                                             // heap owner (free storage to bucket, from bucket to heap)
+        }; // FreeHeader
 
         FreeHeader freeLists[NoBucketSizes];                            // buckets for different allocation sizes
-
-        __spinlock_t extlock;                                                           // protects allocation-buffer extension
-        void * heapBegin;                                                                       // start of heap
-        void * heapEnd;                                                                         // logical end of heap
-        size_t heapRemaining;                                                           // amount of storage not allocated in the current chunk
+        void * heapBuffer;                                                                      // start of free storage in buffer
+        size_t heapReserve;                                                                     // amount of remaining free storage in buffer
+
+        #if defined( __STATISTICS__ ) || defined( __CFA_DEBUG__ )
+        Heap * nextHeapManager;                                                         // intrusive link of existing heaps; traversed to collect statistics or check unfreed storage
+        #endif // __STATISTICS__ || __CFA_DEBUG__
+        Heap * nextFreeHeapManager;                                                     // intrusive link of free heaps from terminated threads; reused by new threads
+
+        #ifdef __CFA_DEBUG__
+        int64_t allocUnfreed;                                                           // running total of allocations minus frees; can be negative
+        #endif // __CFA_DEBUG__
+
+        #ifdef __STATISTICS__
+        HeapStatistics stats;                                                           // local statistic table for this heap
+        #endif // __STATISTICS__
 }; // Heap
 
 #if BUCKETLOCK == LOCKFREE
-static inline {
+inline __attribute__((always_inline))
+static {
         Link(Heap.Storage) * ?`next( Heap.Storage * this ) { return &this->header.kind.real.next; }
         void ?{}( Heap.FreeHeader & ) {}
@@ -229 +300 @@
 #endif // LOCKFREE
 
-static inline size_t getKey( const Heap.FreeHeader & freeheader ) { return freeheader.blockSize; }
+
+struct HeapMaster {
+        SpinLock_t extLock;                                                                     // protects allocation-buffer extension
+        SpinLock_t mgrLock;                                                                     // protects freeHeapManagersList, heapManagersList, heapManagersStorage, heapManagersStorageEnd
+
+        void * heapBegin;                                                                       // start of heap
+        void * heapEnd;                                                                         // logical end of heap
+        size_t heapRemaining;                                                           // amount of storage not allocated in the current chunk
+        size_t pageSize;                                                                        // architecture pagesize
+        size_t heapExpand;                                                                      // sbrk advance
+        size_t mmapStart;                                                                       // cross over point for mmap
+        unsigned int maxBucketsUsed;                                            // maximum number of buckets in use
+
+        Heap * heapManagersList;                                                        // heap-list head
+        Heap * freeHeapManagersList;                                            // free-list head
+
+        // Heap superblocks are not linked; heaps in superblocks are linked via intrusive links.
+        Heap * heapManagersStorage;                                                     // next heap to use in heap superblock
+        Heap * heapManagersStorageEnd;                                          // logical heap outside of superblock's end
+
+        #ifdef __STATISTICS__
+        HeapStatistics stats;                                                           // global stats for thread-local heaps to add there counters when exiting
+        unsigned long int threads_started, threads_exited;      // counts threads that have started and exited
+        unsigned long int reused_heap, new_heap;                        // counts reusability of heaps
+        unsigned int sbrk_calls;
+        unsigned long long int sbrk_storage;
+        int stats_fd;
+        #endif // __STATISTICS__
+}; // HeapMaster
 
 
 #ifdef FASTLOOKUP
-enum { LookupSizes = 65_536 + sizeof(Heap.Storage) }; // number of fast lookup sizes
+enum { LookupSizes = 65_536 + sizeof(Heap.Storage) };   // number of fast lookup sizes
 static unsigned char lookup[LookupSizes];                               // O(1) lookup for small sizes
 #endif // FASTLOOKUP
 
-static const off_t mmapFd = -1;                                                 // fake or actual fd for anonymous file
-#ifdef __CFA_DEBUG__
-static bool heapBoot = 0;                                                               // detect recursion during boot
-#endif // __CFA_DEBUG__
+static volatile bool heapMasterBootFlag = false;                // trigger for first heap
+static HeapMaster heapMaster @= {};                                             // program global
+
+static void heapMasterCtor();
+static void heapMasterDtor();
+static Heap * getHeap();
 
 
@@ -268 +369 @@
 static_assert( NoBucketSizes == sizeof(bucketSizes) / sizeof(bucketSizes[0] ), "size of bucket array wrong" );
 
-// The constructor for heapManager is called explicitly in memory_startup.
-static Heap heapManager __attribute__(( aligned (128) )) @= {}; // size of cache line to prevent false sharing
+
+// extern visibility, used by runtime kernel
+libcfa_public size_t __page_size;                                               // architecture pagesize
+libcfa_public int __map_prot;                                                   // common mmap/mprotect protection
+
+
+// Thread-local storage is allocated lazily when the storage is accessed.
+static __thread size_t PAD1 CALIGN TLSMODEL __attribute__(( unused )); // protect false sharing
+static __thread Heap * volatile heapManager CALIGN TLSMODEL;
+static __thread size_t PAD2 CALIGN TLSMODEL __attribute__(( unused )); // protect further false sharing
+
+
+// declare helper functions for HeapMaster
+void noMemory();                                                                                // forward, called by "builtin_new" when malloc returns 0
+
+
+// generic Bsearchl does not inline, so substitute with hand-coded binary-search.
+inline __attribute__((always_inline))
+static size_t Bsearchl( unsigned int key, const unsigned int vals[], size_t dim ) {
+        size_t l = 0, m, h = dim;
+        while ( l < h ) {
+                m = (l + h) / 2;
+                if ( (unsigned int &)(vals[m]) < key ) {                // cast away const
+                        l = m + 1;
+                } else {
+                        h = m;
+                } // if
+        } // while
+        return l;
+} // Bsearchl
+
+
+void heapMasterCtor() with( heapMaster ) {
+        // Singleton pattern to initialize heap master
+
+        verify( bucketSizes[0] == (16 + sizeof(Heap.Storage)) );
+
+        __page_size = sysconf( _SC_PAGESIZE );
+        __map_prot = PROT_READ | PROT_WRITE | PROT_EXEC;
+
+        ?{}( extLock );
+        ?{}( mgrLock );
+
+        char * end = (char *)sbrk( 0 );
+        heapBegin = heapEnd = sbrk( (char *)ceiling2( (long unsigned int)end, libAlign() ) - end ); // move start of heap to multiple of alignment
+        heapRemaining = 0;
+        heapExpand = malloc_expansion();
+        mmapStart = malloc_mmap_start();
+
+        // find the closest bucket size less than or equal to the mmapStart size
+        maxBucketsUsed = Bsearchl( mmapStart, bucketSizes, NoBucketSizes ); // binary search
+
+        verify( (mmapStart >= pageSize) && (bucketSizes[NoBucketSizes - 1] >= mmapStart) );
+        verify( maxBucketsUsed < NoBucketSizes );                       // subscript failure ?
+        verify( mmapStart <= bucketSizes[maxBucketsUsed] ); // search failure ?
+
+        heapManagersList = 0p;
+        freeHeapManagersList = 0p;
+
+        heapManagersStorage = 0p;
+        heapManagersStorageEnd = 0p;
+
+        #ifdef __STATISTICS__
+        HeapStatisticsCtor( stats );                                            // clear statistic counters
+        threads_started = threads_exited = 0;
+        reused_heap = new_heap = 0;
+        sbrk_calls = sbrk_storage = 0;
+        stats_fd = STDERR_FILENO;
+        #endif // __STATISTICS__
+
+        #ifdef FASTLOOKUP
+        for ( unsigned int i = 0, idx = 0; i < LookupSizes; i += 1 ) {
+                if ( i > bucketSizes[idx] ) idx += 1;
+                lookup[i] = idx;
+                verify( i <= bucketSizes[idx] );
+                verify( (i <= 32 && idx == 0) || (i > bucketSizes[idx - 1]) );
+        } // for
+        #endif // FASTLOOKUP
+
+        heapMasterBootFlag = true;
+} // heapMasterCtor
+
+
+#define NO_MEMORY_MSG "**** Error **** insufficient heap memory available to allocate %zd new bytes."
+
+Heap * getHeap() with( heapMaster ) {
+        Heap * heap;
+        if ( freeHeapManagersList ) {                                           // free heap for reused ?
+                heap = freeHeapManagersList;
+                freeHeapManagersList = heap->nextFreeHeapManager;
+
+                #ifdef __STATISTICS__
+                reused_heap += 1;
+                #endif // __STATISTICS__
+        } else {                                                                                        // free heap not found, create new
+                // Heap size is about 12K, FreeHeader (128 bytes because of cache alignment) * NoBucketSizes (91) => 128 heaps *
+                // 12K ~= 120K byte superblock.  Where 128-heap superblock handles a medium sized multi-processor server.
+                size_t remaining = heapManagersStorageEnd - heapManagersStorage; // remaining free heaps in superblock
+                if ( ! heapManagersStorage || remaining != 0 ) {
+                        // Each block of heaps is a multiple of the number of cores on the computer.
+                        int HeapDim = get_nprocs();                                     // get_nprocs_conf does not work
+                        size_t size = HeapDim * sizeof( Heap );
+
+                        heapManagersStorage = (Heap *)mmap( 0, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0 );
+                        if ( unlikely( heapManagersStorage == (Heap *)MAP_FAILED ) ) { // failed ?
+                                if ( errno == ENOMEM ) abort( NO_MEMORY_MSG, size ); // no memory
+                                // Do not call strerror( errno ) as it may call malloc.
+                                abort( "**** Error **** attempt to allocate block of heaps of size %zu bytes and mmap failed with errno %d.", size, errno );
+                        } // if
+                        heapManagersStorageEnd = &heapManagersStorage[HeapDim]; // outside array
+                } // if
+
+                heap = heapManagersStorage;
+                heapManagersStorage = heapManagersStorage + 1; // bump next heap
+
+                #if defined( __STATISTICS__ ) || defined( __CFA_DEBUG__ )
+                heap->nextHeapManager = heapManagersList;
+                #endif // __STATISTICS__ || __CFA_DEBUG__
+                heapManagersList = heap;
+
+                #ifdef __STATISTICS__
+                new_heap += 1;
+                #endif // __STATISTICS__
+
+                with( *heap ) {
+                        for ( unsigned int j = 0; j < NoBucketSizes; j += 1 ) { // initialize free lists
+                                #ifdef OWNERSHIP
+                                #ifdef RETURNSPIN
+                                ?{}( freeLists[j].returnLock );
+                                #endif // RETURNSPIN
+                                freeLists[j].returnList = 0p;
+                                #endif // OWNERSHIP
+                                freeLists[j].freeList = 0p;
+                                freeLists[j].homeManager = heap;
+                                freeLists[j].blockSize = bucketSizes[j];
+                        } // for
+        
+                        heapBuffer = 0p;
+                        heapReserve = 0;
+                        nextFreeHeapManager = 0p;
+                        #ifdef __CFA_DEBUG__
+                        allocUnfreed = 0;
+                        #endif // __CFA_DEBUG__
+                } // with
+        } // if
+
+        return heap;
+} // getHeap
+
+
+void heapManagerCtor() libcfa_public {
+        if ( unlikely( ! heapMasterBootFlag ) ) heapMasterCtor();
+
+        lock( heapMaster.mgrLock );             // protect heapMaster counters
+
+        // get storage for heap manager
+
+        heapManager = getHeap();
+
+        #ifdef __STATISTICS__
+        HeapStatisticsCtor( heapManager->stats );                       // heap local
+        heapMaster.threads_started += 1;
+        #endif // __STATISTICS__
+
+        unlock( heapMaster.mgrLock );
+} // heapManagerCtor
+
+
+void heapManagerDtor() libcfa_public {
+        lock( heapMaster.mgrLock );
+
+        // place heap on list of free heaps for reusability
+        heapManager->nextFreeHeapManager = heapMaster.freeHeapManagersList;
+        heapMaster.freeHeapManagersList = heapManager;
+
+        #ifdef __STATISTICS__
+        heapMaster.threads_exited += 1;
+        #endif // __STATISTICS__
+
+        // Do not set heapManager to NULL because it is used after Cforall is shutdown but before the program shuts down.
+
+        unlock( heapMaster.mgrLock );
+} // heapManagerDtor
 
 
 //####################### Memory Allocation Routines Helpers ####################
 
-
-#ifdef __CFA_DEBUG__
-static size_t allocUnfreed;                                                             // running total of allocations minus frees
-
-static void prtUnfreed() {
-        if ( allocUnfreed != 0 ) {
-                // DO NOT USE STREAMS AS THEY MAY BE UNAVAILABLE AT THIS POINT.
-                char helpText[512];
-                __cfaabi_bits_print_buffer( STDERR_FILENO, helpText, sizeof(helpText),
-                                                                        "CFA warning (UNIX pid:%ld) : program terminating with %zu(0x%zx) bytes of storage allocated but not freed.\n"
-                                                                        "Possible cause is unfreed storage allocated by the program or system/library routines called from the program.\n",
-                                                                        (long int)getpid(), allocUnfreed, allocUnfreed ); // always print the UNIX pid
-        } // if
-} // prtUnfreed
 
 extern int cfa_main_returned;                                                   // from interpose.cfa
 extern "C" {
+        void memory_startup( void ) {
+                if ( ! heapMasterBootFlag ) heapManagerCtor();  // sanity check
+        } // memory_startup
+
+        void memory_shutdown( void ) {
+                heapManagerDtor();
+        } // memory_shutdown
+
         void heapAppStart() {                                                           // called by __cfaabi_appready_startup
-                allocUnfreed = 0;
+                verify( heapManager );
+                #ifdef __CFA_DEBUG__
+                heapManager->allocUnfreed = 0;                                  // clear prior allocation counts
+                #endif // __CFA_DEBUG__
+
+                #ifdef __STATISTICS__
+                HeapStatisticsCtor( heapManager->stats );               // clear prior statistic counters
+                #endif // __STATISTICS__
         } // heapAppStart
 
         void heapAppStop() {                                                            // called by __cfaabi_appready_startdown
-                fclose( stdin ); fclose( stdout );
-                if ( cfa_main_returned ) prtUnfreed();                  // do not check unfreed storage if exit called
+                fclose( stdin ); fclose( stdout );                              // free buffer storage
+          if ( ! cfa_main_returned ) return;                            // do not check unfreed storage if exit called
+
+                #ifdef __CFA_DEBUG__
+                // allocUnfreed is set to 0 when a heap is created and it accumulates any unfreed storage during its multiple thread
+                // usages.  At the end, add up each heap allocUnfreed value across all heaps to get the total unfreed storage.
+                int64_t allocUnfreed = 0;
+                for ( Heap * heap = heapMaster.heapManagersList; heap; heap = heap->nextHeapManager ) {
+                        allocUnfreed += heap->allocUnfreed;
+                } // for
+
+                allocUnfreed -= malloc_unfreed();                               // subtract any user specified unfreed storage
+                if ( allocUnfreed > 0 ) {
+                        // DO NOT USE STREAMS AS THEY MAY BE UNAVAILABLE AT THIS POINT.
+                        char helpText[512];
+                        __cfaabi_bits_print_buffer( STDERR_FILENO, helpText, sizeof(helpText),
+                                                                                "CFA warning (UNIX pid:%ld) : program terminating with %ju(0x%jx) bytes of storage allocated but not freed.\n"
+                                                                                "Possible cause is unfreed storage allocated by the program or system/library routines called from the program.\n",
+                                                                                (long int)getpid(), allocUnfreed, allocUnfreed ); // always print the UNIX pid
+                } // if
+                #endif // __CFA_DEBUG__
         } // heapAppStop
 } // extern "C"
-#endif // __CFA_DEBUG__
 
 
 #ifdef __STATISTICS__
 static HeapStatistics stats;                                                    // zero filled
-static unsigned int sbrk_calls;
-static unsigned long long int sbrk_storage;
-// Statistics file descriptor (changed by malloc_stats_fd).
-static int stats_fd = STDERR_FILENO;                                    // default stderr
 
 #define prtFmt \
@@ -321 +618 @@
         "  realloc   >0 calls %'u; 0 calls %'u; storage %'llu / %'llu bytes\n" \
         "  free      !null calls %'u; null calls %'u; storage %'llu / %'llu bytes\n" \
-        "  sbrk      calls %'u; storage %'llu bytes\n"                                          \
-        "  mmap      calls %'u; storage %'llu / %'llu bytes\n"                          \
-        "  munmap    calls %'u; storage %'llu / %'llu bytes\n"                          \
+        "  return    pulls %'u; pushes %'u; storage %'llu / %'llu bytes\n" \
+        "  sbrk      calls %'u; storage %'llu bytes\n" \
+        "  mmap      calls %'u; storage %'llu / %'llu bytes\n" \
+        "  munmap    calls %'u; storage %'llu / %'llu bytes\n" \
+        "  threads   started %'lu; exited %'lu\n" \
+        "  heaps     new %'lu; reused %'lu\n"
 
 // Use "write" because streams may be shutdown when calls are made.
-static int printStats() {                                                               // see malloc_stats
+static int printStats( HeapStatistics & stats ) with( heapMaster, stats ) {     // see malloc_stats
         char helpText[sizeof(prtFmt) + 1024];                           // space for message and values
-        return __cfaabi_bits_print_buffer( STDERR_FILENO, helpText, sizeof(helpText), prtFmt,
-                        stats.malloc_calls, stats.malloc_0_calls, stats.malloc_storage_request, stats.malloc_storage_alloc,
-                        stats.aalloc_calls, stats.aalloc_0_calls, stats.aalloc_storage_request, stats.aalloc_storage_alloc,
-                        stats.calloc_calls, stats.calloc_0_calls, stats.calloc_storage_request, stats.calloc_storage_alloc,
-                        stats.memalign_calls, stats.memalign_0_calls, stats.memalign_storage_request, stats.memalign_storage_alloc,
-                        stats.amemalign_calls, stats.amemalign_0_calls, stats.amemalign_storage_request, stats.amemalign_storage_alloc,
-                        stats.cmemalign_calls, stats.cmemalign_0_calls, stats.cmemalign_storage_request, stats.cmemalign_storage_alloc,
-                        stats.resize_calls, stats.resize_0_calls, stats.resize_storage_request, stats.resize_storage_alloc,
-                        stats.realloc_calls, stats.realloc_0_calls, stats.realloc_storage_request, stats.realloc_storage_alloc,
-                        stats.free_calls, stats.free_null_calls, stats.free_storage_request, stats.free_storage_alloc,
+        return __cfaabi_bits_print_buffer( stats_fd, helpText, sizeof(helpText), prtFmt,
+                        malloc_calls, malloc_0_calls, malloc_storage_request, malloc_storage_alloc,
+                        aalloc_calls, aalloc_0_calls, aalloc_storage_request, aalloc_storage_alloc,
+                        calloc_calls, calloc_0_calls, calloc_storage_request, calloc_storage_alloc,
+                        memalign_calls, memalign_0_calls, memalign_storage_request, memalign_storage_alloc,
+                        amemalign_calls, amemalign_0_calls, amemalign_storage_request, amemalign_storage_alloc,
+                        cmemalign_calls, cmemalign_0_calls, cmemalign_storage_request, cmemalign_storage_alloc,
+                        resize_calls, resize_0_calls, resize_storage_request, resize_storage_alloc,
+                        realloc_calls, realloc_0_calls, realloc_storage_request, realloc_storage_alloc,
+                        free_calls, free_null_calls, free_storage_request, free_storage_alloc,
+                        return_pulls, return_pushes, return_storage_request, return_storage_alloc,
                         sbrk_calls, sbrk_storage,
-                        stats.mmap_calls, stats.mmap_storage_request, stats.mmap_storage_alloc,
-                        stats.munmap_calls, stats.munmap_storage_request, stats.munmap_storage_alloc
+                        mmap_calls, mmap_storage_request, mmap_storage_alloc,
+                        munmap_calls, munmap_storage_request, munmap_storage_alloc,
+                        threads_started, threads_exited,
+                        new_heap, reused_heap
                 );
 } // printStats
@@ -358 +661 @@
         "<total type=\"realloc\" >0 count=\"%'u;\" 0 count=\"%'u;\" size=\"%'llu / %'llu\"/> bytes\n" \
         "<total type=\"free\" !null=\"%'u;\" 0 null=\"%'u;\" size=\"%'llu / %'llu\"/> bytes\n" \
+        "<total type=\"return\" pulls=\"%'u;\" 0 pushes=\"%'u;\" size=\"%'llu / %'llu\"/> bytes\n" \
         "<total type=\"sbrk\" count=\"%'u;\" size=\"%'llu\"/> bytes\n" \
         "<total type=\"mmap\" count=\"%'u;\" size=\"%'llu / %'llu\" / > bytes\n" \
         "<total type=\"munmap\" count=\"%'u;\" size=\"%'llu / %'llu\"/> bytes\n" \
+        "<total type=\"threads\" started=\"%'lu;\" exited=\"%'lu\"/>\n" \
+        "<total type=\"heaps\" new=\"%'lu;\" reused=\"%'lu\"/>\n" \
         "</malloc>"
 
-static int printStatsXML( FILE * stream ) {                             // see malloc_info
+static int printStatsXML( HeapStatistics & stats, FILE * stream ) with( heapMaster, stats ) { // see malloc_info
         char helpText[sizeof(prtFmtXML) + 1024];                        // space for message and values
         return __cfaabi_bits_print_buffer( fileno( stream ), helpText, sizeof(helpText), prtFmtXML,
-                        stats.malloc_calls, stats.malloc_0_calls, stats.malloc_storage_request, stats.malloc_storage_alloc,
-                        stats.aalloc_calls, stats.aalloc_0_calls, stats.aalloc_storage_request, stats.aalloc_storage_alloc,
-                        stats.calloc_calls, stats.calloc_0_calls, stats.calloc_storage_request, stats.calloc_storage_alloc,
-                        stats.memalign_calls, stats.memalign_0_calls, stats.memalign_storage_request, stats.memalign_storage_alloc,
-                        stats.amemalign_calls, stats.amemalign_0_calls, stats.amemalign_storage_request, stats.amemalign_storage_alloc,
-                        stats.cmemalign_calls, stats.cmemalign_0_calls, stats.cmemalign_storage_request, stats.cmemalign_storage_alloc,
-                        stats.resize_calls, stats.resize_0_calls, stats.resize_storage_request, stats.resize_storage_alloc,
-                        stats.realloc_calls, stats.realloc_0_calls, stats.realloc_storage_request, stats.realloc_storage_alloc,
-                        stats.free_calls, stats.free_null_calls, stats.free_storage_request, stats.free_storage_alloc,
+                        malloc_calls, malloc_0_calls, malloc_storage_request, malloc_storage_alloc,
+                        aalloc_calls, aalloc_0_calls, aalloc_storage_request, aalloc_storage_alloc,
+                        calloc_calls, calloc_0_calls, calloc_storage_request, calloc_storage_alloc,
+                        memalign_calls, memalign_0_calls, memalign_storage_request, memalign_storage_alloc,
+                        amemalign_calls, amemalign_0_calls, amemalign_storage_request, amemalign_storage_alloc,
+                        cmemalign_calls, cmemalign_0_calls, cmemalign_storage_request, cmemalign_storage_alloc,
+                        resize_calls, resize_0_calls, resize_storage_request, resize_storage_alloc,
+                        realloc_calls, realloc_0_calls, realloc_storage_request, realloc_storage_alloc,
+                        free_calls, free_null_calls, free_storage_request, free_storage_alloc,
+                        return_pulls, return_pushes, return_storage_request, return_storage_alloc,
                         sbrk_calls, sbrk_storage,
-                        stats.mmap_calls, stats.mmap_storage_request, stats.mmap_storage_alloc,
-                        stats.munmap_calls, stats.munmap_storage_request, stats.munmap_storage_alloc
+                        mmap_calls, mmap_storage_request, mmap_storage_alloc,
+                    munmap_calls, munmap_storage_request, munmap_storage_alloc,
+                        threads_started, threads_exited,
+                        new_heap, reused_heap
                 );
 } // printStatsXML
+
+static HeapStatistics & collectStats( HeapStatistics & stats ) with( heapMaster ) {
+        lock( mgrLock );
+
+        stats += heapMaster.stats;
+        for ( Heap * heap = heapManagersList; heap; heap = heap->nextHeapManager ) {
+                stats += heap->stats;
+        } // for
+
+        unlock( mgrLock );
+        return stats;
+} // collectStats
 #endif // __STATISTICS__
 
 
-// statically allocated variables => zero filled.
-static size_t heapExpand;                                                               // sbrk advance
-static size_t mmapStart;                                                                // cross over point for mmap
-static unsigned int maxBucketsUsed;                                             // maximum number of buckets in use
-// extern visibility, used by runtime kernel
-// would be cool to remove libcfa_public but it's needed for libcfathread
-libcfa_public size_t __page_size;                                                       // architecture pagesize
-libcfa_public int __map_prot;                                                           // common mmap/mprotect protection
-
-
-// thunk problem
-size_t Bsearchl( unsigned int key, const unsigned int * vals, size_t dim ) {
-        size_t l = 0, m, h = dim;
-        while ( l < h ) {
-                m = (l + h) / 2;
-                if ( (unsigned int &)(vals[m]) < key ) {                // cast away const
-                        l = m + 1;
-                } else {
-                        h = m;
-                } // if
-        } // while
-        return l;
-} // Bsearchl
-
-
-static inline bool setMmapStart( size_t value ) {               // true => mmapped, false => sbrk
+static bool setMmapStart( size_t value ) with( heapMaster ) { // true => mmapped, false => sbrk
   if ( value < __page_size || bucketSizes[NoBucketSizes - 1] < value ) return false;
         mmapStart = value;                                                                      // set global
 
         // find the closest bucket size less than or equal to the mmapStart size
-        maxBucketsUsed = Bsearchl( (unsigned int)mmapStart, bucketSizes, NoBucketSizes ); // binary search
-        assert( maxBucketsUsed < NoBucketSizes );                       // subscript failure ?
-        assert( mmapStart <= bucketSizes[maxBucketsUsed] ); // search failure ?
+        maxBucketsUsed = Bsearchl( mmapStart, bucketSizes, NoBucketSizes ); // binary search
+        verify( maxBucketsUsed < NoBucketSizes );                       // subscript failure ?
+        verify( mmapStart <= bucketSizes[maxBucketsUsed] ); // search failure ?
         return true;
 } // setMmapStart
@@ -438 +734 @@
 
 
-static inline void checkAlign( size_t alignment ) {
+inline __attribute__((always_inline))
+static void checkAlign( size_t alignment ) {
         if ( unlikely( alignment < libAlign() || ! is_pow2( alignment ) ) ) {
                 abort( "**** Error **** alignment %zu for memory allocation is less than %d and/or not a power of 2.", alignment, libAlign() );
@@ -445 +742 @@
 
 
-static inline void checkHeader( bool check, const char name[], void * addr ) {
+inline __attribute__((always_inline))
+static void checkHeader( bool check, const char name[], void * addr ) {
         if ( unlikely( check ) ) {                                                      // bad address ?
                 abort( "**** Error **** attempt to %s storage %p with address outside the heap.\n"
@@ -470 +768 @@
 
 
-static inline void fakeHeader( Heap.Storage.Header *& header, size_t & alignment ) {
+inline __attribute__((always_inline))
+static void fakeHeader( Heap.Storage.Header *& header, size_t & alignment ) {
         if ( unlikely( AlignmentBit( header ) ) ) {                     // fake header ?
                 alignment = ClearAlignmentBit( header );                // clear flag from value
@@ -483 +782 @@
 
 
-static inline bool headers( const char name[] __attribute__(( unused )), void * addr, Heap.Storage.Header *& header,
-                                                        Heap.FreeHeader *& freeHead, size_t & size, size_t & alignment ) with( heapManager ) {
+inline __attribute__((always_inline))
+static bool headers( const char name[] __attribute__(( unused )), void * addr, Heap.Storage.Header *& header,
+                                                        Heap.FreeHeader *& freeHead, size_t & size, size_t & alignment ) with( heapMaster, *heapManager ) {
         header = HeaderAddr( addr );
 
@@ -509 +809 @@
         checkHeader( header < (Heap.Storage.Header *)heapBegin || (Heap.Storage.Header *)heapEnd < header, name, addr ); // bad address ? (offset could be + or -)
 
+        Heap * homeManager;
         if ( unlikely( freeHead == 0p || // freed and only free-list node => null link
                                    // freed and link points at another free block not to a bucket in the bucket array.
-                                   freeHead < &freeLists[0] || &freeLists[NoBucketSizes] <= freeHead ) ) {
+                                   (homeManager = freeHead->homeManager, freeHead < &homeManager->freeLists[0] ||
+                                        &homeManager->freeLists[NoBucketSizes] <= freeHead ) ) ) {
                 abort( "**** Error **** attempt to %s storage %p with corrupted header.\n"
                            "Possible cause is duplicate free on same block or overwriting of header information.",
@@ -521 +823 @@
 } // headers
 
-// #ifdef __CFA_DEBUG__
-// #if __SIZEOF_POINTER__ == 4
-// #define MASK 0xdeadbeef
-// #else
-// #define MASK 0xdeadbeefdeadbeef
-// #endif
-// #define STRIDE size_t
-
-// static void * Memset( void * addr, STRIDE size ) {           // debug only
-//      if ( size % sizeof(STRIDE) != 0 ) abort( "Memset() : internal error, size %zd not multiple of %zd.", size, sizeof(STRIDE) );
-//      if ( (STRIDE)addr % sizeof(STRIDE) != 0 ) abort( "Memset() : internal error, addr %p not multiple of %zd.", addr, sizeof(STRIDE) );
-
-//      STRIDE * end = (STRIDE *)addr + size / sizeof(STRIDE);
-//      for ( STRIDE * p = (STRIDE *)addr; p < end; p += 1 ) *p = MASK;
-//      return addr;
-// } // Memset
-// #endif // __CFA_DEBUG__
-
-
-#define NO_MEMORY_MSG "insufficient heap memory available for allocating %zd new bytes."
-
-static inline void * extend( size_t size ) with( heapManager ) {
-        lock( extlock __cfaabi_dbg_ctx2 );
+
+static void * master_extend( size_t size ) with( heapMaster ) {
+        lock( extLock );
 
         ptrdiff_t rem = heapRemaining - size;
@@ -549 +831 @@
                 // If the size requested is bigger than the current remaining storage, increase the size of the heap.
 
-                size_t increase = ceiling2( size > heapExpand ? size : heapExpand, __page_size );
+                size_t increase = ceiling2( size > heapExpand ? size : heapExpand, libAlign() );
                 // Do not call abort or strerror( errno ) as they may call malloc.
-                if ( sbrk( increase ) == (void *)-1 ) {                 // failed, no memory ?
-                        unlock( extlock );
-                        __cfaabi_bits_print_nolock( STDERR_FILENO, NO_MEMORY_MSG, size );
-                        _exit( EXIT_FAILURE );                                          // give up
+                if ( unlikely( sbrk( increase ) == (void *)-1 ) ) {     // failed, no memory ?
+                        unlock( extLock );
+                        abort( NO_MEMORY_MSG, size );                           // no memory
                 } // if
 
                 // Make storage executable for thunks.
                 if ( mprotect( (char *)heapEnd + heapRemaining, increase, __map_prot ) ) {
-                        unlock( extlock );
-                        __cfaabi_bits_print_nolock( STDERR_FILENO, "extend() : internal error, mprotect failure, heapEnd:%p size:%zd, errno:%d.\n", heapEnd, increase, errno );
-                        _exit( EXIT_FAILURE );
-                } // if
+                        unlock( extLock );
+                        abort( "**** Error **** attempt to make heap storage executable for thunks and mprotect failed with errno %d.", errno );
+                } // if
+
+                rem = heapRemaining + increase - size;
 
                 #ifdef __STATISTICS__
@@ -568 +850 @@
                 sbrk_storage += increase;
                 #endif // __STATISTICS__
-
-                #ifdef __CFA_DEBUG__
-                // Set new memory to garbage so subsequent uninitialized usages might fail.
-                memset( (char *)heapEnd + heapRemaining, '\xde', increase );
-                //Memset( (char *)heapEnd + heapRemaining, increase );
-                #endif // __CFA_DEBUG__
-
-                rem = heapRemaining + increase - size;
         } // if
 
@@ -581 +855 @@
         heapRemaining = rem;
         heapEnd = (char *)heapEnd + size;
-        unlock( extlock );
+
+        unlock( extLock );
         return block;
-} // extend
-
-
-static inline void * doMalloc( size_t size ) with( heapManager ) {
-        Heap.Storage * block;                                           // pointer to new block of storage
+} // master_extend
+
+
+__attribute__(( noinline ))
+static void * manager_extend( size_t size ) with( *heapManager ) {
+        ptrdiff_t rem = heapReserve - size;
+
+        if ( unlikely( rem < 0 ) ) {                                            // negative
+                // If the size requested is bigger than the current remaining reserve, use the current reserve to populate
+                // smaller freeLists, and increase the reserve.
+
+                rem = heapReserve;                                                              // positive
+
+                if ( rem >= bucketSizes[0] ) {                                  // minimal size ? otherwise ignore
+                        size_t bucket;
+                        #ifdef FASTLOOKUP
+                        if ( likely( rem < LookupSizes ) ) bucket = lookup[rem];
+                        #endif // FASTLOOKUP
+                                bucket = Bsearchl( rem, bucketSizes, heapMaster.maxBucketsUsed );
+                        verify( 0 <= bucket && bucket <= heapMaster.maxBucketsUsed );
+                        Heap.FreeHeader * freeHead = &(freeLists[bucket]);
+
+                        // The remaining storage many not be bucket size, whereas all other allocations are. Round down to previous
+                        // bucket size in this case.
+                        if ( unlikely( freeHead->blockSize > (size_t)rem ) ) freeHead -= 1;
+                        Heap.Storage * block = (Heap.Storage *)heapBuffer;
+
+                        block->header.kind.real.next = freeHead->freeList; // push on stack
+                        freeHead->freeList = block;
+                } // if
+
+                size_t increase = ceiling( size > ( heapMaster.heapExpand / 10 ) ? size : ( heapMaster.heapExpand / 10 ), libAlign() );
+                heapBuffer = master_extend( increase );
+                rem = increase - size;
+        } // if
+
+        Heap.Storage * block = (Heap.Storage *)heapBuffer;
+        heapReserve = rem;
+        heapBuffer = (char *)heapBuffer + size;
+
+        return block;
+} // manager_extend
+
+
+#define BOOT_HEAP_MANAGER \
+        if ( unlikely( ! heapMasterBootFlag ) ) { \
+                heapManagerCtor(); /* trigger for first heap */ \
+        } /* if */
+
+#ifdef __STATISTICS__
+#define STAT_NAME __counter
+#define STAT_PARM , unsigned int STAT_NAME
+#define STAT_ARG( name ) , name
+#define STAT_0_CNT( counter ) stats.counters[counter].calls_0 += 1
+#else
+#define STAT_NAME
+#define STAT_PARM
+#define STAT_ARG( name )
+#define STAT_0_CNT( counter )
+#endif // __STATISTICS__
+
+#define PROLOG( counter, ... ) \
+        BOOT_HEAP_MANAGER; \
+        if ( unlikely( size == 0 ) ||                                           /* 0 BYTE ALLOCATION RETURNS NULL POINTER */ \
+                unlikely( size > ULONG_MAX - sizeof(Heap.Storage) ) ) { /* error check */ \
+                STAT_0_CNT( counter ); \
+                __VA_ARGS__; \
+                return 0p; \
+        } /* if */
+
+
+#define SCRUB_SIZE 1024lu
+// Do not use '\xfe' for scrubbing because dereferencing an address composed of it causes a SIGSEGV *without* a valid IP
+// pointer in the interrupt frame.
+#define SCRUB '\xff'
+
+static void * doMalloc( size_t size STAT_PARM ) libcfa_nopreempt with( *heapManager ) {
+        PROLOG( STAT_NAME );
+
+        verify( heapManager );
+        Heap.Storage * block;                                                           // pointer to new block of storage
 
         // Look up size in the size list.  Make sure the user request includes space for the header that must be allocated
         // along with the block and is a multiple of the alignment size.
-
         size_t tsize = size + sizeof(Heap.Storage);
 
-        if ( likely( tsize < mmapStart ) ) {                            // small size => sbrk
-                size_t posn;
+        #ifdef __STATISTICS__
+        stats.counters[STAT_NAME].calls += 1;
+        stats.counters[STAT_NAME].request += size;
+        #endif // __STATISTICS__
+
+        #ifdef __CFA_DEBUG__
+        allocUnfreed += size;
+        #endif // __CFA_DEBUG__
+
+        if ( likely( tsize < heapMaster.mmapStart ) ) {         // small size => sbrk
+                size_t bucket;
                 #ifdef FASTLOOKUP
-                if ( tsize < LookupSizes ) posn = lookup[tsize];
+                if ( likely( tsize < LookupSizes ) ) bucket = lookup[tsize];
                 else
                 #endif // FASTLOOKUP
-                        posn = Bsearchl( (unsigned int)tsize, bucketSizes, (size_t)maxBucketsUsed );
-                Heap.FreeHeader * freeElem = &freeLists[posn];
-                verify( freeElem <= &freeLists[maxBucketsUsed] ); // subscripting error ?
-                verify( tsize <= freeElem->blockSize );                 // search failure ?
-                tsize = freeElem->blockSize;                                    // total space needed for request
+                        bucket = Bsearchl( tsize, bucketSizes, heapMaster.maxBucketsUsed );
+                verify( 0 <= bucket && bucket <= heapMaster.maxBucketsUsed );
+                Heap.FreeHeader * freeHead = &freeLists[bucket];
+
+                verify( freeHead <= &freeLists[heapMaster.maxBucketsUsed] ); // subscripting error ?
+                verify( tsize <= freeHead->blockSize );                 // search failure ?
+
+                tsize = freeHead->blockSize;                                    // total space needed for request
+                #ifdef __STATISTICS__
+                stats.counters[STAT_NAME].alloc += tsize;
+                #endif // __STATISTICS__
 
                 // Spin until the lock is acquired for this particular size of block.
 
                 #if BUCKETLOCK == SPINLOCK
-                lock( freeElem->lock __cfaabi_dbg_ctx2 );
-                block = freeElem->freeList;                                             // remove node from stack
+                block = freeHead->freeList;                                             // remove node from stack
                 #else
-                block = pop( freeElem->freeList );
+                block = pop( freeHead->freeList );
                 #endif // BUCKETLOCK
                 if ( unlikely( block == 0p ) ) {                                // no free block ?
+                        #ifdef OWNERSHIP
+                        // Freelist for that size is empty, so carve it out of the heap, if there is enough left, or get some more
+                        // and then carve it off.
+                        #ifdef RETURNSPIN
                         #if BUCKETLOCK == SPINLOCK
-                        unlock( freeElem->lock );
+                        lock( freeHead->returnLock );
+                        block = freeHead->returnList;
+                        freeHead->returnList = 0p;
+                        unlock( freeHead->returnLock );
+                        #else
+                        block = __atomic_exchange_n( &freeHead->returnList, nullptr, __ATOMIC_SEQ_CST );
+                        #endif // RETURNSPIN
+
+                        if ( likely( block == 0p ) ) {                  // return list also empty?
+                        #endif // OWNERSHIP
+                                // Do not leave kernel thread as manager_extend accesses heapManager.
+                                disable_interrupts();
+                                block = (Heap.Storage *)manager_extend( tsize ); // mutual exclusion on call
+                                enable_interrupts( false );
+
+                                // OK TO BE PREEMPTED HERE AS heapManager IS NO LONGER ACCESSED.
+
+                                #ifdef __CFA_DEBUG__
+                                // Scrub new memory so subsequent uninitialized usages might fail. Only scrub the first 1024 bytes.
+                                memset( block->data, SCRUB, min( SCRUB_SIZE, tsize - sizeof(Heap.Storage) ) );
+                                #endif // __CFA_DEBUG__
                         #endif // BUCKETLOCK
-
-                        // Freelist for that size was empty, so carve it out of the heap if there's enough left, or get some more
-                        // and then carve it off.
-
-                        block = (Heap.Storage *)extend( tsize );        // mutual exclusion on call
-                #if BUCKETLOCK == SPINLOCK
+                        #ifdef OWNERSHIP
+                        } else {                                                                        // merge returnList into freeHead
+                                #ifdef __STATISTICS__
+                                stats.return_pulls += 1;
+                                #endif // __STATISTICS__
+
+                                // OK TO BE PREEMPTED HERE AS heapManager IS NO LONGER ACCESSED.
+
+                                freeHead->freeList = block->header.kind.real.next;
+                        } // if
+                        #endif // OWNERSHIP
                 } else {
-                        freeElem->freeList = block->header.kind.real.next;
-                        unlock( freeElem->lock );
-                #endif // BUCKETLOCK
-                } // if
-
-                block->header.kind.real.home = freeElem;                // pointer back to free list of apropriate size
+                        // Memory is scrubbed in doFree.
+                        freeHead->freeList = block->header.kind.real.next;
+                } // if
+
+                block->header.kind.real.home = freeHead;                // pointer back to free list of apropriate size
         } else {                                                                                        // large size => mmap
   if ( unlikely( size > ULONG_MAX - __page_size ) ) return 0p;
                 tsize = ceiling2( tsize, __page_size );                 // must be multiple of page size
                 #ifdef __STATISTICS__
-                __atomic_add_fetch( &stats.mmap_calls, 1, __ATOMIC_SEQ_CST );
-                __atomic_add_fetch( &stats.mmap_storage_request, size, __ATOMIC_SEQ_CST );
-                __atomic_add_fetch( &stats.mmap_storage_alloc, tsize, __ATOMIC_SEQ_CST );
+                stats.counters[STAT_NAME].alloc += tsize;
+                stats.mmap_calls += 1;
+                stats.mmap_storage_request += size;
+                stats.mmap_storage_alloc += tsize;
                 #endif // __STATISTICS__
 
-                block = (Heap.Storage *)mmap( 0, tsize, __map_prot, MAP_PRIVATE | MAP_ANONYMOUS, mmapFd, 0 );
-                if ( block == (Heap.Storage *)MAP_FAILED ) { // failed ?
+                disable_interrupts();
+                block = (Heap.Storage *)mmap( 0, tsize, __map_prot, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0 );
+                enable_interrupts( false );
+
+                // OK TO BE PREEMPTED HERE AS heapManager IS NO LONGER ACCESSED.
+
+                if ( unlikely( block == (Heap.Storage *)MAP_FAILED ) ) { // failed ?
                         if ( errno == ENOMEM ) abort( NO_MEMORY_MSG, tsize ); // no memory
                         // Do not call strerror( errno ) as it may call malloc.
-                        abort( "(Heap &)0x%p.doMalloc() : internal error, mmap failure, size:%zu errno:%d.", &heapManager, tsize, errno );
-                } //if
+                        abort( "**** Error **** attempt to allocate large object (> %zu) of size %zu bytes and mmap failed with errno %d.", size, heapMaster.mmapStart, errno );
+                } // if
+                block->header.kind.real.blockSize = MarkMmappedBit( tsize ); // storage size for munmap
+
                 #ifdef __CFA_DEBUG__
-                // Set new memory to garbage so subsequent uninitialized usages might fail.
-                memset( block, '\xde', tsize );
-                //Memset( block, tsize );
+                // Scrub new memory so subsequent uninitialized usages might fail. Only scrub the first 1024 bytes.  The rest of
+                // the storage set to 0 by mmap.
+                memset( block->data, SCRUB, min( SCRUB_SIZE, tsize - sizeof(Heap.Storage) ) );
                 #endif // __CFA_DEBUG__
-                block->header.kind.real.blockSize = MarkMmappedBit( tsize ); // storage size for munmap
         } // if
 
@@ -659 +1058 @@
 
         #ifdef __CFA_DEBUG__
-        __atomic_add_fetch( &allocUnfreed, tsize, __ATOMIC_SEQ_CST );
         if ( traceHeap() ) {
                 char helpText[64];
@@ -667 +1065 @@
         #endif // __CFA_DEBUG__
 
+//      poll_interrupts();                                                                      // call rollforward
+
         return addr;
 } // doMalloc
 
 
-static inline void doFree( void * addr ) with( heapManager ) {
+static void doFree( void * addr ) libcfa_nopreempt with( *heapManager ) {
+        verify( addr );
+
+        // detect free after thread-local storage destruction and use global stats in that case
+
+        Heap.Storage.Header * header;
+        Heap.FreeHeader * freeHead;
+        size_t size, alignment;
+
+        bool mapped = headers( "free", addr, header, freeHead, size, alignment );
+        #if defined( __STATISTICS__ ) || defined( __CFA_DEBUG__ )
+        size_t rsize = header->kind.real.size;                          // optimization
+        #endif // __STATISTICS__ || __CFA_DEBUG__
+
+        #ifdef __STATISTICS__
+        stats.free_storage_request += rsize;
+        stats.free_storage_alloc += size;
+        #endif // __STATISTICS__
+
         #ifdef __CFA_DEBUG__
-        if ( unlikely( heapManager.heapBegin == 0p ) ) {
-                abort( "doFree( %p ) : internal error, called before heap is initialized.", addr );
-        } // if
+        allocUnfreed -= rsize;
         #endif // __CFA_DEBUG__
 
-        Heap.Storage.Header * header;
-        Heap.FreeHeader * freeElem;
-        size_t size, alignment;                                                         // not used (see realloc)
-
-        if ( headers( "free", addr, header, freeElem, size, alignment ) ) { // mmapped ?
+        if ( unlikely( mapped ) ) {                                                     // mmapped ?
                 #ifdef __STATISTICS__
-                __atomic_add_fetch( &stats.munmap_calls, 1, __ATOMIC_SEQ_CST );
-                __atomic_add_fetch( &stats.munmap_storage_request, header->kind.real.size, __ATOMIC_SEQ_CST );
-                __atomic_add_fetch( &stats.munmap_storage_alloc, size, __ATOMIC_SEQ_CST );
+                stats.munmap_calls += 1;
+                stats.munmap_storage_request += rsize;
+                stats.munmap_storage_alloc += size;
                 #endif // __STATISTICS__
-                if ( munmap( header, size ) == -1 ) {
-                        abort( "Attempt to deallocate storage %p not allocated or with corrupt header.\n"
-                                   "Possible cause is invalid pointer.",
-                                   addr );
+
+                // OK TO BE PREEMPTED HERE AS heapManager IS NO LONGER ACCESSED.
+
+                // Does not matter where this storage is freed.
+                if ( unlikely( munmap( header, size ) == -1 ) ) {
+                        // Do not call strerror( errno ) as it may call malloc.
+                        abort( "**** Error **** attempt to deallocate large object %p and munmap failed with errno %d.\n"
+                                   "Possible cause is invalid delete pointer: either not allocated or with corrupt header.",
+                                   addr, errno );
                 } // if
         } else {
                 #ifdef __CFA_DEBUG__
-                // Set free memory to garbage so subsequent usages might fail.
-                memset( ((Heap.Storage *)header)->data, '\xde', freeElem->blockSize - sizeof( Heap.Storage ) );
-                //Memset( ((Heap.Storage *)header)->data, freeElem->blockSize - sizeof( Heap.Storage ) );
+                // memset is NOT always inlined!
+                disable_interrupts();
+                // Scrub old memory so subsequent usages might fail. Only scrub the first/last SCRUB_SIZE bytes.
+                char * data = ((Heap.Storage *)header)->data;   // data address
+                size_t dsize = size - sizeof(Heap.Storage);             // data size
+                if ( dsize <= SCRUB_SIZE * 2 ) {
+                        memset( data, SCRUB, dsize );                           // scrub all
+                } else {
+                        memset( data, SCRUB, SCRUB_SIZE );                      // scrub front
+                        memset( data + dsize - SCRUB_SIZE, SCRUB, SCRUB_SIZE ); // scrub back
+                } // if
+                enable_interrupts( false );
                 #endif // __CFA_DEBUG__
 
-                #ifdef __STATISTICS__
-                __atomic_add_fetch( &stats.free_calls, 1, __ATOMIC_SEQ_CST );
-                __atomic_add_fetch( &stats.free_storage_request, header->kind.real.size, __ATOMIC_SEQ_CST );
-                __atomic_add_fetch( &stats.free_storage_alloc, size, __ATOMIC_SEQ_CST );
-                #endif // __STATISTICS__
-
-                #if BUCKETLOCK == SPINLOCK
-                lock( freeElem->lock __cfaabi_dbg_ctx2 );               // acquire spin lock
-                header->kind.real.next = freeElem->freeList;    // push on stack
-                freeElem->freeList = (Heap.Storage *)header;
-                unlock( freeElem->lock );                                               // release spin lock
-                #else
-                push( freeElem->freeList, *(Heap.Storage *)header );
-                #endif // BUCKETLOCK
+                if ( likely( heapManager == freeHead->homeManager ) ) { // belongs to this thread
+                        header->kind.real.next = freeHead->freeList; // push on stack
+                        freeHead->freeList = (Heap.Storage *)header;
+                } else {                                                                                // return to thread owner
+                        verify( heapManager );
+
+                        #ifdef OWNERSHIP
+                        #ifdef RETURNSPIN
+                        lock( freeHead->returnLock );
+                        header->kind.real.next = freeHead->returnList; // push to bucket return list
+                        freeHead->returnList = (Heap.Storage *)header;
+                        unlock( freeHead->returnLock );
+                        #else                                                                           // lock free
+                        header->kind.real.next = freeHead->returnList; // link new node to top node
+                        // CAS resets header->kind.real.next = freeHead->returnList on failure
+                        while ( ! __atomic_compare_exchange_n( &freeHead->returnList, &header->kind.real.next, header,
+                                                                                                   false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST ) );
+                        #endif // RETURNSPIN
+
+                        #else                                                                           // no OWNERSHIP
+
+                        freeHead = &heap->freeLists[ClearStickyBits( header->kind.real.home ) - &freeHead->homeManager->freeLists[0]];
+                        header->kind.real.next = freeHead->freeList; // push on stack
+                        freeHead->freeList = (Heap.Storage *)header;
+                        #endif // ! OWNERSHIP
+
+                        #ifdef __U_STATISTICS__
+                        stats.return_pushes += 1;
+                        stats.return_storage_request += rsize;
+                        stats.return_storage_alloc += size;
+                        #endif // __U_STATISTICS__
+
+                        // OK TO BE PREEMPTED HERE AS heapManager IS NO LONGER ACCESSED.
+                } // if
         } // if
 
         #ifdef __CFA_DEBUG__
-        __atomic_add_fetch( &allocUnfreed, -size, __ATOMIC_SEQ_CST );
         if ( traceHeap() ) {
                 char helpText[64];
@@ -724 +1169 @@
         } // if
         #endif // __CFA_DEBUG__
+
+//      poll_interrupts();                                                                      // call rollforward
 } // doFree
 
 
-static size_t prtFree( Heap & manager ) with( manager ) {
+size_t prtFree( Heap & manager ) with( manager ) {
         size_t total = 0;
         #ifdef __STATISTICS__
@@ -733 +1180 @@
         __cfaabi_bits_print_nolock( STDERR_FILENO, "\nBin lists (bin size : free blocks on list)\n" );
         #endif // __STATISTICS__
-        for ( unsigned int i = 0; i < maxBucketsUsed; i += 1 ) {
+        for ( unsigned int i = 0; i < heapMaster.maxBucketsUsed; i += 1 ) {
                 size_t size = freeLists[i].blockSize;
                 #ifdef __STATISTICS__
@@ -764 +1211 @@
         __cfaabi_bits_release();
         #endif // __STATISTICS__
-        return (char *)heapEnd - (char *)heapBegin - total;
+        return (char *)heapMaster.heapEnd - (char *)heapMaster.heapBegin - total;
 } // prtFree
 
 
-static void ?{}( Heap & manager ) with( manager ) {
-        __page_size = sysconf( _SC_PAGESIZE );
-        __map_prot = PROT_READ | PROT_WRITE | PROT_EXEC;
-
-        for ( unsigned int i = 0; i < NoBucketSizes; i += 1 ) { // initialize the free lists
-                freeLists[i].blockSize = bucketSizes[i];
-        } // for
-
-        #ifdef FASTLOOKUP
-        unsigned int idx = 0;
-        for ( unsigned int i = 0; i < LookupSizes; i += 1 ) {
-                if ( i > bucketSizes[idx] ) idx += 1;
-                lookup[i] = idx;
-        } // for
-        #endif // FASTLOOKUP
-
-        if ( ! setMmapStart( malloc_mmap_start() ) ) {
-                abort( "Heap : internal error, mmap start initialization failure." );
-        } // if
-        heapExpand = malloc_expansion();
-
-        char * end = (char *)sbrk( 0 );
-        heapBegin = heapEnd = sbrk( (char *)ceiling2( (long unsigned int)end, __page_size ) - end ); // move start of heap to multiple of alignment
-} // Heap
-
-
-static void ^?{}( Heap & ) {
-        #ifdef __STATISTICS__
-        if ( traceHeapTerm() ) {
-                printStats();
-                // prtUnfreed() called in heapAppStop()
-        } // if
-        #endif // __STATISTICS__
-} // ~Heap
-
-
-static void memory_startup( void ) __attribute__(( constructor( STARTUP_PRIORITY_MEMORY ) ));
-void memory_startup( void ) {
-        #ifdef __CFA_DEBUG__
-        if ( heapBoot ) {                                                                       // check for recursion during system boot
-                abort( "boot() : internal error, recursively invoked during system boot." );
-        } // if
-        heapBoot = true;
-        #endif // __CFA_DEBUG__
-
-        //verify( heapManager.heapBegin != 0 );
-        //heapManager{};
-        if ( heapManager.heapBegin == 0p ) heapManager{};       // sanity check
-} // memory_startup
-
-static void memory_shutdown( void ) __attribute__(( destructor( STARTUP_PRIORITY_MEMORY ) ));
-void memory_shutdown( void ) {
-        ^heapManager{};
-} // memory_shutdown
-
-
-static inline void * mallocNoStats( size_t size ) {             // necessary for malloc statistics
-        verify( heapManager.heapBegin != 0p );                          // called before memory_startup ?
-  if ( unlikely( size ) == 0 ) return 0p;                               // 0 BYTE ALLOCATION RETURNS NULL POINTER
-
-#if __SIZEOF_POINTER__ == 8
-        verify( size < ((typeof(size_t))1 << 48) );
-#endif // __SIZEOF_POINTER__ == 8
-        return doMalloc( size );
-} // mallocNoStats
-
-
-static inline void * memalignNoStats( size_t alignment, size_t size ) {
-  if ( unlikely( size ) == 0 ) return 0p;                               // 0 BYTE ALLOCATION RETURNS NULL POINTER
-
-        #ifdef __CFA_DEBUG__
+#ifdef __STATISTICS__
+static void incCalls( intptr_t statName ) libcfa_nopreempt {
+        heapManager->stats.counters[statName].calls += 1;
+} // incCalls
+
+static void incZeroCalls( intptr_t statName ) libcfa_nopreempt {
+        heapManager->stats.counters[statName].calls_0 += 1;
+} // incZeroCalls
+#endif // __STATISTICS__
+
+#ifdef __CFA_DEBUG__
+static void incUnfreed( intptr_t offset ) libcfa_nopreempt {
+        heapManager->allocUnfreed += offset;
+} // incUnfreed
+#endif // __CFA_DEBUG__
+
+
+static void * memalignNoStats( size_t alignment, size_t size STAT_PARM ) {
         checkAlign( alignment );                                                        // check alignment
-        #endif // __CFA_DEBUG__
-
-        // if alignment <= default alignment, do normal malloc as two headers are unnecessary
-  if ( unlikely( alignment <= libAlign() ) ) return mallocNoStats( size );
+
+        // if alignment <= default alignment or size == 0, do normal malloc as two headers are unnecessary
+  if ( unlikely( alignment <= libAlign() || size == 0 ) ) return doMalloc( size STAT_ARG( STAT_NAME ) );
 
         // Allocate enough storage to guarantee an address on the alignment boundary, and sufficient space before it for
@@ -854 +1247 @@
         // subtract libAlign() because it is already the minimum alignment
         // add sizeof(Storage) for fake header
-        char * addr = (char *)mallocNoStats( size + alignment - libAlign() + sizeof(Heap.Storage) );
+        size_t offset = alignment - libAlign() + sizeof(Heap.Storage);
+        char * addr = (char *)doMalloc( size + offset STAT_ARG( STAT_NAME ) );
 
         // address in the block of the "next" alignment address
@@ -860 +1254 @@
 
         // address of header from malloc
-        Heap.Storage.Header * RealHeader = HeaderAddr( addr );
-        RealHeader->kind.real.size = size;                                      // correct size to eliminate above alignment offset
-        // address of fake header * before* the alignment location
+        Heap.Storage.Header * realHeader = HeaderAddr( addr );
+        realHeader->kind.real.size = size;                                      // correct size to eliminate above alignment offset
+        #ifdef __CFA_DEBUG__
+        incUnfreed( -offset );                                                          // adjustment off the offset from call to doMalloc
+        #endif // __CFA_DEBUG__
+
+        // address of fake header *before* the alignment location
         Heap.Storage.Header * fakeHeader = HeaderAddr( user );
+
         // SKULLDUGGERY: insert the offset to the start of the actual storage block and remember alignment
-        fakeHeader->kind.fake.offset = (char *)fakeHeader - (char *)RealHeader;
+        fakeHeader->kind.fake.offset = (char *)fakeHeader - (char *)realHeader;
         // SKULLDUGGERY: odd alignment implies fake header
         fakeHeader->kind.fake.alignment = MarkAlignmentBit( alignment );
@@ -880 +1279 @@
         // then malloc() returns a unique pointer value that can later be successfully passed to free().
         void * malloc( size_t size ) libcfa_public {
-                #ifdef __STATISTICS__
-                if ( likely( size > 0 ) ) {
-                        __atomic_add_fetch( &stats.malloc_calls, 1, __ATOMIC_SEQ_CST );
-                        __atomic_add_fetch( &stats.malloc_storage_request, size, __ATOMIC_SEQ_CST );
-                } else {
-                        __atomic_add_fetch( &stats.malloc_0_calls, 1, __ATOMIC_SEQ_CST );
-                } // if
-                #endif // __STATISTICS__
-
-                return mallocNoStats( size );
+                return doMalloc( size STAT_ARG( MALLOC ) );
         } // malloc
 
@@ -895 +1285 @@
         // Same as malloc() except size bytes is an array of dim elements each of elemSize bytes.
         void * aalloc( size_t dim, size_t elemSize ) libcfa_public {
-                size_t size = dim * elemSize;
-                #ifdef __STATISTICS__
-                if ( likely( size > 0 ) ) {
-                        __atomic_add_fetch( &stats.aalloc_calls, 1, __ATOMIC_SEQ_CST );
-                        __atomic_add_fetch( &stats.aalloc_storage_request, size, __ATOMIC_SEQ_CST );
-                } else {
-                        __atomic_add_fetch( &stats.aalloc_0_calls, 1, __ATOMIC_SEQ_CST );
-                } // if
-                #endif // __STATISTICS__
-
-                return mallocNoStats( size );
+                return doMalloc( dim * elemSize STAT_ARG( AALLOC ) );
         } // aalloc
 
@@ -912 +1292 @@
         void * calloc( size_t dim, size_t elemSize ) libcfa_public {
                 size_t size = dim * elemSize;
-          if ( unlikely( size ) == 0 ) {                        // 0 BYTE ALLOCATION RETURNS NULL POINTER
-                        #ifdef __STATISTICS__
-                        __atomic_add_fetch( &stats.calloc_0_calls, 1, __ATOMIC_SEQ_CST );
-                        #endif // __STATISTICS__
-                        return 0p;
-                } // if
-                #ifdef __STATISTICS__
-                __atomic_add_fetch( &stats.calloc_calls, 1, __ATOMIC_SEQ_CST );
-                __atomic_add_fetch( &stats.calloc_storage_request, dim * elemSize, __ATOMIC_SEQ_CST );
-                #endif // __STATISTICS__
-
-                char * addr = (char *)mallocNoStats( size );
+                char * addr = (char *)doMalloc( size STAT_ARG( CALLOC ) );
+
+          if ( unlikely( addr == NULL ) ) return NULL;          // stop further processing if 0p is returned
 
                 Heap.Storage.Header * header;
-                Heap.FreeHeader * freeElem;
+                Heap.FreeHeader * freeHead;
                 size_t bsize, alignment;
 
@@ -932 +1303 @@
                 bool mapped =
                         #endif // __CFA_DEBUG__
-                        headers( "calloc", addr, header, freeElem, bsize, alignment );
+                        headers( "calloc", addr, header, freeHead, bsize, alignment );
 
                 #ifndef __CFA_DEBUG__
                 // Mapped storage is zero filled, but in debug mode mapped memory is scrubbed in doMalloc, so it has to be reset to zero.
-                if ( ! mapped )
+                if ( likely( ! mapped ) )
                 #endif // __CFA_DEBUG__
                         // <-------0000000000000000000000000000UUUUUUUUUUUUUUUUUUUUUUUUU> bsize (bucket size) U => undefined
@@ -952 +1323 @@
         // call to malloc(), alloc(), calloc() or realloc(). If the area pointed to was moved, a free(oaddr) is done.
         void * resize( void * oaddr, size_t size ) libcfa_public {
-                // If size is equal to 0, either NULL or a pointer suitable to be passed to free() is returned.
-          if ( unlikely( size == 0 ) ) {                                        // special cases
-                        #ifdef __STATISTICS__
-                        __atomic_add_fetch( &stats.resize_0_calls, 1, __ATOMIC_SEQ_CST );
-                        #endif // __STATISTICS__
-                        free( oaddr );
-                        return 0p;
-                } // if
-                #ifdef __STATISTICS__
-                __atomic_add_fetch( &stats.resize_calls, 1, __ATOMIC_SEQ_CST );
-                #endif // __STATISTICS__
-
-          if ( unlikely( oaddr == 0p ) ) {
-                        #ifdef __STATISTICS__
-                        __atomic_add_fetch( &stats.resize_storage_request, size, __ATOMIC_SEQ_CST );
-                        #endif // __STATISTICS__
-                        return mallocNoStats( size );
-                } // if
+          if ( unlikely( oaddr == 0p ) ) {                              // => malloc( size )
+                        return doMalloc( size STAT_ARG( RESIZE ) );
+                } // if
+
+                PROLOG( RESIZE, doFree( oaddr ) );                              // => free( oaddr )
 
                 Heap.Storage.Header * header;
-                Heap.FreeHeader * freeElem;
+                Heap.FreeHeader * freeHead;
                 size_t bsize, oalign;
-                headers( "resize", oaddr, header, freeElem, bsize, oalign );
+                headers( "resize", oaddr, header, freeHead, bsize, oalign );
 
                 size_t odsize = DataStorage( bsize, oaddr, header ); // data storage available in bucket
@@ -980 +1338 @@
                 if ( oalign == libAlign() && size <= odsize && odsize <= size * 2 ) { // allow 50% wasted storage for smaller size
                         ClearZeroFillBit( header );                                     // no alignment and turn off 0 fill
+                        #ifdef __CFA_DEBUG__
+                        incUnfreed( size - header->kind.real.size ); // adjustment off the size difference
+                        #endif // __CFA_DEBUG__
                         header->kind.real.size = size;                          // reset allocation size
+                        #ifdef __STATISTICS__
+                        incCalls( RESIZE );
+                        #endif // __STATISTICS__
                         return oaddr;
                 } // if
 
-                #ifdef __STATISTICS__
-                __atomic_add_fetch( &stats.resize_storage_request, size, __ATOMIC_SEQ_CST );
-                #endif // __STATISTICS__
-
                 // change size, DO NOT preserve STICKY PROPERTIES.
-                free( oaddr );
-                return mallocNoStats( size );                                   // create new area
+                doFree( oaddr );                                                                // free previous storage
+
+                return doMalloc( size STAT_ARG( RESIZE ) );             // create new area
         } // resize
 
@@ -997 +1358 @@
         // the old and new sizes.
         void * realloc( void * oaddr, size_t size ) libcfa_public {
-                // If size is equal to 0, either NULL or a pointer suitable to be passed to free() is returned.
-          if ( unlikely( size == 0 ) ) {                                        // special cases
-                        #ifdef __STATISTICS__
-                        __atomic_add_fetch( &stats.realloc_0_calls, 1, __ATOMIC_SEQ_CST );
-                        #endif // __STATISTICS__
-                        free( oaddr );
-                        return 0p;
-                } // if
-                #ifdef __STATISTICS__
-                __atomic_add_fetch( &stats.realloc_calls, 1, __ATOMIC_SEQ_CST );
-                #endif // __STATISTICS__
-
-          if ( unlikely( oaddr == 0p ) ) {
-                        #ifdef __STATISTICS__
-                        __atomic_add_fetch( &stats.realloc_storage_request, size, __ATOMIC_SEQ_CST );
-                        #endif // __STATISTICS__
-                        return mallocNoStats( size );
-                } // if
+          if ( unlikely( oaddr == 0p ) ) {                                      // => malloc( size )
+                  return doMalloc( size STAT_ARG( REALLOC ) );
+                } // if
+
+                PROLOG( REALLOC, doFree( oaddr ) );                             // => free( oaddr )
 
                 Heap.Storage.Header * header;
-                Heap.FreeHeader * freeElem;
+                Heap.FreeHeader * freeHead;
                 size_t bsize, oalign;
-                headers( "realloc", oaddr, header, freeElem, bsize, oalign );
+                headers( "realloc", oaddr, header, freeHead, bsize, oalign );
 
                 size_t odsize = DataStorage( bsize, oaddr, header ); // data storage available in bucket
@@ -1025 +1373 @@
                 bool ozfill = ZeroFillBit( header );                    // old allocation zero filled
           if ( unlikely( size <= odsize ) && odsize <= size * 2 ) { // allow up to 50% wasted storage
-                        header->kind.real.size = size;                          // reset allocation size
+                        #ifdef __CFA_DEBUG__
+                        incUnfreed( size - header->kind.real.size ); // adjustment off the size difference
+                        #endif // __CFA_DEBUG__
+                        header->kind.real.size = size;                          // reset allocation size
                         if ( unlikely( ozfill ) && size > osize ) {     // previous request zero fill and larger ?
                                 memset( (char *)oaddr + osize, '\0', size - osize ); // initialize added storage
                         } // if
+                        #ifdef __STATISTICS__
+                        incCalls( REALLOC );
+                        #endif // __STATISTICS__
                         return oaddr;
                 } // if
 
-                #ifdef __STATISTICS__
-                __atomic_add_fetch( &stats.realloc_storage_request, size, __ATOMIC_SEQ_CST );
-                #endif // __STATISTICS__
-
                 // change size and copy old content to new storage
 
                 void * naddr;
-                if ( likely( oalign == libAlign() ) ) {                 // previous request not aligned ?
-                        naddr = mallocNoStats( size );                          // create new area
+                if ( likely( oalign <= libAlign() ) ) {                 // previous request not aligned ?
+                        naddr = doMalloc( size STAT_ARG( REALLOC ) ); // create new area
                 } else {
-                        naddr = memalignNoStats( oalign, size );        // create new aligned area
-                } // if
-
-                headers( "realloc", naddr, header, freeElem, bsize, oalign );
+                        naddr = memalignNoStats( oalign, size STAT_ARG( REALLOC ) ); // create new aligned area
+                } // if
+
+                headers( "realloc", naddr, header, freeHead, bsize, oalign );
+                // To preserve prior fill, the entire bucket must be copied versus the size.
                 memcpy( naddr, oaddr, min( osize, size ) );             // copy bytes
-                free( oaddr );
+                doFree( oaddr );                                                                // free previous storage
 
                 if ( unlikely( ozfill ) ) {                                             // previous request zero fill ?
@@ -1067 +1418 @@
         // Same as malloc() except the memory address is a multiple of alignment, which must be a power of two. (obsolete)
         void * memalign( size_t alignment, size_t size ) libcfa_public {
-                #ifdef __STATISTICS__
-                if ( likely( size > 0 ) ) {
-                        __atomic_add_fetch( &stats.memalign_calls, 1, __ATOMIC_SEQ_CST );
-                        __atomic_add_fetch( &stats.memalign_storage_request, size, __ATOMIC_SEQ_CST );
-                } else {
-                        __atomic_add_fetch( &stats.memalign_0_calls, 1, __ATOMIC_SEQ_CST );
-                } // if
-                #endif // __STATISTICS__
-
-                return memalignNoStats( alignment, size );
+                return memalignNoStats( alignment, size STAT_ARG( MEMALIGN ) );
         } // memalign
 
@@ -1082 +1424 @@
         // Same as aalloc() with memory alignment.
         void * amemalign( size_t alignment, size_t dim, size_t elemSize ) libcfa_public {
-                size_t size = dim * elemSize;
-                #ifdef __STATISTICS__
-                if ( likely( size > 0 ) ) {
-                        __atomic_add_fetch( &stats.cmemalign_calls, 1, __ATOMIC_SEQ_CST );
-                        __atomic_add_fetch( &stats.cmemalign_storage_request, size, __ATOMIC_SEQ_CST );
-                } else {
-                        __atomic_add_fetch( &stats.cmemalign_0_calls, 1, __ATOMIC_SEQ_CST );
-                } // if
-                #endif // __STATISTICS__
-
-                return memalignNoStats( alignment, size );
+                return memalignNoStats( alignment, dim * elemSize STAT_ARG( AMEMALIGN ) );
         } // amemalign
 
@@ -1099 +1431 @@
         void * cmemalign( size_t alignment, size_t dim, size_t elemSize ) libcfa_public {
                 size_t size = dim * elemSize;
-          if ( unlikely( size ) == 0 ) {                                        // 0 BYTE ALLOCATION RETURNS NULL POINTER
-                        #ifdef __STATISTICS__
-                        __atomic_add_fetch( &stats.cmemalign_0_calls, 1, __ATOMIC_SEQ_CST );
-                        #endif // __STATISTICS__
-                        return 0p;
-                } // if
-                #ifdef __STATISTICS__
-                __atomic_add_fetch( &stats.cmemalign_calls, 1, __ATOMIC_SEQ_CST );
-                __atomic_add_fetch( &stats.cmemalign_storage_request, dim * elemSize, __ATOMIC_SEQ_CST );
-                #endif // __STATISTICS__
-
-                char * addr = (char *)memalignNoStats( alignment, size );
+                char * addr = (char *)memalignNoStats( alignment, size STAT_ARG( CMEMALIGN ) );
+
+          if ( unlikely( addr == NULL ) ) return NULL;          // stop further processing if 0p is returned
 
                 Heap.Storage.Header * header;
-                Heap.FreeHeader * freeElem;
+                Heap.FreeHeader * freeHead;
                 size_t bsize;
 
@@ -1119 +1442 @@
                 bool mapped =
                         #endif // __CFA_DEBUG__
-                        headers( "cmemalign", addr, header, freeElem, bsize, alignment );
+                        headers( "cmemalign", addr, header, freeHead, bsize, alignment );
 
                 // Mapped storage is zero filled, but in debug mode mapped memory is scrubbed in doMalloc, so it has to be reset to zero.
@@ -1169 +1492 @@
         // 0p, no operation is performed.
         void free( void * addr ) libcfa_public {
+//              verify( heapManager );
+
           if ( unlikely( addr == 0p ) ) {                                       // special case
                         #ifdef __STATISTICS__
-                        __atomic_add_fetch( &stats.free_null_calls, 1, __ATOMIC_SEQ_CST );
+                  if ( heapManager )
+                        incZeroCalls( FREE );
                         #endif // __STATISTICS__
-
-                        // #ifdef __CFA_DEBUG__
-                        // if ( traceHeap() ) {
-                        //      #define nullmsg "Free( 0x0 ) size:0\n"
-                        //      // Do not debug print free( 0p ), as it can cause recursive entry from sprintf.
-                        //      __cfaabi_dbg_write( nullmsg, sizeof(nullmsg) - 1 );
-                        // } // if
-                        // #endif // __CFA_DEBUG__
                         return;
-                } // exit
-
-                doFree( addr );
+                } // if
+
+                #ifdef __STATISTICS__
+                incCalls( FREE );
+                #endif // __STATISTICS__
+
+                doFree( addr );                                                                 // handles heapManager == nullptr
         } // free
 
@@ -1227 +1549 @@
           if ( unlikely( addr == 0p ) ) return 0;                       // null allocation has 0 size
                 Heap.Storage.Header * header;
-                Heap.FreeHeader * freeElem;
+                Heap.FreeHeader * freeHead;
                 size_t bsize, alignment;
 
-                headers( "malloc_usable_size", addr, header, freeElem, bsize, alignment );
+                headers( "malloc_usable_size", addr, header, freeHead, bsize, alignment );
                 return DataStorage( bsize, addr, header );              // data storage in bucket
         } // malloc_usable_size
@@ -1238 +1560 @@
         void malloc_stats( void ) libcfa_public {
                 #ifdef __STATISTICS__
-                printStats();
-                if ( prtFree() ) prtFree( heapManager );
+                HeapStatistics stats;
+                HeapStatisticsCtor( stats );
+                if ( printStats( collectStats( stats ) ) == -1 ) {
+                #else
+                #define MALLOC_STATS_MSG "malloc_stats statistics disabled.\n"
+                if ( write( STDERR_FILENO, MALLOC_STATS_MSG, sizeof( MALLOC_STATS_MSG ) - 1 /* size includes '\0' */ ) == -1 ) {
                 #endif // __STATISTICS__
+                        abort( "**** Error **** write failed in malloc_stats" );
+                } // if
         } // malloc_stats
 
@@ -1247 +1575 @@
         int malloc_stats_fd( int fd __attribute__(( unused )) ) libcfa_public {
                 #ifdef __STATISTICS__
-                int temp = stats_fd;
-                stats_fd = fd;
+                int temp = heapMaster.stats_fd;
+                heapMaster.stats_fd = fd;
                 return temp;
                 #else
@@ -1262 +1590 @@
           if ( options != 0 ) { errno = EINVAL; return -1; }
                 #ifdef __STATISTICS__
-                return printStatsXML( stream );
+                HeapStatistics stats;
+                HeapStatisticsCtor( stats );
+                return printStatsXML( collectStats( stats ), stream ); // returns bytes written or -1
                 #else
                 return 0;                                                                               // unsupported
@@ -1275 +1605 @@
                 choose( option ) {
                   case M_TOP_PAD:
-                        heapExpand = ceiling2( value, __page_size );
+                        heapMaster.heapExpand = ceiling2( value, __page_size );
                         return 1;
                   case M_MMAP_THRESHOLD:
@@ -1319 +1649 @@
 // Must have CFA linkage to overload with C linkage realloc.
 void * resize( void * oaddr, size_t nalign, size_t size ) libcfa_public {
-        // If size is equal to 0, either NULL or a pointer suitable to be passed to free() is returned.
-  if ( unlikely( size == 0 ) ) {                                                // special cases
-                #ifdef __STATISTICS__
-                __atomic_add_fetch( &stats.resize_0_calls, 1, __ATOMIC_SEQ_CST );
-                #endif // __STATISTICS__
-                free( oaddr );
-                return 0p;
+  if ( unlikely( oaddr == 0p ) ) {                                              // => malloc( size )
+                return memalignNoStats( nalign, size STAT_ARG( RESIZE ) );
         } // if
 
-        if ( unlikely( nalign < libAlign() ) ) nalign = libAlign(); // reset alignment to minimum
-        #ifdef __CFA_DEBUG__
-        else checkAlign( nalign );                                                      // check alignment
-        #endif // __CFA_DEBUG__
-
-  if ( unlikely( oaddr == 0p ) ) {
-                #ifdef __STATISTICS__
-                __atomic_add_fetch( &stats.resize_calls, 1, __ATOMIC_SEQ_CST );
-                __atomic_add_fetch( &stats.resize_storage_request, size, __ATOMIC_SEQ_CST );
-                #endif // __STATISTICS__
-                return memalignNoStats( nalign, size );
-        } // if
+        PROLOG( RESIZE, doFree( oaddr ) );                                      // => free( oaddr )
 
         // Attempt to reuse existing alignment.
@@ -1347 +1661 @@
 
         if ( unlikely( isFakeHeader ) ) {
+                checkAlign( nalign );                                                   // check alignment
                 oalign = ClearAlignmentBit( header );                   // old alignment
                 if ( unlikely( (uintptr_t)oaddr % nalign == 0   // lucky match ?
@@ -1353 +1668 @@
                         ) ) {
                         HeaderAddr( oaddr )->kind.fake.alignment = MarkAlignmentBit( nalign ); // update alignment (could be the same)
-                        Heap.FreeHeader * freeElem;
+                        Heap.FreeHeader * freeHead;
                         size_t bsize, oalign;
-                        headers( "resize", oaddr, header, freeElem, bsize, oalign );
+                        headers( "resize", oaddr, header, freeHead, bsize, oalign );
                         size_t odsize = DataStorage( bsize, oaddr, header ); // data storage available in bucket
 
@@ -1361 +1676 @@
                                 HeaderAddr( oaddr )->kind.fake.alignment = MarkAlignmentBit( nalign ); // update alignment (could be the same)
                                 ClearZeroFillBit( header );                             // turn off 0 fill
+                                #ifdef __CFA_DEBUG__
+                                incUnfreed( size - header->kind.real.size ); // adjustment off the size difference
+                                #endif // __CFA_DEBUG__
                                 header->kind.real.size = size;                  // reset allocation size
+                                #ifdef __STATISTICS__
+                                incCalls( RESIZE );
+                                #endif // __STATISTICS__
                                 return oaddr;
                         } // if
@@ -1370 +1691 @@
         } // if
 
-        #ifdef __STATISTICS__
-        __atomic_add_fetch( &stats.resize_storage_request, size, __ATOMIC_SEQ_CST );
-        #endif // __STATISTICS__
-
         // change size, DO NOT preserve STICKY PROPERTIES.
-        free( oaddr );
-        return memalignNoStats( nalign, size );                         // create new aligned area
+        doFree( oaddr );                                                                        // free previous storage
+        return memalignNoStats( nalign, size STAT_ARG( RESIZE ) ); // create new aligned area
 } // resize
 
 
 void * realloc( void * oaddr, size_t nalign, size_t size ) libcfa_public {
-        // If size is equal to 0, either NULL or a pointer suitable to be passed to free() is returned.
-  if ( unlikely( size == 0 ) ) {                                                // special cases
-                #ifdef __STATISTICS__
-                __atomic_add_fetch( &stats.realloc_0_calls, 1, __ATOMIC_SEQ_CST );
-                #endif // __STATISTICS__
-                free( oaddr );
-                return 0p;
+  if ( unlikely( oaddr == 0p ) ) {                                              // => malloc( size )
+                return memalignNoStats( nalign, size STAT_ARG( REALLOC ) );
         } // if
 
-        if ( unlikely( nalign < libAlign() ) ) nalign = libAlign(); // reset alignment to minimum
-        #ifdef __CFA_DEBUG__
-        else checkAlign( nalign );                                                      // check alignment
-        #endif // __CFA_DEBUG__
-
-  if ( unlikely( oaddr == 0p ) ) {
-                #ifdef __STATISTICS__
-                __atomic_add_fetch( &stats.realloc_calls, 1, __ATOMIC_SEQ_CST );
-                __atomic_add_fetch( &stats.realloc_storage_request, size, __ATOMIC_SEQ_CST );
-                #endif // __STATISTICS__
-                return memalignNoStats( nalign, size );
-        } // if
+        PROLOG( REALLOC, doFree( oaddr ) );                                     // => free( oaddr )
 
         // Attempt to reuse existing alignment.
@@ -1408 +1709 @@
         size_t oalign;
         if ( unlikely( isFakeHeader ) ) {
+                checkAlign( nalign );                                                   // check alignment
                 oalign = ClearAlignmentBit( header );                   // old alignment
                 if ( unlikely( (uintptr_t)oaddr % nalign == 0   // lucky match ?
@@ -1421 +1723 @@
         } // if
 
-        #ifdef __STATISTICS__
-        __atomic_add_fetch( &stats.realloc_calls, 1, __ATOMIC_SEQ_CST );
-        __atomic_add_fetch( &stats.realloc_storage_request, size, __ATOMIC_SEQ_CST );
-        #endif // __STATISTICS__
-
-        Heap.FreeHeader * freeElem;
+        Heap.FreeHeader * freeHead;
         size_t bsize;
-        headers( "realloc", oaddr, header, freeElem, bsize, oalign );
+        headers( "realloc", oaddr, header, freeHead, bsize, oalign );
 
         // change size and copy old content to new storage
@@ -1435 +1732 @@
         bool ozfill = ZeroFillBit( header );                            // old allocation zero filled
 
-        void * naddr = memalignNoStats( nalign, size );         // create new aligned area
-
-        headers( "realloc", naddr, header, freeElem, bsize, oalign );
+        void * naddr = memalignNoStats( nalign, size STAT_ARG( REALLOC ) ); // create new aligned area
+
+        headers( "realloc", naddr, header, freeHead, bsize, oalign );
         memcpy( naddr, oaddr, min( osize, size ) );                     // copy bytes
-        free( oaddr );
+        doFree( oaddr );                                                                        // free previous storage
 
         if ( unlikely( ozfill ) ) {                                                     // previous request zero fill ?
@@ -1451 +1748 @@
 
 
+void * reallocarray( void * oaddr, size_t nalign, size_t dim, size_t elemSize ) __THROW {
+        return realloc( oaddr, nalign, dim * elemSize );
+} // reallocarray
+
+
 // Local Variables: //
 // tab-width: 4 //

libcfa/src/heap.hfa

@@ -10 +10 @@
 // Created On       : Tue May 26 11:23:55 2020
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Thu Apr 21 22:52:25 2022
-// Update Count     : 21
+// Last Modified On : Tue Oct  4 19:08:55 2022
+// Update Count     : 23
 // 
 
@@ -30 +30 @@
 bool checkFreeOff();
 
-// supported mallopt options
-#ifndef M_MMAP_THRESHOLD
-#define M_MMAP_THRESHOLD (-1)
-#endif // M_MMAP_THRESHOLD
-
-#ifndef M_TOP_PAD
-#define M_TOP_PAD (-2)
-#endif // M_TOP_PAD
-
 extern "C" {
         // New allocation operations.
@@ -49 +40 @@
         size_t malloc_size( void * addr );
         int malloc_stats_fd( int fd );
-        size_t malloc_usable_size( void * addr );
         size_t malloc_expansion();                                                      // heap expansion size (bytes)
         size_t malloc_mmap_start();                                                     // crossover allocation size from sbrk to mmap

libcfa/src/math.hfa

@@ -10 +10 @@
 // Created On       : Mon Apr 18 23:37:04 2016
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Thu Apr 15 11:47:56 2021
-// Update Count     : 132
+// Last Modified On : Sat Oct  8 08:40:42 2022
+// Update Count     : 136
 //
 
@@ -22 +22 @@
 
 #include "common.hfa"
+#include "bits/debug.hfa"
 
 //---------------------- General ----------------------
 
-static inline {
+static inline __attribute__((always_inline)) {
         float ?%?( float x, float y ) { return fmodf( x, y ); }
         float fmod( float x, float y ) { return fmodf( x, y ); }
@@ -63 +64 @@
 //---------------------- Exponential ----------------------
 
-static inline {
+static inline __attribute__((always_inline)) {
         float exp( float x ) { return expf( x ); }
         // extern "C" { double exp( double ); }
@@ -92 +93 @@
 //---------------------- Logarithm ----------------------
 
-static inline {
+static inline __attribute__((always_inline)) {
         float log( float x ) { return logf( x ); }
         // extern "C" { double log( double ); }
@@ -147 +148 @@
 } // distribution
 
+static inline unsigned long long log2_u32_32( unsigned long long val ) {
+        enum {
+                TABLE_BITS = 6,
+                TABLE_SIZE = (1 << TABLE_BITS) + 2,
+        };
+        // for(i; TABLE_SIZE) {
+        //      table[i] = (unsigned long long)(log2(1.0 + i / pow(2, TABLE_BITS)) * pow(2, 32)));
+        // }
+        static const unsigned long long table[] = {
+                0x0000000000, 0x0005b9e5a1, 0x000b5d69ba, 0x0010eb389f,
+                0x001663f6fa, 0x001bc84240, 0x002118b119, 0x002655d3c4,
+                0x002b803473, 0x00309857a0, 0x00359ebc5b, 0x003a93dc98,
+                0x003f782d72, 0x00444c1f6b, 0x0049101eac, 0x004dc4933a,
+                0x005269e12f, 0x00570068e7, 0x005b888736, 0x006002958c,
+                0x00646eea24, 0x0068cdd829, 0x006d1fafdc, 0x007164beb4,
+                0x00759d4f80, 0x0079c9aa87, 0x007dea15a3, 0x0081fed45c,
+                0x0086082806, 0x008a064fd5, 0x008df988f4, 0x0091e20ea1,
+                0x0095c01a39, 0x009993e355, 0x009d5d9fd5, 0x00a11d83f4,
+                0x00a4d3c25e, 0x00a8808c38, 0x00ac241134, 0x00afbe7fa0,
+                0x00b3500472, 0x00b6d8cb53, 0x00ba58feb2, 0x00bdd0c7c9,
+                0x00c1404ead, 0x00c4a7ba58, 0x00c80730b0, 0x00cb5ed695,
+                0x00ceaecfea, 0x00d1f73f9c, 0x00d53847ac, 0x00d8720935,
+                0x00dba4a47a, 0x00ded038e6, 0x00e1f4e517, 0x00e512c6e5,
+                0x00e829fb69, 0x00eb3a9f01, 0x00ee44cd59, 0x00f148a170,
+                0x00f446359b, 0x00f73da38d, 0x00fa2f045e, 0x00fd1a708b,
+                0x0100000000, 0x0102dfca16,
+        };
+        _Static_assert((sizeof(table) / sizeof(table[0])) == TABLE_SIZE, "TABLE_SIZE should be accurate");
+        // starting from val = (2 ** i)*(1 + f) where 0 <= f < 1
+        // log identities mean log2(val) = log2((2 ** i)*(1 + f)) = log2(2**i) + log2(1+f)
+        //
+        // getting i is easy to do using builtin_clz (count leading zero)
+        //
+        // we want to calculate log2(1+f) independently to have a many bits of precision as possible.
+        //     val = (2 ** i)*(1 + f) = 2 ** i   +   f * 2 ** i
+        // isolating f we get
+        //     val - 2 ** i = f * 2 ** i
+        //     (val - 2 ** i) / 2 ** i = f
+        //
+        // we want to interpolate from the table to get the values
+        // and compromise by doing quadratic interpolation (rather than higher degree interpolation)
+        //
+        // for the interpolation we want to shift everything the fist sample point
+        // so our parabola becomes x = 0
+        // this further simplifies the equations
+        //
+        // the consequence is that we need f in 2 forms:
+        //  - finding the index of x0
+        //  - finding the distance between f and x0
+        //
+        // since sample points are equidistant we can significantly simplify the equations
+
+        // get i
+        const unsigned long long bits = sizeof(val) * __CHAR_BIT__;
+        const unsigned long long lz = __builtin_clzl(val);
+        const unsigned long long i = bits - 1 - lz;
+
+        // get the fractinal part as a u32.32
+        const unsigned long long frac = (val << (lz + 1)) >> 32;
+
+        // get high order bits for the index into the table
+        const unsigned long long idx0 = frac >> (32 - TABLE_BITS);
+
+        // get the x offset, i.e., the difference between the first sample point and the actual fractional part
+        const long long udx = frac - (idx0 << (32 - TABLE_BITS));
+        /* paranoid */ verify((idx0 + 2) < TABLE_SIZE);
+
+        const long long y0 = table[idx0 + 0];
+        const long long y1 = table[idx0 + 1];
+        const long long y2 = table[idx0 + 2];
+
+        // from there we can quadraticly interpolate to get the data, using the lagrange polynomial
+        // normally it would look like:
+        //     double r0 = y0 * ((x - x1) / (x0 - x1)) * ((x - x2) / (x0 - x2));
+        //     double r1 = y1 * ((x - x0) / (x1 - x0)) * ((x - x2) / (x1 - x2));
+        //     double r2 = y2 * ((x - x0) / (x2 - x0)) * ((x - x1) / (x2 - x1));
+        // but since the spacing between sample points is fixed, we can simplify it and extract common expressions
+        const long long f1 = (y1 - y0);
+        const long long f2 = (y2 - y0);
+        const long long a = f2 - (f1 * 2l);
+        const long long b = (f1 * 2l) - a;
+
+        // Now we can compute it in the form (ax + b)x + c (which avoid repeating steps)
+        long long sum = ((a*udx) >> (32 - TABLE_BITS))  + b;
+        sum = (sum*udx) >> (32 - TABLE_BITS + 1);
+        sum = y0 + sum;
+
+        return (i << 32) + (sum);
+}
+
 //---------------------- Trigonometric ----------------------
 
-static inline {
+static inline __attribute__((always_inline)) {
         float sin( float x ) { return sinf( x ); }
         // extern "C" { double sin( double ); }
@@ -204 +295 @@
 //---------------------- Hyperbolic ----------------------
 
-static inline {
+static inline __attribute__((always_inline)) {
         float sinh( float x ) { return sinhf( x ); }
         // extern "C" { double sinh( double ); }
@@ -250 +341 @@
 //---------------------- Error / Gamma ----------------------
 
-static inline {
+static inline __attribute__((always_inline)) {
         float erf( float x ) { return erff( x ); }
         // extern "C" { double erf( double ); }
@@ -279 +370 @@
 //---------------------- Nearest Integer ----------------------
 
-static inline {
+inline __attribute__((always_inline)) static {
         signed char floor( signed char n, signed char align ) { return n / align * align; }
         unsigned char floor( unsigned char n, unsigned char align ) { return n / align * align; }
@@ -307 +398 @@
         // forall( T | { T ?+?( T, T ); T ?-?( T, T ); T ?%?( T, T ); } )
         // T ceiling_div( T n, T align ) { verify( is_pow2( align ) );return (n + (align - 1)) / align; }
-        
+
         // gcc notices the div/mod pair and saves both so only one div.
         signed char ceiling( signed char n, signed char align ) { return floor( n + (n % align != 0 ? align - 1 : 0), align ); }
@@ -376 +467 @@
 //---------------------- Manipulation ----------------------
 
-static inline {
+static inline __attribute__((always_inline)) {
         float copysign( float x, float y ) { return copysignf( x, y ); }
         // extern "C" { double copysign( double, double ); }
@@ -418 +509 @@
 //---------------------------------------
 
-static inline {
+static inline __attribute__((always_inline)) {
         forall( T | { void ?{}( T &, one_t ); T ?+?( T, T ); T ?-?( T, T );T ?*?( T, T ); } )
         T lerp( T x, T y, T a ) { return x * ((T){1} - a) + y * a; }

libcfa/src/parseargs.cfa

@@ +1 @@
+//
+// Cforall Version 1.0.0 Copyright (C) 2022 University of Waterloo
+//
+// The contents of this file are covered under the licence agreement in the
+// file "LICENCE" distributed with Cforall.
+//
+// parseargs.cfa
+// implementation of arguments parsing (argc, argv)
+//
+// Author           : Thierry Delisle
+// Created On       : Wed Oct 12 15:28:01 2022
+// Last Modified By :
+// Last Modified On :
+// Update Count     :
+//
+
 #include "parseargs.hfa"
 
+#include <assert.h>
+#include <ctype.h>
 #include <stdint.h>
 #include <string.h>
 #include <errno.h>
 #include <unistd.h>
-#include <assert.h>
 
 extern "C" {
@@ -33 +50 @@
 extern char ** cfa_args_envp __attribute__((weak));
 
-static void usage(char * cmd, cfa_option options[], size_t opt_count, const char * usage, FILE * out)  __attribute__ ((noreturn));
+forall([N])
+static void usage(char * cmd, const array( cfa_option, N ) & options, const char * usage, FILE * out)  __attribute__ ((noreturn));
 //-----------------------------------------------------------------------------
 // checking
-static void check_args(cfa_option options[], size_t opt_count) {
-        for(i; opt_count) {
-                for(j; opt_count) {
+forall([N])
+static void check_args( const array( cfa_option, N ) & options ) {
+        for(i; N) {
+                for(j; N) {
                         if(i == j) continue;
 
@@ -53 +72 @@
 //-----------------------------------------------------------------------------
 // Parsing args
-void parse_args( cfa_option options[], size_t opt_count, const char * usage, char ** & left ) {
-        if( 0p != &cfa_args_argc ) {
-                parse_args(cfa_args_argc, cfa_args_argv, options, opt_count, usage, left );
-        }
-        else {
-                char * temp = "";
-                parse_args(0, &temp, options, opt_count, usage, left );
-        }
-}
-
-void parse_args(
-        int argc,
-        char * argv[],
-        cfa_option options[],
-        size_t opt_count,
-        const char * usage,
-        char ** & left
-) {
-        check_args(options, opt_count);
-
-        int maxv = 'h';
-        assert( opt_count > 0 );
-        char optstring[opt_count * 3] = { '\0' };
-        {
-                int idx = 0;
-                for(i; opt_count) {
-                        if (options[i].short_name) {
-                                maxv = max(options[i].short_name, maxv);
-                                optstring[idx] = options[i].short_name;
-                                idx++;
-                                if(    ((intptr_t)options[i].parse) != ((intptr_t)parse_settrue)
-                                && ((intptr_t)options[i].parse) != ((intptr_t)parse_setfalse) ) {
-                                        optstring[idx] = ':';
+forall([opt_count]) {
+        void parse_args( const array( cfa_option, opt_count ) & options, const char * usage, char ** & left ) {
+                if( 0p != &cfa_args_argc ) {
+                        parse_args(cfa_args_argc, cfa_args_argv, options, usage, left );
+                }
+                else {
+                        char * temp = "";
+                        parse_args(0, &temp, options, usage, left );
+                }
+        }
+
+        void parse_args(
+                int argc,
+                char * argv[],
+                const array( cfa_option, opt_count ) & options,
+                const char * usage,
+                char ** & left
+        ) {
+                check_args(options);
+
+                int maxv = 'h';
+                char optstring[(opt_count * 3) + 2] = { '\0' };
+                {
+                        int idx = 0;
+                        for(i; opt_count) {
+                                if (options[i].short_name) {
+                                        maxv = max(options[i].short_name, maxv);
+                                        optstring[idx] = options[i].short_name;
+                                        idx++;
+                                        if(    ((intptr_t)options[i].parse) != ((intptr_t)parse_settrue)
+                                        && ((intptr_t)options[i].parse) != ((intptr_t)parse_setfalse) ) {
+                                                optstring[idx] = ':';
+                                                idx++;
+                                        }
+                                }
+                        }
+                        optstring[idx+0] = 'h';
+                        optstring[idx+1] = '\0';
+                }
+
+                struct option optarr[opt_count + 2];
+                {
+                        int idx = 0;
+                        for(i; opt_count) {
+                                if(options[i].long_name) {
+                                        // we don't have the mutable keyword here, which is really what we would want
+                                        int & val_ref = (int &)(const int &)options[i].val;
+                                        val_ref = (options[i].short_name != '\0') ? ((int)options[i].short_name) : ++maxv;
+
+                                        optarr[idx].name = options[i].long_name;
+                                        optarr[idx].flag = 0p;
+                                        optarr[idx].val  = options[i].val;
+                                        if(    ((intptr_t)options[i].parse) == ((intptr_t)parse_settrue)
+                                        || ((intptr_t)options[i].parse) == ((intptr_t)parse_setfalse) ) {
+                                                optarr[idx].has_arg = no_argument;
+                                        } else {
+                                                optarr[idx].has_arg = required_argument;
+                                        }
                                         idx++;
                                 }
                         }
-                }
-                optstring[idx+0] = 'h';
-                optstring[idx+1] = '\0';
-        }
-
-        struct option optarr[opt_count + 2];
-        {
-                int idx = 0;
-                for(i; opt_count) {
-                        if(options[i].long_name) {
-                                options[i].val = (options[i].short_name != '\0') ? ((int)options[i].short_name) : ++maxv;
-                                optarr[idx].name = options[i].long_name;
-                                optarr[idx].flag = 0p;
-                                optarr[idx].val  = options[i].val;
-                                if(    ((intptr_t)options[i].parse) == ((intptr_t)parse_settrue)
-                                    || ((intptr_t)options[i].parse) == ((intptr_t)parse_setfalse) ) {
-                                        optarr[idx].has_arg = no_argument;
-                                } else {
-                                        optarr[idx].has_arg = required_argument;
-                                }
-                                idx++;
+                        optarr[idx+0].[name, has_arg, flag, val] = ["help", no_argument, 0, 'h'];
+                        optarr[idx+1].[name, has_arg, flag, val] = [0, no_argument, 0, 0];
+                }
+
+                FILE * out = stderr;
+                NEXT_ARG:
+                for() {
+                        int idx = 0;
+                        int opt = getopt_long(argc, argv, optstring, optarr, &idx);
+                        switch(opt) {
+                                case -1:
+                                        if(&left != 0p) left = argv + optind;
+                                        return;
+                                case 'h':
+                                        out = stdout;
+                                case '?':
+                                        usage(argv[0], options, usage, out);
+                                default:
+                                        for(i; opt_count) {
+                                                if(opt == options[i].val) {
+                                                        const char * arg = optarg ? optarg : "";
+                                                        if( arg[0] == '=' ) { arg++; }
+                                                        // work around for some weird bug
+                                                        void * variable = options[i].variable;
+                                                        bool (*parse_func)(const char *, void * ) = options[i].parse;
+                                                        bool success = parse_func( arg, variable );
+                                                        if(success) continue NEXT_ARG;
+
+                                                        fprintf(out, "Argument '%s' for option %c could not be parsed\n\n", arg, (char)opt);
+                                                        usage(argv[0], options, usage, out);
+                                                }
+                                        }
+                                        abort("Internal parse arg error\n");
                         }
-                }
-                optarr[idx+0].[name, has_arg, flag, val] = ["help", no_argument, 0, 'h'];
-                optarr[idx+1].[name, has_arg, flag, val] = [0, no_argument, 0, 0];
-        }
-
-        FILE * out = stderr;
-        NEXT_ARG:
-        for() {
-                int idx = 0;
-                int opt = getopt_long(argc, argv, optstring, optarr, &idx);
-                switch(opt) {
-                        case -1:
-                                if(&left != 0p) left = argv + optind;
-                                return;
-                        case 'h':
-                                out = stdout;
-                        case '?':
-                                usage(argv[0], options, opt_count, usage, out);
-                        default:
-                                for(i; opt_count) {
-                                        if(opt == options[i].val) {
-                                                const char * arg = optarg ? optarg : "";
-                                                if( arg[0] == '=' ) { arg++; }
-                                                bool success = options[i].parse( arg, options[i].variable );
-                                                if(success) continue NEXT_ARG;
-
-                                                fprintf(out, "Argument '%s' for option %c could not be parsed\n\n", arg, (char)opt);
-                                                usage(argv[0], options, opt_count, usage, out);
-                                        }
-                                }
-                                abort("Internal parse arg error\n");
-                }
-
-        }
+
+                }
+        }
+}
+
+static inline int next_newline(const char * str) {
+        int ret;
+        const char * ptr = strstr(str, "\n");
+        if(!ptr) return MAX;
+
+        /* paranoid */ verify( str <= ptr);
+        intptr_t low = (intptr_t)str;
+        intptr_t hi  = (intptr_t)ptr;
+        ret = hi - low;
+
+        return ret;
 }
 
@@ -150 +188 @@
 // Print usage
 static void printopt(FILE * out, int width, int max, char sn, const char * ln, const char * help) {
+        // check how wide we should be printing
+        // this includes all options and the help message
         int hwidth = max - (11 + width);
         if(hwidth <= 0) hwidth = max;
 
-        char sname[4] = { ' ', ' ', ' ', '\0' };
-        if(sn != '\0') {
-                sname[0] = '-';
-                sname[1] = sn;
-                sname[2] = ',';
-        }
-
-        fprintf(out, "  %s --%-*s   %.*s\n", sname, width, ln, hwidth, help);
-        for() {
-                help += min(strlen(help), hwidth);
-                if('\0' == *help) break;
-                fprintf(out, "%*s%.*s\n", width + 11, "", hwidth, help);
-        }
+        // check which pieces we have
+        bool has_ln = ln && strcmp("", ln);
+        bool has_help = help && strcmp("", help);
+
+        // print the small name if present
+        if(sn != '\0') fprintf(out, "  -%c", sn);
+        else fprintf(out, "    ");
+
+        // print a comma if we have both short and long names
+        if(sn != '\0' && has_ln) fprintf(out, ", ");
+        else fprintf(out, "  ");
+
+        // print the long name if present
+        if(has_ln)        fprintf(out, "--%-*s", width, ln);
+        else if(has_help) fprintf(out, "  %-*s", width, "");
+
+        if(has_help) {
+                // print the help
+                // We need to wrap at the max width, and also indent newlines so everything is nice and pretty
+
+                // for each line to print
+                for() {
+                        //find out if there is a newline
+                        int nextnl = next_newline(help);
+                        int real = min(min(strlen(help), hwidth), nextnl);
+
+                        fprintf(out, "   %.*s", real, help);
+                        // printf("%d %d\n", real, nextnl);
+                        help += real;
+                        if( nextnl == real ) help++;
+                        if('\0' == *help) break;
+                        fprintf(out, "\n%*s", width + 8, "");
+                }
+        }
+        fprintf(out, "\n");
 }
 
 void print_args_usage(cfa_option options[], size_t opt_count, const char * usage, bool error)  __attribute__ ((noreturn)) {
-        usage(cfa_args_argv[0], options, opt_count, usage, error ? stderr : stdout);
+        const array( cfa_option, opt_count ) & arr = (const array( cfa_option, opt_count ) &) *options;
+        usage(cfa_args_argv[0], arr, usage, error ? stderr : stdout);
 }
 
 void print_args_usage(int , char * argv[], cfa_option options[], size_t opt_count, const char * usage, bool error)  __attribute__ ((noreturn)) {
-        usage(argv[0], options, opt_count, usage, error ? stderr : stdout);
-}
-
-static void usage(char * cmd, cfa_option options[], size_t opt_count, const char * help, FILE * out) __attribute__((noreturn)) {
+        const array( cfa_option, opt_count ) & arr = (const array( cfa_option, opt_count ) &) *options;
+        usage(argv[0], arr, usage, error ? stderr : stdout);
+}
+
+forall( [N] ) {
+        void print_args_usage( const array(cfa_option, N ) & options, const char * usage, bool error) {
+                usage(cfa_args_argv[0], options, usage, error ? stderr : stdout);
+        }
+
+        void print_args_usage(int argc, char * argv[], const array( cfa_option, N ) & options, const char * usage, bool error) {
+                usage(argv[0], options, usage, error ? stderr : stdout);
+        }
+}
+
+forall([N])
+static void usage(char * cmd, const array( cfa_option, N ) & options, const char * help, FILE * out) __attribute__((noreturn)) {
         int width = 0;
         {
-                for(i; opt_count) {
+                for(i; N) {
                         if(options[i].long_name) {
                                 int w = strlen(options[i].long_name);
@@ -198 +273 @@
         fprintf(out, "Usage:\n  %s %s\n", cmd, help);
 
-        for(i; opt_count) {
+        for(i; N) {
                 printopt(out, width, max_width, options[i].short_name, options[i].long_name, options[i].help);
         }
@@ -272 +347 @@
 bool parse(const char * arg, int & value) {
         char * end;
-        int r = strtoll(arg, &end, 10);
+
+        errno = 0;
+        long long int r = strtoll(arg, &end, 0);
+        if(errno) return false;
         if(*end != '\0') return false;
+        if(r > (int)MAX) return false;
+        if(r < (int)MIN) return false;
 
         value = r;
@@ -279 +359 @@
 }
 
+static unsigned long long int strict_strtoull( const char * arg, int base) {
+        errno = 0;
+        {
+                const char * in = arg;
+                for() {
+                        if('\0' == *in) {
+                                errno = EINVAL;
+                                return 0;
+                        }
+                        if(!isspace(*in)) break;
+                        in++;
+                }
+                if(!isdigit(*in)) {
+                        errno = EINVAL;
+                        return 0;
+                }
+        }
+
+        *char end;
+        unsigned long long int r = strtoull(arg, &end, base);
+        if(*end != '\0') errno = EINVAL;
+        if(errno) return 0;
+        return r;
+}
+
 bool parse(const char * arg, unsigned & value) {
-        char * end;
-        unsigned long long int r = strtoull(arg, &end, 10);
-        if(*end != '\0') return false;
+        unsigned long long int r = strict_strtoull(arg, 0);
+        if(errno) return false;
         if(r > (unsigned)MAX) return false;
 
@@ -290 +394 @@
 
 bool parse(const char * arg, unsigned long & value) {
-        char * end;
-        unsigned long long int r = strtoull(arg, &end, 10);
-        if(*end != '\0') return false;
+        unsigned long long int r = strict_strtoull(arg, 0);
+        if(errno) return false;
         if(r > (unsigned long)MAX) return false;
 
@@ -300 +403 @@
 
 bool parse(const char * arg, unsigned long long & value) {
-        char * end;
-        unsigned long long int r = strtoull(arg, &end, 10);
-        if(*end != '\0') return false;
-        if(r > (unsigned long long)MAX) return false;
-
-        value = r;
-        return true;
+        unsigned long long int r = strict_strtoull(arg, 0);
+        if(errno) return false;
+        if(r > (unsigned long long)MAX) return false;
+
+        value = r;
+        return true;
 }
 

libcfa/src/parseargs.hfa

@@ +1 @@
+//
+// Cforall Version 1.0.0 Copyright (C) 2022 University of Waterloo
+//
+// The contents of this file are covered under the licence agreement in the
+// file "LICENCE" distributed with Cforall.
+//
+// parseargs.cfa -- PUBLIC
+// API for arguments parsing (argc, argv)
+//
+// Author           : Thierry Delisle
+// Created On       : Wed Oct 12 15:28:01 2022
+// Last Modified By :
+// Last Modified On :
+// Update Count     :
+//
 #pragma once
 
+#include <array.hfa>
+
 struct cfa_option {
-      int val; // reserved
-      char short_name;
-      const char * long_name;
-      const char * help;
-      void * variable;
-      bool (*parse)(const char *, void * );
+        int val; // reserved
+        char short_name;
+        const char * long_name;
+        const char * help;
+        void * variable;
+        bool (*parse)(const char *, void * );
 };
 
@@ -16 +33 @@
 forall(T & | { bool parse(const char *, T & ); })
 static inline void ?{}( cfa_option & this, char short_name, const char * long_name, const char * help, T & variable ) {
-      this.val        = 0;
-      this.short_name = short_name;
-      this.long_name  = long_name;
-      this.help       = help;
-      this.variable   = (void*)&variable;
-      this.parse      = (bool (*)(const char *, void * ))parse;
+        this.val        = 0;
+        this.short_name = short_name;
+        this.long_name  = long_name;
+        this.help       = help;
+        this.variable   = (void*)&variable;
+        this.parse      = (bool (*)(const char *, void * ))parse;
 }
 
 forall(T &)
 static inline void ?{}( cfa_option & this, char short_name, const char * long_name, const char * help, T & variable, bool (*parse)(const char *, T & )) {
-      this.val        = 0;
-      this.short_name = short_name;
-      this.long_name  = long_name;
-      this.help       = help;
-      this.variable   = (void*)&variable;
-      this.parse      = (bool (*)(const char *, void * ))parse;
+        this.val        = 0;
+        this.short_name = short_name;
+        this.long_name  = long_name;
+        this.help       = help;
+        this.variable   = (void*)&variable;
+        this.parse      = (bool (*)(const char *, void * ))parse;
 }
 
@@ -37 +54 @@
 void parse_args( int argc, char * argv[], cfa_option options[], size_t opt_count, const char * usage, char ** & left );
 
+forall( [N] ) {
+        void parse_args( const array( cfa_option, N ) & options, const char * usage, char ** & left );
+        void parse_args( int argc, char * argv[], const array( cfa_option, N ) & options, const char * usage, char ** & left );
+}
+
 void print_args_usage(cfa_option options[], size_t opt_count, const char * usage, bool error)  __attribute__ ((noreturn));
 void print_args_usage(int argc, char * argv[], cfa_option options[], size_t opt_count, const char * usage, bool error)  __attribute__ ((noreturn));
+
+forall( [N] ) {
+        void print_args_usage( const array(cfa_option, N ) & options, const char * usage, bool error)  __attribute__ ((noreturn));
+        void print_args_usage(int argc, char * argv[], const array( cfa_option, N ) & options, const char * usage, bool error)  __attribute__ ((noreturn));
+}
 
 bool parse_yesno    (const char *, bool & );

libcfa/src/startup.cfa

@@ -10 +10 @@
 // Created On       : Tue Jul 24 16:21:57 2018
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Mon Jan 17 16:41:54 2022
-// Update Count     : 55
+// Last Modified On : Thu Oct  6 13:51:57 2022
+// Update Count     : 57
 //
 
@@ -24 +24 @@
 
 extern "C" {
+        void __cfaabi_memory_startup( void ) __attribute__(( constructor( STARTUP_PRIORITY_MEMORY ) ));
+        void __cfaabi_memory_startup( void ) {
+                extern void memory_startup();
+                memory_startup();
+        } // __cfaabi_memory_startup
+
+        void __cfaabi_memory_shutdown( void ) __attribute__(( destructor( STARTUP_PRIORITY_MEMORY ) ));
+        void __cfaabi_memory_shutdown( void ) {
+                extern void memory_shutdown();
+                memory_shutdown();
+        } // __cfaabi_memory_shutdown
+
         void __cfaabi_appready_startup( void ) __attribute__(( constructor( STARTUP_PRIORITY_APPREADY ) ));
         void __cfaabi_appready_startup( void ) {
                 tzset();                                                                                // initialize time global variables
-                #ifdef __CFA_DEBUG__
                 extern void heapAppStart();
                 heapAppStart();
-                #endif // __CFA_DEBUG__
         } // __cfaabi_appready_startup
 
         void __cfaabi_appready_shutdown( void ) __attribute__(( destructor( STARTUP_PRIORITY_APPREADY ) ));
         void __cfaabi_appready_shutdown( void ) {
-                #ifdef __CFA_DEBUG__
                 extern void heapAppStop();
                 heapAppStop();
-                #endif // __CFA_DEBUG__
         } // __cfaabi_appready_shutdown
 

libcfa/src/stdhdr/assert.h

@@ -10 +10 @@
 // Created On       : Mon Jul  4 23:25:26 2016
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Tue Feb  4 12:58:49 2020
-// Update Count     : 15
+// Last Modified On : Sun Oct  9 21:28:22 2022
+// Update Count     : 16
 //
 
@@ -27 +27 @@
         #define assertf( expr, fmt, ... ) ((expr) ? ((void)0) : __assert_fail_f(__VSTRINGIFY__(expr), __FILE__, __LINE__, __PRETTY_FUNCTION__, fmt, ## __VA_ARGS__ ))
 
+        void __assert_warn_f( const char assertion[], const char file[], unsigned int line, const char function[], const char fmt[], ... ) __attribute__((format( printf, 5, 6) ));
         void __assert_fail_f( const char assertion[], const char file[], unsigned int line, const char function[], const char fmt[], ... ) __attribute__((noreturn, format( printf, 5, 6) ));
 #endif
 
-#if !defined(NDEBUG) && (defined(__CFA_DEBUG__) || defined(__CFA_VERIFY__))
+#if ! defined(NDEBUG) && (defined(__CFA_DEBUG__) || defined(__CFA_VERIFY__))
+        #define __CFA_WITH_VERIFY__
         #define verify(x) assert(x)
         #define verifyf(x, ...) assertf(x, __VA_ARGS__)
         #define verifyfail(...)
-        #define __CFA_WITH_VERIFY__
+        #define warnf( expr, fmt, ... ) ({ static bool check_once##__LINE__ = false; if( false == check_once##__LINE__ && false == (expr)) { check_once##__LINE__ = true; __assert_warn_f(__VSTRINGIFY__(expr), __FILE__, __LINE__, __PRETTY_FUNCTION__, fmt, ## __VA_ARGS__ ); } })
 #else
         #define verify(x)
         #define verifyf(x, ...)
         #define verifyfail(...)
+        #define warnf( expr, fmt, ... )
 #endif
 

libcfa/src/time.hfa

@@ -10 +10 @@
 // Created On       : Wed Mar 14 23:18:57 2018
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Wed Apr 21 06:32:31 2021
-// Update Count     : 667
+// Last Modified On : Sat Oct  8 09:07:48 2022
+// Update Count     : 668
 //
 
@@ -27 +27 @@
 //######################### Duration #########################
 
-static inline {
+static inline __attribute__((always_inline)) {
         void ?{}( Duration & dur, timeval t ) with( dur ) { tn = (int64_t)t.tv_sec * TIMEGRAN + t.tv_usec * 1000; }
         void ?{}( Duration & dur, timespec t ) with( dur ) { tn = (int64_t)t.tv_sec * TIMEGRAN + t.tv_nsec; }
@@ -115 +115 @@
 //######################### C timeval #########################
 
-static inline {
+static inline __attribute__((always_inline)) {
         void ?{}( timeval & t ) {}
         void ?{}( timeval & t, time_t sec, suseconds_t usec ) { t.tv_sec = sec; t.tv_usec = usec; }
@@ -130 +130 @@
 //######################### C timespec #########################
 
-static inline {
+static inline __attribute__((always_inline)) {
         void ?{}( timespec & t ) {}
         void ?{}( timespec & t, time_t sec, __syscall_slong_t nsec ) { t.tv_sec = sec; t.tv_nsec = nsec; }
@@ -145 +145 @@
 //######################### C itimerval #########################
 
-static inline {
+static inline __attribute__((always_inline)) {
         void ?{}( itimerval & itv, Duration alarm ) with( itv ) {
                 // itimerval contains durations but but uses time data-structure timeval.
@@ -162 +162 @@
 
 void ?{}( Time & time, int year, int month = 1, int day = 1, int hour = 0, int min = 0, int sec = 0, int64_t nsec = 0 );
-static inline {
+static inline __attribute__((always_inline)) {
         void ?{}( Time & time, timeval t ) with( time ) { tn = (int64_t)t.tv_sec * TIMEGRAN + t.tv_usec * 1000; }
         void ?{}( Time & time, timespec t ) with( time ) { tn = (int64_t)t.tv_sec * TIMEGRAN + t.tv_nsec; }
@@ -194 +194 @@
 
 char * yy_mm_dd( Time time, char * buf );
-static inline char * ?`ymd( Time time, char * buf ) {   // short form
+static inline __attribute__((always_inline)) char * ?`ymd( Time time, char * buf ) {    // short form
         return yy_mm_dd( time, buf );
 } // ymd
 
 char * mm_dd_yy( Time time, char * buf );
-static inline char * ?`mdy( Time time, char * buf ) {   // short form
+static inline __attribute__((always_inline)) char * ?`mdy( Time time, char * buf ) {    // short form
         return mm_dd_yy( time, buf );
 } // mdy
 
 char * dd_mm_yy( Time time, char * buf );
-static inline char * ?`dmy( Time time, char * buf ) {   // short form
+static inline __attribute__((always_inline)) char * ?`dmy( Time time, char * buf ) {    // short form
         return dd_mm_yy( time, buf );;
 } // dmy
@@ -212 +212 @@
 //------------------------- timeval (cont) -------------------------
 
-static inline void ?{}( timeval & t, Time time ) with( t, time ) {
+static inline __attribute__((always_inline)) void ?{}( timeval & t, Time time ) with( t, time ) {
         tv_sec = tn / TIMEGRAN;                                                         // seconds
         tv_usec = tn % TIMEGRAN / (TIMEGRAN / 1_000_000LL);     // microseconds
@@ -219 +219 @@
 //------------------------- timespec (cont) -------------------------
 
-static inline void ?{}( timespec & t, Time time ) with( t, time ) {
+static inline __attribute__((always_inline)) void ?{}( timespec & t, Time time ) with( t, time ) {
         tv_sec = tn / TIMEGRAN;                                                         // seconds
         tv_nsec = tn % TIMEGRAN;                                                        // nanoseconds