Changeset c8a0210 for libcfa/src

Timestamp:

Apr 16, 2021, 2:28:09 PM (4 years ago)

Author:

Thierry Delisle <tdelisle@…>

Branches:

ADT, arm-eh, ast-experimental, enum, forall-pointer-decay, jacob/cs343-translation, master, new-ast-unique-expr, pthread-emulation, qualifiedEnum

Children:

665edf40

Parents:

857a1c6 (diff), 5f6a172 (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the (diff) links above to see all the changes relative to each parent.

Message:

Merge branch 'master' of plg.uwaterloo.ca:software/cfa/cfa-cc

Location:

libcfa/src

Files:

• bits/weakso_locks.cfa (modified) (1 diff)
• clock.hfa (modified) (2 diffs)
• concurrency/coroutine.cfa (modified) (2 diffs)
• concurrency/coroutine.hfa (modified) (2 diffs)
• concurrency/invoke.h (modified) (1 diff)
• concurrency/io/call.cfa.in (modified) (18 diffs)
• concurrency/kernel.cfa (modified) (12 diffs)
• concurrency/kernel.hfa (modified) (6 diffs)
• concurrency/kernel/startup.cfa (modified) (8 diffs)
• concurrency/kernel_private.hfa (modified) (5 diffs)
• concurrency/preemption.cfa (modified) (2 diffs)
• concurrency/ready_queue.cfa (modified) (18 diffs)
• concurrency/ready_subqueue.hfa (modified) (1 diff)
• concurrency/stats.cfa (modified) (3 diffs)
• concurrency/stats.hfa (modified) (4 diffs)
• concurrency/thread.cfa (modified) (5 diffs)
• concurrency/thread.hfa (modified) (3 diffs)
• exception.c (modified) (3 diffs)
• exception.h (modified) (4 diffs)
• exception.hfa (modified) (3 diffs)
• fstream.cfa (modified) (1 diff)
• fstream.hfa (modified) (1 diff)
• iostream.cfa (modified) (7 diffs)
• iostream.hfa (modified) (4 diffs)
• math.hfa (modified) (2 diffs)
• time.hfa (modified) (2 diffs)
• virtual.c (modified) (2 diffs)

libcfa/src/bits/weakso_locks.cfa

@@ -25 +25 @@
 void unlock( blocking_lock & ) {}
 void on_notify( blocking_lock &, struct $thread * ) {}
-size_t on_wait( blocking_lock & ) {}
+size_t on_wait( blocking_lock & ) { return 0; }
 void on_wakeup( blocking_lock &, size_t ) {}
 size_t wait_count( blocking_lock & ) { return 0; }

libcfa/src/clock.hfa

@@ -10 +10 @@
 // Created On       : Thu Apr 12 14:36:06 2018
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Mon Jan  6 12:49:58 2020
-// Update Count     : 9
+// Last Modified On : Wed Apr 14 17:48:25 2021
+// Update Count     : 20
 //
 
@@ -32 +32 @@
 
 static inline {
-        void resetClock( Clock & clk, Duration adj ) with( clk ) {
+        void reset( Clock & clk, Duration adj ) with( clk ) {
                 offset = adj + __timezone`s;                                    // timezone (global) is (UTC - local time) in seconds
-        } // resetClock
+        } // reset
 
-        void ?{}( Clock & clk, Duration adj ) { resetClock( clk, adj ); }
+        void ?{}( Clock & clk ) { reset( clk, (Duration){ 0 } ); }
+        void ?{}( Clock & clk, Duration adj ) { reset( clk, adj ); }
 
-        Duration getResNsec() {
+        // System-wide clock that measures real, i.e., wall-clock) time. This clock is affected by discontinuous jumps in
+        // the system time. For example, manual changes of the clock, and incremental adjustments performed by adjtime(3)
+        // and NTP (daylight saving (Fall back).
+        Duration resolutionNsec() {
                 struct timespec res;
                 clock_getres( CLOCK_REALTIME, &res );
                 return ((int64_t)res.tv_sec * TIMEGRAN + res.tv_nsec)`ns;
-        } // getRes
+        } // resolutionNsec
 
-        Duration getRes() {
+        Duration resolution() {
                 struct timespec res;
                 clock_getres( CLOCK_REALTIME_COARSE, &res );
                 return ((int64_t)res.tv_sec * TIMEGRAN + res.tv_nsec)`ns;
-        } // getRes
+        } // resolution
 
-        Time getTimeNsec() {                                                            // with nanoseconds
+        Time timeNsec() {                                                                       // with nanoseconds
                 timespec curr;
                 clock_gettime( CLOCK_REALTIME, &curr );
                 return (Time){ curr };
-        } // getTimeNsec
+        } // timeNsec
 
-        Time getTime() {                                                                        // without nanoseconds
+        Time time() {                                                                           // without nanoseconds
                 timespec curr;
                 clock_gettime( CLOCK_REALTIME_COARSE, &curr );
                 curr.tv_nsec = 0;
                 return (Time){ curr };
-        } // getTime
+        } // time
 
-        Time getTime( Clock & clk ) with( clk ) {
-                return getTime() + offset;
-        } // getTime
+        Time time( Clock & clk ) with( clk ) {
+                return time() + offset;
+        } // time
 
         Time ?()( Clock & clk ) with( clk ) {                           // alternative syntax
-                return getTime() + offset;
-        } // getTime
+                return time() + offset;
+        } // ?()
 
-        timeval getTime( Clock & clk ) {
+        timeval time( Clock & clk ) {
                 return (timeval){ clk() };
-        } // getTime
+        } // time
 
-        tm getTime( Clock & clk ) with( clk ) {
+        tm time( Clock & clk ) with( clk ) {
                 tm ret;
-                localtime_r( getTime( clk ).tv_sec, &ret );
+                localtime_r( time( clk ).tv_sec, &ret );
                 return ret;
-        } // getTime
+        } // time
 
-        Time getCPUTime() {
+        // CFA processor CPU-time watch that ticks when the processor (kernel thread) is running. This watch is affected by
+        // discontinuous jumps when the OS is not running the kernal thread. A duration is returned because the value is
+        // relative and cannot be converted to real-time (wall-clock) time.
+        Duration processor() {
                 timespec ts;
                 clock_gettime( CLOCK_THREAD_CPUTIME_ID, &ts );
-                return (Time){ ts };
-    } // getCPUTime
+                return (Duration){ ts };
+        } // processor
+
+        // Program CPU-time watch measures CPU time consumed by all processors (kernel threads) in the UNIX process.  This
+        // watch is affected by discontinuous jumps when the OS is not running the kernel threads. A duration is returned
+        // because the value is relative and cannot be converted to real-time (wall-clock) time.
+        Duration program() {
+                timespec ts;
+                clock_gettime( CLOCK_PROCESS_CPUTIME_ID, &ts );
+                return (Duration){ ts };
+        } // program
+
+        // Monotonic stopwatch starting at machine boot and includes system suspension. This watch is unaffected by
+        // discontinuous jumps resulting from manual changes of the clock, and incremental adjustments performed by
+        // adjtime(3) and NTP (Fall back). A duration is returned because the value is relative and cannot be converted to
+        // real-time (wall-clock) time.
+        Duration boot() {
+                timespec ts;
+                clock_gettime( CLOCK_BOOTTIME, &ts );
+                return (Duration){ ts };
+        } // boot
 } // distribution
 

libcfa/src/concurrency/coroutine.cfa

@@ -46 +46 @@
 
 //-----------------------------------------------------------------------------
-FORALL_DATA_INSTANCE(CoroutineCancelled, (coroutine_t &), (coroutine_t))
-
-forall(T &)
-void mark_exception(CoroutineCancelled(T) *) {}
+EHM_VIRTUAL_TABLE(SomeCoroutineCancelled, std_coroutine_cancelled);
 
 forall(T &)
@@ -71 +68 @@
 
         // TODO: Remove explitate vtable set once trac#186 is fixed.
-        CoroutineCancelled(T) except;
-        except.virtual_table = &get_exception_vtable(&except);
+        SomeCoroutineCancelled except;
+        except.virtual_table = &std_coroutine_cancelled;
         except.the_coroutine = &cor;
         except.the_exception = except;
-        throwResume except;
+        // Why does this need a cast?
+        throwResume (SomeCoroutineCancelled &)except;
 
         except->virtual_table->free( except );

libcfa/src/concurrency/coroutine.hfa

@@ -22 +22 @@
 //-----------------------------------------------------------------------------
 // Exception thrown from resume when a coroutine stack is cancelled.
-FORALL_DATA_EXCEPTION(CoroutineCancelled, (coroutine_t &), (coroutine_t)) (
+EHM_EXCEPTION(SomeCoroutineCancelled)(
+        void * the_coroutine;
+        exception_t * the_exception;
+);
+
+EHM_EXTERN_VTABLE(SomeCoroutineCancelled, std_coroutine_cancelled);
+
+EHM_FORALL_EXCEPTION(CoroutineCancelled, (coroutine_t &), (coroutine_t)) (
         coroutine_t * the_coroutine;
         exception_t * the_exception;
@@ -37 +44 @@
 // Anything that implements this trait can be resumed.
 // Anything that is resumed is a coroutine.
-trait is_coroutine(T & | IS_RESUMPTION_EXCEPTION(CoroutineCancelled, (T))) {
+trait is_coroutine(T & | IS_RESUMPTION_EXCEPTION(SomeCoroutineCancelled)) {
         void main(T & this);
         $coroutine * get_coroutine(T & this);

libcfa/src/concurrency/invoke.h

@@ -148 +148 @@
                 struct $thread * prev;
                 volatile unsigned long long ts;
-                int preferred;
         };
 

libcfa/src/concurrency/io/call.cfa.in

@@ -201 +201 @@
 
                 sqe->opcode = IORING_OP_{op};
-                sqe->user_data = (__u64)(uintptr_t)&future;
+                sqe->user_data = (uintptr_t)&future;
                 sqe->flags = sflags;
                 sqe->ioprio = 0;
@@ -215 +215 @@
                 asm volatile("": : :"memory");
 
-                verify( sqe->user_data == (__u64)(uintptr_t)&future );
+                verify( sqe->user_data == (uintptr_t)&future );
                 cfa_io_submit( ctx, &idx, 1, 0 != (submit_flags & CFA_IO_LAZY) );
         #endif
@@ -238 +238 @@
                 'fd'  : 'fd',
                 'off' : 'offset',
-                'addr': '(__u64)iov',
+                'addr': '(uintptr_t)iov',
                 'len' : 'iovcnt',
         }, define = 'CFA_HAVE_PREADV2'),
@@ -245 +245 @@
                 'fd'  : 'fd',
                 'off' : 'offset',
-                'addr': '(__u64)iov',
+                'addr': '(uintptr_t)iov',
                 'len' : 'iovcnt'
         }, define = 'CFA_HAVE_PWRITEV2'),
@@ -257 +257 @@
                 'addr': 'fd',
                 'len': 'op',
-                'off': '(__u64)event'
+                'off': '(uintptr_t)event'
         }),
         # CFA_HAVE_IORING_OP_SYNC_FILE_RANGE
@@ -269 +269 @@
         Call('SENDMSG', 'ssize_t sendmsg(int sockfd, const struct msghdr *msg, int flags)', {
                 'fd': 'sockfd',
-                'addr': '(__u64)(struct msghdr *)msg',
+                'addr': '(uintptr_t)(struct msghdr *)msg',
                 'len': '1',
                 'msg_flags': 'flags'
@@ -276 +276 @@
         Call('RECVMSG', 'ssize_t recvmsg(int sockfd, struct msghdr *msg, int flags)', {
                 'fd': 'sockfd',
-                'addr': '(__u64)(struct msghdr *)msg',
+                'addr': '(uintptr_t)(struct msghdr *)msg',
                 'len': '1',
                 'msg_flags': 'flags'
@@ -283 +283 @@
         Call('SEND', 'ssize_t send(int sockfd, const void *buf, size_t len, int flags)', {
                 'fd': 'sockfd',
-                'addr': '(__u64)buf',
+                'addr': '(uintptr_t)buf',
                 'len': 'len',
                 'msg_flags': 'flags'
@@ -290 +290 @@
         Call('RECV', 'ssize_t recv(int sockfd, void *buf, size_t len, int flags)', {
                 'fd': 'sockfd',
-                'addr': '(__u64)buf',
+                'addr': '(uintptr_t)buf',
                 'len': 'len',
                 'msg_flags': 'flags'
@@ -297 +297 @@
         Call('ACCEPT', 'int accept4(int sockfd, struct sockaddr *addr, socklen_t *addrlen, int flags)', {
                 'fd': 'sockfd',
-                'addr': '(__u64)addr',
-                'addr2': '(__u64)addrlen',
+                'addr': '(uintptr_t)addr',
+                'addr2': '(uintptr_t)addrlen',
                 'accept_flags': 'flags'
         }),
@@ -304 +304 @@
         Call('CONNECT', 'int connect(int sockfd, const struct sockaddr *addr, socklen_t addrlen)', {
                 'fd': 'sockfd',
-                'addr': '(__u64)addr',
+                'addr': '(uintptr_t)addr',
                 'off': 'addrlen'
         }),
@@ -310 +310 @@
         Call('FALLOCATE', 'int fallocate(int fd, int mode, off_t offset, off_t len)', {
                 'fd': 'fd',
-                'addr': '(__u64)len',
+                'addr': '(uintptr_t)len',
                 'len': 'mode',
                 'off': 'offset'
@@ -323 +323 @@
         # CFA_HAVE_IORING_OP_MADVISE
         Call('MADVISE', 'int madvise(void *addr, size_t length, int advice)', {
-                'addr': '(__u64)addr',
+                'addr': '(uintptr_t)addr',
                 'len': 'length',
                 'fadvise_advice': 'advice'
@@ -330 +330 @@
         Call('OPENAT', 'int openat(int dirfd, const char *pathname, int flags, mode_t mode)', {
                 'fd': 'dirfd',
-                'addr': '(__u64)pathname',
+                'addr': '(uintptr_t)pathname',
                 'len': 'mode',
                 'open_flags': 'flags;'
@@ -339 +339 @@
                 'addr': 'pathname',
                 'len': 'sizeof(*how)',
-                'off': '(__u64)how',
+                'off': '(uintptr_t)how',
         }, define = 'CFA_HAVE_OPENAT2'),
         # CFA_HAVE_IORING_OP_CLOSE
@@ -348 +348 @@
         Call('STATX', 'int statx(int dirfd, const char *pathname, int flags, unsigned int mask, struct statx *statxbuf)', {
                 'fd': 'dirfd',
-                'off': '(__u64)statxbuf',
+                'off': '(uintptr_t)statxbuf',
                 'addr': 'pathname',
                 'len': 'mask',
@@ -356 +356 @@
         Call('READ', 'ssize_t read(int fd, void * buf, size_t count)', {
                 'fd': 'fd',
-                'addr': '(__u64)buf',
+                'addr': '(uintptr_t)buf',
                 'len': 'count'
         }),
@@ -362 +362 @@
         Call('WRITE', 'ssize_t write(int fd, void * buf, size_t count)', {
                 'fd': 'fd',
-                'addr': '(__u64)buf',
+                'addr': '(uintptr_t)buf',
                 'len': 'count'
         }),

libcfa/src/concurrency/kernel.cfa

@@ -113 +113 @@
 static void __wake_one(cluster * cltr);
 
-static void push  (__cluster_idles & idles, processor & proc);
-static void remove(__cluster_idles & idles, processor & proc);
-static [unsigned idle, unsigned total, * processor] query( & __cluster_idles idles );
+static void mark_idle (__cluster_proc_list & idles, processor & proc);
+static void mark_awake(__cluster_proc_list & idles, processor & proc);
+static [unsigned idle, unsigned total, * processor] query_idles( & __cluster_proc_list idles );
 
 extern void __cfa_io_start( processor * );
@@ -189 +189 @@
 
                                 // Push self to idle stack
-                                push(this->cltr->idles, * this);
+                                mark_idle(this->cltr->procs, * this);
 
                                 // Confirm the ready-queue is empty
@@ -195 +195 @@
                                 if( readyThread ) {
                                         // A thread was found, cancel the halt
-                                        remove(this->cltr->idles, * this);
+                                        mark_awake(this->cltr->procs, * this);
 
                                         #if !defined(__CFA_NO_STATISTICS__)
@@ -225 +225 @@
 
                                 // We were woken up, remove self from idle
-                                remove(this->cltr->idles, * this);
+                                mark_awake(this->cltr->procs, * this);
 
                                 // DON'T just proceed, start looking again
@@ -359 +359 @@
                                 #if !defined(__CFA_NO_STATISTICS__)
                                         __tls_stats()->ready.threads.threads++;
+                                        __push_stat( __tls_stats(), __tls_stats()->ready.threads.threads, false, "Processor", this );
                                 #endif
                                 // This is case 2, the racy case, someone tried to run this thread before it finished blocking
@@ -376 +377 @@
         #if !defined(__CFA_NO_STATISTICS__)
                 __tls_stats()->ready.threads.threads--;
+                __push_stat( __tls_stats(), __tls_stats()->ready.threads.threads, false, "Processor", this );
         #endif
 
@@ -455 +457 @@
                 if( kernelTLS().this_stats ) {
                         __tls_stats()->ready.threads.threads++;
+                        __push_stat( __tls_stats(), __tls_stats()->ready.threads.threads, false, "Processor", kernelTLS().this_processor );
                 }
                 else {
                         __atomic_fetch_add(&cl->stats->ready.threads.threads, 1, __ATOMIC_RELAXED);
+                        __push_stat( cl->stats, cl->stats->ready.threads.threads, true, "Cluster", cl );
                 }
         #endif
@@ -470 +474 @@
 
         ready_schedule_lock();
-                $thread * thrd = pop( this );
+                $thread * thrd = pop_fast( this );
         ready_schedule_unlock();
 
@@ -613 +617 @@
         unsigned idle;
         unsigned total;
-        [idle, total, p] = query(this->idles);
+        [idle, total, p] = query_idles(this->procs);
 
         // If no one is sleeping, we are done
@@ -650 +654 @@
 }
 
-static void push  (__cluster_idles & this, processor & proc) {
+static void mark_idle(__cluster_proc_list & this, processor & proc) {
         /* paranoid */ verify( ! __preemption_enabled() );
         lock( this );
                 this.idle++;
                 /* paranoid */ verify( this.idle <= this.total );
-
-                insert_first(this.list, proc);
+                remove(proc);
+                insert_first(this.idles, proc);
         unlock( this );
         /* paranoid */ verify( ! __preemption_enabled() );
 }
 
-static void remove(__cluster_idles & this, processor & proc) {
+static void mark_awake(__cluster_proc_list & this, processor & proc) {
         /* paranoid */ verify( ! __preemption_enabled() );
         lock( this );
                 this.idle--;
                 /* paranoid */ verify( this.idle >= 0 );
-
                 remove(proc);
+                insert_last(this.actives, proc);
         unlock( this );
         /* paranoid */ verify( ! __preemption_enabled() );
 }
 
-static [unsigned idle, unsigned total, * processor] query( & __cluster_idles this ) {
+static [unsigned idle, unsigned total, * processor] query_idles( & __cluster_proc_list this ) {
+        /* paranoid */ verify( ! __preemption_enabled() );
+        /* paranoid */ verify( ready_schedule_islocked() );
+
         for() {
                 uint64_t l = __atomic_load_n(&this.lock, __ATOMIC_SEQ_CST);
@@ -678 +685 @@
                 unsigned idle    = this.idle;
                 unsigned total   = this.total;
-                processor * proc = &this.list`first;
+                processor * proc = &this.idles`first;
                 // Compiler fence is unnecessary, but gcc-8 and older incorrectly reorder code without it
                 asm volatile("": : :"memory");
@@ -684 +691 @@
                 return [idle, total, proc];
         }
+
+        /* paranoid */ verify( ready_schedule_islocked() );
+        /* paranoid */ verify( ! __preemption_enabled() );
 }
 

libcfa/src/concurrency/kernel.hfa

@@ -69 +69 @@
         struct cluster * cltr;
 
-        // Id within the cluster
-        unsigned cltr_id;
+        // Ready Queue state per processor
+        struct {
+                unsigned short its;
+                unsigned short itr;
+                unsigned id;
+                unsigned target;
+                unsigned long long int cutoff;
+        } rdq;
 
         // Set to true to notify the processor should terminate
@@ -140 +146 @@
 // Cluster Tools
 
-// Intrusives lanes which are used by the relaxed ready queue
+// Intrusives lanes which are used by the ready queue
 struct __attribute__((aligned(128))) __intrusive_lane_t;
 void  ?{}(__intrusive_lane_t & this);
 void ^?{}(__intrusive_lane_t & this);
 
-// Counter used for wether or not the lanes are all empty
-struct __attribute__((aligned(128))) __snzi_node_t;
-struct __snzi_t {
-        unsigned mask;
-        int root;
-        __snzi_node_t * nodes;
-};
-
-void  ?{}( __snzi_t & this, unsigned depth );
-void ^?{}( __snzi_t & this );
+// Aligned timestamps which are used by the relaxed ready queue
+struct __attribute__((aligned(128))) __timestamp_t;
+void  ?{}(__timestamp_t & this);
+void ^?{}(__timestamp_t & this);
 
 //TODO adjust cache size to ARCHITECTURE
 // Structure holding the relaxed ready queue
 struct __ready_queue_t {
-        // Data tracking how many/which lanes are used
-        // Aligned to 128 for cache locality
-        __snzi_t snzi;
-
         // Data tracking the actual lanes
         // On a seperate cacheline from the used struct since
@@ -171 +167 @@
                 __intrusive_lane_t * volatile data;
 
+                // Array of times
+                __timestamp_t * volatile tscs;
+
                 // Number of lanes (empty or not)
                 volatile size_t count;
@@ -180 +179 @@
 
 // Idle Sleep
-struct __cluster_idles {
+struct __cluster_proc_list {
         // Spin lock protecting the queue
         volatile uint64_t lock;
@@ -191 +190 @@
 
         // List of idle processors
-        dlist(processor, processor) list;
+        dlist(processor, processor) idles;
+
+        // List of active processors
+        dlist(processor, processor) actives;
 };
 
@@ -207 +209 @@
 
         // List of idle processors
-        __cluster_idles idles;
+        __cluster_proc_list procs;
 
         // List of threads

libcfa/src/concurrency/kernel/startup.cfa

@@ -268 +268 @@
                         __print_stats( st, mainProcessor->print_stats, "Processor ", mainProcessor->name, (void*)mainProcessor );
                 }
+                #if defined(CFA_STATS_ARRAY)
+                        __flush_stat( st, "Processor", mainProcessor );
+                #endif
         #endif
 
@@ -348 +351 @@
                         __print_stats( &local_stats, proc->print_stats, "Processor ", proc->name, (void*)proc );
                 }
+                #if defined(CFA_STATS_ARRAY)
+                        __flush_stat( &local_stats, "Processor", proc );
+                #endif
         #endif
 
@@ -463 +469 @@
         this.name = name;
         this.cltr = &_cltr;
+        this.rdq.its = 0;
+        this.rdq.itr = 0;
+        this.rdq.id  = -1u;
+        this.rdq.target = -1u;
+        this.rdq.cutoff = -1ull;
         do_terminate = false;
         preemption_alarm = 0p;
@@ -483 +494 @@
         #endif
 
-        lock( this.cltr->idles );
-                int target = this.cltr->idles.total += 1u;
-        unlock( this.cltr->idles );
-
-        id = doregister((__processor_id_t*)&this);
-
+        // Register and Lock the RWlock so no-one pushes/pops while we are changing the queue
+        uint_fast32_t last_size = ready_mutate_register((__processor_id_t*)&this);
+                this.cltr->procs.total += 1u;
+                insert_last(this.cltr->procs.actives, this);
+
+                // Adjust the ready queue size
+                ready_queue_grow( cltr );
+
+        // Unlock the RWlock
+        ready_mutate_unlock( last_size );
+
+        __cfadbg_print_safe(runtime_core, "Kernel : core %p created\n", &this);
+}
+
+// Not a ctor, it just preps the destruction but should not destroy members
+static void deinit(processor & this) {
         // Lock the RWlock so no-one pushes/pops while we are changing the queue
         uint_fast32_t last_size = ready_mutate_lock();
+                this.cltr->procs.total -= 1u;
+                remove(this);
 
                 // Adjust the ready queue size
-                this.cltr_id = ready_queue_grow( cltr, target );
-
-        // Unlock the RWlock
-        ready_mutate_unlock( last_size );
-
-        __cfadbg_print_safe(runtime_core, "Kernel : core %p created\n", &this);
-}
-
-// Not a ctor, it just preps the destruction but should not destroy members
-static void deinit(processor & this) {
-        lock( this.cltr->idles );
-                int target = this.cltr->idles.total -= 1u;
-        unlock( this.cltr->idles );
-
-        // Lock the RWlock so no-one pushes/pops while we are changing the queue
-        uint_fast32_t last_size = ready_mutate_lock();
-
-                // Adjust the ready queue size
-                ready_queue_shrink( this.cltr, target );
-
-        // Unlock the RWlock
-        ready_mutate_unlock( last_size );
-
-        // Finally we don't need the read_lock any more
-        unregister((__processor_id_t*)&this);
+                ready_queue_shrink( this.cltr );
+
+        // Unlock the RWlock and unregister: we don't need the read_lock any more
+        ready_mutate_unregister((__processor_id_t*)&this, last_size );
 
         close(this.idle);
@@ -560 +562 @@
 //-----------------------------------------------------------------------------
 // Cluster
-static void ?{}(__cluster_idles & this) {
+static void ?{}(__cluster_proc_list & this) {
         this.lock  = 0;
         this.idle  = 0;
         this.total = 0;
-        (this.list){};
 }
 
@@ -590 +591 @@
 
                 // Adjust the ready queue size
-                ready_queue_grow( &this, 0 );
+                ready_queue_grow( &this );
 
         // Unlock the RWlock
@@ -605 +606 @@
 
                 // Adjust the ready queue size
-                ready_queue_shrink( &this, 0 );
+                ready_queue_shrink( &this );
 
         // Unlock the RWlock
@@ -615 +616 @@
                         __print_stats( this.stats, this.print_stats, "Cluster", this.name, (void*)&this );
                 }
+                #if defined(CFA_STATS_ARRAY)
+                        __flush_stat( this.stats, "Cluster", &this );
+                #endif
                 free( this.stats );
         #endif

libcfa/src/concurrency/kernel_private.hfa

@@ -83 +83 @@
 // Cluster lock API
 //=======================================================================
-// Cells use by the reader writer lock
-// while not generic it only relies on a opaque pointer
-struct __attribute__((aligned(128))) __scheduler_lock_id_t {
-        // Spin lock used as the underlying lock
-        volatile bool lock;
-
-        // Handle pointing to the proc owning this cell
-        // Used for allocating cells and debugging
-        __processor_id_t * volatile handle;
-
-        #ifdef __CFA_WITH_VERIFY__
-                // Debug, check if this is owned for reading
-                bool owned;
-        #endif
-};
-
-static_assert( sizeof(struct __scheduler_lock_id_t) <= __alignof(struct __scheduler_lock_id_t));
-
 // Lock-Free registering/unregistering of threads
 // Register a processor to a given cluster and get its unique id in return
-unsigned doregister( struct __processor_id_t * proc );
+void register_proc_id( struct __processor_id_t * );
 
 // Unregister a processor from a given cluster using its id, getting back the original pointer
-void     unregister( struct __processor_id_t * proc );
-
-//-----------------------------------------------------------------------
-// Cluster idle lock/unlock
-static inline void lock(__cluster_idles & this) {
-        for() {
-                uint64_t l = this.lock;
-                if(
-                        (0 == (l % 2))
-                        && __atomic_compare_exchange_n(&this.lock, &l, l + 1, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)
-                ) return;
-                Pause();
-        }
-}
-
-static inline void unlock(__cluster_idles & this) {
-        /* paranoid */ verify( 1 == (this.lock % 2) );
-        __atomic_fetch_add( &this.lock, 1, __ATOMIC_SEQ_CST );
-}
+void unregister_proc_id( struct __processor_id_t * proc );
 
 //=======================================================================
@@ -152 +116 @@
         __atomic_store_n(ll, (bool)false, __ATOMIC_RELEASE);
 }
+
+// Cells use by the reader writer lock
+// while not generic it only relies on a opaque pointer
+struct __attribute__((aligned(128))) __scheduler_lock_id_t {
+        // Spin lock used as the underlying lock
+        volatile bool lock;
+
+        // Handle pointing to the proc owning this cell
+        // Used for allocating cells and debugging
+        __processor_id_t * volatile handle;
+
+        #ifdef __CFA_WITH_VERIFY__
+                // Debug, check if this is owned for reading
+                bool owned;
+        #endif
+};
+
+static_assert( sizeof(struct __scheduler_lock_id_t) <= __alignof(struct __scheduler_lock_id_t));
 
 //-----------------------------------------------------------------------
@@ -247 +229 @@
 void ready_mutate_unlock( uint_fast32_t /* value returned by lock */ );
 
+//-----------------------------------------------------------------------
+// Lock-Free registering/unregistering of threads
+// Register a processor to a given cluster and get its unique id in return
+// For convenience, also acquires the lock
+static inline uint_fast32_t ready_mutate_register( struct __processor_id_t * proc ) {
+        register_proc_id( proc );
+        return ready_mutate_lock();
+}
+
+// Unregister a processor from a given cluster using its id, getting back the original pointer
+// assumes the lock is acquired
+static inline void ready_mutate_unregister( struct __processor_id_t * proc, uint_fast32_t last_s ) {
+        ready_mutate_unlock( last_s );
+        unregister_proc_id( proc );
+}
+
+//-----------------------------------------------------------------------
+// Cluster idle lock/unlock
+static inline void lock(__cluster_proc_list & this) {
+        /* paranoid */ verify( ! __preemption_enabled() );
+
+        // Start by locking the global RWlock so that we know no-one is
+        // adding/removing processors while we mess with the idle lock
+        ready_schedule_lock();
+
+        // Simple counting lock, acquired, acquired by incrementing the counter
+        // to an odd number
+        for() {
+                uint64_t l = this.lock;
+                if(
+                        (0 == (l % 2))
+                        && __atomic_compare_exchange_n(&this.lock, &l, l + 1, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)
+                ) return;
+                Pause();
+        }
+
+        /* paranoid */ verify( ! __preemption_enabled() );
+}
+
+static inline void unlock(__cluster_proc_list & this) {
+        /* paranoid */ verify( ! __preemption_enabled() );
+
+        /* paranoid */ verify( 1 == (this.lock % 2) );
+        // Simple couting lock, release by incrementing to an even number
+        __atomic_fetch_add( &this.lock, 1, __ATOMIC_SEQ_CST );
+
+        // Release the global lock, which we acquired when locking
+        ready_schedule_unlock();
+
+        /* paranoid */ verify( ! __preemption_enabled() );
+}
+
 //=======================================================================
 // Ready-Queue API
 //-----------------------------------------------------------------------
-// pop thread from the ready queue of a cluster
-// returns 0p if empty
-__attribute__((hot)) bool query(struct cluster * cltr);
-
-//-----------------------------------------------------------------------
 // push thread onto a ready queue for a cluster
 // returns true if the list was previously empty, false otherwise
-__attribute__((hot)) bool push(struct cluster * cltr, struct $thread * thrd);
+__attribute__((hot)) void push(struct cluster * cltr, struct $thread * thrd);
 
 //-----------------------------------------------------------------------
@@ -263 +292 @@
 // returns 0p if empty
 // May return 0p spuriously
-__attribute__((hot)) struct $thread * pop(struct cluster * cltr);
+__attribute__((hot)) struct $thread * pop_fast(struct cluster * cltr);
 
 //-----------------------------------------------------------------------
@@ -272 +301 @@
 
 //-----------------------------------------------------------------------
-// remove thread from the ready queue of a cluster
-// returns bool if it wasn't found
-bool remove_head(struct cluster * cltr, struct $thread * thrd);
-
-//-----------------------------------------------------------------------
 // Increase the width of the ready queue (number of lanes) by 4
-unsigned ready_queue_grow  (struct cluster * cltr, int target);
+void ready_queue_grow  (struct cluster * cltr);
 
 //-----------------------------------------------------------------------
 // Decrease the width of the ready queue (number of lanes) by 4
-void ready_queue_shrink(struct cluster * cltr, int target);
+void ready_queue_shrink(struct cluster * cltr);
 
 

libcfa/src/concurrency/preemption.cfa

@@ -712 +712 @@
 static void * alarm_loop( __attribute__((unused)) void * args ) {
         __processor_id_t id;
-        id.id = doregister(&id);
+        register_proc_id(&id);
         __cfaabi_tls.this_proc_id = &id;
 
@@ -773 +773 @@
 EXIT:
         __cfaabi_dbg_print_safe( "Kernel : Preemption thread stopping\n" );
-        unregister(&id);
+        register_proc_id(&id);
 
         return 0p;

libcfa/src/concurrency/ready_queue.cfa

@@ -17 +17 @@
 // #define __CFA_DEBUG_PRINT_READY_QUEUE__
 
-// #define USE_SNZI
 // #define USE_MPSC
+
+#define USE_RELAXED_FIFO
+// #define USE_WORK_STEALING
 
 #include "bits/defs.hfa"
@@ -29 +31 @@
 #include <unistd.h>
 
-#include "snzi.hfa"
 #include "ready_subqueue.hfa"
 
@@ -40 +41 @@
 #endif
 
-#define BIAS 4
+#if   defined(USE_RELAXED_FIFO)
+        #define BIAS 4
+        #define READYQ_SHARD_FACTOR 4
+        #define SEQUENTIAL_SHARD 1
+#elif defined(USE_WORK_STEALING)
+        #define READYQ_SHARD_FACTOR 2
+        #define SEQUENTIAL_SHARD 2
+#else
+        #error no scheduling strategy selected
+#endif
+
+static inline [unsigned, bool] idx_from_r(unsigned r, unsigned preferred);
+static inline struct $thread * try_pop(struct cluster * cltr, unsigned w);
+static inline struct $thread * try_pop(struct cluster * cltr, unsigned i, unsigned j);
+static inline struct $thread * search(struct cluster * cltr);
+
 
 // returns the maximum number of processors the RWLock support
@@ -94 +110 @@
 //=======================================================================
 // Lock-Free registering/unregistering of threads
-unsigned doregister( struct __processor_id_t * proc ) with(*__scheduler_lock) {
+void register_proc_id( struct __processor_id_t * proc ) with(*__scheduler_lock) {
         __cfadbg_print_safe(ready_queue, "Kernel : Registering proc %p for RW-Lock\n", proc);
 
@@ -108 +124 @@
                         /*paranoid*/ verify(0 == (__alignof__(data[i]) % cache_line_size));
                         /*paranoid*/ verify((((uintptr_t)&data[i]) % cache_line_size) == 0);
-                        return i;
+                        proc->id = i;
                 }
         }
@@ -135 +151 @@
         /*paranoid*/ verify(__alignof__(data[n]) == (2 * cache_line_size));
         /*paranoid*/ verify((((uintptr_t)&data[n]) % cache_line_size) == 0);
-        return n;
-}
-
-void unregister( struct __processor_id_t * proc ) with(*__scheduler_lock) {
+        proc->id = n;
+}
+
+void unregister_proc_id( struct __processor_id_t * proc ) with(*__scheduler_lock) {
         unsigned id = proc->id;
         /*paranoid*/ verify(id < ready);
@@ -193 +209 @@
 
 //=======================================================================
-// Cforall Reqdy Queue used for scheduling
+// Cforall Ready Queue used for scheduling
 //=======================================================================
 void ?{}(__ready_queue_t & this) with (this) {
         lanes.data  = 0p;
+        lanes.tscs  = 0p;
         lanes.count = 0;
 }
 
 void ^?{}(__ready_queue_t & this) with (this) {
-        verify( 1 == lanes.count );
-        #ifdef USE_SNZI
-                verify( !query( snzi ) );
-        #endif
+        verify( SEQUENTIAL_SHARD == lanes.count );
         free(lanes.data);
+        free(lanes.tscs);
 }
 
 //-----------------------------------------------------------------------
-__attribute__((hot)) bool query(struct cluster * cltr) {
-        #ifdef USE_SNZI
-                return query(cltr->ready_queue.snzi);
-        #endif
-        return true;
-}
-
-static inline [unsigned, bool] idx_from_r(unsigned r, unsigned preferred) {
-        unsigned i;
-        bool local;
-        #if defined(BIAS)
+#if defined(USE_RELAXED_FIFO)
+        //-----------------------------------------------------------------------
+        // get index from random number with or without bias towards queues
+        static inline [unsigned, bool] idx_from_r(unsigned r, unsigned preferred) {
+                unsigned i;
+                bool local;
                 unsigned rlow  = r % BIAS;
                 unsigned rhigh = r / BIAS;
@@ -225 +235 @@
                         // (BIAS - 1) out of BIAS chances
                         // Use perferred queues
-                        i = preferred + (rhigh % 4);
+                        i = preferred + (rhigh % READYQ_SHARD_FACTOR);
                         local = true;
                 }
@@ -234 +244 @@
                         local = false;
                 }
-        #else
-                i = r;
-                local = false;
-        #endif
-        return [i, local];
-}
-
-//-----------------------------------------------------------------------
-__attribute__((hot)) bool push(struct cluster * cltr, struct $thread * thrd) with (cltr->ready_queue) {
-        __cfadbg_print_safe(ready_queue, "Kernel : Pushing %p on cluster %p\n", thrd, cltr);
-
-        const bool external = (!kernelTLS().this_processor) || (cltr != kernelTLS().this_processor->cltr);
-
-        // write timestamp
-        thrd->link.ts = rdtscl();
-
-        bool first = false;
-        __attribute__((unused)) bool local;
-        __attribute__((unused)) int preferred;
-        #if defined(BIAS)
-                preferred =
-                        //*
-                        external ? -1 : kernelTLS().this_processor->cltr_id;
-                        /*/
-                        thrd->link.preferred * 4;
-                        //*/
-        #endif
-
-        // Try to pick a lane and lock it
-        unsigned i;
-        do {
-                // Pick the index of a lane
-                // unsigned r = __tls_rand();
-                unsigned r = __tls_rand_fwd();
-                [i, local] = idx_from_r(r, preferred);
-
-                i %= __atomic_load_n( &lanes.count, __ATOMIC_RELAXED );
-
+                return [i, local];
+        }
+
+        __attribute__((hot)) void push(struct cluster * cltr, struct $thread * thrd) with (cltr->ready_queue) {
+                __cfadbg_print_safe(ready_queue, "Kernel : Pushing %p on cluster %p\n", thrd, cltr);
+
+                const bool external = (!kernelTLS().this_processor) || (cltr != kernelTLS().this_processor->cltr);
+                /* paranoid */ verify(external || kernelTLS().this_processor->rdq.id < lanes.count );
+
+                // write timestamp
+                thrd->link.ts = rdtscl();
+
+                bool local;
+                int preferred = external ? -1 : kernelTLS().this_processor->rdq.id;
+
+                // Try to pick a lane and lock it
+                unsigned i;
+                do {
+                        // Pick the index of a lane
+                        unsigned r = __tls_rand_fwd();
+                        [i, local] = idx_from_r(r, preferred);
+
+                        i %= __atomic_load_n( &lanes.count, __ATOMIC_RELAXED );
+
+                        #if !defined(__CFA_NO_STATISTICS__)
+                                if(external) {
+                                        if(local) __atomic_fetch_add(&cltr->stats->ready.pick.ext.local, 1, __ATOMIC_RELAXED);
+                                        __atomic_fetch_add(&cltr->stats->ready.pick.ext.attempt, 1, __ATOMIC_RELAXED);
+                                }
+                                else {
+                                        if(local) __tls_stats()->ready.pick.push.local++;
+                                        __tls_stats()->ready.pick.push.attempt++;
+                                }
+                        #endif
+
+                #if defined(USE_MPSC)
+                        // mpsc always succeeds
+                } while( false );
+                #else
+                        // If we can't lock it retry
+                } while( !__atomic_try_acquire( &lanes.data[i].lock ) );
+                #endif
+
+                // Actually push it
+                push(lanes.data[i], thrd);
+
+                #if !defined(USE_MPSC)
+                        // Unlock and return
+                        __atomic_unlock( &lanes.data[i].lock );
+                #endif
+
+                // Mark the current index in the tls rng instance as having an item
+                __tls_rand_advance_bck();
+
+                __cfadbg_print_safe(ready_queue, "Kernel : Pushed %p on cluster %p (idx: %u, mask %llu, first %d)\n", thrd, cltr, i, used.mask[0], lane_first);
+
+                // Update statistics
                 #if !defined(__CFA_NO_STATISTICS__)
                         if(external) {
-                                if(local) __atomic_fetch_add(&cltr->stats->ready.pick.ext.local, 1, __ATOMIC_RELAXED);
-                                __atomic_fetch_add(&cltr->stats->ready.pick.ext.attempt, 1, __ATOMIC_RELAXED);
+                                if(local) __atomic_fetch_add(&cltr->stats->ready.pick.ext.lsuccess, 1, __ATOMIC_RELAXED);
+                                __atomic_fetch_add(&cltr->stats->ready.pick.ext.success, 1, __ATOMIC_RELAXED);
                         }
                         else {
-                                if(local) __tls_stats()->ready.pick.push.local++;
-                                __tls_stats()->ready.pick.push.attempt++;
+                                if(local) __tls_stats()->ready.pick.push.lsuccess++;
+                                __tls_stats()->ready.pick.push.success++;
                         }
                 #endif
-
-        #if defined(USE_MPSC)
-                // mpsc always succeeds
-        } while( false );
-        #else
-                // If we can't lock it retry
-        } while( !__atomic_try_acquire( &lanes.data[i].lock ) );
-        #endif
-
-        // Actually push it
-        #ifdef USE_SNZI
-                bool lane_first =
-        #endif
-
-        push(lanes.data[i], thrd);
-
-        #ifdef USE_SNZI
-                // If this lane used to be empty we need to do more
-                if(lane_first) {
-                        // Check if the entire queue used to be empty
-                        first = !query(snzi);
-
-                        // Update the snzi
-                        arrive( snzi, i );
-                }
-        #endif
-
-        #if !defined(USE_MPSC)
-                // Unlock and return
-                __atomic_unlock( &lanes.data[i].lock );
-        #endif
-
-        // Mark the current index in the tls rng instance as having an item
-        __tls_rand_advance_bck();
-
-        __cfadbg_print_safe(ready_queue, "Kernel : Pushed %p on cluster %p (idx: %u, mask %llu, first %d)\n", thrd, cltr, i, used.mask[0], lane_first);
+        }
+
+        // Pop from the ready queue from a given cluster
+        __attribute__((hot)) $thread * pop_fast(struct cluster * cltr) with (cltr->ready_queue) {
+                /* paranoid */ verify( lanes.count > 0 );
+                /* paranoid */ verify( kernelTLS().this_processor );
+                /* paranoid */ verify( kernelTLS().this_processor->rdq.id < lanes.count );
+
+                unsigned count = __atomic_load_n( &lanes.count, __ATOMIC_RELAXED );
+                int preferred = kernelTLS().this_processor->rdq.id;
+
+
+                // As long as the list is not empty, try finding a lane that isn't empty and pop from it
+                for(25) {
+                        // Pick two lists at random
+                        unsigned ri = __tls_rand_bck();
+                        unsigned rj = __tls_rand_bck();
+
+                        unsigned i, j;
+                        __attribute__((unused)) bool locali, localj;
+                        [i, locali] = idx_from_r(ri, preferred);
+                        [j, localj] = idx_from_r(rj, preferred);
+
+                        #if !defined(__CFA_NO_STATISTICS__)
+                                if(locali && localj) {
+                                        __tls_stats()->ready.pick.pop.local++;
+                                }
+                        #endif
+
+                        i %= count;
+                        j %= count;
+
+                        // try popping from the 2 picked lists
+                        struct $thread * thrd = try_pop(cltr, i, j);
+                        if(thrd) {
+                                #if !defined(__CFA_NO_STATISTICS__)
+                                        if( locali || localj ) __tls_stats()->ready.pick.pop.lsuccess++;
+                                #endif
+                                return thrd;
+                        }
+                }
+
+                // All lanes where empty return 0p
+                return 0p;
+        }
+
+        __attribute__((hot)) struct $thread * pop_slow(struct cluster * cltr) {
+                return search(cltr);
+        }
+#endif
+#if defined(USE_WORK_STEALING)
+        __attribute__((hot)) void push(struct cluster * cltr, struct $thread * thrd) with (cltr->ready_queue) {
+                __cfadbg_print_safe(ready_queue, "Kernel : Pushing %p on cluster %p\n", thrd, cltr);
+
+                const bool external = (!kernelTLS().this_processor) || (cltr != kernelTLS().this_processor->cltr);
+                /* paranoid */ verify(external || kernelTLS().this_processor->rdq.id < lanes.count );
+
+                // write timestamp
+                thrd->link.ts = rdtscl();
+
+                // Try to pick a lane and lock it
+                unsigned i;
+                do {
+                        if(unlikely(external)) {
+                                i = __tls_rand() % lanes.count;
+                        }
+                        else {
+                                processor * proc = kernelTLS().this_processor;
+                                unsigned r = proc->rdq.its++;
+                                i =  proc->rdq.id + (r % READYQ_SHARD_FACTOR);
+                        }
+
+
+                #if defined(USE_MPSC)
+                        // mpsc always succeeds
+                } while( false );
+                #else
+                        // If we can't lock it retry
+                } while( !__atomic_try_acquire( &lanes.data[i].lock ) );
+                #endif
+
+                // Actually push it
+                push(lanes.data[i], thrd);
+
+                #if !defined(USE_MPSC)
+                        // Unlock and return
+                        __atomic_unlock( &lanes.data[i].lock );
+                #endif
+
+                __cfadbg_print_safe(ready_queue, "Kernel : Pushed %p on cluster %p (idx: %u, mask %llu, first %d)\n", thrd, cltr, i, used.mask[0], lane_first);
+        }
+
+        // Pop from the ready queue from a given cluster
+        __attribute__((hot)) $thread * pop_fast(struct cluster * cltr) with (cltr->ready_queue) {
+                /* paranoid */ verify( lanes.count > 0 );
+                /* paranoid */ verify( kernelTLS().this_processor );
+                /* paranoid */ verify( kernelTLS().this_processor->rdq.id < lanes.count );
+
+                processor * proc = kernelTLS().this_processor;
+
+                if(proc->rdq.target == -1u) {
+                        proc->rdq.target = __tls_rand() % lanes.count;
+                        unsigned it1  = proc->rdq.itr;
+                        unsigned it2  = proc->rdq.itr + 1;
+                        unsigned idx1 = proc->rdq.id + (it1 % READYQ_SHARD_FACTOR);
+                        unsigned idx2 = proc->rdq.id + (it1 % READYQ_SHARD_FACTOR);
+                        unsigned long long tsc1 = ts(lanes.data[idx1]);
+                        unsigned long long tsc2 = ts(lanes.data[idx2]);
+                        proc->rdq.cutoff = min(tsc1, tsc2);
+                }
+                else if(lanes.tscs[proc->rdq.target].tv < proc->rdq.cutoff) {
+                        $thread * t = try_pop(cltr, proc->rdq.target);
+                        proc->rdq.target = -1u;
+                        if(t) return t;
+                }
+
+                for(READYQ_SHARD_FACTOR) {
+                        unsigned i = proc->rdq.id + (--proc->rdq.itr % READYQ_SHARD_FACTOR);
+                        if($thread * t = try_pop(cltr, i)) return t;
+                }
+                return 0p;
+        }
+
+        __attribute__((hot)) struct $thread * pop_slow(struct cluster * cltr) with (cltr->ready_queue) {
+                for(25) {
+                        unsigned i = __tls_rand() % lanes.count;
+                        $thread * t = try_pop(cltr, i);
+                        if(t) return t;
+                }
+
+                return search(cltr);
+        }
+#endif
+
+//=======================================================================
+// Various Ready Queue utilities
+//=======================================================================
+// these function work the same or almost the same
+// whether they are using work-stealing or relaxed fifo scheduling
+
+//-----------------------------------------------------------------------
+// try to pop from a lane given by index w
+static inline struct $thread * try_pop(struct cluster * cltr, unsigned w) with (cltr->ready_queue) {
+        // Get relevant elements locally
+        __intrusive_lane_t & lane = lanes.data[w];
+
+        // If list looks empty retry
+        if( is_empty(lane) ) return 0p;
+
+        // If we can't get the lock retry
+        if( !__atomic_try_acquire(&lane.lock) ) return 0p;
+
+        // If list is empty, unlock and retry
+        if( is_empty(lane) ) {
+                __atomic_unlock(&lane.lock);
+                return 0p;
+        }
+
+        // Actually pop the list
+        struct $thread * thrd;
+        thrd = pop(lane);
+
+        /* paranoid */ verify(thrd);
+        /* paranoid */ verify(lane.lock);
+
+        // Unlock and return
+        __atomic_unlock(&lane.lock);
 
         // Update statistics
         #if !defined(__CFA_NO_STATISTICS__)
-                if(external) {
-                        if(local) __atomic_fetch_add(&cltr->stats->ready.pick.ext.lsuccess, 1, __ATOMIC_RELAXED);
-                        __atomic_fetch_add(&cltr->stats->ready.pick.ext.success, 1, __ATOMIC_RELAXED);
-                }
-                else {
-                        if(local) __tls_stats()->ready.pick.push.lsuccess++;
-                        __tls_stats()->ready.pick.push.success++;
-                }
+                __tls_stats()->ready.pick.pop.success++;
         #endif
 
-        // return whether or not the list was empty before this push
-        return first;
-}
-
-static struct $thread * try_pop(struct cluster * cltr, unsigned i, unsigned j);
-static struct $thread * try_pop(struct cluster * cltr, unsigned i);
-
-// Pop from the ready queue from a given cluster
-__attribute__((hot)) $thread * pop(struct cluster * cltr) with (cltr->ready_queue) {
-        /* paranoid */ verify( lanes.count > 0 );
-        unsigned count = __atomic_load_n( &lanes.count, __ATOMIC_RELAXED );
-        int preferred;
-        #if defined(BIAS)
-                // Don't bother trying locally too much
-                preferred = kernelTLS().this_processor->cltr_id;
+        #if defined(USE_WORK_STEALING)
+                lanes.tscs[w].tv = thrd->link.ts;
         #endif
 
-
-        // As long as the list is not empty, try finding a lane that isn't empty and pop from it
-        #ifdef USE_SNZI
-                while( query(snzi) ) {
-        #else
-                for(25) {
-        #endif
-                // Pick two lists at random
-                // unsigned ri = __tls_rand();
-                // unsigned rj = __tls_rand();
-                unsigned ri = __tls_rand_bck();
-                unsigned rj = __tls_rand_bck();
-
-                unsigned i, j;
-                __attribute__((unused)) bool locali, localj;
-                [i, locali] = idx_from_r(ri, preferred);
-                [j, localj] = idx_from_r(rj, preferred);
-
-                #if !defined(__CFA_NO_STATISTICS__)
-                        if(locali && localj) {
-                                __tls_stats()->ready.pick.pop.local++;
-                        }
-                #endif
-
-                i %= count;
-                j %= count;
-
-                // try popping from the 2 picked lists
-                struct $thread * thrd = try_pop(cltr, i, j);
-                if(thrd) {
-                        #if defined(BIAS) && !defined(__CFA_NO_STATISTICS__)
-                                if( locali || localj ) __tls_stats()->ready.pick.pop.lsuccess++;
-                        #endif
-                        return thrd;
-                }
-        }
-
-        // All lanes where empty return 0p
-        return 0p;
-}
-
-__attribute__((hot)) struct $thread * pop_slow(struct cluster * cltr) with (cltr->ready_queue) {
+        // return the popped thread
+        return thrd;
+}
+
+//-----------------------------------------------------------------------
+// try to pop from any lanes making sure you don't miss any threads push
+// before the start of the function
+static inline struct $thread * search(struct cluster * cltr) with (cltr->ready_queue) {
         /* paranoid */ verify( lanes.count > 0 );
         unsigned count = __atomic_load_n( &lanes.count, __ATOMIC_RELAXED );
@@ -405 +511 @@
 }
 
-
 //-----------------------------------------------------------------------
-// Given 2 indexes, pick the list with the oldest push an try to pop from it
-static inline struct $thread * try_pop(struct cluster * cltr, unsigned i, unsigned j) with (cltr->ready_queue) {
-        #if !defined(__CFA_NO_STATISTICS__)
-                __tls_stats()->ready.pick.pop.attempt++;
-        #endif
-
-        // Pick the bet list
-        int w = i;
-        if( __builtin_expect(!is_empty(lanes.data[j]), true) ) {
-                w = (ts(lanes.data[i]) < ts(lanes.data[j])) ? i : j;
-        }
-
-        return try_pop(cltr, w);
-}
-
-static inline struct $thread * try_pop(struct cluster * cltr, unsigned w) with (cltr->ready_queue) {
-        // Get relevant elements locally
-        __intrusive_lane_t & lane = lanes.data[w];
-
-        // If list looks empty retry
-        if( is_empty(lane) ) return 0p;
-
-        // If we can't get the lock retry
-        if( !__atomic_try_acquire(&lane.lock) ) return 0p;
-
-
-        // If list is empty, unlock and retry
-        if( is_empty(lane) ) {
-                __atomic_unlock(&lane.lock);
-                return 0p;
-        }
-
-        // Actually pop the list
-        struct $thread * thrd;
-        thrd = pop(lane);
-
-        /* paranoid */ verify(thrd);
-        /* paranoid */ verify(lane.lock);
-
-        #ifdef USE_SNZI
-                // If this was the last element in the lane
-                if(emptied) {
-                        depart( snzi, w );
-                }
-        #endif
-
-        // Unlock and return
-        __atomic_unlock(&lane.lock);
-
-        // Update statistics
-        #if !defined(__CFA_NO_STATISTICS__)
-                __tls_stats()->ready.pick.pop.success++;
-        #endif
-
-        // Update the thread bias
-        thrd->link.preferred = w / 4;
-
-        // return the popped thread
-        return thrd;
-}
-//-----------------------------------------------------------------------
-
-bool remove_head(struct cluster * cltr, struct $thread * thrd) with (cltr->ready_queue) {
-        for(i; lanes.count) {
-                __intrusive_lane_t & lane = lanes.data[i];
-
-                bool removed = false;
-
-                __atomic_acquire(&lane.lock);
-                        if(head(lane)->link.next == thrd) {
-                                $thread * pthrd;
-                                pthrd = pop(lane);
-
-                                /* paranoid */ verify( pthrd == thrd );
-
-                                removed = true;
-                                #ifdef USE_SNZI
-                                        if(emptied) {
-                                                depart( snzi, i );
-                                        }
-                                #endif
-                        }
-                __atomic_unlock(&lane.lock);
-
-                if( removed ) return true;
-        }
-        return false;
-}
-
-//-----------------------------------------------------------------------
-
+// Check that all the intrusive queues in the data structure are still consistent
 static void check( __ready_queue_t & q ) with (q) {
         #if defined(__CFA_WITH_VERIFY__) && !defined(USE_MPSC)
@@ -522 +537 @@
 }
 
+//-----------------------------------------------------------------------
+// Given 2 indexes, pick the list with the oldest push an try to pop from it
+static inline struct $thread * try_pop(struct cluster * cltr, unsigned i, unsigned j) with (cltr->ready_queue) {
+        #if !defined(__CFA_NO_STATISTICS__)
+                __tls_stats()->ready.pick.pop.attempt++;
+        #endif
+
+        // Pick the bet list
+        int w = i;
+        if( __builtin_expect(!is_empty(lanes.data[j]), true) ) {
+                w = (ts(lanes.data[i]) < ts(lanes.data[j])) ? i : j;
+        }
+
+        return try_pop(cltr, w);
+}
+
 // Call this function of the intrusive list was moved using memcpy
 // fixes the list so that the pointers back to anchors aren't left dangling
@@ -541 +572 @@
 }
 
+static void assign_list(unsigned & value, dlist(processor, processor) & list, unsigned count) {
+        processor * it = &list`first;
+        for(unsigned i = 0; i < count; i++) {
+                /* paranoid */ verifyf( it, "Unexpected null iterator, at index %u of %u\n", i, count);
+                it->rdq.id = value;
+                it->rdq.target = -1u;
+                value += READYQ_SHARD_FACTOR;
+                it = &(*it)`next;
+        }
+}
+
+static void reassign_cltr_id(struct cluster * cltr) {
+        unsigned preferred = 0;
+        assign_list(preferred, cltr->procs.actives, cltr->procs.total - cltr->procs.idle);
+        assign_list(preferred, cltr->procs.idles  , cltr->procs.idle );
+}
+
+static void fix_times( struct cluster * cltr ) with( cltr->ready_queue ) {
+        #if defined(USE_WORK_STEALING)
+                lanes.tscs = alloc(lanes.count, lanes.tscs`realloc);
+                for(i; lanes.count) {
+                        lanes.tscs[i].tv = ts(lanes.data[i]);
+                }
+        #endif
+}
+
 // Grow the ready queue
-unsigned ready_queue_grow(struct cluster * cltr, int target) {
-        unsigned preferred;
+void ready_queue_grow(struct cluster * cltr) {
         size_t ncount;
+        int target = cltr->procs.total;
 
         /* paranoid */ verify( ready_mutate_islocked() );
@@ -554 +611 @@
         // grow the ready queue
         with( cltr->ready_queue ) {
-                #ifdef USE_SNZI
-                        ^(snzi){};
-                #endif
-
                 // Find new count
                 // Make sure we always have atleast 1 list
                 if(target >= 2) {
-                        ncount = target * 4;
-                        preferred = ncount - 4;
+                        ncount = target * READYQ_SHARD_FACTOR;
                 } else {
-                        ncount = 1;
-                        preferred = 0;
+                        ncount = SEQUENTIAL_SHARD;
                 }
 
@@ -583 +634 @@
                 // Update original
                 lanes.count = ncount;
-
-                #ifdef USE_SNZI
-                        // Re-create the snzi
-                        snzi{ log2( lanes.count / 8 ) };
-                        for( idx; (size_t)lanes.count ) {
-                                if( !is_empty(lanes.data[idx]) ) {
-                                        arrive(snzi, idx);
-                                }
-                        }
-                #endif
-        }
+        }
+
+        fix_times(cltr);
+
+        reassign_cltr_id(cltr);
 
         // Make sure that everything is consistent
@@ -601 +646 @@
 
         /* paranoid */ verify( ready_mutate_islocked() );
-        return preferred;
 }
 
 // Shrink the ready queue
-void ready_queue_shrink(struct cluster * cltr, int target) {
+void ready_queue_shrink(struct cluster * cltr) {
         /* paranoid */ verify( ready_mutate_islocked() );
         __cfadbg_print_safe(ready_queue, "Kernel : Shrinking ready queue\n");
@@ -612 +656 @@
         /* paranoid */ check( cltr->ready_queue );
 
+        int target = cltr->procs.total;
+
         with( cltr->ready_queue ) {
-                #ifdef USE_SNZI
-                        ^(snzi){};
-                #endif
-
                 // Remember old count
                 size_t ocount = lanes.count;
@@ -622 +664 @@
                 // Find new count
                 // Make sure we always have atleast 1 list
-                lanes.count = target >= 2 ? target * 4: 1;
+                lanes.count = target >= 2 ? target * READYQ_SHARD_FACTOR: SEQUENTIAL_SHARD;
                 /* paranoid */ verify( ocount >= lanes.count );
-                /* paranoid */ verify( lanes.count == target * 4 || target < 2 );
+                /* paranoid */ verify( lanes.count == target * READYQ_SHARD_FACTOR || target < 2 );
 
                 // for printing count the number of displaced threads
@@ -667 +709 @@
                         fix(lanes.data[idx]);
                 }
-
-                #ifdef USE_SNZI
-                        // Re-create the snzi
-                        snzi{ log2( lanes.count / 8 ) };
-                        for( idx; (size_t)lanes.count ) {
-                                if( !is_empty(lanes.data[idx]) ) {
-                                        arrive(snzi, idx);
-                                }
-                        }
-                #endif
-        }
+        }
+
+        fix_times(cltr);
+
+        reassign_cltr_id(cltr);
 
         // Make sure that everything is consistent

libcfa/src/concurrency/ready_subqueue.hfa

@@ -246 +246 @@
         #endif
 }
+
+// Aligned timestamps which are used by the relaxed ready queue
+struct __attribute__((aligned(128))) __timestamp_t {
+        volatile unsigned long long tv;
+};
+
+void  ?{}(__timestamp_t & this) { this.tv = 0; }
+void ^?{}(__timestamp_t & this) {}

libcfa/src/concurrency/stats.cfa

@@ -5 +5 @@
 #include <inttypes.h>
 #include "bits/debug.hfa"
+#include "bits/locks.hfa"
 #include "stats.hfa"
 
@@ -44 +45 @@
                         stats->io.calls.errors.busy = 0;
                         stats->io.poller.sleeps     = 0;
+                #endif
+
+                #if defined(CFA_STATS_ARRAY)
+                        stats->array.values = alloc(CFA_STATS_ARRAY);
+                        stats->array.cnt = 0;
                 #endif
         }
@@ -151 +157 @@
                 #endif
         }
+
+        #if defined(CFA_STATS_ARRAY)
+                extern "C" {
+                        #include <stdio.h>
+                        #include <errno.h>
+                        #include <sys/stat.h>
+                        #include <fcntl.h>
+                }
+
+                void __flush_stat( struct __stats_t * this, const char * name, void * handle) {
+                        int ret = mkdir(".cfadata", 0755);
+                        if(ret < 0 && errno != EEXIST) abort("Failed to create directory .cfadata: %d\n", errno);
+
+                        char filename[100];
+                        snprintf(filename, 100, ".cfadata/%s%p.data", name, handle);
+
+                        int fd = open(filename, O_WRONLY | O_APPEND | O_CREAT, 0644);
+                        if(fd < 0) abort("Failed to create file %s: %d\n", filename, errno);
+
+                        for(i; this->array.cnt) {
+                                char line[100];
+                                size_t n = snprintf(line, 100, "%llu, %lld\n", this->array.values[i].ts, this->array.values[i].value);
+                                write(fd, line, n);
+                        }
+
+                        this->array.cnt = 0;
+                        close(fd);
+                }
+
+                static __spinlock_t stats_lock;
+
+                void __push_stat( struct __stats_t * this, int64_t value, bool external, const char * name, void * handle ) {
+                        if(external) lock(stats_lock __cfaabi_dbg_ctx2);
+
+                        if( this->array.cnt >= CFA_STATS_ARRAY ) __flush_stat( this, name, handle );
+
+                        size_t idx = this->array.cnt;
+                        this->array.cnt++;
+
+                        if(external) unlock(stats_lock);
+
+                        this->array.values[idx].ts = rdtscl();
+                        this->array.values[idx].value = value;
+                }
+        #endif
 #endif

libcfa/src/concurrency/stats.hfa

@@ -1 +1 @@
 #pragma once
+
+// #define CFA_STATS_ARRAY 10000
 
 #include <stdint.h>
@@ -109 +111 @@
         #endif
 
+        #if defined(CFA_STATS_ARRAY)
+                struct __stats_elem_t {
+                        long long int ts;
+                        int64_t value;
+                };
+        #endif
+
         struct __attribute__((aligned(128))) __stats_t {
                 __stats_readQ_t ready;
@@ -114 +123 @@
                         __stats_io_t    io;
                 #endif
+
+                #if defined(CFA_STATS_ARRAY)
+                        struct {
+                                __stats_elem_t * values;
+                                volatile size_t cnt;
+                        } array;
+                #endif
+
         };
 
@@ -119 +136 @@
         void __tally_stats( struct __stats_t *, struct __stats_t * );
         void __print_stats( struct __stats_t *, int, const char *, const char *, void * );
+        #if defined(CFA_STATS_ARRAY)
+                void __push_stat ( struct __stats_t *, int64_t value, bool external, const char * name, void * handle);
+                void __flush_stat( struct __stats_t *, const char *, void * );
+        #else
+                static inline void __push_stat ( struct __stats_t *, int64_t, bool, const char *, void * ) {}
+                static inline void __flush_stat( struct __stats_t *, const char *, void * ) {}
+        #endif
 #endif
 

libcfa/src/concurrency/thread.cfa

@@ -39 +39 @@
         link.next = 0p;
         link.prev = 0p;
-        link.preferred = -1;
         #if defined( __CFA_WITH_VERIFY__ )
                 canary = 0x0D15EA5E0D15EA5Ep;
@@ -62 +61 @@
 }
 
-FORALL_DATA_INSTANCE(ThreadCancelled, (thread_t &), (thread_t))
+EHM_VIRTUAL_TABLE(SomeThreadCancelled, std_thread_cancelled);
 
 forall(T &)
@@ -73 +72 @@
 forall(T &)
 const char * msg(ThreadCancelled(T) *) {
-        return "ThreadCancelled";
+        return "ThreadCancelled(...)";
 }
 
 forall(T &)
 static void default_thread_cancel_handler(ThreadCancelled(T) & ) {
+        // Improve this error message, can I do formatting?
         abort( "Unhandled thread cancellation.\n" );
 }
 
-forall(T & | is_thread(T) | IS_EXCEPTION(ThreadCancelled, (T)))
+static void default_thread_cancel_handler(SomeThreadCancelled & ) {
+        // Improve this error message, can I do formatting?
+        abort( "Unhandled thread cancellation.\n" );
+}
+
+forall(T & | is_thread(T) | IS_EXCEPTION(SomeThreadCancelled))
 void ?{}( thread_dtor_guard_t & this,
-                T & thrd, void(*cancelHandler)(ThreadCancelled(T) &)) {
-        $monitor * m = get_monitor(thrd);
+                T & thrd, void(*cancelHandler)(SomeThreadCancelled &)) {
+        $monitor * m = get_monitor(thrd);
         $thread * desc = get_thread(thrd);
 
         // Setup the monitor guard
         void (*dtor)(T& mutex this) = ^?{};
-        bool join = cancelHandler != (void(*)(ThreadCancelled(T)&))0;
+        bool join = cancelHandler != (void(*)(SomeThreadCancelled&))0;
         (this.mg){&m, (void(*)())dtor, join};
 
@@ -103 +108 @@
         }
         desc->state = Cancelled;
-        void(*defaultResumptionHandler)(ThreadCancelled(T) &) = 
+        void(*defaultResumptionHandler)(SomeThreadCancelled &) =
                 join ? cancelHandler : default_thread_cancel_handler;
 
-        ThreadCancelled(T) except;
         // TODO: Remove explitate vtable set once trac#186 is fixed.
-        except.virtual_table = &get_exception_vtable(&except);
+        SomeThreadCancelled except;
+        except.virtual_table = &std_thread_cancelled;
         except.the_thread = &thrd;
         except.the_exception = __cfaehm_cancellation_exception( cancellation );
-        throwResume except;
+        // Why is this cast required?
+        throwResume (SomeThreadCancelled &)except;
 
         except.the_exception->virtual_table->free( except.the_exception );
@@ -158 +164 @@
 
 //-----------------------------------------------------------------------------
-forall(T & | is_thread(T) | IS_RESUMPTION_EXCEPTION(ThreadCancelled, (T)))
+forall(T & | is_thread(T) | IS_RESUMPTION_EXCEPTION(SomeThreadCancelled))
 T & join( T & this ) {
         thread_dtor_guard_t guard = { this, defaultResumptionHandler };

libcfa/src/concurrency/thread.hfa

@@ -32 +32 @@
 };
 
-FORALL_DATA_EXCEPTION(ThreadCancelled, (thread_t &), (thread_t)) (
+EHM_EXCEPTION(SomeThreadCancelled) (
+        void * the_thread;
+        exception_t * the_exception;
+);
+
+EHM_EXTERN_VTABLE(SomeThreadCancelled, std_thread_cancelled);
+
+EHM_FORALL_EXCEPTION(ThreadCancelled, (thread_t &), (thread_t)) (
         thread_t * the_thread;
         exception_t * the_exception;
@@ -79 +86 @@
 };
 
-forall( T & | is_thread(T) | IS_EXCEPTION(ThreadCancelled, (T)) )
-void ?{}( thread_dtor_guard_t & this, T & thrd, void(*)(ThreadCancelled(T) &) );
+forall( T & | is_thread(T) | IS_EXCEPTION(SomeThreadCancelled) )
+void ?{}( thread_dtor_guard_t & this, T & thrd, void(*)(SomeThreadCancelled &) );
 void ^?{}( thread_dtor_guard_t & this );
 
@@ -125 +132 @@
 //----------
 // join
-forall( T & | is_thread(T) | IS_RESUMPTION_EXCEPTION(ThreadCancelled, (T)) )
+forall( T & | is_thread(T) | IS_RESUMPTION_EXCEPTION(SomeThreadCancelled) )
 T & join( T & this );
 

libcfa/src/exception.c

@@ -10 +10 @@
 // Created On       : Mon Jun 26 15:13:00 2017
 // Last Modified By : Andrew Beach
-// Last Modified On : Tue Oct 27 16:27:00 2020
-// Update Count     : 35
+// Last Modified On : Wed Feb 24 13:40:00 2021
+// Update Count     : 36
 //
 
@@ -26 +26 @@
 #include "concurrency/invoke.h"
 #include "stdhdr/assert.h"
+#include "virtual.h"
 
 #if defined( __ARM_ARCH )
@@ -46 +47 @@
 const _Unwind_Exception_Class __cfaehm_exception_class = 0x4c50575500414643;
 
-// Base exception vtable is abstract, you should not have base exceptions.
-struct __cfaehm_base_exception_t_vtable
-                ___cfaehm_base_exception_t_vtable_instance = {
-        .parent = NULL,
-        .size = 0,
-        .copy = NULL,
-        .free = NULL,
-        .msg = NULL
+// Base Exception type id:
+struct __cfa__parent_vtable __cfatid_exception_t = {
+        NULL,
 };
 

libcfa/src/exception.h

@@ -10 +10 @@
 // Created On       : Mon Jun 26 15:11:00 2017
 // Last Modified By : Andrew Beach
-// Last Modified On : Tue Oct 27 14:45:00 2020
-// Update Count     : 11
+// Last Modified On : Thr Apr  8 15:20:00 2021
+// Update Count     : 12
 //
 
@@ -29 +29 @@
 struct __cfaehm_base_exception_t;
 typedef struct __cfaehm_base_exception_t exception_t;
+struct __cfa__parent_vtable;
 struct __cfaehm_base_exception_t_vtable {
-        const struct __cfaehm_base_exception_t_vtable * parent;
+        const struct __cfa__parent_vtable * __cfavir_typeid;
         size_t size;
         void (*copy)(struct __cfaehm_base_exception_t *this,
@@ -40 +41 @@
         struct __cfaehm_base_exception_t_vtable const * virtual_table;
 };
-extern struct __cfaehm_base_exception_t_vtable
-        ___cfaehm_base_exception_t_vtable_instance;
+extern struct __cfa__parent_vtable __cfatid_exception_t;
 
 
@@ -104 +104 @@
         /* The first field must be a pointer to a virtual table.
          * That virtual table must be a decendent of the base exception virtual table.
+         * The virtual table must point at the prober type-id.
+         * None of these can be enforced in an assertion.
          */
-        virtualT const & get_exception_vtable(exceptT *);
-        // Always returns the virtual table for this type (associated types hack).
 };
 

libcfa/src/exception.hfa

@@ -10 +10 @@
 // Created On       : Thu Apr  7 10:25:00 2020
 // Last Modified By : Andrew Beach
-// Last Modified On : Tue Aug  4 16:22:00 2020
-// Update Count     : 3
+// Last Modified On : Thr Apr  8 15:16:00 2021
+// Update Count     : 4
 //
 
@@ -18 +18 @@
 // -----------------------------------------------------------------------------------------------
 
-// TRIVIAL_EXCEPTION_DECLARATION(exception_name);
-// Declare a trivial exception, one that adds no fields or features.
-// This will make the exception visible and may go in a .hfa or .cfa file.
-#define TRIVIAL_EXCEPTION_DECLARATION(...) \
-        _EXC_DISPATCH(_TRIVIAL_EXCEPTION_DECLARATION, __VA_ARGS__)
+// EHM_EXCEPTION(exception_name)(fields...);
+// Create an exception (a virtual structure that inherits from exception_t)
+// with the given name and fields.
+#define EHM_EXCEPTION(exception_name) \
+        _EHM_TYPE_ID_STRUCT(exception_name, ); \
+        _EHM_TYPE_ID_VALUE(exception_name, ); \
+        _EHM_VIRTUAL_TABLE_STRUCT(exception_name, , ); \
+        _EHM_EXCEPTION_STRUCT(exception_name, , )
 
-// TRIVIAL_EXCEPTION_INSTANCE(exception_name);
-// Create the trival exception. This must be used exactly once and should be used in a .cfa file,
-// as it creates the unique instance of the virtual table.
-#define TRIVIAL_EXCEPTION_INSTANCE(...) _EXC_DISPATCH(_TRIVIAL_EXCEPTION_INSTANCE, __VA_ARGS__)
+// EHM_EXTERN_VTABLE(exception_name, table_name);
+// Forward declare a virtual table called table_name for exception_name type.
+#define EHM_EXTERN_VTABLE(exception_name, table_name) \
+        _EHM_EXTERN_VTABLE(exception_name, , table_name)
 
-// TRIVIAL_EXCEPTION(exception_name[, parent_name]);
-// Does both of the above, a short hand if the exception is only used in one .cfa file.
-// For legacy reasons this is the only one that official supports having a parent other than the
-// base exception. This feature may be removed or changed.
-#define TRIVIAL_EXCEPTION(...) \
-        _EXC_DISPATCH(_TRIVIAL_EXCEPTION_DECLARATION, __VA_ARGS__); \
-        _EXC_DISPATCH(_TRIVIAL_EXCEPTION_INSTANCE, __VA_ARGS__)
+// EHM_VIRTUAL_TABLE(exception_name, table_name);
+// Define a virtual table called table_name for exception_name type.
+#define EHM_VIRTUAL_TABLE(exception_name, table_name) \
+        _EHM_DEFINE_COPY(exception_name, ) \
+        _EHM_DEFINE_MSG(exception_name, ) \
+        _EHM_VIRTUAL_TABLE(exception_name, , table_name)
 
-// FORALL_TRIVIAL_EXCEPTION(exception_name, (assertions...), (parameters...));
-// Forward declare a polymorphic but otherwise trivial exception type. You must provide the entire
-// assertion list (exactly what would go in the forall clause) and parameters list (only the
-// parameter names from the assertion list, same order and comma seperated). This should be
-// visible where ever use the exception. This just generates the polymorphic framework, see
-// POLY_VTABLE_DECLARATION to allow instantiations.
-#define FORALL_TRIVIAL_EXCEPTION(exception_name, assertions, parameters) \
-        _FORALL_TRIVIAL_EXCEPTION(exception_name, __cfaehm_base_exception_t, assertions, parameters, )
+// EHM_FORALL_EXCEPTION(exception_name, (assertions), (parameters))(fields...);
+// As EHM_EXCEPTION but for polymorphic types instead of monomorphic ones.
+// The assertions list should include all polymorphic parameters and
+// assertions inside a parentisized list. Parameters should include all the
+// polymorphic parameter names inside a parentisized list (same order).
+#define EHM_FORALL_EXCEPTION(exception_name, assertions, parameters) \
+        _EHM_TYPE_ID_STRUCT(exception_name, forall assertions); \
+        _EHM_VIRTUAL_TABLE_STRUCT(exception_name, forall assertions, parameters); \
+        _EHM_EXCEPTION_STRUCT(exception_name, forall assertions, parameters)
 
-// FORALL_TRIVIAL_INSTANCE(exception_name, (assertions...), (parameters...))
-// Create the forall trivial exception. The assertion list and parameters must match.
-// There must be exactly one use of this in a program for each exception type. This just
-// generates the polymorphic framework, see POLY_VTABLE_INSTANCE to allow instantiations.
-#define FORALL_TRIVIAL_INSTANCE(exception_name, assertions, parameters) \
-        _FORALL_CTOR0_INSTANCE(exception_name, assertions, parameters)
+// EHM_FORALL_EXTERN_VTABLE(exception_name, (arguments), table_name);
+// As EHM_EXTERN_VTABLE but for polymorphic types instead of monomorphic ones.
+// Arguments should be the parentisized list of polymorphic arguments.
+#define EHM_FORALL_EXTERN_VTABLE(exception_name, arguments, table_name) \
+        _EHM_EXTERN_VTABLE(exception_name, arguments, table_name)
 
-// DATA_EXCEPTION(exception_name)(fields...);
-// Forward declare an exception that adds fields but no features. The added fields go in the
-// second argument list. The virtual table instance must be provided later (see VTABLE_INSTANCE).
-#define DATA_EXCEPTION(...) _EXC_DISPATCH(_DATA_EXCEPTION, __VA_ARGS__)
+// EHM_FORALL_VIRTUAL_TABLE(exception_name, (arguments), table_name);
+// As EHM_VIRTUAL_TABLE but for polymorphic types instead of monomorphic ones.
+// Arguments should be the parentisized list of polymorphic arguments.
+#define EHM_FORALL_VIRTUAL_TABLE(exception_name, arguments, table_name) \
+        _EHM_TYPE_ID_VALUE(exception_name, arguments); \
+        _EHM_DEFINE_COPY(exception_name, arguments) \
+        _EHM_DEFINE_MSG(exception_name, arguments) \
+        _EHM_VIRTUAL_TABLE(exception_name, arguments, table_name)
 
-// FORALL_DATA_EXCEPTION(exception_name, (assertions...), (parameters...))(fields...);
-// Define a polymorphic exception that adds fields but no additional features. The assertion list
-// and matching parameters must match. Then you can give the list of fields. This should be
-// visible where ever you use the exception. This just generates the polymorphic framework, see
-// POLY_VTABLE_DECLARATION to allow instantiations.
-#define FORALL_DATA_EXCEPTION(exception_name, assertions, parameters) \
-        _FORALL_DATA_EXCEPTION(exception_name, __cfaehm_base_exception_t, assertions, parameters, )
+#define EHM_TYPE_ID(exception_name) _EHM_TYPE_ID_TYPE(exception_name)
 
-// FORALL_DATA_INSTANCE(exception_name, (assertions...), (parameters...))
-// Create a polymorphic data exception. The assertion list and parameters must match. This should
-// appear once in each program. This just generates the polymorphic framework, see
-// POLY_VTABLE_INSTANCE to allow instantiations.
-#define FORALL_DATA_INSTANCE(exception_name, assertions, parameters) \
-        _FORALL_CTOR0_INSTANCE(exception_name, assertions, parameters)
-
-// VTABLE_DECLARATION(exception_name)([new_features...]);
-// Declare a virtual table type for an exception with exception_name. You may also add features
-// (fields on the virtual table) by including them in the second list.
-#define VTABLE_DECLARATION(...) _EXC_DISPATCH(_VTABLE_DECLARATION, __VA_ARGS__)
-
-// VTABLE_INSTANCE(exception_name)(msg [, others...]);
-// Create the instance of the virtual table. There must be exactly one instance of a virtual table
-// for each exception type. This fills in most of the fields of the virtual table (uses ?=? and
-// ^?{}) but you must provide the message function and any other fields added in the declaration.
-#define VTABLE_INSTANCE(...) _EXC_DISPATCH(_VTABLE_INSTANCE, __VA_ARGS__)
-
-// FORALL_VTABLE_DECLARATION(exception_name, (assertions...), (parameters...))([new_features...]);
-// Declare a polymorphic virtual table type for an exception with exception_name, the given
-// assertions and parameters. You may also add features (fields on the virtual table). This just
-// generates the polymorphic framework, see POLY_VTABLE_DECLARATION to allow instantiations.
-#define FORALL_VTABLE_DECLARATION(exception_name, assertions, parameters) \
-        _FORALL_VTABLE_DECLARATION(exception_name, __cfaehm_base_exception_t, assertions, parameters, )
-
-// POLY_VTABLE_DECLARATION(exception_name, types...);
-// Declares that an instantiation for this exception exists for the given types. This should be
-// visible anywhere you use the instantiation of the exception is used.
-#define POLY_VTABLE_DECLARATION(exception_name, ...) \
-        VTABLE_TYPE(exception_name)(__VA_ARGS__) const & get_exception_vtable(exception_name(__VA_ARGS__) *); \
-        extern VTABLE_TYPE(exception_name)(__VA_ARGS__) VTABLE_NAME(exception_name)
-
-// POLY_VTABLE_INSTANCE(exception_name, types...)(msg [, others...]);
-// Creates an instantiation for the given exception for the given types. This should occur only
-// once in the entire program. You must fill in all features, message and any others given in the
-// initial declaration.
-#define POLY_VTABLE_INSTANCE(exception_name, ...) \
-        _POLY_VTABLE_INSTANCE(exception_name, __cfaehm_base_exception_t, __VA_ARGS__)
-
-// VTABLE_TYPE(exception_name) | VTABLE_NAME(exception_name)
-// Get the name of the vtable type or the name of the vtable instance for an exception type.
-#define VTABLE_TYPE(exception_name) struct _GLUE2(exception_name,_vtable)
-#define VTABLE_NAME(exception_name) _GLUE3(_,exception_name,_vtable_instance)
-
-// VTABLE_FIELD(exception_name);
-// FORALL_VTABLE_FIELD(exception_name, (parameters-or-types));
-// The declaration of the virtual table field. Should be the first declaration in a virtual type.
-#define VTABLE_FIELD(exception_name) VTABLE_TYPE(exception_name) const * virtual_table
-#define FORALL_VTABLE_FIELD(exception_name, parameters) \
-        VTABLE_TYPE(exception_name) parameters const * virtual_table
-
-// VTABLE_INIT(object_reference, exception_name);
-// Sets a virtual table field on an object to the virtual table instance for the type.
-#define VTABLE_INIT(this, exception_name) (this).virtual_table = &VTABLE_NAME(exception_name)
-
-// VTABLE_ASSERTION(exception_name, (parameters...))
-// The assertion that there is an instantiation of the vtable for the exception and types.
-#define VTABLE_ASSERTION(exception_name, parameters) \
-        { VTABLE_TYPE(exception_name) parameters VTABLE_NAME(exception_name); }
+#define EHM_MATCH_ALL __cfa__parent_vtable
 
 // IS_EXCEPTION(exception_name [, (...parameters)])
@@ -135 +77 @@
 #define IS_TERMINATION_EXCEPTION(...) _IS_EXCEPTION(is_termination_exception, __VA_ARGS__, , ~)
 
-// All internal helper macros begin with an underscore.
-#define _CLOSE(...) __VA_ARGS__ }
-#define _GLUE2(left, right) left##right
-#define _GLUE3(left, middle, right) left##middle##right
-#define _EXC_DISPATCH(to, ...) to(__VA_ARGS__,__cfaehm_base_exception_t,)
-#define _UNPACK(...) __VA_ARGS__
+// Macros starting with a leading underscore are internal.
 
-#define _TRIVIAL_EXCEPTION_DECLARATION(exception_name, parent_name, ...) \
-        _VTABLE_DECLARATION(exception_name, parent_name)(); \
-        struct exception_name { \
-                VTABLE_FIELD(exception_name); \
-        }; \
-        void ?{}(exception_name & this); \
-        const char * _GLUE2(exception_name,_msg)(exception_name * this)
+// Create an exception type definition. must be tailing, can be polymorphic.
+#define _EHM_EXCEPTION_STRUCT(exception_name, forall_clause, parameters) \
+        forall_clause struct exception_name { \
+                _EHM_VTABLE_TYPE(exception_name) parameters const * virtual_table; \
+                _CLOSE
 
-#define _TRIVIAL_EXCEPTION_INSTANCE(exception_name, parent_name, ...) \
-        void ?{}(exception_name & this) { \
-                VTABLE_INIT(this, exception_name); \
-        } \
-        const char * _GLUE2(exception_name,_msg)(exception_name * this) { \
-                return #exception_name; \
-        } \
-        _VTABLE_INSTANCE(exception_name, parent_name,)(_GLUE2(exception_name,_msg))
+// Create a (possibly polymorphic) virtual table forward declaration.
+#define _EHM_EXTERN_VTABLE(exception_name, arguments, table_name) \
+        extern const _EHM_VTABLE_TYPE(exception_name) arguments table_name
 
-#define _FORALL_TRIVIAL_EXCEPTION(exception_name, parent_name, assertions, \
-                parameters, parent_parameters) \
-        _FORALL_VTABLE_DECLARATION(exception_name, parent_name, assertions, \
-                parameters, parent_parameters)(); \
-        forall assertions struct exception_name { \
-                FORALL_VTABLE_FIELD(exception_name, parameters); \
-        }; \
-        _FORALL_CTOR0_DECLARATION(exception_name, assertions, parameters)
-
-#define _FORALL_CTOR0_DECLARATION(exception_name, assertions, parameters) \
-        forall(_UNPACK assertions | \
-                is_exception(exception_name parameters, VTABLE_TYPE(exception_name) parameters)) \
-        void ?{}(exception_name parameters & this)
-
-#define _FORALL_CTOR0_INSTANCE(exception_name, assertions, parameters) \
-        _FORALL_CTOR0_DECLARATION(exception_name, assertions, parameters) { \
-                (this).virtual_table = &get_exception_vtable(&this); \
+// Create a (possibly polymorphic) virtual table definition.
+#define _EHM_VIRTUAL_TABLE(exception_type, arguments, table_name) \
+        const _EHM_VTABLE_TYPE(exception_type) arguments table_name @= { \
+                .__cfavir_typeid : &_EHM_TYPE_ID_NAME(exception_type), \
+                .size : sizeof(struct exception_type arguments), \
+                .copy : copy, \
+                .^?{} : ^?{}, \
+                .msg : msg, \
         }
 
-#define _DATA_EXCEPTION(exception_name, parent_name, ...) \
-        _VTABLE_DECLARATION(exception_name, parent_name)(); \
-        struct exception_name { \
-                VTABLE_FIELD(exception_name); \
-                _CLOSE
+// Create a (possibly polymorphic) copy function from an assignment operator.
+#define _EHM_DEFINE_FORALL_COPY(exception_name, forall_clause, parameters) \
+        forall_clause void copy(exception_name parameters * this, \
+                        exception_name parameters * that) { \
+                *this = *that; \
+        }
 
-#define _FORALL_DATA_EXCEPTION(exception_name, parent_name, \
-                assertions, parameters, parent_parameters) \
-        _FORALL_VTABLE_DECLARATION(exception_name, parent_name, \
-                assertions, parameters, parent_parameters)(); \
-        _FORALL_CTOR0_DECLARATION(exception_name, assertions, parameters); \
-        forall assertions struct exception_name { \
-                FORALL_VTABLE_FIELD(exception_name, parameters); \
-                _CLOSE
+#define _EHM_DEFINE_COPY(exception_name, arguments) \
+        void copy(exception_name arguments * this, exception_name arguments * that) { \
+                *this = *that; \
+        }
 
-#define _VTABLE_DECLARATION(exception_name, parent_name, ...) \
-        struct exception_name; \
-        VTABLE_TYPE(exception_name); \
-        VTABLE_TYPE(exception_name) const & get_exception_vtable(exception_name *); \
-        extern VTABLE_TYPE(exception_name) VTABLE_NAME(exception_name); \
-        VTABLE_TYPE(exception_name) { \
-                VTABLE_TYPE(parent_name) const * parent; \
-                size_t size; \
-                void (*copy)(exception_name * this, exception_name * other); \
-                void (*^?{})(exception_name & this); \
-                const char * (*msg)(exception_name * this); \
-                _CLOSE
+// Create a (possibly polymorphic) msg function
+#define _EHM_DEFINE_FORALL_MSG(exception_name, forall_clause, parameters) \
+        forall_clause const char * msg(exception_name parameters * this) { \
+                return #exception_name #parameters; \
+        }
 
-#define _VTABLE_INSTANCE(exception_name, parent_name, ...) \
-        VTABLE_TYPE(exception_name) const & get_exception_vtable(exception_name *) { \
-                return VTABLE_NAME(exception_name); \
-        } \
-        void _GLUE2(exception_name,_copy)(exception_name * this, exception_name * other) { \
-                *this = *other; \
-        } \
-        VTABLE_TYPE(exception_name) VTABLE_NAME(exception_name) @= { \
-                &VTABLE_NAME(parent_name), sizeof(exception_name), \
-                _GLUE2(exception_name,_copy), ^?{}, \
-                _CLOSE
+#define _EHM_DEFINE_MSG(exception_name, arguments) \
+        const char * msg(exception_name arguments * this) { \
+                return #exception_name #arguments; \
+        }
 
-#define _FORALL_VTABLE_DECLARATION(exception_name, parent_name, assertions, \
-                parameters, parent_parameters) \
-        forall assertions struct exception_name; \
-        forall assertions VTABLE_TYPE(exception_name) { \
-                VTABLE_TYPE(parent_name) parent_parameters const * parent; \
+// Produces the C compatable name of the virtual table type for a virtual type.
+#define _EHM_VTABLE_TYPE(type_name) struct _GLUE2(type_name,_vtable)
+
+// Create the vtable type for exception name.
+#define _EHM_VIRTUAL_TABLE_STRUCT(exception_name, forall_clause, parameters) \
+        forall_clause struct exception_name; \
+        forall_clause _EHM_VTABLE_TYPE(exception_name) { \
+                _EHM_TYPE_ID_TYPE(exception_name) parameters const * __cfavir_typeid; \
                 size_t size; \
                 void (*copy)(exception_name parameters * this, exception_name parameters * other); \
                 void (*^?{})(exception_name parameters & this); \
                 const char * (*msg)(exception_name parameters * this); \
-                _CLOSE
+        }
 
-#define _POLY_VTABLE_INSTANCE(exception_name, parent_name, ...) \
-        extern VTABLE_TYPE(exception_name)(__VA_ARGS__) VTABLE_NAME(exception_name); \
-        VTABLE_TYPE(exception_name)(__VA_ARGS__) const & get_exception_vtable( \
-                        exception_name(__VA_ARGS__) *) { \
-                return VTABLE_NAME(exception_name); \
-        } \
-        void _GLUE2(exception_name,_copy)( \
-                        exception_name(__VA_ARGS__) * this, exception_name(__VA_ARGS__) * other) { \
-                *this = *other; \
-        } \
-        VTABLE_TYPE(exception_name)(__VA_ARGS__) VTABLE_NAME(exception_name) @= { \
-                &VTABLE_NAME(parent_name), sizeof(exception_name(__VA_ARGS__)), \
-                _GLUE2(exception_name,_copy), ^?{}, \
-                _CLOSE
+// Define the function required to satify the trait for exceptions.
+#define _EHM_TRAIT_FUNCTION(exception_name, forall_clause, parameters) \
+        forall_clause inline void mark_exception( \
+                exception_name parameters const &, \
+                _EHM_VTABLE_TYPE(exception_name) parameters const &) {} \
+
+#define _EHM_TRAIT_FUNCTION2(exception_name, forall_clause, parameters) \
+        forall_clause _EHM_VTABLE_TYPE(exception_name) parameters const & \
+                        get_exception_vtable(exception_name parameters const & this)
+
+#define __EHM_TRAIT_FUNCTION(exception_name, forall_clause, parameters) \
+        forall_clause inline _EHM_VTABLE_TYPE(exception_name) parameters const & \
+                        get_exception_vtable(exception_name parameters const & this) { \
+                /* This comes before the structure definition, but we know the offset. */ \
+                /* return (_EHM_VTABLE_TYPE(exception_name) parameters const &)this; */ \
+                assert(false); \
+        }
+
+// Generates a new type-id structure. This is used to mangle the name of the
+// type-id instance so it also includes polymorphic information. Must be the
+// direct decendent of exception_t.
+// The second field is used to recover type information about the exception.
+#define _EHM_TYPE_ID_STRUCT(exception_name, forall_clause) \
+        forall_clause _EHM_TYPE_ID_TYPE(exception_name) { \
+                __cfa__parent_vtable const * parent; \
+        }
+
+// Generate a new type-id value.
+#define _EHM_TYPE_ID_VALUE(exception_name, arguments) \
+        __attribute__(( section(".gnu.linkonce." "__cfatid_" #exception_name) )) \
+        _EHM_TYPE_ID_TYPE(exception_name) arguments const \
+                        _EHM_TYPE_ID_NAME(exception_name) = { \
+                &__cfatid_exception_t, \
+        }
+
+// _EHM_TYPE_ID_STRUCT and _EHM_TYPE_ID_VALUE are the two that would need to
+// be updated to extend the hierarchy if we are still using macros when that
+// is added.
+
+// Produce the C compatable name of the type-id type for an exception type.
+#define _EHM_TYPE_ID_TYPE(exception_name) \
+        struct _GLUE2(__cfatid_struct_, exception_name)
+
+// Produce the name of the instance of the type-id for an exception type.
+#define _EHM_TYPE_ID_NAME(exception_name) _GLUE2(__cfatid_,exception_name)
 
 #define _IS_EXCEPTION(kind, exception_name, parameters, ...) \
-        kind(exception_name parameters, VTABLE_TYPE(exception_name) parameters)
+        kind(exception_name parameters, _EHM_VTABLE_TYPE(exception_name) parameters)
+
+// Internal helper macros:
+#define _CLOSE(...) __VA_ARGS__ }
+#define _GLUE2(left, right) left##right

libcfa/src/fstream.cfa

@@ -321 +321 @@
 
 
+EHM_VIRTUAL_TABLE(Open_Failure, Open_Failure_main_table);
 void ?{}( Open_Failure & this, ofstream & ostream ) {
-        VTABLE_INIT(this, Open_Failure);
+        this.virtual_table = &Open_Failure_main_table;
         this.ostream = &ostream;
         this.tag = 1;
 }
 void ?{}( Open_Failure & this, ifstream & istream ) {
-        VTABLE_INIT(this, Open_Failure);
+        this.virtual_table = &Open_Failure_main_table;
         this.istream = &istream;
         this.tag = 0;
 }
-const char * Open_Failure_msg(Open_Failure * this) {
-        return "Open_Failure";
-}
-VTABLE_INSTANCE(Open_Failure)(Open_Failure_msg);
 void throwOpen_Failure( ofstream & ostream ) {
         Open_Failure exc = { ostream };

libcfa/src/fstream.hfa

@@ -133 +133 @@
 
 
-DATA_EXCEPTION(Open_Failure)(
+EHM_EXCEPTION(Open_Failure)(
         union {
                 ofstream * ostream;

libcfa/src/iostream.cfa

@@ -10 +10 @@
 // Created On       : Wed May 27 17:56:53 2015
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Tue Mar  2 14:51:30 2021
-// Update Count     : 1151
+// Last Modified On : Tue Apr 13 13:05:24 2021
+// Update Count     : 1324
 //
 
@@ -195 +195 @@
                         int len = snprintf( buf, size, format, ##__VA_ARGS__, val ); \
                         fmt( os, "%s", buf ); \
-                        if ( isfinite( val ) ) {                                        /* if number, always print decimal point */ \
+                        if ( isfinite( val ) ) { /* if number, print decimal point when no fraction or exponent */ \
                                 for ( int i = 0;; i += 1 ) { \
                                         if ( i == len ) { fmt( os, "." ); break; } \
-                                        if ( buf[i] == '.' ) break; \
+                                        if ( buf[i] == '.' || buf[i] == 'e' || buf[i] == 'E' ) break; /* decimal point or scientific ? */ \
                                 } /* for */ \
                         } /* if */ \
@@ -525 +525 @@
 } // distribution
 
-IntegralFMTImpl( signed char, "%    *hh ", "%    *.*hh " )
-IntegralFMTImpl( unsigned char, "%    *hh ", "%    *.*hh " )
-IntegralFMTImpl( signed short int, "%    *h ", "%    *.*h " )
-IntegralFMTImpl( unsigned short int, "%    *h ", "%    *.*h " )
-IntegralFMTImpl( signed int, "%    * ", "%    *.* " )
-IntegralFMTImpl( unsigned int, "%    * ", "%    *.* " )
-IntegralFMTImpl( signed long int, "%    *l ", "%    *.*l " )
-IntegralFMTImpl( unsigned long int, "%    *l ", "%    *.*l " )
-IntegralFMTImpl( signed long long int, "%    *ll ", "%    *.*ll " )
-IntegralFMTImpl( unsigned long long int, "%    *ll ", "%    *.*ll " )
-
-#if 0
-#if defined( __SIZEOF_INT128__ )
-// Default prefix for non-decimal prints is 0b, 0, 0x.
-#define IntegralFMTImpl128( T, SIGNED, CODE, IFMTNP, IFMTP ) \
-forall( ostype & | ostream( ostype ) ) \
-static void base10_128( ostype & os, _Ostream_Manip(T) f ) { \
-        if ( f.val > UINT64_MAX ) { \
-                unsigned long long int lsig = f.val % P10_UINT64; \
-                f.val /= P10_UINT64; /* msig */ \
-                base10_128( os, f ); /* recursion */ \
-                _Ostream_Manip(unsigned long long int) fmt @= { lsig, 0, 19, 'u', { .all : 0 } }; \
-                fmt.flags.nobsdp = true; \
-                /* printf( "fmt1 %c %lld %d\n", fmt.base, fmt.val, fmt.all ); */ \
-                sepOff( os ); \
-                (ostype &)(os | fmt); \
-        } else { \
-                /* printf( "fmt2 %c %lld %d\n", f.base, (unsigned long long int)f.val, f.all ); */ \
-                _Ostream_Manip(SIGNED long long int) fmt @= { (SIGNED long long int)f.val, f.wd, f.pc, f.base, { .all : f.all } }; \
-                (ostype &)(os | fmt); \
-        } /* if */ \
-} /* base10_128 */ \
-forall( ostype & | ostream( ostype ) ) { \
-        ostype & ?|?( ostype & os, _Ostream_Manip(T) f ) { \
-                if ( $sepPrt( os ) ) fmt( os, "%s", $sepGetCur( os ) ); \
-\
-                if ( f.base == 'b' | f.base == 'B' | f.base == 'o' | f.base == 'x' | f.base == 'X' ) { \
-                        unsigned long long int msig = (unsigned long long int)(f.val >> 64); \
-                        unsigned long long int lsig = (unsigned long long int)(f.val); \
-                        _Ostream_Manip(SIGNED long long int) fmt @= { msig, f.wd, f.pc, f.base, { .all : f.all } }; \
-                        _Ostream_Manip(unsigned long long int) fmt2 @= { lsig, 0, 0, f.base, { .all : 0 } }; \
-                        if ( msig == 0 ) { \
-                                fmt.val = lsig; \
-                                (ostype &)(os | fmt); \
-                        } else { \
-                                fmt2.flags.pad0 = fmt2.flags.nobsdp = true;     \
-                                if ( f.base == 'b' | f.base == 'B' ) { \
-                                        if ( fmt.flags.pc && fmt.pc > 64 ) fmt.pc -= 64; else { fmt.flags.pc = false; fmt.pc = 0; } \
-                                        if ( fmt.flags.left ) { \
-                                                fmt.flags.left = false; \
-                                                fmt.wd = 0; \
-                                                /* printf( "L %llo %llo %llo %d %d '%c' %x\n", msig, lsig, fmt.val, fmt.wd, fmt.pc, fmt.base, fmt.all ); */ \
-                                                fmt2.flags.left = true; \
-                                                int msigd = high1( msig ); \
-                                                fmt2.wd = f.wd - (fmt.pc > msigd ? fmt.pc : msigd); \
-                                                if ( ! fmt.flags.nobsdp ) fmt2.wd -= 2; /* compensate for 0b base specifier */ \
-                                                if ( (int)fmt2.wd < 64 ) fmt2.wd = 64; /* cast deals with negative value */ \
-                                                fmt2.flags.pc = true; fmt2.pc = 64; \
-                                        } else { \
-                                                if ( fmt.wd > 64 ) fmt.wd -= 64; \
-                                                else fmt.wd = 1; \
-                                                /* printf( "R %llo %llo %llo %d %d '%c' %x\n", msig, lsig, fmt.val, fmt.wd, fmt.pc, fmt.base, fmt.all ); */ \
-                                                fmt2.wd = 64; \
-                                        } /* if */ \
-                                        /* printf( "C %llo %d %d '%c' %x\n", fmt2.val, fmt2.wd, fmt2.pc, fmt2.base, fmt2.all ); */ \
-                                        (ostype &)(os | fmt | "" | fmt2); \
-                                } else if ( f.base == 'o' ) { \
-                                        if ( fmt.flags.pc && fmt.pc > 22 ) fmt.pc -= 22; else { fmt.flags.pc = false; fmt.pc = 0; } \
-                                        fmt.val = (unsigned long long int)fmt.val >> 2; \
-                                        fmt2.val = ((msig & 0x3) << 1) + ((lsig & 0x8000000000000000U) != 0); \
-                                        if ( fmt.flags.left ) { \
-                                                fmt.flags.left = false; \
-                                                fmt.wd = 0; \
-                                                /* printf( "L %llo %llo %llo %d %d '%c' %x %llo %d %d '%c' %x\n", msig, lsig, fmt.val, fmt.wd, fmt.pc, fmt.base, fmt.all, fmt2.val, fmt2.wd, fmt2.pc, fmt2.base, fmt2.all ); */ \
-                                                (ostype &)(os | fmt | "" | fmt2); \
-                                                sepOff( os ); \
-                                                fmt2.flags.left = true; \
-                                                int msigd = ceiling_div( high1( fmt.val ), 3 ); \
-                                                fmt2.wd = f.wd - (fmt.pc > msigd ? fmt.pc : msigd); \
-                                                if ( ! fmt.flags.nobsdp ) fmt2.wd -= 1; /* compensate for 0 base specifier */ \
-                                                if ( (int)fmt2.wd < 21 ) fmt2.wd = 21; /* cast deals with negative value */ \
-                                                fmt2.flags.pc = true; fmt2.pc = 21; \
-                                        } else { \
-                                                if ( fmt.wd > 22 ) fmt.wd -= 22; \
-                                                else fmt.wd = 1; \
-                                                /* printf( "R %llo %llo %llo %d %d '%c' %x %llo %d %d '%c' %x\n", msig, lsig, fmt.val, fmt.wd, fmt.pc, fmt.base, fmt.all, fmt2.val, fmt2.wd, fmt2.pc, fmt2.base, fmt2.all ); */ \
-                                                (ostype &)(os | fmt | "" | fmt2); \
-                                                sepOff( os ); \
-                                                fmt2.wd = 21; \
-                                        } /* if */ \
-                                        fmt2.val = lsig & 0x7fffffffffffffffU; \
-                                        /* printf( "\nC %llo %d %d '%c' %x\n", fmt2.val, fmt2.wd, fmt2.pc, fmt2.base, fmt2.all ); */ \
-                                        (ostype &)(os | fmt2); \
-                                } else { /* f.base == 'x'  | f.base == 'X' */ \
-                                        if ( fmt.flags.pc && fmt.pc > 16 ) fmt.pc -= 16; else { fmt.flags.pc = false; fmt.pc = 0; } \
-                                        if ( fmt.flags.left ) { \
-                                                fmt.flags.left = false; \
-                                                fmt.wd = 0; \
-                                                /* printf( "L %llo %llo %llo %d %d '%c' %x\n", msig, lsig, fmt.val, fmt.wd, fmt.pc, fmt.base, fmt.all ); */ \
-                                                fmt2.flags.left = true; \
-                                                int msigd = high1( msig ); \
-                                                fmt2.wd = f.wd - (fmt.pc > msigd ? fmt.pc : msigd); \
-                                                if ( ! fmt.flags.nobsdp ) fmt2.wd -= 2; /* compensate for 0x base specifier */ \
-                                                if ( (int)fmt2.wd < 16 ) fmt2.wd = 16; /* cast deals with negative value */ \
-                                                fmt2.flags.pc = true; fmt2.pc = 16; \
-                                        } else { \
-                                                if ( fmt.wd > 16 ) fmt.wd -= 16; \
-                                                else fmt.wd = 1; \
-                                                /* printf( "R %llo %llo %llo %d %d '%c' %x\n", msig, lsig, fmt.val, fmt.wd, fmt.pc, fmt.base, fmt.all ); */ \
-                                                fmt2.wd = 16; \
-                                        } /* if */ \
-                                        /* printf( "C %llo %d %d '%c' %x\n", fmt2.val, fmt2.wd, fmt2.pc, fmt2.base, fmt2.all ); */ \
-                                        (ostype &)(os | fmt | "" | fmt2); \
-                                } /* if */ \
-                        } /* if */ \
-                } else { \
-                        if ( CODE == 'd' ) { \
-                                if ( f.val < 0 )  { fmt( os, "-" ); sepOff( os ); f.val = -f.val; f.flags.sign = false; } \
-                        } /* if */ \
-                        base10_128( os, f ); \
-                } /* if */ \
-                return os; \
-        } /* ?|? */ \
-        void ?|?( ostype & os, _Ostream_Manip(T) f ) { (ostype &)(os | f); ends( os ); } \
-} // distribution
-
-IntegralFMTImpl128( int128, signed, 'd', "%    *ll ", "%    *.*ll " )
-IntegralFMTImpl128( unsigned int128, unsigned, 'u', "%    *ll ", "%    *.*ll " )
-#endif // __SIZEOF_INT128__
-#endif // 0
-
-#if 1
+IntegralFMTImpl( signed char, "     *hh ", "     *.*hh " )
+IntegralFMTImpl( unsigned char, "     *hh ", "     *.*hh " )
+IntegralFMTImpl( signed short int, "     *h ", "     *.*h " )
+IntegralFMTImpl( unsigned short int, "     *h ", "     *.*h " )
+IntegralFMTImpl( signed int, "     * ", "     *.* " )
+IntegralFMTImpl( unsigned int, "     * ", "     *.* " )
+IntegralFMTImpl( signed long int, "     *l ", "     *.*l " )
+IntegralFMTImpl( unsigned long int, "     *l ", "     *.*l " )
+IntegralFMTImpl( signed long long int, "     *ll ", "     *.*ll " )
+IntegralFMTImpl( unsigned long long int, "     *ll ", "     *.*ll " )
+
+
 #if defined( __SIZEOF_INT128__ )
 // Default prefix for non-decimal prints is 0b, 0, 0x.
@@ -746 +626 @@
 IntegralFMTImpl128( unsigned int128 )
 #endif // __SIZEOF_INT128__
-#endif // 0
 
 // *********************************** floating point ***********************************
 
-#define PrintWithDP2( os, format, val, ... ) \
+static const char *suffixes[] = {
+        "y", "z", "a", "f", "p", "n", "u", "m", "",
+        "K", "M", "G", "T", "P", "E", "Z", "Y"
+}; 
+#define SUFFIXES_START (-24) /* Smallest power for which there is a suffix defined. */
+#define SUFFIXES_END (SUFFIXES_START + (int)((sizeof(suffixes) / sizeof(char *) - 1) * 3))
+
+#define PrintWithDP2( os, format, ... ) \
         { \
-                enum { size = 48 }; \
-                char buf[size]; \
-                int bufbeg = 0, i, len = snprintf( buf, size, format, ##__VA_ARGS__, val ); \
-                if ( isfinite( val ) && (f.base != 'g' || f.pc != 0) ) { /* if number, print decimal point */ \
-                        for ( i = 0; i < len && buf[i] != '.' && buf[i] != 'e' && buf[i] != 'E'; i += 1 ); /* decimal point or scientific ? */ \
-                        if ( i == len && ! f.flags.nobsdp ) { \
-                                if ( ! f.flags.left ) { \
-                                        buf[i] = '.'; buf[i + 1] = '\0'; \
-                                        if ( buf[0] == ' ' ) bufbeg = 1;        /* decimal point within width */ \
-                                } else { \
-                                        for ( i = 0; i < len && buf[i] != ' '; i += 1 ); /* trailing blank ? */ \
-                                        buf[i] = '.'; \
-                                        if ( i == len ) buf[i + 1] = '\0'; \
+                if ( ! f.flags.eng ) { \
+                        len = snprintf( buf, size, format, ##__VA_ARGS__ ); \
+                        if ( isfinite( f.val ) && ( f.pc != 0 || ! f.flags.nobsdp ) ) { /* if number, print decimal point when no fraction or exponent */ \
+                                for ( i = 0; i < len && buf[i] != '.' && buf[i] != 'e' && buf[i] != 'E'; i += 1 ); /* decimal point or scientific ? */ \
+                                if ( i == len ) { \
+                                        if ( ! f.flags.left ) { \
+                                                buf[i] = '.'; buf[i + 1] = '\0'; \
+                                                if ( buf[0] == ' ' ) bufbeg = 1; /* decimal point within width */ \
+                                        } else { \
+                                                for ( i = 0; i < len && buf[i] != ' '; i += 1 ); /* trailing blank ? */ \
+                                                buf[i] = '.'; \
+                                                if ( i == len ) buf[i + 1] = '\0'; \
+                                        } /* if */ \
                                 } /* if */ \
                         } /* if */ \
+                } else { \
+                        int exp10, len2; \
+                        eng( f.val, f.pc, exp10 );                                      /* changes arguments */ \
+                        if ( ! f.flags.left && f.wd > 1 ) { \
+                                /* Exponent size (number of digits, 'e', optional minus sign) */ \
+                                f.wd -= lrint( floor( log10( abs( exp10 ) ) ) ) + 1 + 1 + (exp10 < 0 ? 1 : 0); \
+                                if ( f.wd < 1 ) f.wd = 1; \
+                        } /* if */ \
+                        len = snprintf( buf, size, format, ##__VA_ARGS__ ); \
+                        if ( f.flags.left ) { \
+                                for ( len -= 1; len > 0 && buf[len] == ' '; len -= 1 ); \
+                                len += 1; \
+                        } /* if */ \
+                        if ( ! f.flags.nobsdp || (exp10 < SUFFIXES_START) || (exp10 > SUFFIXES_END) ) { \
+                                len2 = snprintf( &buf[len], size - len, "e%d", exp10 ); \
+                        } else { \
+                                len2 = snprintf( &buf[len], size - len, "%s", suffixes[(exp10 - SUFFIXES_START) / 3] ); \
+                        } /* if */ \
+                        if ( f.flags.left && len + len2 < f.wd ) buf[len + len2] = ' '; \
                 } /* if */ \
                 fmt( os, "%s", &buf[bufbeg] ); \
@@ -773 +678 @@
 #define FloatingPointFMTImpl( T, DFMTNP, DFMTP ) \
 forall( ostype & | ostream( ostype ) ) { \
+        static void eng( T &value, int & pc, int & exp10 ) { \
+                exp10 = lrint( floor( log10( abs( value ) ) ) ); /* round to desired precision */ \
+                if ( exp10 < 0 ) exp10 -= 2; \
+                exp10 = floor( exp10, 3 ); \
+                value *= pow( 10.0, -exp10 ); \
+                if ( pc <= 3 ) pc = 3; \
+        } /* eng */ \
+\
         ostype & ?|?( ostype & os, _Ostream_Manip(T) f ) { \
+                enum { size = 48 }; \
+                char buf[size]; \
+                int bufbeg = 0, i, len; \
+\
                 if ( $sepPrt( os ) ) fmt( os, "%s", $sepGetCur( os ) ); \
-                char fmtstr[sizeof(DFMTP)];                                             /* sizeof includes '\0' */ \
+                char fmtstr[sizeof(DFMTP) + 8];                                 /* sizeof includes '\0' */ \
                 if ( ! f.flags.pc ) memcpy( &fmtstr, DFMTNP, sizeof(DFMTNP) ); \
                 else memcpy( &fmtstr, DFMTP, sizeof(DFMTP) ); \
@@ -789 +706 @@
                         fmtstr[sizeof(DFMTNP)-2] = f.base;                      /* sizeof includes '\0' */ \
                         /* printf( "%g %d %s\n", f.val, f.wd, &fmtstr[star]); */ \
-                        PrintWithDP2( os, &fmtstr[star], f.val, f.wd ) \
+                        PrintWithDP2( os, &fmtstr[star], f.wd, f.val ) \
                 } else {                                                                                /* precision */ \
                         fmtstr[sizeof(DFMTP)-2] = f.base;                       /* sizeof includes '\0' */ \
                         /* printf( "%g %d %d %s\n", f.val, f.wd, f.pc, &fmtstr[star] ); */ \
-                        PrintWithDP2( os, &fmtstr[star], f.val, f.wd, f.pc ) \
+                        PrintWithDP2( os, &fmtstr[star], f.wd, f.pc, f.val ) \
                 } /* if */ \
                 return os; \
@@ -801 +718 @@
 } // distribution
 
-FloatingPointFMTImpl( double, "%    * ", "%    *.* " )
-FloatingPointFMTImpl( long double, "%    *L ", "%    *.*L " )
+FloatingPointFMTImpl( double, "     * ", "     *.* " )
+FloatingPointFMTImpl( long double, "     *L ", "     *.*L " )
 
 // *********************************** character ***********************************

libcfa/src/iostream.hfa

@@ -10 +10 @@
 // Created On       : Wed May 27 17:56:53 2015
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Tue Mar  2 14:05:08 2021
-// Update Count     : 369
+// Last Modified On : Tue Apr 13 13:05:11 2021
+// Update Count     : 384
 //
 
@@ -158 +158 @@
 struct _Ostream_Manip {
         T val;                                                                                          // polymorphic base-type
-        unsigned int wd, pc;                                                            // width, precision
+        int wd, pc;                                                                                     // width, precision
         char base;                                                                                      // numeric base / floating-point style
         union {
                 unsigned char all;
                 struct {
+                        unsigned char eng:1;                                            // engineering notation
                         unsigned char neg:1;                                            // val is negative
                         unsigned char pc:1;                                                     // precision specified
@@ -222 +223 @@
         _Ostream_Manip(T) hex( T val ) { return (_Ostream_Manip(T))@{ val, 1, 0, 'a', { .all : 0 } }; } \
         _Ostream_Manip(T) sci( T val ) { return (_Ostream_Manip(T))@{ val, 1, 0, 'e', { .all : 0 } }; } \
-        _Ostream_Manip(T) wd( unsigned int w, T val ) { return (_Ostream_Manip(T))@{ val, w, 0, 'f', { .all : 0 } }; } \
+        _Ostream_Manip(T) eng( T val ) { return (_Ostream_Manip(T))@{ val, 1, 0, 'g', { .flags.eng : true } }; } \
+        _Ostream_Manip(T) wd( unsigned int w, T val ) { return (_Ostream_Manip(T))@{ val, w, 0, 'g', { .all : 0 } }; } \
         _Ostream_Manip(T) wd( unsigned int w, unsigned char pc, T val ) { return (_Ostream_Manip(T))@{ val, w, pc, 'f', { .flags.pc : true } }; } \
         _Ostream_Manip(T) ws( unsigned int w, unsigned char pc, T val ) { return (_Ostream_Manip(T))@{ val, w, pc, 'g', { .flags.pc : true } }; } \
-        _Ostream_Manip(T) & wd( unsigned int w, _Ostream_Manip(T) & fmt ) { fmt.wd = w; return fmt; } \
-        _Ostream_Manip(T) & wd( unsigned int w, unsigned char pc, _Ostream_Manip(T) & fmt ) { fmt.wd = w; fmt.pc = pc; fmt.flags.pc = true; return fmt; } \
+        _Ostream_Manip(T) & wd( unsigned int w, _Ostream_Manip(T) & fmt ) { if ( fmt.flags.eng ) fmt.base = 'f'; fmt.wd = w; return fmt; } \
+        _Ostream_Manip(T) & wd( unsigned int w, unsigned char pc, _Ostream_Manip(T) & fmt ) { if ( fmt.flags.eng ) fmt.base = 'f'; fmt.wd = w; fmt.pc = pc; fmt.flags.pc = true; return fmt; } \
+        _Ostream_Manip(T) & ws( unsigned int w, unsigned char pc, _Ostream_Manip(T) & fmt ) { fmt.wd = w; fmt.pc = pc; fmt.flags.pc = true; return fmt; } \
         _Ostream_Manip(T) & left( _Ostream_Manip(T) & fmt ) { fmt.flags.left = true; return fmt; } \
         _Ostream_Manip(T) upcase( T val ) { return (_Ostream_Manip(T))@{ val, 1, 0, 'G', { .all : 0 } }; } \
@@ -235 +238 @@
         _Ostream_Manip(T) nodp( T val ) { return (_Ostream_Manip(T))@{ val, 1, 0, 'g', { .flags.nobsdp : true } }; } \
         _Ostream_Manip(T) & nodp( _Ostream_Manip(T) & fmt ) { fmt.flags.nobsdp = true; return fmt; } \
+        _Ostream_Manip(T) unit( T val ) { return (_Ostream_Manip(T))@{ val, 1, 0, 'g', { .flags.nobsdp : true } }; } \
+        _Ostream_Manip(T) & unit( _Ostream_Manip(T) & fmt ) { fmt.flags.nobsdp = true; return fmt; } \
 } /* distribution */ \
 forall( ostype & | ostream( ostype ) ) { \

libcfa/src/math.hfa

@@ -5 +5 @@
 // file "LICENCE" distributed with Cforall.
 //
-// math --
+// math.hfa --
 //
 // Author           : Peter A. Buhr
 // Created On       : Mon Apr 18 23:37:04 2016
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Mon Aug 24 08:56:20 2020
-// Update Count     : 126
+// Last Modified On : Thu Apr 15 11:47:56 2021
+// Update Count     : 132
 //
 
@@ -100 +100 @@
         long double _Complex log( long double _Complex x ) { return clogl( x ); }
 
+        // O(1) polymorphic integer log2, using clz, which returns the number of leading 0-bits, starting at the most
+        // significant bit (single instruction on x86)
+        int log2( unsigned int n ) { return n == 0 ? -1 : sizeof(n) * __CHAR_BIT__ - 1 - __builtin_clz( n ); }
+        long int log2( unsigned long int n ) { return n == 0 ? -1 : sizeof(n) * __CHAR_BIT__ - 1 - __builtin_clzl( n ); }
+        long long int log2( unsigned long long int n ) { return n == 0 ? -1 : sizeof(n) * __CHAR_BIT__ - 1 - __builtin_clzll( n ); }
         float log2( float x ) { return log2f( x ); }
         // extern "C" { double log2( double ); }

libcfa/src/time.hfa

@@ -10 +10 @@
 // Created On       : Wed Mar 14 23:18:57 2018
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Wed Jun 17 16:13:00 2020
-// Update Count     : 663
+// Last Modified On : Wed Apr 14 09:30:30 2021
+// Update Count     : 664
 //
 
@@ -29 +29 @@
 static inline {
         Duration ?=?( Duration & dur, __attribute__((unused)) zero_t ) { return dur{ 0 }; }
+
+        void ?{}( Duration & dur, timeval t ) with( dur ) { tn = (int64_t)t.tv_sec * TIMEGRAN + t.tv_usec * 1000; }
+        Duration ?=?( Duration & dur, timeval t ) with( dur ) {
+                tn = (int64_t)t.tv_sec * TIMEGRAN + t.tv_usec * (TIMEGRAN / 1_000_000LL);
+                return dur;
+        } // ?=?
+
+        void ?{}( Duration & dur, timespec t ) with( dur ) { tn = (int64_t)t.tv_sec * TIMEGRAN + t.tv_nsec; }
+        Duration ?=?( Duration & dur, timespec t ) with( dur ) {
+                tn = (int64_t)t.tv_sec * TIMEGRAN + t.tv_nsec;
+                return dur;
+        } // ?=?
 
         Duration +?( Duration rhs ) with( rhs ) { return (Duration)@{ +tn }; }

libcfa/src/virtual.c

@@ -15 +15 @@
 
 #include "virtual.h"
+#include "assert.h"
 
 int __cfa__is_parent( struct __cfa__parent_vtable const * parent,
         struct __cfa__parent_vtable const * child ) {
+        assert( child );
         do {
                 if ( parent == child )
@@ -28 +30 @@
 void * __cfa__virtual_cast( struct __cfa__parent_vtable const * parent,
         struct __cfa__parent_vtable const * const * child ) {
+        assert( child );
         return (__cfa__is_parent(parent, *child)) ? (void *)child : (void *)0;
 }