Changeset 8d66610 for libcfa/src/concurrency

Timestamp:

May 21, 2021, 4:48:10 PM (5 years ago)

Author:

Thierry Delisle <tdelisle@…>

Branches:

ADT, arm-eh, ast-experimental, enum, forall-pointer-decay, jacob/cs343-translation, master, new-ast-unique-expr, pthread-emulation, qualifiedEnum, stuck-waitfor-destruct

Children:

f1bce515

Parents:

5407cdc (diff), 7404cdc (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the (diff) links above to see all the changes relative to each parent.

Message:

Merge branch 'master' of plg.uwaterloo.ca:software/cfa/cfa-cc

Location:

libcfa/src/concurrency

Files:

• alarm.cfa (modified) (6 diffs)
• alarm.hfa (modified) (1 diff)
• clib/cfathread.cfa (modified) (2 diffs)
• clib/cfathread.h (modified) (1 diff)
• invoke.h (modified) (3 diffs)
• io.cfa (modified) (3 diffs)
• kernel.cfa (modified) (16 diffs)
• kernel.hfa (modified) (10 diffs)
• kernel/fwd.hfa (modified) (2 diffs)
• kernel/startup.cfa (modified) (13 diffs)
• kernel_private.hfa (modified) (10 diffs)
• locks.cfa (modified) (5 diffs)
• locks.hfa (modified) (2 diffs)
• preemption.cfa (modified) (15 diffs)
• ready_queue.cfa (modified) (26 diffs)
• ready_subqueue.hfa (modified) (3 diffs)
• stats.cfa (modified) (6 diffs)
• stats.hfa (modified) (3 diffs)
• thread.cfa (modified) (1 diff)

libcfa/src/concurrency/alarm.cfa

@@ -38 +38 @@
 
 void __kernel_set_timer( Duration alarm ) {
-        verifyf(alarm >= 1`us || alarm == 0, "Setting timer to < 1us (%jins)", alarm`ns);
-        setitimer( ITIMER_REAL, &(itimerval){ alarm }, 0p );
+        alarm = max(alarm, 1`us);
+        itimerval otv @= { 0 };
+        getitimer( ITIMER_REAL, &otv );
+        Duration od = { otv.it_value };
+        if(od == 0 || od > alarm) {
+                setitimer( ITIMER_REAL, &(itimerval){ alarm }, 0p );
+        }
 }
 
@@ -46 +51 @@
 //=============================================================================================
 
-void ?{}( alarm_node_t & this, $thread * thrd, Time alarm, Duration period) with( this ) {
+void ?{}( alarm_node_t & this, $thread * thrd, Duration alarm, Duration period) with( this ) {
+        this.initial = alarm;
+        this.period  = period;
         this.thrd = thrd;
-        this.alarm = alarm;
-        this.period = period;
+        this.timeval = __kernel_get_time() + alarm;
         set = false;
         type = User;
 }
 
-void ?{}( alarm_node_t & this, processor * proc, Time alarm, Duration period ) with( this ) {
+void ?{}( alarm_node_t & this, processor * proc, Duration alarm, Duration period ) with( this ) {
+        this.initial = alarm;
+        this.period  = period;
         this.proc = proc;
-        this.alarm = alarm;
-        this.period = period;
+        this.timeval = __kernel_get_time() + alarm;
         set = false;
         type = Kernel;
 }
-void ?{}( alarm_node_t & this, Alarm_Callback callback, Time alarm, Duration period ) with( this ) {
-        this.alarm = alarm;
-        this.period = period;
+void ?{}( alarm_node_t & this, Alarm_Callback callback, Duration alarm, Duration period ) with( this ) {
         this.callback = callback;
+        this.initial = alarm;
+        this.period  = period;
+        this.timeval = __kernel_get_time() + alarm;
         set = false;
         type = Callback;
@@ -77 +85 @@
 void insert( alarm_list_t * this, alarm_node_t * n ) {
         alarm_node_t * it = & (*this)`first;
-        while( it && (n->alarm > it->alarm) ) {
+        while( it && (n->timeval > it->timeval) ) {
                 it = & (*it)`next;
         }
@@ -105 +113 @@
         lock( event_kernel->lock __cfaabi_dbg_ctx2 );
         {
-                verify( validate( alarms ) );
-                bool first = ! & alarms`first;
+                /* paranoid */ verify( validate( alarms ) );
 
-                __cfadbg_print_safe( preemption, " KERNEL: alarm inserting %p (%lu).\n", this, this->alarm.tn );
+                Time curr = __kernel_get_time();
+                __cfadbg_print_safe( preemption, " KERNEL: alarm inserting %p (%lu -> %lu).\n", this, curr.tn, this->timeval.tn );
                 insert( &alarms, this );
-                if( first ) {
-                        __kernel_set_timer( alarms`first.alarm - __kernel_get_time() );
-                }
+                __kernel_set_timer( this->timeval - curr);
+                this->set = true;
         }
         unlock( event_kernel->lock );
-        this->set = true;
         enable_interrupts();
 }
@@ -124 +130 @@
         {
                 verify( validate( event_kernel->alarms ) );
-                remove( *this );
+                if (this->set) remove( *this );
+                this->set = false;
         }
         unlock( event_kernel->lock );
         enable_interrupts();
-        this->set = false;
 }
 
@@ -136 +142 @@
 
 void sleep( Duration duration ) {
-        alarm_node_t node = { active_thread(), __kernel_get_time() + duration, 0`s };
+        alarm_node_t node = { active_thread(), duration, 0`s };
 
         register_self( &node );

libcfa/src/concurrency/alarm.hfa

@@ -46 +46 @@
 
 struct alarm_node_t {
-        Time alarm;                             // time when alarm goes off
-        Duration period;                        // if > 0 => period of alarm
+        Duration initial;       // time when alarm goes off
+        Duration period;        // if > 0 => period of alarm
 
-        DLISTED_MGD_IMPL_IN(alarm_node_t)
+        inline dlink(alarm_node_t);
 
         union {
-                $thread * thrd;                                 // thrd who created event
-                processor * proc;                               // proc who created event
-                Alarm_Callback callback;                // callback to handle event
+                $thread * thrd;                 // thrd who created event
+                processor * proc;                       // proc who created event
+                Alarm_Callback callback;        // callback to handle event
         };
 
-        bool set                :1;             // whether or not the alarm has be registered
-        enum alarm_type type;           // true if this is not a user defined alarm
+        Time timeval;           // actual time at which the alarm goes off
+        enum alarm_type type;   // true if this is not a user defined alarm
+        bool set                :1;     // whether or not the alarm has be registered
 };
-DLISTED_MGD_IMPL_OUT(alarm_node_t)
+P9_EMBEDDED( alarm_node_t, dlink(alarm_node_t) )
 
-void ?{}( alarm_node_t & this, $thread * thrd, Time alarm, Duration period );
-void ?{}( alarm_node_t & this, processor   * proc, Time alarm, Duration period );
-void ?{}( alarm_node_t & this, Alarm_Callback callback, Time alarm, Duration period );
+void ?{}( alarm_node_t & this, $thread * thrd, Duration alarm, Duration period );
+void ?{}( alarm_node_t & this, processor * proc, Duration alarm, Duration period );
+void ?{}( alarm_node_t & this, Alarm_Callback callback, Duration alarm, Duration period );
 void ^?{}( alarm_node_t & this );
 
-typedef dlist(alarm_node_t, alarm_node_t) alarm_list_t;
+typedef dlist(alarm_node_t) alarm_list_t;
 
 void insert( alarm_list_t * this, alarm_node_t * n );

libcfa/src/concurrency/clib/cfathread.cfa

@@ -27 +27 @@
       extern void __cfactx_invoke_thread(void (*main)(void *), void * this);
 }
+
+extern Time __kernel_get_time();
 
 //================================================================================
@@ -265 +267 @@
         int cfathread_cond_timedwait(cfathread_cond_t *restrict cond, cfathread_mutex_t *restrict mut, const struct timespec *restrict abstime) __attribute__((nonnull (1,2,3))) {
                 Time t = { *abstime };
-                if( wait( (*cond)->impl, (*mut)->impl, t ) ) {
+                timespec curr;
+                clock_gettime( CLOCK_REALTIME, &curr );
+                Time c = { curr };
+                if( wait( (*cond)->impl, (*mut)->impl, t - c ) ) {
                         return 0;
                 }

libcfa/src/concurrency/clib/cfathread.h

@@ -80 +80 @@
 
         typedef struct cfathread_cond_attr {
+                // WARNING: adding support for pthread_condattr_setclock would require keeping track of the clock
+                // and reading it in cond_timedwait
         } cfathread_condattr_t;
         typedef struct cfathread_condition * cfathread_cond_t;

libcfa/src/concurrency/invoke.h

@@ -146 +146 @@
         struct __thread_desc_link {
                 struct $thread * next;
-                struct $thread * prev;
                 volatile unsigned long long ts;
-                unsigned preferred;
         };
 
@@ -155 +153 @@
                 // context that is switch during a __cfactx_switch
                 struct __stack_context_t context;
+
+                // Link lists fields
+                // instrusive link field for threads
+                struct __thread_desc_link link;
 
                 // current execution status for coroutine
@@ -170 +172 @@
                 struct cluster * curr_cluster;
 
-                // Link lists fields
-                // instrusive link field for threads
-                struct __thread_desc_link link;
+                // preferred ready-queue
+                unsigned preferred;
 
                 // coroutine body used to store context

libcfa/src/concurrency/io.cfa

@@ -138 +138 @@
                 /* paranoid */ verify( proc->io.ctx );
 
+                __attribute__((unused)) cluster * cltr = proc->cltr;
                 $io_context & ctx = *proc->io.ctx;
+
+                // for(i; 2) {
+                //      unsigned idx = proc->rdq.id + i;
+                //      cltr->ready_queue.lanes.tscs[idx].tv = -1ull;
+                // }
 
                 __ioarbiter_flush( ctx );
@@ -151 +157 @@
                                 // Update statistics
                                 __STATS__( false, io.calls.errors.busy ++; )
+                                // for(i; 2) {
+                                //      unsigned idx = proc->rdq.id + i;
+                                //      cltr->ready_queue.lanes.tscs[idx].tv = rdtscl();
+                                // }
                                 return;
                         default:
@@ -172 +182 @@
 
                 ctx.proc->io.pending = false;
+
+                ready_schedule_lock();
+                __cfa_io_drain( proc );
+                ready_schedule_unlock();
+                // for(i; 2) {
+                //      unsigned idx = proc->rdq.id + i;
+                //      cltr->ready_queue.lanes.tscs[idx].tv = rdtscl();
+                // }
         }
 

libcfa/src/concurrency/kernel.cfa

@@ -163 +163 @@
         #if !defined(__CFA_NO_STATISTICS__)
                 if( this->print_halts ) {
-                        __cfaabi_bits_print_safe( STDOUT_FILENO, "Processor : %d - %s (%p)\n", this->id, this->name, (void*)this);
+                        __cfaabi_bits_print_safe( STDOUT_FILENO, "Processor : %d - %s (%p)\n", this->unique_id, this->name, (void*)this);
                 }
         #endif
@@ -170 +170 @@
                 // Setup preemption data
                 preemption_scope scope = { this };
-
-                __STATS( unsigned long long last_tally = rdtscl(); )
 
                 // if we need to run some special setup, now is the time to do it.
@@ -184 +182 @@
                 MAIN_LOOP:
                 for() {
+                        #define OLD_MAIN 1
+                        #if OLD_MAIN
                         // Check if there is pending io
                         __maybe_io_drain( this );
@@ -223 +223 @@
                                 #if !defined(__CFA_NO_STATISTICS__)
                                         if(this->print_halts) {
-                                                __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 0\n", this->id, rdtscl());
+                                                __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 0\n", this->unique_id, rdtscl());
                                         }
                                 #endif
@@ -236 +236 @@
                                 #if !defined(__CFA_NO_STATISTICS__)
                                         if(this->print_halts) {
-                                                __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 1\n", this->id, rdtscl());
+                                                __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 1\n", this->unique_id, rdtscl());
                                         }
                                 #endif
@@ -258 +258 @@
                         if( __atomic_load_n(&this->do_terminate, __ATOMIC_SEQ_CST) ) break MAIN_LOOP;
 
-                        #if !defined(__CFA_NO_STATISTICS__)
-                                unsigned long long curr = rdtscl();
-                                if(curr > (last_tally + 500000000)) {
-                                        __tally_stats(this->cltr->stats, __cfaabi_tls.this_stats);
-                                        last_tally = curr;
-                                }
-                        #endif
-
                         if(this->io.pending && !this->io.dirty) {
                                 __cfa_io_flush( this );
                         }
 
-                //      SEARCH: {
-                //              /* paranoid */ verify( ! __preemption_enabled() );
-                //              /* paranoid */ verify( kernelTLS().this_proc_id );
-
-                //              // First, lock the scheduler since we are searching for a thread
-
-                //              // Try to get the next thread
-                //              ready_schedule_lock();
-                //              readyThread = pop_fast( this->cltr );
-                //              ready_schedule_unlock();
-                //              if(readyThread) {  break SEARCH; }
-
-                //              // If we can't find a thread, might as well flush any outstanding I/O
-                //              if(this->io.pending) { __cfa_io_flush( this ); }
-
-                //              // Spin a little on I/O, just in case
-                //              for(25) {
-                //                      __maybe_io_drain( this );
-                //                      ready_schedule_lock();
-                //                      readyThread = pop_fast( this->cltr );
-                //                      ready_schedule_unlock();
-                //                      if(readyThread) {  break SEARCH; }
-                //              }
-
-                //              // no luck, try stealing a few times
-                //              for(25) {
-                //                      if( __maybe_io_drain( this ) ) {
-                //                              ready_schedule_lock();
-                //                              readyThread = pop_fast( this->cltr );
-                //                      } else {
-                //                              ready_schedule_lock();
-                //                              readyThread = pop_slow( this->cltr );
-                //                      }
-                //                      ready_schedule_unlock();
-                //                      if(readyThread) {  break SEARCH; }
-                //              }
-
-                //              // still no luck, search for a thread
-                //              ready_schedule_lock();
-                //              readyThread = pop_search( this->cltr );
-                //              ready_schedule_unlock();
-                //              if(readyThread) { break SEARCH; }
-
-                //              // Don't block if we are done
-                //              if( __atomic_load_n(&this->do_terminate, __ATOMIC_SEQ_CST) ) break MAIN_LOOP;
-
-                //              __STATS( __tls_stats()->ready.sleep.halts++; )
-
-                //              // Push self to idle stack
-                //              mark_idle(this->cltr->procs, * this);
-
-                //              // Confirm the ready-queue is empty
-                //              __maybe_io_drain( this );
-                //              ready_schedule_lock();
-                //              readyThread = pop_search( this->cltr );
-                //              ready_schedule_unlock();
-
-                //              if( readyThread ) {
-                //                      // A thread was found, cancel the halt
-                //                      mark_awake(this->cltr->procs, * this);
-
-                //                      __STATS( __tls_stats()->ready.sleep.cancels++; )
-
-                //                      // continue the main loop
-                //                      break SEARCH;
-                //              }
-
-                //              __STATS( if(this->print_halts) __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 0\n", this->id, rdtscl()); )
-                //              __cfadbg_print_safe(runtime_core, "Kernel : core %p waiting on eventfd %d\n", this, this->idle);
-
-                //              // __disable_interrupts_hard();
-                //              eventfd_t val;
-                //              eventfd_read( this->idle, &val );
-                //              // __enable_interrupts_hard();
-
-                //              __STATS( if(this->print_halts) __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 1\n", this->id, rdtscl()); )
-
-                //              // We were woken up, remove self from idle
-                //              mark_awake(this->cltr->procs, * this);
-
-                //              // DON'T just proceed, start looking again
-                //              continue MAIN_LOOP;
-                //      }
-
-                // RUN_THREAD:
-                //      /* paranoid */ verify( kernelTLS().this_proc_id );
-                //      /* paranoid */ verify( ! __preemption_enabled() );
-                //      /* paranoid */ verify( readyThread );
-
-                //      // Reset io dirty bit
-                //      this->io.dirty = false;
-
-                //      // We found a thread run it
-                //      __run_thread(this, readyThread);
-
-                //      // Are we done?
-                //      if( __atomic_load_n(&this->do_terminate, __ATOMIC_SEQ_CST) ) break MAIN_LOOP;
-
-                //      #if !defined(__CFA_NO_STATISTICS__)
-                //              unsigned long long curr = rdtscl();
-                //              if(curr > (last_tally + 500000000)) {
-                //                      __tally_stats(this->cltr->stats, __cfaabi_tls.this_stats);
-                //                      last_tally = curr;
-                //              }
-                //      #endif
-
-                //      if(this->io.pending && !this->io.dirty) {
-                //              __cfa_io_flush( this );
-                //      }
-
-                //      // Check if there is pending io
-                //      __maybe_io_drain( this );
+                        #else
+                                #warning new kernel loop
+                        SEARCH: {
+                                /* paranoid */ verify( ! __preemption_enabled() );
+
+                                // First, lock the scheduler since we are searching for a thread
+                                ready_schedule_lock();
+
+                                // Try to get the next thread
+                                readyThread = pop_fast( this->cltr );
+                                if(readyThread) { ready_schedule_unlock(); break SEARCH; }
+
+                                // If we can't find a thread, might as well flush any outstanding I/O
+                                if(this->io.pending) { __cfa_io_flush( this ); }
+
+                                // Spin a little on I/O, just in case
+                                        for(5) {
+                                        __maybe_io_drain( this );
+                                        readyThread = pop_fast( this->cltr );
+                                        if(readyThread) { ready_schedule_unlock(); break SEARCH; }
+                                }
+
+                                // no luck, try stealing a few times
+                                        for(5) {
+                                        if( __maybe_io_drain( this ) ) {
+                                                readyThread = pop_fast( this->cltr );
+                                        } else {
+                                                readyThread = pop_slow( this->cltr );
+                                        }
+                                        if(readyThread) { ready_schedule_unlock(); break SEARCH; }
+                                }
+
+                                // still no luck, search for a thread
+                                readyThread = pop_search( this->cltr );
+                                if(readyThread) { ready_schedule_unlock(); break SEARCH; }
+
+                                // Don't block if we are done
+                                if( __atomic_load_n(&this->do_terminate, __ATOMIC_SEQ_CST) ) break MAIN_LOOP;
+
+                                __STATS( __tls_stats()->ready.sleep.halts++; )
+
+                                // Push self to idle stack
+                                ready_schedule_unlock();
+                                mark_idle(this->cltr->procs, * this);
+                                ready_schedule_lock();
+
+                                // Confirm the ready-queue is empty
+                                __maybe_io_drain( this );
+                                readyThread = pop_search( this->cltr );
+                                ready_schedule_unlock();
+
+                                if( readyThread ) {
+                                        // A thread was found, cancel the halt
+                                        mark_awake(this->cltr->procs, * this);
+
+                                        __STATS( __tls_stats()->ready.sleep.cancels++; )
+
+                                        // continue the main loop
+                                        break SEARCH;
+                                }
+
+                                        __STATS( if(this->print_halts) __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 0\n", this->unique_id, rdtscl()); )
+                                __cfadbg_print_safe(runtime_core, "Kernel : core %p waiting on eventfd %d\n", this, this->idle);
+
+                                // __disable_interrupts_hard();
+                                eventfd_t val;
+                                eventfd_read( this->idle, &val );
+                                // __enable_interrupts_hard();
+
+                                        __STATS( if(this->print_halts) __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 1\n", this->unique_id, rdtscl()); )
+
+                                // We were woken up, remove self from idle
+                                mark_awake(this->cltr->procs, * this);
+
+                                // DON'T just proceed, start looking again
+                                continue MAIN_LOOP;
+                        }
+
+                RUN_THREAD:
+                        /* paranoid */ verify( ! __preemption_enabled() );
+                        /* paranoid */ verify( readyThread );
+
+                        // Reset io dirty bit
+                        this->io.dirty = false;
+
+                        // We found a thread run it
+                        __run_thread(this, readyThread);
+
+                        // Are we done?
+                        if( __atomic_load_n(&this->do_terminate, __ATOMIC_SEQ_CST) ) break MAIN_LOOP;
+
+                        if(this->io.pending && !this->io.dirty) {
+                                __cfa_io_flush( this );
+                        }
+
+                        ready_schedule_lock();
+                        __maybe_io_drain( this );
+                        ready_schedule_unlock();
+                        #endif
                 }
 
@@ -390 +369 @@
 
         post( this->terminated );
-
 
         if(this == mainProcessor) {
@@ -553 +531 @@
 static void __schedule_thread( $thread * thrd ) {
         /* paranoid */ verify( ! __preemption_enabled() );
-        /* paranoid */ verify( kernelTLS().this_proc_id );
         /* paranoid */ verify( ready_schedule_islocked());
         /* paranoid */ verify( thrd );
@@ -567 +544 @@
         /* paranoid */ verify( 0x0D15EA5E0D15EA5Ep == thrd->canary );
 
-
+        const bool local = thrd->state != Start;
         if (thrd->preempted == __NO_PREEMPTION) thrd->state = Ready;
 
@@ -575 +552 @@
 
         // push the thread to the cluster ready-queue
-        push( cl, thrd );
+        push( cl, thrd, local );
 
         // variable thrd is no longer safe to use
@@ -611 +588 @@
 static inline $thread * __next_thread(cluster * this) with( *this ) {
         /* paranoid */ verify( ! __preemption_enabled() );
-        /* paranoid */ verify( kernelTLS().this_proc_id );
 
         ready_schedule_lock();
@@ -617 +593 @@
         ready_schedule_unlock();
 
-        /* paranoid */ verify( kernelTLS().this_proc_id );
         /* paranoid */ verify( ! __preemption_enabled() );
         return thrd;
@@ -625 +600 @@
 static inline $thread * __next_thread_slow(cluster * this) with( *this ) {
         /* paranoid */ verify( ! __preemption_enabled() );
-        /* paranoid */ verify( kernelTLS().this_proc_id );
 
         ready_schedule_lock();
@@ -638 +612 @@
         ready_schedule_unlock();
 
-        /* paranoid */ verify( kernelTLS().this_proc_id );
         /* paranoid */ verify( ! __preemption_enabled() );
         return thrd;
@@ -895 +868 @@
                 unsigned tail = *ctx->cq.tail;
                 if(head == tail) return false;
+                #if OLD_MAIN
                 ready_schedule_lock();
                 ret = __cfa_io_drain( proc );
                 ready_schedule_unlock();
+                #else
+                        ret = __cfa_io_drain( proc );
+        #endif
         #endif
         return ret;
@@ -926 +903 @@
         }
 
+        static void crawl_list( cluster * cltr, dlist(processor) & list, unsigned count ) {
+                /* paranoid */ verify( cltr->stats );
+
+                processor * it = &list`first;
+                for(unsigned i = 0; i < count; i++) {
+                        /* paranoid */ verifyf( it, "Unexpected null iterator, at index %u of %u\n", i, count);
+                        /* paranoid */ verify( it->local_data->this_stats );
+                        __tally_stats( cltr->stats, it->local_data->this_stats );
+                        it = &(*it)`next;
+                }
+        }
+
+        void crawl_cluster_stats( cluster & this ) {
+                // Stop the world, otherwise stats could get really messed-up
+                // this doesn't solve all problems but does solve many
+                // so it's probably good enough
+                uint_fast32_t last_size = ready_mutate_lock();
+
+                        crawl_list(&this, this.procs.actives, this.procs.total - this.procs.idle);
+                        crawl_list(&this, this.procs.idles  , this.procs.idle );
+
+                // Unlock the RWlock
+                ready_mutate_unlock( last_size );
+        }
+
+
         void print_stats_now( cluster & this, int flags ) {
+                crawl_cluster_stats( this );
                 __print_stats( this.stats, this.print_stats, "Cluster", this.name, (void*)&this );
         }

libcfa/src/concurrency/kernel.hfa

@@ -49 +49 @@
 
 // Processor id, required for scheduling threads
-struct __processor_id_t {
-        unsigned id:24;
-
-        #if !defined(__CFA_NO_STATISTICS__)
-                struct __stats_t * stats;
-        #endif
-};
+
 
 coroutine processorCtx_t {
@@ -63 +57 @@
 // Wrapper around kernel threads
 struct __attribute__((aligned(128))) processor {
-        // Main state
-        inline __processor_id_t;
-
         // Cluster from which to get threads
         struct cluster * cltr;
@@ -90 +81 @@
         pthread_t kernel_thread;
 
+        // Unique id for the processor (not per cluster)
+        unsigned unique_id;
+
         struct {
                 $io_context * ctx;
@@ -113 +107 @@
 
         // Link lists fields
-        DLISTED_MGD_IMPL_IN(processor)
+        inline dlink(processor);
 
         // special init fields
@@ -123 +117 @@
         } init;
 
+        struct KernelThreadData * local_data;
+
         #if !defined(__CFA_NO_STATISTICS__)
                 int print_stats;
@@ -133 +129 @@
 #endif
 };
+P9_EMBEDDED( processor, dlink(processor) )
 
 void  ?{}(processor & this, const char name[], struct cluster & cltr);
@@ -140 +137 @@
 static inline void  ?{}(processor & this, struct cluster & cltr) { this{ "Anonymous Processor", cltr}; }
 static inline void  ?{}(processor & this, const char name[])     { this{name, *mainCluster}; }
-
-DLISTED_MGD_IMPL_OUT(processor)
 
 //-----------------------------------------------------------------------------
@@ -152 +147 @@
 
 // Aligned timestamps which are used by the relaxed ready queue
-struct __attribute__((aligned(128))) __timestamp_t;
-void  ?{}(__timestamp_t & this);
-void ^?{}(__timestamp_t & this);
+struct __attribute__((aligned(128))) __timestamp_t {
+        volatile unsigned long long tv;
+};
+
+static inline void  ?{}(__timestamp_t & this) { this.tv = 0; }
+static inline void ^?{}(__timestamp_t & this) {}
 
 //TODO adjust cache size to ARCHITECTURE
@@ -177 +175 @@
 void  ?{}(__ready_queue_t & this);
 void ^?{}(__ready_queue_t & this);
+#if !defined(__CFA_NO_STATISTICS__)
+        unsigned cnt(const __ready_queue_t & this, unsigned idx);
+#endif
 
 // Idle Sleep
@@ -190 +191 @@
 
         // List of idle processors
-        dlist(processor, processor) idles;
+        dlist(processor) idles;
 
         // List of active processors
-        dlist(processor, processor) actives;
+        dlist(processor) actives;
 };
 

libcfa/src/concurrency/kernel/fwd.hfa

@@ -38 +38 @@
                         struct $thread          * volatile this_thread;
                         struct processor        * volatile this_processor;
-                        struct __processor_id_t * volatile this_proc_id;
-                        struct __stats_t        * volatile this_stats;
+                        volatile bool sched_lock;
 
                         struct {
@@ -56 +55 @@
                                 uint64_t bck_seed;
                         } ready_rng;
+
+                        struct __stats_t        * volatile this_stats;
+
+
+                        #ifdef __CFA_WITH_VERIFY__
+                                // Debug, check if the rwlock is owned for reading
+                                bool in_sched_lock;
+                                unsigned sched_id;
+                        #endif
                 } __cfaabi_tls __attribute__ ((tls_model ( "initial-exec" )));
 

libcfa/src/concurrency/kernel/startup.cfa

@@ -77 +77 @@
 static void doregister( struct cluster & cltr );
 static void unregister( struct cluster & cltr );
+static void register_tls( processor * this );
+static void unregister_tls( processor * this );
 static void ?{}( $coroutine & this, current_stack_info_t * info);
 static void ?{}( $thread & this, current_stack_info_t * info);
@@ -123 +125 @@
         NULL,                                                                                           // cannot use 0p
         NULL,
+        false,
+        { 1, false, false },
+        0,
+        { 0, 0 },
         NULL,
-        NULL,
-        { 1, false, false },
+        #ifdef __CFA_WITH_VERIFY__
+                false,
+                0,
+        #endif
 };
 
@@ -210 +218 @@
         (*mainProcessor){};
 
+        register_tls( mainProcessor );
+
         //initialize the global state variables
         __cfaabi_tls.this_processor = mainProcessor;
-        __cfaabi_tls.this_proc_id   = (__processor_id_t*)mainProcessor;
         __cfaabi_tls.this_thread    = mainThread;
 
@@ -219 +228 @@
                 __init_stats( __cfaabi_tls.this_stats );
         #endif
+        mainProcessor->local_data = &__cfaabi_tls;
 
         // Enable preemption
@@ -273 +283 @@
         #endif
 
+        mainProcessor->local_data = 0p;
+
+        unregister_tls( mainProcessor );
+
         // Destroy the main processor and its context in reverse order of construction
         // These were manually constructed so we need manually destroy them
@@ -316 +330 @@
         processor * proc = (processor *) arg;
         __cfaabi_tls.this_processor = proc;
-        __cfaabi_tls.this_proc_id   = (__processor_id_t*)proc;
         __cfaabi_tls.this_thread    = 0p;
         __cfaabi_tls.preemption_state.[enabled, disable_count] = [false, 1];
+        proc->local_data = &__cfaabi_tls;
+
+        register_tls( proc );
+
         // SKULLDUGGERY: We want to create a context for the processor coroutine
         // which is needed for the 2-step context switch. However, there is no reason
@@ -355 +372 @@
                 #endif
         #endif
+
+        proc->local_data = 0p;
+
+        unregister_tls( proc );
 
         return 0p;
@@ -446 +467 @@
         self_mon_p = &self_mon;
         link.next = 0p;
-        link.prev = 0p;
-        link.preferred = -1u;
+        link.ts   = 0;
+        preferred = -1u;
         last_proc = 0p;
         #if defined( __CFA_WITH_VERIFY__ )
@@ -475 +496 @@
         this.rdq.id  = -1u;
         this.rdq.target = -1u;
-        this.rdq.cutoff = -1ull;
+        this.rdq.cutoff = 0ull;
         do_terminate = false;
         preemption_alarm = 0p;
@@ -485 +506 @@
 
         this.init.thrd = initT;
+
+        this.local_data = 0p;
 
         this.idle = eventfd(0, 0);
@@ -496 +519 @@
         #endif
 
-        // Register and Lock the RWlock so no-one pushes/pops while we are changing the queue
-        uint_fast32_t last_size = ready_mutate_register((__processor_id_t*)&this);
-                this.cltr->procs.total += 1u;
-                insert_last(this.cltr->procs.actives, this);
-
-                // Adjust the ready queue size
-                ready_queue_grow( cltr );
-
-        // Unlock the RWlock
-        ready_mutate_unlock( last_size );
-
         __cfadbg_print_safe(runtime_core, "Kernel : core %p created\n", &this);
 }
@@ -512 +524 @@
 // Not a ctor, it just preps the destruction but should not destroy members
 static void deinit(processor & this) {
-        // Lock the RWlock so no-one pushes/pops while we are changing the queue
-        uint_fast32_t last_size = ready_mutate_lock();
-                this.cltr->procs.total -= 1u;
-                remove(this);
-
-                // Adjust the ready queue size
-                ready_queue_shrink( this.cltr );
-
-        // Unlock the RWlock and unregister: we don't need the read_lock any more
-        ready_mutate_unregister((__processor_id_t*)&this, last_size );
-
         close(this.idle);
 }
@@ -656 +657 @@
         cltr->nthreads -= 1;
         unlock(cltr->thread_list_lock);
+}
+
+static void register_tls( processor * this ) {
+        // Register and Lock the RWlock so no-one pushes/pops while we are changing the queue
+        uint_fast32_t last_size;
+        [this->unique_id, last_size] = ready_mutate_register();
+
+                this->cltr->procs.total += 1u;
+                insert_last(this->cltr->procs.actives, *this);
+
+                // Adjust the ready queue size
+                ready_queue_grow( this->cltr );
+
+        // Unlock the RWlock
+        ready_mutate_unlock( last_size );
+}
+
+
+static void unregister_tls( processor * this ) {
+        // Lock the RWlock so no-one pushes/pops while we are changing the queue
+        uint_fast32_t last_size = ready_mutate_lock();
+                this->cltr->procs.total -= 1u;
+                remove(*this);
+
+                // clear the cluster so nothing gets pushed to local queues
+                cluster * cltr = this->cltr;
+                this->cltr = 0p;
+
+                // Adjust the ready queue size
+                ready_queue_shrink( cltr );
+
+        // Unlock the RWlock and unregister: we don't need the read_lock any more
+        ready_mutate_unregister( this->unique_id, last_size );
 }
 

libcfa/src/concurrency/kernel_private.hfa

@@ -25 +25 @@
 // Scheduler
 
-struct __attribute__((aligned(128))) __scheduler_lock_id_t;
 
 extern "C" {
@@ -80 +79 @@
 // Lock-Free registering/unregistering of threads
 // Register a processor to a given cluster and get its unique id in return
-void register_proc_id( struct __processor_id_t * );
+unsigned register_proc_id( void );
 
 // Unregister a processor from a given cluster using its id, getting back the original pointer
-void unregister_proc_id( struct __processor_id_t * proc );
+void unregister_proc_id( unsigned );
 
 //=======================================================================
@@ -112 +111 @@
 }
 
-// Cells use by the reader writer lock
-// while not generic it only relies on a opaque pointer
-struct __attribute__((aligned(128))) __scheduler_lock_id_t {
-        // Spin lock used as the underlying lock
-        volatile bool lock;
-
-        // Handle pointing to the proc owning this cell
-        // Used for allocating cells and debugging
-        __processor_id_t * volatile handle;
-
-        #ifdef __CFA_WITH_VERIFY__
-                // Debug, check if this is owned for reading
-                bool owned;
-        #endif
-};
-
-static_assert( sizeof(struct __scheduler_lock_id_t) <= __alignof(struct __scheduler_lock_id_t));
+
+
+
 
 //-----------------------------------------------------------------------
@@ -147 +132 @@
 
         // writer lock
-        volatile bool lock;
+        volatile bool write_lock;
 
         // data pointer
-        __scheduler_lock_id_t * data;
+        volatile bool * volatile * data;
 };
 
@@ -163 +148 @@
 static inline void ready_schedule_lock(void) with(*__scheduler_lock) {
         /* paranoid */ verify( ! __preemption_enabled() );
-        /* paranoid */ verify( kernelTLS().this_proc_id );
-
-        unsigned iproc = kernelTLS().this_proc_id->id;
-        /*paranoid*/ verify(data[iproc].handle == kernelTLS().this_proc_id);
-        /*paranoid*/ verify(iproc < ready);
+        /* paranoid */ verify( ! kernelTLS().in_sched_lock );
+        /* paranoid */ verify( data[kernelTLS().sched_id] == &kernelTLS().sched_lock );
+        /* paranoid */ verify( !kernelTLS().this_processor || kernelTLS().this_processor->unique_id == kernelTLS().sched_id );
 
         // Step 1 : make sure no writer are in the middle of the critical section
-        while(__atomic_load_n(&lock, (int)__ATOMIC_RELAXED))
+        while(__atomic_load_n(&write_lock, (int)__ATOMIC_RELAXED))
                 Pause();
 
@@ -179 +162 @@
 
         // Step 2 : acquire our local lock
-        __atomic_acquire( &data[iproc].lock );
-        /*paranoid*/ verify(data[iproc].lock);
+        __atomic_acquire( &kernelTLS().sched_lock );
+        /*paranoid*/ verify(kernelTLS().sched_lock);
 
         #ifdef __CFA_WITH_VERIFY__
                 // Debug, check if this is owned for reading
-                data[iproc].owned = true;
+                kernelTLS().in_sched_lock = true;
         #endif
 }
@@ -190 +173 @@
 static inline void ready_schedule_unlock(void) with(*__scheduler_lock) {
         /* paranoid */ verify( ! __preemption_enabled() );
-        /* paranoid */ verify( kernelTLS().this_proc_id );
-
-        unsigned iproc = kernelTLS().this_proc_id->id;
-        /*paranoid*/ verify(data[iproc].handle == kernelTLS().this_proc_id);
-        /*paranoid*/ verify(iproc < ready);
-        /*paranoid*/ verify(data[iproc].lock);
-        /*paranoid*/ verify(data[iproc].owned);
+        /* paranoid */ verify( data[kernelTLS().sched_id] == &kernelTLS().sched_lock );
+        /* paranoid */ verify( !kernelTLS().this_processor || kernelTLS().this_processor->unique_id == kernelTLS().sched_id );
+        /* paranoid */ verify( kernelTLS().sched_lock );
+        /* paranoid */ verify( kernelTLS().in_sched_lock );
         #ifdef __CFA_WITH_VERIFY__
                 // Debug, check if this is owned for reading
-                data[iproc].owned = false;
+                kernelTLS().in_sched_lock = false;
         #endif
-        __atomic_unlock(&data[iproc].lock);
+        __atomic_unlock(&kernelTLS().sched_lock);
 }
 
@@ -207 +187 @@
         static inline bool ready_schedule_islocked(void) {
                 /* paranoid */ verify( ! __preemption_enabled() );
-                /*paranoid*/ verify( kernelTLS().this_proc_id );
-                __processor_id_t * proc = kernelTLS().this_proc_id;
-                return __scheduler_lock->data[proc->id].owned;
+                /* paranoid */ verify( (!kernelTLS().in_sched_lock) || kernelTLS().sched_lock );
+                return kernelTLS().sched_lock;
         }
 
         static inline bool ready_mutate_islocked() {
-                return __scheduler_lock->lock;
+                return __scheduler_lock->write_lock;
         }
 #endif
@@ -228 +207 @@
 // Register a processor to a given cluster and get its unique id in return
 // For convenience, also acquires the lock
-static inline uint_fast32_t ready_mutate_register( struct __processor_id_t * proc ) {
-        register_proc_id( proc );
-        return ready_mutate_lock();
+static inline [unsigned, uint_fast32_t] ready_mutate_register() {
+        unsigned id = register_proc_id();
+        uint_fast32_t last = ready_mutate_lock();
+        return [id, last];
 }
 
 // Unregister a processor from a given cluster using its id, getting back the original pointer
 // assumes the lock is acquired
-static inline void ready_mutate_unregister( struct __processor_id_t * proc, uint_fast32_t last_s ) {
+static inline void ready_mutate_unregister( unsigned id, uint_fast32_t last_s ) {
         ready_mutate_unlock( last_s );
-        unregister_proc_id( proc );
+        unregister_proc_id( id );
 }
 
@@ -281 +261 @@
 // push thread onto a ready queue for a cluster
 // returns true if the list was previously empty, false otherwise
-__attribute__((hot)) void push(struct cluster * cltr, struct $thread * thrd);
+__attribute__((hot)) void push(struct cluster * cltr, struct $thread * thrd, bool local);
 
 //-----------------------------------------------------------------------

libcfa/src/concurrency/locks.cfa

@@ -188 +188 @@
                 alarm_node_t alarm_node;
                 condition_variable(L) * cond;
-                info_thread(L) * i;
+                info_thread(L) * info_thd;
         };
 
-        void ?{}( alarm_node_wrap(L) & this, Time alarm, Duration period, Alarm_Callback callback, condition_variable(L) * c, info_thread(L) * i ) {
+        void ?{}( alarm_node_wrap(L) & this, Duration alarm, Duration period, Alarm_Callback callback, condition_variable(L) * c, info_thread(L) * i ) {
                 this.alarm_node{ callback, alarm, period };
                 this.cond = c;
-                this.i = i;
+                this.info_thd = i;
         }
 
@@ -206 +206 @@
                 //      may still be called after a thread has been removed from the queue but
                 //      before the alarm is unregistered
-                if ( listed(i) ) {      // is thread on queue
-                        i->signalled = false;
+                if ( listed(info_thd) ) {       // is thread on queue
+                        info_thd->signalled = false;
                         // remove this thread O(1)
-                        remove( cond->blocked_threads, *i );
+                        remove( cond->blocked_threads, *info_thd );
                         cond->count--;
-                        if( i->lock ) {
+                        if( info_thd->lock ) {
                                 // call lock's on_notify if a lock was passed
-                                on_notify(*i->lock, i->t);
+                                on_notify(*info_thd->lock, info_thd->t);
                         } else {
                                 // otherwise wake thread
-                                unpark( i->t );
+                                unpark( info_thd->t );
                         }
                 }
@@ -313 +313 @@
 
         // helper for wait()'s' with a timeout
-        void queue_info_thread_timeout( condition_variable(L) & this, info_thread(L) & info, Time t ) with(this) {
+        void queue_info_thread_timeout( condition_variable(L) & this, info_thread(L) & info, Duration t, Alarm_Callback callback ) with(this) {
                 lock( lock __cfaabi_dbg_ctx2 );
                 size_t recursion_count = queue_and_get_recursion(this, &info);
-                alarm_node_wrap(L) node_wrap = { t, 0`s, alarm_node_wrap_cast, &this, &info };
+                alarm_node_wrap(L) node_wrap = { t, 0`s, callback, &this, &info };
                 register_self( &node_wrap.alarm_node );
                 unlock( lock );
@@ -332 +332 @@
         #define WAIT_TIME( u, l, t ) \
                 info_thread( L ) i = { active_thread(), u, l }; \
-                queue_info_thread_timeout(this, i, t ); \
+                queue_info_thread_timeout(this, i, t, alarm_node_wrap_cast ); \
                 return i.signalled;
 
@@ -340 +340 @@
         void wait( condition_variable(L) & this, L & l, uintptr_t info ) with(this) { WAIT( info, &l ) }
 
-        bool wait( condition_variable(L) & this, Duration duration                        ) with(this) { WAIT_TIME( 0   , 0p , __kernel_get_time() + duration ) }
-        bool wait( condition_variable(L) & this, uintptr_t info, Duration duration        ) with(this) { WAIT_TIME( info, 0p , __kernel_get_time() + duration ) }
-        bool wait( condition_variable(L) & this, Time time                                ) with(this) { WAIT_TIME( 0   , 0p , time ) }
-        bool wait( condition_variable(L) & this, uintptr_t info, Time time                ) with(this) { WAIT_TIME( info, 0p , time ) }
-        bool wait( condition_variable(L) & this, L & l, Duration duration                 ) with(this) { WAIT_TIME( 0   , &l , __kernel_get_time() + duration ) }
-        bool wait( condition_variable(L) & this, L & l, uintptr_t info, Duration duration ) with(this) { WAIT_TIME( info, &l , __kernel_get_time() + duration ) }
-        bool wait( condition_variable(L) & this, L & l, Time time                         ) with(this) { WAIT_TIME( 0   , &l , time ) }
-        bool wait( condition_variable(L) & this, L & l, uintptr_t info, Time time         ) with(this) { WAIT_TIME( info, &l , time ) }
+        bool wait( condition_variable(L) & this, Duration duration                        ) with(this) { WAIT_TIME( 0   , 0p , duration ) }
+        bool wait( condition_variable(L) & this, uintptr_t info, Duration duration        ) with(this) { WAIT_TIME( info, 0p , duration ) }
+        bool wait( condition_variable(L) & this, L & l, Duration duration                 ) with(this) { WAIT_TIME( 0   , &l , duration ) }
+        bool wait( condition_variable(L) & this, L & l, uintptr_t info, Duration duration ) with(this) { WAIT_TIME( info, &l , duration ) }
 }
 

libcfa/src/concurrency/locks.hfa

@@ -290 +290 @@
         bool wait( condition_variable(L) & this, Duration duration );
         bool wait( condition_variable(L) & this, uintptr_t info, Duration duration );
-        bool wait( condition_variable(L) & this, Time time );
-        bool wait( condition_variable(L) & this, uintptr_t info, Time time );
 
         void wait( condition_variable(L) & this, L & l );
@@ -297 +295 @@
         bool wait( condition_variable(L) & this, L & l, Duration duration );
         bool wait( condition_variable(L) & this, L & l, uintptr_t info, Duration duration );
-        bool wait( condition_variable(L) & this, L & l, Time time );
-        bool wait( condition_variable(L) & this, L & l, uintptr_t info, Time time );
-}
+}

libcfa/src/concurrency/preemption.cfa

@@ -18 +18 @@
 
 #include "preemption.hfa"
+
 #include <assert.h>
 
@@ -26 +27 @@
 #include <limits.h>                                                                             // PTHREAD_STACK_MIN
 
+#include "bits/debug.hfa"
 #include "bits/signal.hfa"
 #include "kernel_private.hfa"
@@ -105 +107 @@
 static inline alarm_node_t * get_expired( alarm_list_t * alarms, Time currtime ) {
         if( ! & (*alarms)`first ) return 0p;                                            // If no alarms return null
-        if( (*alarms)`first.alarm >= currtime ) return 0p;      // If alarms head not expired return null
+        if( (*alarms)`first.timeval >= currtime ) return 0p;    // If alarms head not expired return null
         return pop(alarms);                                                                     // Otherwise just pop head
 }
@@ -141 +143 @@
                 if( period > 0 ) {
                         __cfadbg_print_buffer_local( preemption, " KERNEL: alarm period is %lu.\n", period`ns );
-                        node->alarm = currtime + period;    // Alarm is periodic, add currtime to it (used cached current time)
+                        node->timeval = currtime + period;  // Alarm is periodic, add currtime to it (used cached current time)
                         insert( alarms, node );             // Reinsert the node for the next time it triggers
                 }
@@ -148 +150 @@
         // If there are still alarms pending, reset the timer
         if( & (*alarms)`first ) {
-                Duration delta = (*alarms)`first.alarm - currtime;
-                Duration capped = max(delta, 50`us);
-                __kernel_set_timer( capped );
+                Duration delta = (*alarms)`first.timeval - currtime;
+                __kernel_set_timer( delta );
         }
 }
@@ -160 +161 @@
         // Alarms need to be enabled
         if ( duration > 0 && ! alarm->set ) {
-                alarm->alarm = __kernel_get_time() + duration;
-                alarm->period = duration;
+                alarm->initial = duration;
+                alarm->period  = duration;
                 register_self( alarm );
         }
@@ -167 +168 @@
         else if ( duration == 0 && alarm->set ) {
                 unregister_self( alarm );
-                alarm->alarm = 0;
-                alarm->period = 0;
+                alarm->initial = 0;
+                alarm->period  = 0;
         }
         // If alarm is different from previous, change it
         else if ( duration > 0 && alarm->period != duration ) {
                 unregister_self( alarm );
-                alarm->alarm = __kernel_get_time() + duration;
-                alarm->period = duration;
+                alarm->initial = duration;
+                alarm->period  = duration;
                 register_self( alarm );
         }
@@ -599 +600 @@
 
         // Notify the alarm thread of the shutdown
-        sigval val = { 1 };
+        sigval val;
+        val.sival_int = 0;
         pthread_sigqueue( alarm_thread, SIGALRM, val );
 
@@ -619 +621 @@
 // Used by thread to control when they want to receive preemption signals
 void ?{}( preemption_scope & this, processor * proc ) {
-        (this.alarm){ proc, (Time){ 0 }, 0`s };
+        (this.alarm){ proc, 0`s, 0`s };
         this.proc = proc;
         this.proc->preemption_alarm = &this.alarm;
@@ -687 +689 @@
 // Waits on SIGALRM and send SIGUSR1 to whom ever needs it
 static void * alarm_loop( __attribute__((unused)) void * args ) {
-        __processor_id_t id;
-        register_proc_id(&id);
-        __cfaabi_tls.this_proc_id = &id;
-
+        unsigned id = register_proc_id();
 
         // Block sigalrms to control when they arrive
@@ -707 +706 @@
                 siginfo_t info;
                 int sig = sigwaitinfo( &mask, &info );
+
+                __cfadbg_print_buffer_decl ( preemption, " KERNEL: sigwaitinfo returned %d, c: %d, v: %d\n", sig, info.si_code, info.si_value.sival_int );
+                __cfadbg_print_buffer_local( preemption, " KERNEL: SI_QUEUE %d, SI_TIMER %d, SI_KERNEL %d\n", SI_QUEUE, SI_TIMER, SI_KERNEL );
 
                 if( sig < 0 ) {
@@ -714 +716 @@
                                 case EAGAIN :
                                 case EINTR :
-                                        {__cfaabi_dbg_print_buffer_decl( " KERNEL: Spurious wakeup %d.\n", err );}
+                                        {__cfadbg_print_buffer_local( preemption, " KERNEL: Spurious wakeup %d.\n", err );}
                                         continue;
                                 case EINVAL :
@@ -726 +728 @@
                 assertf(sig == SIGALRM, "Kernel Internal Error, sigwait: Unexpected signal %d (%d : %d)\n", sig, info.si_code, info.si_value.sival_int);
 
-                // __cfaabi_dbg_print_safe( "Kernel : Caught alarm from %d with %d\n", info.si_code, info.si_value.sival_int );
                 // Switch on the code (a.k.a. the sender) to
                 switch( info.si_code )
                 {
+                // Signal was not sent by the kernel but by an other thread
+                case SI_QUEUE:
+                        // other threads may signal the alarm thread to shut it down
+                        // or to manual cause the preemption tick
+                        // use info.si_value and handle the case here
+                        switch( info.si_value.sival_int ) {
+                        case 0:
+                                goto EXIT;
+                        default:
+                                abort( "SI_QUEUE with val %d", info.si_value.sival_int);
+                        }
+                        // fallthrough
                 // Timers can apparently be marked as sent for the kernel
                 // In either case, tick preemption
@@ -739 +752 @@
                         unlock( event_kernel->lock );
                         break;
-                // Signal was not sent by the kernel but by an other thread
-                case SI_QUEUE:
-                        // For now, other thread only signal the alarm thread to shut it down
-                        // If this needs to change use info.si_value and handle the case here
-                        goto EXIT;
                 }
         }
@@ -749 +757 @@
 EXIT:
         __cfaabi_dbg_print_safe( "Kernel : Preemption thread stopping\n" );
-        register_proc_id(&id);
+        unregister_proc_id(id);
 
         return 0p;

libcfa/src/concurrency/ready_queue.cfa

@@ -17 +17 @@
 // #define __CFA_DEBUG_PRINT_READY_QUEUE__
 
-// #define USE_MPSC
 
 #define USE_RELAXED_FIFO
@@ -93 +92 @@
         this.alloc = 0;
         this.ready = 0;
-        this.lock  = false;
         this.data  = alloc(this.max);
-
-        /*paranoid*/ verify( 0 == (((uintptr_t)(this.data    )) % 64) );
-        /*paranoid*/ verify( 0 == (((uintptr_t)(this.data + 1)) % 64) );
+        this.write_lock  = false;
+
         /*paranoid*/ verify(__atomic_is_lock_free(sizeof(this.alloc), &this.alloc));
         /*paranoid*/ verify(__atomic_is_lock_free(sizeof(this.ready), &this.ready));
@@ -106 +103 @@
 }
 
-void ?{}( __scheduler_lock_id_t & this, __processor_id_t * proc ) {
-        this.handle = proc;
-        this.lock   = false;
-        #ifdef __CFA_WITH_VERIFY__
-                this.owned  = false;
-        #endif
-}
 
 //=======================================================================
 // Lock-Free registering/unregistering of threads
-void register_proc_id( struct __processor_id_t * proc ) with(*__scheduler_lock) {
+unsigned register_proc_id( void ) with(*__scheduler_lock) {
         __cfadbg_print_safe(ready_queue, "Kernel : Registering proc %p for RW-Lock\n", proc);
+        bool * handle = (bool *)&kernelTLS().sched_lock;
 
         // Step - 1 : check if there is already space in the data
@@ -124 +115 @@
         // Check among all the ready
         for(uint_fast32_t i = 0; i < s; i++) {
-                __processor_id_t * null = 0p; // Re-write every loop since compare thrashes it
-                if( __atomic_load_n(&data[i].handle, (int)__ATOMIC_RELAXED) == null
-                        && __atomic_compare_exchange_n( &data[i].handle, &null, proc, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) {
-                        /*paranoid*/ verify(i < ready);
-                        /*paranoid*/ verify(0 == (__alignof__(data[i]) % cache_line_size));
-                        /*paranoid*/ verify((((uintptr_t)&data[i]) % cache_line_size) == 0);
-                        proc->id = i;
+                bool * volatile * cell = (bool * volatile *)&data[i]; // Cforall is bugged and the double volatiles causes problems
+                /* paranoid */ verify( handle != *cell );
+
+                bool * null = 0p; // Re-write every loop since compare thrashes it
+                if( __atomic_load_n(cell, (int)__ATOMIC_RELAXED) == null
+                        && __atomic_compare_exchange_n( cell, &null, handle, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) {
+                        /* paranoid */ verify(i < ready);
+                        /* paranoid */ verify( (kernelTLS().sched_id = i, true) );
+                        return i;
                 }
         }
@@ -141 +134 @@
 
         // Step - 3 : Mark space as used and then publish it.
-        __scheduler_lock_id_t * storage = (__scheduler_lock_id_t *)&data[n];
-        (*storage){ proc };
+        data[n] = handle;
         while() {
                 unsigned copy = n;
@@ -154 +146 @@
 
         // Return new spot.
-        /*paranoid*/ verify(n < ready);
-        /*paranoid*/ verify(__alignof__(data[n]) == (2 * cache_line_size));
-        /*paranoid*/ verify((((uintptr_t)&data[n]) % cache_line_size) == 0);
-        proc->id = n;
-}
-
-void unregister_proc_id( struct __processor_id_t * proc ) with(*__scheduler_lock) {
-        unsigned id = proc->id;
-        /*paranoid*/ verify(id < ready);
-        /*paranoid*/ verify(proc == __atomic_load_n(&data[id].handle, __ATOMIC_RELAXED));
-        __atomic_store_n(&data[id].handle, 0p, __ATOMIC_RELEASE);
+        /* paranoid */ verify(n < ready);
+        /* paranoid */ verify( (kernelTLS().sched_id = n, true) );
+        return n;
+}
+
+void unregister_proc_id( unsigned id ) with(*__scheduler_lock) {
+        /* paranoid */ verify(id < ready);
+        /* paranoid */ verify(id == kernelTLS().sched_id);
+        /* paranoid */ verify(data[id] == &kernelTLS().sched_lock);
+
+        bool * volatile * cell = (bool * volatile *)&data[id]; // Cforall is bugged and the double volatiles causes problems
+
+        __atomic_store_n(cell, 0p, __ATOMIC_RELEASE);
 
         __cfadbg_print_safe(ready_queue, "Kernel : Unregister proc %p\n", proc);
@@ -174 +168 @@
 uint_fast32_t ready_mutate_lock( void ) with(*__scheduler_lock) {
         /* paranoid */ verify( ! __preemption_enabled() );
+        /* paranoid */ verify( ! kernelTLS().sched_lock );
 
         // Step 1 : lock global lock
         // It is needed to avoid processors that register mid Critical-Section
         //   to simply lock their own lock and enter.
-        __atomic_acquire( &lock );
+        __atomic_acquire( &write_lock );
 
         // Step 2 : lock per-proc lock
@@ -186 +181 @@
         uint_fast32_t s = ready;
         for(uint_fast32_t i = 0; i < s; i++) {
-                __atomic_acquire( &data[i].lock );
+                volatile bool * llock = data[i];
+                if(llock) __atomic_acquire( llock );
         }
 
@@ -203 +199 @@
         // Alternative solution : return s in write_lock and pass it to write_unlock
         for(uint_fast32_t i = 0; i < last_s; i++) {
-                verify(data[i].lock);
-                __atomic_store_n(&data[i].lock, (bool)false, __ATOMIC_RELEASE);
+                volatile bool * llock = data[i];
+                if(llock) __atomic_store_n(llock, (bool)false, __ATOMIC_RELEASE);
         }
 
         // Step 2 : release global lock
-        /*paranoid*/ assert(true == lock);
-        __atomic_store_n(&lock, (bool)false, __ATOMIC_RELEASE);
+        /*paranoid*/ assert(true == write_lock);
+        __atomic_store_n(&write_lock, (bool)false, __ATOMIC_RELEASE);
 
         /* paranoid */ verify( ! __preemption_enabled() );
@@ -253 +249 @@
         }
 
-        __attribute__((hot)) void push(struct cluster * cltr, struct $thread * thrd) with (cltr->ready_queue) {
+        __attribute__((hot)) void push(struct cluster * cltr, struct $thread * thrd, bool push_local) with (cltr->ready_queue) {
                 __cfadbg_print_safe(ready_queue, "Kernel : Pushing %p on cluster %p\n", thrd, cltr);
 
-                const bool external = (!kernelTLS().this_processor) || (cltr != kernelTLS().this_processor->cltr);
+                const bool external = !push_local || (!kernelTLS().this_processor) || (cltr != kernelTLS().this_processor->cltr);
                 /* paranoid */ verify(external || kernelTLS().this_processor->rdq.id < lanes.count );
-
-                // write timestamp
-                thrd->link.ts = rdtscl();
 
                 bool local;
@@ -280 +273 @@
                         #endif
 
-                #if defined(USE_MPSC)
-                        // mpsc always succeeds
-                } while( false );
-                #else
                         // If we can't lock it retry
                 } while( !__atomic_try_acquire( &lanes.data[i].lock ) );
-                #endif
 
                 // Actually push it
                 push(lanes.data[i], thrd);
 
-                #if !defined(USE_MPSC)
-                        // Unlock and return
-                        __atomic_unlock( &lanes.data[i].lock );
-                #endif
+                // Unlock and return
+                __atomic_unlock( &lanes.data[i].lock );
 
                 // Mark the current index in the tls rng instance as having an item
@@ -350 +336 @@
 #endif
 #if defined(USE_WORK_STEALING)
-        __attribute__((hot)) void push(struct cluster * cltr, struct $thread * thrd) with (cltr->ready_queue) {
+        __attribute__((hot)) void push(struct cluster * cltr, struct $thread * thrd, bool push_local) with (cltr->ready_queue) {
                 __cfadbg_print_safe(ready_queue, "Kernel : Pushing %p on cluster %p\n", thrd, cltr);
 
-                const bool external = (!kernelTLS().this_processor) || (cltr != kernelTLS().this_processor->cltr);
+                // #define USE_PREFERRED
+                #if !defined(USE_PREFERRED)
+                const bool external = !push_local || (!kernelTLS().this_processor) || (cltr != kernelTLS().this_processor->cltr);
                 /* paranoid */ verify(external || kernelTLS().this_processor->rdq.id < lanes.count );
-
-                // write timestamp
-                thrd->link.ts = rdtscl();
+                #else
+                        unsigned preferred = thrd->preferred;
+                        const bool external = push_local || (!kernelTLS().this_processor) || preferred == -1u || thrd->curr_cluster != cltr;
+                        /* paranoid */ verifyf(external || preferred < lanes.count, "Invalid preferred queue %u for %u lanes", preferred, lanes.count );
+
+                        unsigned r = preferred % READYQ_SHARD_FACTOR;
+                        const unsigned start = preferred - r;
+                #endif
 
                 // Try to pick a lane and lock it
@@ -371 +364 @@
                         }
                         else {
-                                processor * proc = kernelTLS().this_processor;
-                                unsigned r = proc->rdq.its++;
-                                i =  proc->rdq.id + (r % READYQ_SHARD_FACTOR);
+                                #if !defined(USE_PREFERRED)
+                                        processor * proc = kernelTLS().this_processor;
+                                        unsigned r = proc->rdq.its++;
+                                        i =  proc->rdq.id + (r % READYQ_SHARD_FACTOR);
+                                #else
+                                        i = start + (r++ % READYQ_SHARD_FACTOR);
+                                #endif
                         }
-
-
-                #if defined(USE_MPSC)
-                        // mpsc always succeeds
-                } while( false );
-                #else
                         // If we can't lock it retry
                 } while( !__atomic_try_acquire( &lanes.data[i].lock ) );
-                #endif
 
                 // Actually push it
                 push(lanes.data[i], thrd);
 
-                #if !defined(USE_MPSC)
-                        // Unlock and return
-                        __atomic_unlock( &lanes.data[i].lock );
-                #endif
+                // Unlock and return
+                __atomic_unlock( &lanes.data[i].lock );
 
                 #if !defined(__CFA_NO_STATISTICS__)
@@ -410 +398 @@
 
                 if(proc->rdq.target == -1u) {
+                        unsigned long long min = ts(lanes.data[proc->rdq.id]);
+                        for(int i = 0; i < READYQ_SHARD_FACTOR; i++) {
+                                unsigned long long tsc = ts(lanes.data[proc->rdq.id + i]);
+                                if(tsc < min) min = tsc;
+                        }
+                        proc->rdq.cutoff = min;
                         proc->rdq.target = __tls_rand() % lanes.count;
-                        unsigned it1  = proc->rdq.itr;
-                        unsigned it2  = proc->rdq.itr + 1;
-                        unsigned idx1 = proc->rdq.id + (it1 % READYQ_SHARD_FACTOR);
-                        unsigned idx2 = proc->rdq.id + (it2 % READYQ_SHARD_FACTOR);
-                        unsigned long long tsc1 = ts(lanes.data[idx1]);
-                        unsigned long long tsc2 = ts(lanes.data[idx2]);
-                        proc->rdq.cutoff = min(tsc1, tsc2);
-                        if(proc->rdq.cutoff == 0) proc->rdq.cutoff = -1ull;
                 }
                 else {
                         unsigned target = proc->rdq.target;
                         proc->rdq.target = -1u;
-                        if(lanes.tscs[target].tv < proc->rdq.cutoff) {
+                        const unsigned long long bias = 0; //2_500_000_000;
+                        const unsigned long long cutoff = proc->rdq.cutoff > bias ? proc->rdq.cutoff - bias : proc->rdq.cutoff;
+                        if(lanes.tscs[target].tv < cutoff && ts(lanes.data[target]) < cutoff) {
                                 $thread * t = try_pop(cltr, target __STATS(, __tls_stats()->ready.pop.help));
                                 if(t) return t;
@@ -430 +418 @@
 
                 for(READYQ_SHARD_FACTOR) {
-                        unsigned i = proc->rdq.id + (--proc->rdq.itr % READYQ_SHARD_FACTOR);
+                        unsigned i = proc->rdq.id + (proc->rdq.itr++ % READYQ_SHARD_FACTOR);
                         if($thread * t = try_pop(cltr, i __STATS(, __tls_stats()->ready.pop.local))) return t;
                 }
@@ -462 +450 @@
         // If list looks empty retry
         if( is_empty(lane) ) {
-                __STATS( stats.espec++; )
                 return 0p;
         }
@@ -468 +455 @@
         // If we can't get the lock retry
         if( !__atomic_try_acquire(&lane.lock) ) {
-                __STATS( stats.elock++; )
                 return 0p;
         }
@@ -475 +461 @@
         if( is_empty(lane) ) {
                 __atomic_unlock(&lane.lock);
-                __STATS( stats.eempty++; )
                 return 0p;
         }
@@ -481 +466 @@
         // Actually pop the list
         struct $thread * thrd;
-        thrd = pop(lane);
+        unsigned long long tsv;
+        [thrd, tsv] = pop(lane);
 
         /* paranoid */ verify(thrd);
+        /* paranoid */ verify(tsv);
         /* paranoid */ verify(lane.lock);
 
@@ -493 +480 @@
 
         #if defined(USE_WORK_STEALING)
-                lanes.tscs[w].tv = thrd->link.ts;
+                lanes.tscs[w].tv = tsv;
         #endif
+
+        thrd->preferred = w;
 
         // return the popped thread
@@ -522 +511 @@
 // Check that all the intrusive queues in the data structure are still consistent
 static void check( __ready_queue_t & q ) with (q) {
-        #if defined(__CFA_WITH_VERIFY__) && !defined(USE_MPSC)
+        #if defined(__CFA_WITH_VERIFY__)
                 {
                         for( idx ; lanes.count ) {
@@ -528 +517 @@
                                 assert(!lanes.data[idx].lock);
 
-                                assert(head(sl)->link.prev == 0p );
-                                assert(head(sl)->link.next->link.prev == head(sl) );
-                                assert(tail(sl)->link.next == 0p );
-                                assert(tail(sl)->link.prev->link.next == tail(sl) );
-
-                                if(is_empty(sl)) {
-                                        assert(tail(sl)->link.prev == head(sl));
-                                        assert(head(sl)->link.next == tail(sl));
-                                } else {
-                                        assert(tail(sl)->link.prev != head(sl));
-                                        assert(head(sl)->link.next != tail(sl));
-                                }
+                                        if(is_empty(sl)) {
+                                                assert( sl.anchor.next == 0p );
+                                                assert( sl.anchor.ts   == 0  );
+                                                assert( mock_head(sl)  == sl.prev );
+                                        } else {
+                                                assert( sl.anchor.next != 0p );
+                                                assert( sl.anchor.ts   != 0  );
+                                                assert( mock_head(sl)  != sl.prev );
+                                        }
                         }
                 }
@@ -560 +546 @@
 // fixes the list so that the pointers back to anchors aren't left dangling
 static inline void fix(__intrusive_lane_t & ll) {
-        #if !defined(USE_MPSC)
-                // if the list is not empty then follow he pointer and fix its reverse
-                if(!is_empty(ll)) {
-                        head(ll)->link.next->link.prev = head(ll);
-                        tail(ll)->link.prev->link.next = tail(ll);
-                }
-                // Otherwise just reset the list
-                else {
-                        verify(tail(ll)->link.next == 0p);
-                        tail(ll)->link.prev = head(ll);
-                        head(ll)->link.next = tail(ll);
-                        verify(head(ll)->link.prev == 0p);
-                }
-        #endif
-}
-
-static void assign_list(unsigned & value, dlist(processor, processor) & list, unsigned count) {
+                        if(is_empty(ll)) {
+                                verify(ll.anchor.next == 0p);
+                                ll.prev = mock_head(ll);
+                        }
+}
+
+static void assign_list(unsigned & value, dlist(processor) & list, unsigned count) {
         processor * it = &list`first;
         for(unsigned i = 0; i < count; i++) {
@@ -597 +573 @@
                 lanes.tscs = alloc(lanes.count, lanes.tscs`realloc);
                 for(i; lanes.count) {
-                        lanes.tscs[i].tv = ts(lanes.data[i]);
+                        unsigned long long tsc = ts(lanes.data[i]);
+                        lanes.tscs[i].tv = tsc != 0 ? tsc : rdtscl();
                 }
         #endif
@@ -686 +663 @@
                         while(!is_empty(lanes.data[idx])) {
                                 struct $thread * thrd;
-                                thrd = pop(lanes.data[idx]);
-
-                                push(cltr, thrd);
+                                unsigned long long _;
+                                [thrd, _] = pop(lanes.data[idx]);
+
+                                push(cltr, thrd, true);
 
                                 // for printing count the number of displaced threads
@@ -725 +703 @@
         /* paranoid */ verify( ready_mutate_islocked() );
 }
+
+#if !defined(__CFA_NO_STATISTICS__)
+        unsigned cnt(const __ready_queue_t & this, unsigned idx) {
+                /* paranoid */ verify(this.lanes.count > idx);
+                return this.lanes.data[idx].cnt;
+        }
+#endif

libcfa/src/concurrency/ready_subqueue.hfa

@@ -7 +7 @@
 // Intrusives lanes which are used by the relaxed ready queue
 struct __attribute__((aligned(128))) __intrusive_lane_t {
-
-        #if defined(USE_MPSC)
-                mpsc_queue($thread) queue;
-                __attribute__((aligned(128)))
-        #else
-                // anchor for the head and the tail of the queue
-                __attribute__((aligned(128))) struct __sentinel_t {
-                        // Link lists fields
-                        // instrusive link field for threads
-                        // must be exactly as in $thread
-                        __thread_desc_link link;
-                } before, after;
-        #endif
+        struct $thread * prev;
 
         // spin lock protecting the queue
         volatile bool lock;
 
-        // Optional statistic counters
-        #if !defined(__CFA_NO_SCHED_STATS__)
-                struct __attribute__((aligned(64))) {
-                        // difference between number of push and pops
-                        ssize_t diff;
+        __thread_desc_link anchor;
 
-                        // total number of pushes and pops
-                        size_t  push;
-                        size_t  pop ;
-                } stat;
+        #if !defined(__CFA_NO_STATISTICS__)
+                unsigned cnt;
         #endif
 };
 
-void  ?{}(__intrusive_lane_t & this);
-void ^?{}(__intrusive_lane_t & this);
-
 // Get the head pointer (one before the first element) from the anchor
-static inline $thread * head(const __intrusive_lane_t & this) {
-        #if defined(USE_MPSC)
-                return this.queue.head;
-        #else
-                $thread * rhead = ($thread *)(
-                        (uintptr_t)( &this.before ) - offsetof( $thread, link )
-                );
-                /* paranoid */ verify(rhead);
-                return rhead;
-        #endif
-}
-
-// Get the tail pointer (one after the last element) from the anchor
-static inline $thread * tail(const __intrusive_lane_t & this) {
-        #if defined(USE_MPSC)
-                return this.queue.tail;
-        #else
-                $thread * rtail = ($thread *)(
-                        (uintptr_t)( &this.after ) - offsetof( $thread, link )
-                );
-                /* paranoid */ verify(rtail);
-                return rtail;
-        #endif
+static inline $thread * mock_head(const __intrusive_lane_t & this) {
+        $thread * rhead = ($thread *)(
+                (uintptr_t)( &this.anchor ) - __builtin_offsetof( $thread, link )
+        );
+        return rhead;
 }
 
@@ -69 +30 @@
 void ?{}( __intrusive_lane_t & this ) {
         this.lock = false;
+        this.prev = mock_head(this);
+        this.anchor.next = 0p;
+        this.anchor.ts   = 0;
+        #if !defined(__CFA_NO_STATISTICS__)
+                this.cnt  = 0;
+        #endif
 
-        #if !defined(USE_MPSC)
-                this.before.link.prev = 0p;
-                this.before.link.next = tail(this);
-                this.before.link.ts   = 0;
-
-                this.after .link.prev = head(this);
-                this.after .link.next = 0p;
-                this.after .link.ts   = 0;
-
-                #if !defined(__CFA_NO_SCHED_STATS__)
-                        this.stat.diff = 0;
-                        this.stat.push = 0;
-                        this.stat.pop  = 0;
-                #endif
-
-                // We add a boat-load of assertions here because the anchor code is very fragile
-                /* paranoid */ verify(((uintptr_t)( head(this) ) + offsetof( $thread, link )) == (uintptr_t)(&this.before));
-                /* paranoid */ verify(((uintptr_t)( tail(this) ) + offsetof( $thread, link )) == (uintptr_t)(&this.after ));
-                /* paranoid */ verify(head(this)->link.prev == 0p );
-                /* paranoid */ verify(head(this)->link.next == tail(this) );
-                /* paranoid */ verify(tail(this)->link.next == 0p );
-                /* paranoid */ verify(tail(this)->link.prev == head(this) );
-                /* paranoid */ verify(&head(this)->link.prev == &this.before.link.prev );
-                /* paranoid */ verify(&head(this)->link.next == &this.before.link.next );
-                /* paranoid */ verify(&tail(this)->link.prev == &this.after .link.prev );
-                /* paranoid */ verify(&tail(this)->link.next == &this.after .link.next );
-                /* paranoid */ verify(__alignof__(__intrusive_lane_t) == 128);
-                /* paranoid */ verify(__alignof__(this) == 128);
-                /* paranoid */ verifyf(((intptr_t)(&this) % 128) == 0, "Expected address to be aligned %p %% 128 == %zd", &this, ((intptr_t)(&this) % 128));
-        #endif
+        // We add a boat-load of assertions here because the anchor code is very fragile
+        /* paranoid */ _Static_assert( offsetof( $thread, link ) == offsetof(__intrusive_lane_t, anchor) );
+        /* paranoid */ verify( offsetof( $thread, link ) == offsetof(__intrusive_lane_t, anchor) );
+        /* paranoid */ verify( ((uintptr_t)( mock_head(this) ) + offsetof( $thread, link )) == (uintptr_t)(&this.anchor) );
+        /* paranoid */ verify( &mock_head(this)->link.next == &this.anchor.next );
+        /* paranoid */ verify( &mock_head(this)->link.ts   == &this.anchor.ts   );
+        /* paranoid */ verify( mock_head(this)->link.next == 0p );
+        /* paranoid */ verify( mock_head(this)->link.ts   == 0  );
+        /* paranoid */ verify( mock_head(this) == this.prev );
+        /* paranoid */ verify( __alignof__(__intrusive_lane_t) == 128 );
+        /* paranoid */ verify( __alignof__(this) == 128 );
+        /* paranoid */ verifyf( ((intptr_t)(&this) % 128) == 0, "Expected address to be aligned %p %% 128 == %zd", &this, ((intptr_t)(&this) % 128) );
 }
 
 // Dtor is trivial
 void ^?{}( __intrusive_lane_t & this ) {
-        #if !defined(USE_MPSC)
-                // Make sure the list is empty
-                /* paranoid */ verify(head(this)->link.prev == 0p );
-                /* paranoid */ verify(head(this)->link.next == tail(this) );
-                /* paranoid */ verify(tail(this)->link.next == 0p );
-                /* paranoid */ verify(tail(this)->link.prev == head(this) );
-        #endif
+        // Make sure the list is empty
+        /* paranoid */ verify( this.anchor.next == 0p );
+        /* paranoid */ verify( this.anchor.ts   == 0  );
+        /* paranoid */ verify( mock_head(this)  == this.prev );
 }
 
 // Push a thread onto this lane
 // returns true of lane was empty before push, false otherwise
-bool push(__intrusive_lane_t & this, $thread * node) {
-        #if defined(USE_MPSC)
-                inline $thread * volatile & ?`next ( $thread * this )  __attribute__((const)) {
-                        return this->link.next;
-                }
-                push(this.queue, node);
-        #else
-                #if defined(__CFA_WITH_VERIFY__)
-                        /* paranoid */ verify(this.lock);
-                        /* paranoid */ verify(node->link.ts != 0);
-                        /* paranoid */ verify(node->link.next == 0p);
-                        /* paranoid */ verify(node->link.prev == 0p);
-                        /* paranoid */ verify(tail(this)->link.next == 0p);
-                        /* paranoid */ verify(head(this)->link.prev == 0p);
+static inline void push( __intrusive_lane_t & this, $thread * node ) {
+        /* paranoid */ verify( this.lock );
+        /* paranoid */ verify( node->link.next == 0p );
+        /* paranoid */ verify( node->link.ts   == 0  );
+        /* paranoid */ verify( this.prev->link.next == 0p );
+        /* paranoid */ verify( this.prev->link.ts   == 0  );
+        if( this.anchor.next == 0p ) {
+                /* paranoid */ verify( this.anchor.next == 0p );
+                /* paranoid */ verify( this.anchor.ts   == 0  );
+                /* paranoid */ verify( this.prev == mock_head( this ) );
+        } else {
+                /* paranoid */ verify( this.anchor.next != 0p );
+                /* paranoid */ verify( this.anchor.ts   != 0  );
+                /* paranoid */ verify( this.prev != mock_head( this ) );
+        }
 
-                        if(this.before.link.ts == 0l) {
-                                /* paranoid */ verify(tail(this)->link.prev == head(this));
-                                /* paranoid */ verify(head(this)->link.next == tail(this));
-                        } else {
-                                /* paranoid */ verify(tail(this)->link.prev != head(this));
-                                /* paranoid */ verify(head(this)->link.next != tail(this));
-                        }
-                #endif
-
-                // Get the relevant nodes locally
-                $thread * tail = tail(this);
-                $thread * prev = tail->link.prev;
-
-                // Do the push
-                node->link.next = tail;
-                node->link.prev = prev;
-                prev->link.next = node;
-                tail->link.prev = node;
-
-                // Update stats
-                #if !defined(__CFA_NO_SCHED_STATS__)
-                        this.stat.diff++;
-                        this.stat.push++;
-                #endif
-
-                verify(node->link.next == tail(this));
-
-                // Check if the queue used to be empty
-                if(this.before.link.ts == 0l) {
-                        this.before.link.ts = node->link.ts;
-                        /* paranoid */ verify(node->link.prev == head(this));
-                        return true;
-                }
-                return false;
+        // Get the relevant nodes locally
+        this.prev->link.next = node;
+        this.prev->link.ts   = rdtscl();
+        this.prev = node;
+        #if !defined(__CFA_NO_STATISTICS__)
+                this.cnt++;
         #endif
 }
@@ -170 +89 @@
 // returns popped
 // returns true of lane was empty before push, false otherwise
-$thread * pop(__intrusive_lane_t & this) {
-        /* paranoid */ verify(this.lock);
-        #if defined(USE_MPSC)
-                inline $thread * volatile & ?`next ( $thread * this )  __attribute__((const)) {
-                        return this->link.next;
-                }
-                return pop(this.queue);
-        #else
-                /* paranoid */ verify(this.before.link.ts != 0ul);
+static inline [* $thread, unsigned long long] pop( __intrusive_lane_t & this ) {
+        /* paranoid */ verify( this.lock );
+        /* paranoid */ verify( this.anchor.next != 0p );
+        /* paranoid */ verify( this.anchor.ts   != 0  );
 
-                // Get anchors locally
-                $thread * head = head(this);
-                $thread * tail = tail(this);
+        // Get the relevant nodes locally
+        unsigned long long ts = this.anchor.ts;
+        $thread * node = this.anchor.next;
+        this.anchor.next = node->link.next;
+        this.anchor.ts   = node->link.ts;
+        bool is_empty = this.anchor.ts == 0;
+        node->link.next = 0p;
+        node->link.ts   = 0;
+        #if !defined(__CFA_NO_STATISTICS__)
+                this.cnt--;
+        #endif
 
-                // Get the relevant nodes locally
-                $thread * node = head->link.next;
-                $thread * next = node->link.next;
+        // Update head time stamp
+        if(is_empty) this.prev = mock_head( this );
 
-                /* paranoid */ verify(node != tail);
-                /* paranoid */ verify(node);
-
-                // Do the pop
-                head->link.next = next;
-                next->link.prev = head;
-                node->link.next = 0p;
-                node->link.prev = 0p;
-
-                // Update head time stamp
-                this.before.link.ts = next->link.ts;
-
-                // Update stats
-                #ifndef __CFA_NO_SCHED_STATS__
-                        this.stat.diff--;
-                        this.stat.pop ++;
-                #endif
-
-                // Check if we emptied list and return accordingly
-                /* paranoid */ verify(tail(this)->link.next == 0p);
-                /* paranoid */ verify(head(this)->link.prev == 0p);
-                if(next == tail) {
-                        /* paranoid */ verify(this.before.link.ts == 0);
-                        /* paranoid */ verify(tail(this)->link.prev == head(this));
-                        /* paranoid */ verify(head(this)->link.next == tail(this));
-                        return node;
-                }
-                else {
-                        /* paranoid */ verify(next->link.ts != 0);
-                        /* paranoid */ verify(tail(this)->link.prev != head(this));
-                        /* paranoid */ verify(head(this)->link.next != tail(this));
-                        /* paranoid */ verify(this.before.link.ts != 0);
-                        return node;
-                }
-        #endif
+        /* paranoid */ verify( node->link.next == 0p );
+        /* paranoid */ verify( node->link.ts   == 0  );
+        return [node, ts];
 }
 
 // Check whether or not list is empty
 static inline bool is_empty(__intrusive_lane_t & this) {
-        #if defined(USE_MPSC)
-                return this.queue.head == 0p;
-        #else
-                // Cannot verify here since it may not be locked
-                return this.before.link.ts == 0;
-        #endif
+        return this.anchor.ts == 0;
 }
 
 // Return the timestamp
 static inline unsigned long long ts(__intrusive_lane_t & this) {
-        #if defined(USE_MPSC)
-                $thread * tl = this.queue.head;
-                if(!tl) return -1ull;
-                return tl->link.ts;
-        #else
-                // Cannot verify here since it may not be locked
-                return this.before.link.ts;
-        #endif
+        // Cannot verify here since it may not be locked
+        return this.anchor.ts;
 }
-
-// Aligned timestamps which are used by the relaxed ready queue
-struct __attribute__((aligned(128))) __timestamp_t {
-        volatile unsigned long long tv;
-};
-
-void  ?{}(__timestamp_t & this) { this.tv = 0; }
-void ^?{}(__timestamp_t & this) {}

libcfa/src/concurrency/stats.cfa

@@ -19 +19 @@
                 stats->ready.pop.local .attempt = 0;
                 stats->ready.pop.local .success = 0;
-                stats->ready.pop.local .elock   = 0;
-                stats->ready.pop.local .eempty  = 0;
-                stats->ready.pop.local .espec   = 0;
                 stats->ready.pop.help  .attempt = 0;
                 stats->ready.pop.help  .success = 0;
-                stats->ready.pop.help  .elock   = 0;
-                stats->ready.pop.help  .eempty  = 0;
-                stats->ready.pop.help  .espec   = 0;
                 stats->ready.pop.steal .attempt = 0;
                 stats->ready.pop.steal .success = 0;
-                stats->ready.pop.steal .elock   = 0;
-                stats->ready.pop.steal .eempty  = 0;
-                stats->ready.pop.steal .espec   = 0;
                 stats->ready.pop.search.attempt = 0;
                 stats->ready.pop.search.success = 0;
-                stats->ready.pop.search.elock   = 0;
-                stats->ready.pop.search.eempty  = 0;
-                stats->ready.pop.search.espec   = 0;
                 stats->ready.threads.migration = 0;
                 stats->ready.threads.extunpark = 0;
                 stats->ready.threads.threads   = 0;
+                stats->ready.threads.cthreads  = 0;
                 stats->ready.sleep.halts   = 0;
                 stats->ready.sleep.cancels = 0;
@@ -59 +48 @@
                         stats->io.calls.completed   = 0;
                         stats->io.calls.errors.busy = 0;
-                        stats->io.poller.sleeps     = 0;
                 #endif
 
@@ -68 +56 @@
         }
 
+        static inline void tally_one( volatile uint64_t * agg, volatile uint64_t * val) {
+                uint64_t add = __atomic_exchange_n(val, 0_l64u, __ATOMIC_RELAXED);
+                __atomic_fetch_add(agg, add, __ATOMIC_RELAXED);
+        }
+
+        static inline void tally_one( volatile int64_t * agg, volatile int64_t * val) {
+                int64_t add = __atomic_exchange_n(val, 0_l64, __ATOMIC_RELAXED);
+                __atomic_fetch_add(agg, add, __ATOMIC_RELAXED);
+        }
+
         void __tally_stats( struct __stats_t * cltr, struct __stats_t * proc ) {
-                __atomic_fetch_add( &cltr->ready.push.local.attempt, proc->ready.push.local.attempt, __ATOMIC_SEQ_CST ); proc->ready.push.local.attempt = 0;
-                __atomic_fetch_add( &cltr->ready.push.local.success, proc->ready.push.local.success, __ATOMIC_SEQ_CST ); proc->ready.push.local.success = 0;
-                __atomic_fetch_add( &cltr->ready.push.share.attempt, proc->ready.push.share.attempt, __ATOMIC_SEQ_CST ); proc->ready.push.share.attempt = 0;
-                __atomic_fetch_add( &cltr->ready.push.share.success, proc->ready.push.share.success, __ATOMIC_SEQ_CST ); proc->ready.push.share.success = 0;
-                __atomic_fetch_add( &cltr->ready.push.extrn.attempt, proc->ready.push.extrn.attempt, __ATOMIC_SEQ_CST ); proc->ready.push.extrn.attempt = 0;
-                __atomic_fetch_add( &cltr->ready.push.extrn.success, proc->ready.push.extrn.success, __ATOMIC_SEQ_CST ); proc->ready.push.extrn.success = 0;
-                __atomic_fetch_add( &cltr->ready.pop.local .attempt, proc->ready.pop.local .attempt, __ATOMIC_SEQ_CST ); proc->ready.pop.local .attempt = 0;
-                __atomic_fetch_add( &cltr->ready.pop.local .success, proc->ready.pop.local .success, __ATOMIC_SEQ_CST ); proc->ready.pop.local .success = 0;
-                __atomic_fetch_add( &cltr->ready.pop.local .elock  , proc->ready.pop.local .elock  , __ATOMIC_SEQ_CST ); proc->ready.pop.local .elock   = 0;
-                __atomic_fetch_add( &cltr->ready.pop.local .eempty , proc->ready.pop.local .eempty , __ATOMIC_SEQ_CST ); proc->ready.pop.local .eempty  = 0;
-                __atomic_fetch_add( &cltr->ready.pop.local .espec  , proc->ready.pop.local .espec  , __ATOMIC_SEQ_CST ); proc->ready.pop.local .espec   = 0;
-                __atomic_fetch_add( &cltr->ready.pop.help  .attempt, proc->ready.pop.help  .attempt, __ATOMIC_SEQ_CST ); proc->ready.pop.help  .attempt = 0;
-                __atomic_fetch_add( &cltr->ready.pop.help  .success, proc->ready.pop.help  .success, __ATOMIC_SEQ_CST ); proc->ready.pop.help  .success = 0;
-                __atomic_fetch_add( &cltr->ready.pop.help  .elock  , proc->ready.pop.help  .elock  , __ATOMIC_SEQ_CST ); proc->ready.pop.help  .elock   = 0;
-                __atomic_fetch_add( &cltr->ready.pop.help  .eempty , proc->ready.pop.help  .eempty , __ATOMIC_SEQ_CST ); proc->ready.pop.help  .eempty  = 0;
-                __atomic_fetch_add( &cltr->ready.pop.help  .espec  , proc->ready.pop.help  .espec  , __ATOMIC_SEQ_CST ); proc->ready.pop.help  .espec   = 0;
-                __atomic_fetch_add( &cltr->ready.pop.steal .attempt, proc->ready.pop.steal .attempt, __ATOMIC_SEQ_CST ); proc->ready.pop.steal .attempt = 0;
-                __atomic_fetch_add( &cltr->ready.pop.steal .success, proc->ready.pop.steal .success, __ATOMIC_SEQ_CST ); proc->ready.pop.steal .success = 0;
-                __atomic_fetch_add( &cltr->ready.pop.steal .elock  , proc->ready.pop.steal .elock  , __ATOMIC_SEQ_CST ); proc->ready.pop.steal .elock   = 0;
-                __atomic_fetch_add( &cltr->ready.pop.steal .eempty , proc->ready.pop.steal .eempty , __ATOMIC_SEQ_CST ); proc->ready.pop.steal .eempty  = 0;
-                __atomic_fetch_add( &cltr->ready.pop.steal .espec  , proc->ready.pop.steal .espec  , __ATOMIC_SEQ_CST ); proc->ready.pop.steal .espec   = 0;
-                __atomic_fetch_add( &cltr->ready.pop.search.attempt, proc->ready.pop.search.attempt, __ATOMIC_SEQ_CST ); proc->ready.pop.search.attempt = 0;
-                __atomic_fetch_add( &cltr->ready.pop.search.success, proc->ready.pop.search.success, __ATOMIC_SEQ_CST ); proc->ready.pop.search.success = 0;
-                __atomic_fetch_add( &cltr->ready.pop.search.elock  , proc->ready.pop.search.elock  , __ATOMIC_SEQ_CST ); proc->ready.pop.search.elock   = 0;
-                __atomic_fetch_add( &cltr->ready.pop.search.eempty , proc->ready.pop.search.eempty , __ATOMIC_SEQ_CST ); proc->ready.pop.search.eempty  = 0;
-                __atomic_fetch_add( &cltr->ready.pop.search.espec  , proc->ready.pop.search.espec  , __ATOMIC_SEQ_CST ); proc->ready.pop.search.espec   = 0;
-                __atomic_fetch_add( &cltr->ready.threads.migration , proc->ready.threads.migration , __ATOMIC_SEQ_CST ); proc->ready.threads.migration  = 0;
-                __atomic_fetch_add( &cltr->ready.threads.extunpark , proc->ready.threads.extunpark , __ATOMIC_SEQ_CST ); proc->ready.threads.extunpark  = 0;
-                __atomic_fetch_add( &cltr->ready.threads.threads   , proc->ready.threads.threads   , __ATOMIC_SEQ_CST ); proc->ready.threads.threads    = 0;
-                __atomic_fetch_add( &cltr->ready.sleep.halts       , proc->ready.sleep.halts       , __ATOMIC_SEQ_CST ); proc->ready.sleep.halts        = 0;
-                __atomic_fetch_add( &cltr->ready.sleep.cancels     , proc->ready.sleep.cancels     , __ATOMIC_SEQ_CST ); proc->ready.sleep.cancels      = 0;
-                __atomic_fetch_add( &cltr->ready.sleep.wakes       , proc->ready.sleep.wakes       , __ATOMIC_SEQ_CST ); proc->ready.sleep.wakes        = 0;
-                __atomic_fetch_add( &cltr->ready.sleep.exits       , proc->ready.sleep.exits       , __ATOMIC_SEQ_CST ); proc->ready.sleep.exits        = 0;
+                tally_one( &cltr->ready.push.local.attempt, &proc->ready.push.local.attempt );
+                tally_one( &cltr->ready.push.local.success, &proc->ready.push.local.success );
+                tally_one( &cltr->ready.push.share.attempt, &proc->ready.push.share.attempt );
+                tally_one( &cltr->ready.push.share.success, &proc->ready.push.share.success );
+                tally_one( &cltr->ready.push.extrn.attempt, &proc->ready.push.extrn.attempt );
+                tally_one( &cltr->ready.push.extrn.success, &proc->ready.push.extrn.success );
+                tally_one( &cltr->ready.pop.local .attempt, &proc->ready.pop.local .attempt );
+                tally_one( &cltr->ready.pop.local .success, &proc->ready.pop.local .success );
+                tally_one( &cltr->ready.pop.help  .attempt, &proc->ready.pop.help  .attempt );
+                tally_one( &cltr->ready.pop.help  .success, &proc->ready.pop.help  .success );
+                tally_one( &cltr->ready.pop.steal .attempt, &proc->ready.pop.steal .attempt );
+                tally_one( &cltr->ready.pop.steal .success, &proc->ready.pop.steal .success );
+                tally_one( &cltr->ready.pop.search.attempt, &proc->ready.pop.search.attempt );
+                tally_one( &cltr->ready.pop.search.success, &proc->ready.pop.search.success );
+                tally_one( &cltr->ready.threads.migration , &proc->ready.threads.migration  );
+                tally_one( &cltr->ready.threads.extunpark , &proc->ready.threads.extunpark  );
+                tally_one( &cltr->ready.threads.threads   , &proc->ready.threads.threads    );
+                tally_one( &cltr->ready.threads.cthreads  , &proc->ready.threads.cthreads   );
+                tally_one( &cltr->ready.sleep.halts       , &proc->ready.sleep.halts        );
+                tally_one( &cltr->ready.sleep.cancels     , &proc->ready.sleep.cancels      );
+                tally_one( &cltr->ready.sleep.wakes       , &proc->ready.sleep.wakes        );
+                tally_one( &cltr->ready.sleep.exits       , &proc->ready.sleep.exits        );
 
                 #if defined(CFA_HAVE_LINUX_IO_URING_H)
-                        __atomic_fetch_add( &cltr->io.alloc.fast       , proc->io.alloc.fast       , __ATOMIC_SEQ_CST ); proc->io.alloc.fast        = 0;
-                        __atomic_fetch_add( &cltr->io.alloc.slow       , proc->io.alloc.slow       , __ATOMIC_SEQ_CST ); proc->io.alloc.slow        = 0;
-                        __atomic_fetch_add( &cltr->io.alloc.fail       , proc->io.alloc.fail       , __ATOMIC_SEQ_CST ); proc->io.alloc.fail        = 0;
-                        __atomic_fetch_add( &cltr->io.alloc.revoke     , proc->io.alloc.revoke     , __ATOMIC_SEQ_CST ); proc->io.alloc.revoke      = 0;
-                        __atomic_fetch_add( &cltr->io.alloc.block      , proc->io.alloc.block      , __ATOMIC_SEQ_CST ); proc->io.alloc.block       = 0;
-                        __atomic_fetch_add( &cltr->io.submit.fast      , proc->io.submit.fast      , __ATOMIC_SEQ_CST ); proc->io.submit.fast       = 0;
-                        __atomic_fetch_add( &cltr->io.submit.slow      , proc->io.submit.slow      , __ATOMIC_SEQ_CST ); proc->io.submit.slow       = 0;
-                        __atomic_fetch_add( &cltr->io.flush.external   , proc->io.flush.external   , __ATOMIC_SEQ_CST ); proc->io.flush.external    = 0;
-                        __atomic_fetch_add( &cltr->io.calls.flush      , proc->io.calls.flush      , __ATOMIC_SEQ_CST ); proc->io.calls.flush       = 0;
-                        __atomic_fetch_add( &cltr->io.calls.submitted  , proc->io.calls.submitted  , __ATOMIC_SEQ_CST ); proc->io.calls.submitted   = 0;
-                        __atomic_fetch_add( &cltr->io.calls.drain      , proc->io.calls.drain      , __ATOMIC_SEQ_CST ); proc->io.calls.drain       = 0;
-                        __atomic_fetch_add( &cltr->io.calls.completed  , proc->io.calls.completed  , __ATOMIC_SEQ_CST ); proc->io.calls.completed   = 0;
-                        __atomic_fetch_add( &cltr->io.calls.errors.busy, proc->io.calls.errors.busy, __ATOMIC_SEQ_CST ); proc->io.calls.errors.busy = 0;
-                        __atomic_fetch_add( &cltr->io.poller.sleeps    , proc->io.poller.sleeps    , __ATOMIC_SEQ_CST ); proc->io.poller.sleeps     = 0;
+                        tally_one( &cltr->io.alloc.fast       , &proc->io.alloc.fast        );
+                        tally_one( &cltr->io.alloc.slow       , &proc->io.alloc.slow        );
+                        tally_one( &cltr->io.alloc.fail       , &proc->io.alloc.fail        );
+                        tally_one( &cltr->io.alloc.revoke     , &proc->io.alloc.revoke      );
+                        tally_one( &cltr->io.alloc.block      , &proc->io.alloc.block       );
+                        tally_one( &cltr->io.submit.fast      , &proc->io.submit.fast       );
+                        tally_one( &cltr->io.submit.slow      , &proc->io.submit.slow       );
+                        tally_one( &cltr->io.flush.external   , &proc->io.flush.external    );
+                        tally_one( &cltr->io.calls.flush      , &proc->io.calls.flush       );
+                        tally_one( &cltr->io.calls.submitted  , &proc->io.calls.submitted   );
+                        tally_one( &cltr->io.calls.drain      , &proc->io.calls.drain       );
+                        tally_one( &cltr->io.calls.completed  , &proc->io.calls.completed   );
+                        tally_one( &cltr->io.calls.errors.busy, &proc->io.calls.errors.busy );
                 #endif
         }
@@ -130 +116 @@
                 if( flags & CFA_STATS_READY_Q ) {
 
-                        sstr | "----- " | type | "\"" | name | "\" (" | "" | id | "" | ") - Ready Q Stats -----";
+                        sstr | "----- " | type | " \"" | name | "\" (" | "" | id | "" | ") - Ready Q Stats -----";
 
                         uint64_t totalR = ready.pop.local.success + ready.pop.help.success + ready.pop.steal.success + ready.pop.search.success;
                         uint64_t totalS = ready.push.local.success + ready.push.share.success + ready.push.extrn.success;
-                        sstr | "- totals   : " | eng3(totalR) | "run," | eng3(totalS) | "schd (" | eng3(ready.push.extrn.success) | "ext," | eng3(ready.threads.migration) | "mig," | eng3(ready.threads.extunpark) | " eupk)";
+                        sstr | "- totals   : " | eng3(totalR) | "run," | eng3(totalS) | "schd (" | eng3(ready.push.extrn.success) | "ext,"
+                             | eng3(ready.threads.migration) | "mig," | eng3(ready.threads.extunpark) | " eupk," | ready.threads.threads | " t," | ready.threads.cthreads | " cthr)";
 
                         double push_len = ((double)ready.push.local.attempt + ready.push.share.attempt + ready.push.extrn.attempt) / totalS;
@@ -147 +134 @@
                         double rLcl_pc = (100.0 * (double)ready.pop.local .success) / totalR;
                         sstr | "- local    : " | eng3(ready.pop.local .success) | "-"| ws(3, 3, rLcl_pc) | '%'
-                             | " (" | eng3(ready.pop.local .attempt) | " try," | eng3(ready.pop.local .espec) | " spc," | eng3(ready.pop.local .elock) | " lck," | eng3(ready.pop.local .eempty) | " ept)";
+                             | " (" | eng3(ready.pop.local .attempt) | " try)";
                         double rHlp_pc = (100.0 * (double)ready.pop.help  .success) / totalR;
                         sstr | "- help     : " | eng3(ready.pop.help  .success) | "-"| ws(3, 3, rHlp_pc) | '%'
-                             | " (" | eng3(ready.pop.help  .attempt) | " try," | eng3(ready.pop.help  .espec) | " spc," | eng3(ready.pop.help  .elock) | " lck," | eng3(ready.pop.help  .eempty) | " ept)";
+                             | " (" | eng3(ready.pop.help  .attempt) | " try)";
                         double rStl_pc = (100.0 * (double)ready.pop.steal .success) / totalR;
                         sstr | "- steal    : " | eng3(ready.pop.steal .success) | "-"| ws(3, 3, rStl_pc) | '%'
-                             | " (" | eng3(ready.pop.steal .attempt) | " try," | eng3(ready.pop.steal .espec) | " spc," | eng3(ready.pop.steal .elock) | " lck," | eng3(ready.pop.steal .eempty) | " ept)";
+                             | " (" | eng3(ready.pop.steal .attempt) | " try)";
                         double rSch_pc = (100.0 * (double)ready.pop.search.success) / totalR;
                         sstr | "- search   : " | eng3(ready.pop.search.success) | "-"| ws(3, 3, rSch_pc) | '%'
-                             | " (" | eng3(ready.pop.search.attempt) | " try," | eng3(ready.pop.search.espec) | " spc," | eng3(ready.pop.search.elock) | " lck," | eng3(ready.pop.search.eempty) | " ept)";
+                             | " (" | eng3(ready.pop.search.attempt) | " try)";
 
                         sstr | "- Idle Slp : " | eng3(ready.sleep.halts) | "halt," | eng3(ready.sleep.cancels) | "cancel," | eng3(ready.sleep.wakes) | "wake," | eng3(ready.sleep.exits) | "exit";
@@ -164 +151 @@
                 #if defined(CFA_HAVE_LINUX_IO_URING_H)
                         if( flags & CFA_STATS_IO ) {
-                                sstr | "----- " | type | "\"" | name | "\" (" | "" | id | "" | ") - I/O Stats -----";
+                                sstr | "----- " | type | " \"" | name | "\" (" | "" | id | "" | ") - I/O Stats -----";
 
                                 uint64_t total_allocs = io.alloc.fast + io.alloc.slow;
-                                double avgfasta = (100.0 * (double)io.alloc.fast) / total_allocs;
-                                sstr | "- total allocations : " | eng3(io.alloc.fast) | "fast," | eng3(io.alloc.slow) | "slow (" | ws(3, 3, avgfasta) | "%)";
-                                sstr | "-     failures      : " | eng3(io.alloc.fail) | "oom, " | eng3(io.alloc.revoke) | "rvk, " | eng3(io.alloc.block) | "blk";
 
                                 uint64_t total_submits = io.submit.fast + io.submit.slow;
-                                double avgfasts = (100.0 * (double)io.submit.fast) / total_submits;
-                                sstr | "- total submits     : " | eng3(io.submit.fast) | "fast," | eng3(io.submit.slow) | "slow (" | ws(3, 3, avgfasts) | "%)";
-                                sstr | "- flush external    : " | eng3(io.flush.external);
-
-                                sstr | "- io_uring_enter    : " | eng3(io.calls.flush) | " (" | eng3(io.calls.drain) | ", " | eng3(io.calls.errors.busy) | " EBUSY)";
+                                sstr | "- totals : allc" | eng3(io.alloc .fast) | nonl;
+                                if(io.alloc.slow) {
+                                        double avgfasta = (100.0 * (double)io.alloc.fast) / total_allocs;
+                                        sstr | "fast," | eng3(io.alloc .slow) | "slow (" | ws(3, 3, avgfasta) | "%)" | nonl;
+                                }
+                                sstr | " - subm" | eng3(io.submit.fast) | nonl;
+                                if(io.alloc.slow) {
+                                        double avgfasts = (100.0 * (double)io.submit.fast) / total_submits;
+                                        sstr | "fast," | eng3(io.submit.slow) | "slow (" | ws(3, 3, avgfasts) | "%)" | nonl;
+                                }
+                                sstr | nl;
+
+                                if(io.alloc.fail || io.alloc.revoke || io.alloc.block)
+                                        sstr | "-     failures      : " | eng3(io.alloc.fail) | "oom, " | eng3(io.alloc.revoke) | "rvk, " | eng3(io.alloc.block) | "blk";
+                                if(io.flush.external)
+                                        sstr | "- flush external    : " | eng3(io.flush.external);
 
                                 double avgsubs = ((double)io.calls.submitted) / io.calls.flush;
                                 double avgcomp = ((double)io.calls.completed) / io.calls.drain;
-                                sstr | "-     submits       : " | eng3(io.calls.submitted) | "(" | ws(3, 3, avgsubs) | "/flush)";
-                                sstr | "-     completes     : " | eng3(io.calls.completed) | "(" | ws(3, 3, avgcomp) | "/drain)";
-
-                                sstr | "- poller sleeping   : " | eng3(io.poller.sleeps);
+                                sstr | "- syscll : "
+                                     |   " sub " | eng3(io.calls.flush) | "/" | eng3(io.calls.submitted) | "(" | ws(3, 3, avgsubs) | "/flush)"
+                                     | " - cmp " | eng3(io.calls.drain) | "/" | eng3(io.calls.completed) | "(" | ws(3, 3, avgcomp) | "/drain)"
+                                     | " - " | eng3(io.calls.errors.busy) | " EBUSY";
                                 sstr | nl;
                         }

libcfa/src/concurrency/stats.hfa

@@ -2 +2 @@
 
 // #define CFA_STATS_ARRAY 10000
+// #define __CFA_NO_STATISTICS__
 
 #include <stdint.h>
@@ -22 +23 @@
                 // number of successes at poping
                 volatile uint64_t success;
-
-                // number of attempts failed due to the lock being held
-                volatile uint64_t elock;
-
-                // number of attempts failed due to the queue being empty (lock held)
-                volatile uint64_t eempty;
-
-                // number of attempts failed due to the queue looking empty (lock not held)
-                volatile uint64_t espec;
         };
 
@@ -71 +63 @@
                         volatile uint64_t migration;
                         volatile uint64_t extunpark;
-                        volatile  int64_t threads; // number of threads in the system, includes only local change
+                        volatile  int64_t threads;  // number of threads in the system, includes only local change
+                        volatile  int64_t cthreads; // number of threads in the system, includes only local change
                 } threads;
                 struct {

libcfa/src/concurrency/thread.cfa

@@ -38 +38 @@
         curr_cluster = &cl;
         link.next = 0p;
-        link.prev = 0p;
-        link.preferred = -1u;
+        link.ts   = 0;
+        preferred = -1u;
         last_proc = 0p;
         #if defined( __CFA_WITH_VERIFY__ )