Changes in / [730f4f1:4fa44e7]

Files:

• libcfa/src/Makefile.am (modified) (1 diff)
• libcfa/src/Makefile.in (modified) (4 diffs)
• libcfa/src/bits/debug.hfa (modified) (1 diff)
• libcfa/src/bits/defs.hfa (modified) (1 diff)
• libcfa/src/concurrency/invoke.h (modified) (3 diffs)
• libcfa/src/concurrency/kernel.cfa (modified) (15 diffs)
• libcfa/src/concurrency/kernel.hfa (modified) (3 diffs)
• libcfa/src/concurrency/kernel_private.hfa (modified) (2 diffs)
• libcfa/src/concurrency/monitor.cfa (modified) (5 diffs)
• libcfa/src/concurrency/preemption.cfa (modified) (1 diff)
• libcfa/src/concurrency/ready_queue.cfa (added)
• libcfa/src/concurrency/thread.cfa (modified) (1 diff)
• libcfa/src/stdhdr/assert.h (modified) (1 diff)
• tests/concurrent/examples/datingService.cfa (modified) (2 diffs)
• tests/concurrent/waitfor/when.cfa (modified) (1 diff)

libcfa/src/Makefile.am

@@ -48 +48 @@
 thread_headers_nosrc = concurrency/invoke.h
 thread_headers = concurrency/coroutine.hfa concurrency/thread.hfa concurrency/kernel.hfa concurrency/monitor.hfa concurrency/mutex.hfa
-thread_libsrc = concurrency/CtxSwitch-@ARCHITECTURE@.S concurrency/alarm.cfa concurrency/invoke.c concurrency/io.cfa concurrency/preemption.cfa ${thread_headers:.hfa=.cfa}
+thread_libsrc = concurrency/CtxSwitch-@ARCHITECTURE@.S concurrency/alarm.cfa concurrency/invoke.c concurrency/io.cfa concurrency/preemption.cfa concurrency/ready_queue.cfa ${thread_headers:.hfa=.cfa}
 else
 headers =

libcfa/src/Makefile.in

@@ -166 +166 @@
         concurrency/CtxSwitch-@ARCHITECTURE@.S concurrency/alarm.cfa \
         concurrency/invoke.c concurrency/io.cfa \
-        concurrency/preemption.cfa concurrency/coroutine.cfa \
-        concurrency/thread.cfa concurrency/kernel.cfa \
-        concurrency/monitor.cfa concurrency/mutex.cfa
+        concurrency/preemption.cfa concurrency/ready_queue.cfa \
+        concurrency/coroutine.cfa concurrency/thread.cfa \
+        concurrency/kernel.cfa concurrency/monitor.cfa \
+        concurrency/mutex.cfa
 @BUILDLIB_TRUE@am__objects_3 = concurrency/coroutine.lo \
 @BUILDLIB_TRUE@ concurrency/thread.lo concurrency/kernel.lo \
@@ -176 +177 @@
 @BUILDLIB_TRUE@ concurrency/alarm.lo concurrency/invoke.lo \
 @BUILDLIB_TRUE@ concurrency/io.lo concurrency/preemption.lo \
-@BUILDLIB_TRUE@ $(am__objects_3)
+@BUILDLIB_TRUE@ concurrency/ready_queue.lo $(am__objects_3)
 am_libcfathread_la_OBJECTS = $(am__objects_4)
 libcfathread_la_OBJECTS = $(am_libcfathread_la_OBJECTS)
@@ -477 +478 @@
 @BUILDLIB_FALSE@thread_headers = 
 @BUILDLIB_TRUE@thread_headers = concurrency/coroutine.hfa concurrency/thread.hfa concurrency/kernel.hfa concurrency/monitor.hfa concurrency/mutex.hfa
-@BUILDLIB_TRUE@thread_libsrc = concurrency/CtxSwitch-@ARCHITECTURE@.S concurrency/alarm.cfa concurrency/invoke.c concurrency/io.cfa concurrency/preemption.cfa ${thread_headers:.hfa=.cfa}
+@BUILDLIB_TRUE@thread_libsrc = concurrency/CtxSwitch-@ARCHITECTURE@.S concurrency/alarm.cfa concurrency/invoke.c concurrency/io.cfa concurrency/preemption.cfa concurrency/ready_queue.cfa ${thread_headers:.hfa=.cfa}
 
 #----------------------------------------------------------------------------------------------------------------
@@ -615 +616 @@
         concurrency/$(DEPDIR)/$(am__dirstamp)
 concurrency/preemption.lo: concurrency/$(am__dirstamp) \
+        concurrency/$(DEPDIR)/$(am__dirstamp)
+concurrency/ready_queue.lo: concurrency/$(am__dirstamp) \
         concurrency/$(DEPDIR)/$(am__dirstamp)
 concurrency/coroutine.lo: concurrency/$(am__dirstamp) \

libcfa/src/bits/debug.hfa

@@ -52 +52 @@
                 || defined(__CFA_DEBUG_PRINT_IO__) || defined(__CFA_DEBUG_PRINT_IO_CORE__) \
                 || defined(__CFA_DEBUG_PRINT_MONITOR__) || defined(__CFA_DEBUG_PRINT_PREEMPTION__) \
-                || defined(__CFA_DEBUG_PRINT_RUNTIME_CORE__) || defined(__CFA_DEBUG_PRINT_EXCEPTION__)
+                || defined(__CFA_DEBUG_PRINT_RUNTIME_CORE__) || defined(__CFA_DEBUG_PRINT_EXCEPTION__) \
+                || defined(__CFA_DEBUG_PRINT_READY_QUEUE__)
         #include <stdio.h>
         #include <unistd.h>

libcfa/src/bits/defs.hfa

@@ -54 +54 @@
     return ( (unsigned long long)lo)|( ((unsigned long long)hi)<<32 );
 }
+
+// #define __CFA_NO_BIT_TEST_AND_SET__
+
+#if defined( __i386 )
+static inline bool __atomic_bts(volatile unsigned long int * target, unsigned long int bit ) {
+        #if defined(__CFA_NO_BIT_TEST_AND_SET__)
+        unsigned long int mask = 1ul << bit;
+        unsigned long int ret = __atomic_fetch_or(target, mask, (int)__ATOMIC_RELAXED);
+        return (ret & mask) != 0;
+    #else
+        int result = 0;
+        asm volatile(
+            "LOCK btsl %[bit], %[target]\n\t"
+            : "=@ccc" (result)
+            : [target] "m" (*target), [bit] "r" (bit)
+        );
+        return result != 0;
+    #endif
+}
+
+static inline bool __atomic_btr(volatile unsigned long int * target, unsigned long int bit ) {
+        #if defined(__CFA_NO_BIT_TEST_AND_SET__)
+        unsigned long int mask = 1ul << bit;
+        unsigned long int ret = __atomic_fetch_and(target, ~mask, (int)__ATOMIC_RELAXED);
+        return (ret & mask) != 0;
+        #else
+        int result = 0;
+        asm volatile(
+            "LOCK btrl %[bit], %[target]\n\t"
+            :"=@ccc" (result)
+            : [target] "m" (*target), [bit] "r" (bit)
+        );
+        return result != 0;
+    #endif
+}
+#elif defined( __x86_64 )
+static inline bool __atomic_bts(volatile unsigned long long int * target, unsigned long long int bit ) {
+        #if defined(__CFA_NO_BIT_TEST_AND_SET__)
+        unsigned long long int mask = 1ul << bit;
+        unsigned long long int ret = __atomic_fetch_or(target, mask, (int)__ATOMIC_RELAXED);
+        return (ret & mask) != 0;
+    #else
+        int result = 0;
+        asm volatile(
+            "LOCK btsq %[bit], %[target]\n\t"
+            : "=@ccc" (result)
+            : [target] "m" (*target), [bit] "r" (bit)
+        );
+        return result != 0;
+    #endif
+}
+
+static inline bool __atomic_btr(volatile unsigned long long int * target, unsigned long long int bit ) {
+        #if defined(__CFA_NO_BIT_TEST_AND_SET__)
+        unsigned long long int mask = 1ul << bit;
+        unsigned long long int ret = __atomic_fetch_and(target, ~mask, (int)__ATOMIC_RELAXED);
+        return (ret & mask) != 0;
+        #else
+        int result = 0;
+        asm volatile(
+            "LOCK btrq %[bit], %[target]\n\t"
+            :"=@ccc" (result)
+            : [target] "m" (*target), [bit] "r" (bit)
+        );
+        return result != 0;
+    #endif
+}
+#elif defined( __ARM_ARCH )
+    #error __atomic_bts and __atomic_btr not implemented for arm
+#else
+        #error uknown hardware architecture
+#endif

libcfa/src/concurrency/invoke.h

@@ -161 +161 @@
         };
 
+        // Link lists fields
+        // instrusive link field for threads
+        struct __thread_desc_link {
+                struct $thread * next;
+                struct $thread * prev;
+                unsigned long long ts;
+        };
+
         struct $thread {
                 // Core threading fields
@@ -192 +200 @@
                 // Link lists fields
                 // instrusive link field for threads
-                struct $thread * next;
+                struct __thread_desc_link link;
 
                 struct {
@@ -218 +226 @@
         #ifdef __cforall
         extern "Cforall" {
+
                 static inline $thread *& get_next( $thread & this ) __attribute__((const)) {
-                        return this.next;
+                        return this.link.next;
                 }
 

libcfa/src/concurrency/kernel.cfa

@@ -120 +120 @@
 static void __run_thread(processor * this, $thread * dst);
 static $thread * __halt(processor * this);
-static bool __wake_one(cluster * cltr, bool was_empty);
+static bool __wake_one(cluster * cltr);
 static bool __wake_proc(processor *);
 
@@ -197 +197 @@
         self_mon.recursion = 1;
         self_mon_p = &self_mon;
-        next = 0p;
+        link.next = 0p;
+        link.prev = 0p;
 
         node.next = 0p;
@@ -223 +224 @@
         this.name = name;
         this.cltr = &cltr;
+        id = -1u;
         terminated{ 0 };
         destroyer = 0p;
@@ -260 +262 @@
         this.preemption_rate = preemption_rate;
         ready_queue{};
-        ready_queue_lock{};
+        ready_lock{};
 
         #if !defined(__CFA_NO_STATISTICS__)
@@ -295 +297 @@
         __cfadbg_print_safe(runtime_core, "Kernel : core %p starting\n", this);
 
+        // register the processor unless it's the main thread which is handled in the boot sequence
+        if(this != mainProcessor) {
+                this->id = doregister2(this->cltr, this);
+                ready_queue_grow( this->cltr );
+        }
+
         doregister(this->cltr, this);
 
@@ -318 +326 @@
                                 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
                                 /* paranoid */ verifyf( readyThread->state == Ready || readyThread->preempted != __NO_PREEMPTION, "state : %d, preempted %d\n", readyThread->state, readyThread->preempted);
-                                /* paranoid */ verifyf( readyThread->next == 0p, "Expected null got %p", readyThread->next );
+                                /* paranoid */ verifyf( readyThread->link.next == 0p, "Expected null got %p", readyThread->link.next );
 
                                 // We found a thread run it
@@ -334 +342 @@
         V( this->terminated );
 
+        // unregister the processor unless it's the main thread which is handled in the boot sequence
+        if(this != mainProcessor) {
+                ready_queue_shrink( this->cltr );
+                unregister2(this->cltr, this);
+        }
+        else {
+                // HACK : the coroutine context switch expects this_thread to be set
+                // and it make sense for it to be set in all other cases except here
+                // fake it
+                kernelTLS.this_thread = mainThread;
+        }
+
         __cfadbg_print_safe(runtime_core, "Kernel : core %p terminated\n", this);
 
-        // HACK : the coroutine context switch expects this_thread to be set
-        // and it make sense for it to be set in all other cases except here
-        // fake it
-        if( this == mainProcessor ) kernelTLS.this_thread = mainThread;
+        stats_tls_tally(this->cltr);
 }
 
@@ -591 +608 @@
 // Scheduler routines
 // KERNEL ONLY
-void __schedule_thread( $thread * thrd ) with( *thrd->curr_cluster ) {
+void __schedule_thread( $thread * thrd ) {
+        /* paranoid */ verify( thrd );
+        /* paranoid */ verify( thrd->state != Halted );
         /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
         /* paranoid */ #if defined( __CFA_WITH_VERIFY__ )
-        /* paranoid */ if( thrd->state == Blocked || thrd->state == Start ) assertf( thrd->preempted == __NO_PREEMPTION,
-                          "Error inactive thread marked as preempted, state %d, preemption %d\n", thrd->state, thrd->preempted );
-        /* paranoid */ if( thrd->preempted != __NO_PREEMPTION ) assertf(thrd->state == Active || thrd->state == Rerun,
-                          "Error preempted thread marked as not currently running, state %d, preemption %d\n", thrd->state, thrd->preempted );
+        /* paranoid */  if( thrd->state == Blocked || thrd->state == Start ) assertf( thrd->preempted == __NO_PREEMPTION,
+                                        "Error inactive thread marked as preempted, state %d, preemption %d\n", thrd->state, thrd->preempted );
+        /* paranoid */  if( thrd->preempted != __NO_PREEMPTION ) assertf(thrd->state == Active || thrd->state == Rerun,
+                                        "Error preempted thread marked as not currently running, state %d, preemption %d\n", thrd->state, thrd->preempted );
         /* paranoid */ #endif
-        /* paranoid */ verifyf( thrd->next == 0p, "Expected null got %p", thrd->next );
+        /* paranoid */ verifyf( thrd->link.next == 0p, "Expected null got %p", thrd->link.next );
 
         if (thrd->preempted == __NO_PREEMPTION) thrd->state = Ready;
 
-        lock  ( ready_queue_lock __cfaabi_dbg_ctx2 );
-        bool was_empty = !(ready_queue != 0);
-        append( ready_queue, thrd );
-        unlock( ready_queue_lock );
-
-        __wake_one(thrd->curr_cluster, was_empty);
+        ready_schedule_lock(thrd->curr_cluster, kernelTLS.this_processor);
+                push( thrd->curr_cluster, thrd );
+
+                __wake_one(thrd->curr_cluster);
+        ready_schedule_unlock(thrd->curr_cluster, kernelTLS.this_processor);
 
         /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
@@ -617 +635 @@
         /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
 
-        lock( ready_queue_lock __cfaabi_dbg_ctx2 );
-        $thread * head = pop_head( ready_queue );
-        unlock( ready_queue_lock );
+        ready_schedule_lock(this, kernelTLS.this_processor);
+                $thread * head = pop( this );
+        ready_schedule_unlock(this, kernelTLS.this_processor);
 
         /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
@@ -703 +721 @@
         // If that is the case, abandon the preemption.
         bool preempted = false;
-        if(thrd->next == 0p) {
+        if(thrd->link.next == 0p) {
                 preempted = true;
                 thrd->preempted = reason;
@@ -763 +781 @@
                 pending_preemption = false;
                 kernel_thread = pthread_self();
+                id = -1u;
 
                 runner{ &this };
@@ -772 +791 @@
         mainProcessor = (processor *)&storage_mainProcessor;
         (*mainProcessor){};
+
+        mainProcessor->id = doregister2(mainCluster, mainProcessor);
 
         //initialize the global state variables
@@ -826 +847 @@
         kernel_stop_preemption();
 
+        unregister2(mainCluster, mainProcessor);
+
         // Destroy the main processor and its context in reverse order of construction
         // These were manually constructed so we need manually destroy them
         void ^?{}(processor & this) with( this ){
                 /* paranoid */ verify( this.do_terminate == true );
+                __cfaabi_dbg_print_safe("Kernel : destroyed main processor context %p\n", &runner);
         }
 
@@ -835 +859 @@
 
         // Final step, destroy the main thread since it is no longer needed
+
         // Since we provided a stack to this taxk it will not destroy anything
         /* paranoid */ verify(mainThread->self_cor.stack.storage == (__stack_t*)(((uintptr_t)&storage_mainThreadCtx)| 0x1));
@@ -887 +912 @@
 
 // Wake a thread from the front if there are any
-static bool __wake_one(cluster * this, __attribute__((unused)) bool force) {
-        // if we don't want to force check if we know it's false
-        // if( !this->idles.head && !force ) return false;
-
+static bool __wake_one(cluster * this) {
         // First, lock the cluster idle
         lock( this->idle_lock __cfaabi_dbg_ctx2 );

libcfa/src/concurrency/kernel.hfa

@@ -60 +60 @@
         // Cluster from which to get threads
         struct cluster * cltr;
+        unsigned int id;
 
         // Name of the processor
@@ -92 +93 @@
 
         // Link lists fields
-        struct __dbg_node_proc {
-                struct processor * next;
-                struct processor * prev;
+        struct __dbg_node_cltr {
+                processor * next;
+                processor * prev;
         } node;
 
@@ -119 +120 @@
 // #define CFA_CLUSTER_IO_POLLER_KERNEL_SIDE 1 << 1
 
+
+//-----------------------------------------------------------------------------
+// Cluster Tools
+
+// Cells use by the reader writer lock
+// while not generic it only relies on a opaque pointer
+struct __processor_id;
+
+// Reader-Writer lock protecting the ready-queue
+// while this lock is mostly generic some aspects
+// have been hard-coded to for the ready-queue for
+// simplicity and performance
+struct __clusterRWLock_t {
+        // total cachelines allocated
+        unsigned int max;
+
+        // cachelines currently in use
+        volatile unsigned int alloc;
+
+        // cachelines ready to itereate over
+        // (!= to alloc when thread is in second half of doregister)
+        volatile unsigned int ready;
+
+        // writer lock
+        volatile bool lock;
+
+        // data pointer
+        __processor_id * data;
+};
+
+void  ?{}(__clusterRWLock_t & this);
+void ^?{}(__clusterRWLock_t & this);
+
+// Intrusives lanes which are used by the relaxed ready queue
+struct __attribute__((aligned(128))) __intrusive_lane_t {
+        // spin lock protecting the queue
+        volatile bool lock;
+
+        // anchor for the head and the tail of the queue
+        struct __sentinel_t {
+                // Link lists fields
+                // instrusive link field for threads
+                // must be exactly as in $thread
+                __thread_desc_link link;
+        } before, after;
+
+#if defined(__CFA_WITH_VERIFY__)
+        // id of last processor to acquire the lock
+        // needed only to check for mutual exclusion violations
+        unsigned int last_id;
+
+        // number of items on this list
+        // needed only to check for deadlocks
+        unsigned int count;
+#endif
+
+        // Optional statistic counters
+        #if !defined(__CFA_NO_SCHED_STATS__)
+                struct __attribute__((aligned(64))) {
+                        // difference between number of push and pops
+                        ssize_t diff;
+
+                        // total number of pushes and pops
+                        size_t  push;
+                        size_t  pop ;
+                } stat;
+        #endif
+};
+
+void  ?{}(__intrusive_lane_t & this);
+void ^?{}(__intrusive_lane_t & this);
+
+typedef unsigned long long __cfa_readyQ_mask_t;
+
+// enum {
+//      __cfa_ready_queue_mask_size = (64 - sizeof(size_t)) / sizeof(size_t),
+//      __cfa_max_ready_queues = __cfa_ready_queue_mask_size * 8 * sizeof(size_t)
+// };
+
+#define __cfa_lane_mask_size ((64 - sizeof(size_t)) / sizeof(__cfa_readyQ_mask_t))
+#define __cfa_max_lanes (__cfa_lane_mask_size * 8 * sizeof(__cfa_readyQ_mask_t))
+
+//TODO adjust cache size to ARCHITECTURE
+// Structure holding the relaxed ready queue
+struct __attribute__((aligned(128))) __ready_queue_t {
+        // Data tracking how many/which lanes are used
+        // Aligned to 128 for cache locality
+        struct {
+                // number of non-empty lanes
+                volatile size_t count;
+
+                // bit mask, set bits indentify which lanes are non-empty
+                volatile __cfa_readyQ_mask_t mask[ __cfa_lane_mask_size ];
+        } used;
+
+        // Data tracking the actual lanes
+        // On a seperate cacheline from the used struct since
+        // used can change on each push/pop but this data
+        // only changes on shrink/grow
+        struct __attribute__((aligned(64))) {
+                // Arary of lanes
+                __intrusive_lane_t * volatile data;
+
+                // Number of lanes (empty or not)
+                volatile size_t count;
+        } lanes;
+
+        // Statistics
+        #if !defined(__CFA_NO_STATISTICS__)
+                __attribute__((aligned(64))) struct {
+                        struct {
+                                // Push statistic
+                                struct {
+                                        // number of attemps at pushing something
+                                        volatile size_t attempt;
+
+                                        // number of successes at pushing
+                                        volatile size_t success;
+                                } push;
+
+                                // Pop statistic
+                                struct {
+                                        // number of reads of the mask
+                                        // picking an empty __cfa_readyQ_mask_t counts here
+                                        // but not as an attempt
+                                        volatile size_t maskrds;
+
+                                        // number of attemps at poping something
+                                        volatile size_t attempt;
+
+                                        // number of successes at poping
+                                        volatile size_t success;
+                                } pop;
+                        } pick;
+
+                        // stats on the "used" struct of the queue
+                        // tracks average number of queues that are not empty
+                        // when pushing / poping
+                        struct {
+                                volatile size_t value;
+                                volatile size_t count;
+                        } used;
+                } global_stats;
+
+        #endif
+};
+
+void  ?{}(__ready_queue_t & this);
+void ^?{}(__ready_queue_t & this);
+
 //-----------------------------------------------------------------------------
 // Cluster
 struct cluster {
         // Ready queue locks
-        __spinlock_t ready_queue_lock;
+        __clusterRWLock_t ready_lock;
 
         // Ready queue for threads
-        __queue_t($thread) ready_queue;
+        __ready_queue_t ready_queue;
 
         // Name of the cluster

libcfa/src/concurrency/kernel_private.hfa

@@ -84 +84 @@
 //-----------------------------------------------------------------------------
 // Utils
-#define KERNEL_STORAGE(T,X) static char storage_##X[sizeof(T)]
+#define KERNEL_STORAGE(T,X) __attribute((aligned(__alignof__(T)))) static char storage_##X[sizeof(T)]
 
 static inline uint32_t __tls_rand() {
@@ -103 +103 @@
 void unregister( struct cluster * cltr, struct processor * proc );
 
+//=======================================================================
+// Cluster lock API
+//=======================================================================
+struct __attribute__((aligned(64))) __processor_id {
+        processor * volatile handle;
+        volatile bool lock;
+};
+
+// Lock-Free registering/unregistering of threads
+// Register a processor to a given cluster and get its unique id in return
+unsigned doregister2( struct cluster * cltr, struct processor * proc );
+
+// Unregister a processor from a given cluster using its id, getting back the original pointer
+void     unregister2( struct cluster * cltr, struct processor * proc );
+
+//=======================================================================
+// Reader-writer lock implementation
+// Concurrent with doregister/unregister,
+//    i.e., threads can be added at any point during or between the entry/exit
+
+//-----------------------------------------------------------------------
+// simple spinlock underlying the RWLock
+// Blocking acquire
+static inline void __atomic_acquire(volatile bool * ll) {
+        while( __builtin_expect(__atomic_exchange_n(ll, (bool)true, __ATOMIC_SEQ_CST), false) ) {
+                while(__atomic_load_n(ll, (int)__ATOMIC_RELAXED))
+                        asm volatile("pause");
+        }
+        /* paranoid */ verify(*ll);
+}
+
+// Non-Blocking acquire
+static inline bool __atomic_try_acquire(volatile bool * ll) {
+        return !__atomic_exchange_n(ll, (bool)true, __ATOMIC_SEQ_CST);
+}
+
+// Release
+static inline void __atomic_unlock(volatile bool * ll) {
+        /* paranoid */ verify(*ll);
+        __atomic_store_n(ll, (bool)false, __ATOMIC_RELEASE);
+}
+
+//-----------------------------------------------------------------------
+// Reader side : acquire when using the ready queue to schedule but not
+//  creating/destroying queues
+static inline void ready_schedule_lock( struct cluster * cltr, struct processor * proc) with(cltr->ready_lock) {
+        unsigned iproc = proc->id;
+        /*paranoid*/ verify(data[iproc].handle == proc);
+        /*paranoid*/ verify(iproc < ready);
+
+        // Step 1 : make sure no writer are in the middle of the critical section
+        while(__atomic_load_n(&lock, (int)__ATOMIC_RELAXED))
+                asm volatile("pause");
+
+        // Fence needed because we don't want to start trying to acquire the lock
+        // before we read a false.
+        // Not needed on x86
+        // std::atomic_thread_fence(std::memory_order_seq_cst);
+
+        // Step 2 : acquire our local lock
+        __atomic_acquire( &data[iproc].lock );
+        /*paranoid*/ verify(data[iproc].lock);
+}
+
+static inline void ready_schedule_unlock( struct cluster * cltr, struct processor * proc) with(cltr->ready_lock) {
+        unsigned iproc = proc->id;
+        /*paranoid*/ verify(data[iproc].handle == proc);
+        /*paranoid*/ verify(iproc < ready);
+        /*paranoid*/ verify(data[iproc].lock);
+        __atomic_unlock(&data[iproc].lock);
+}
+
+//-----------------------------------------------------------------------
+// Writer side : acquire when changing the ready queue, e.g. adding more
+//  queues or removing them.
+uint_fast32_t ready_mutate_lock( struct cluster & cltr );
+
+void ready_mutate_unlock( struct cluster & cltr, uint_fast32_t /* value returned by lock */ );
+
+//=======================================================================
+// Ready-Queue API
+//-----------------------------------------------------------------------
+// push thread onto a ready queue for a cluster
+// returns true if the list was previously empty, false otherwise
+__attribute__((hot)) bool push(struct cluster * cltr, struct $thread * thrd);
+
+//-----------------------------------------------------------------------
+// pop thread from the ready queue of a cluster
+// returns 0p if empty
+__attribute__((hot)) struct $thread * pop(struct cluster * cltr);
+
+//-----------------------------------------------------------------------
+// Increase the width of the ready queue (number of lanes) by 4
+void ready_queue_grow  (struct cluster * cltr);
+
+//-----------------------------------------------------------------------
+// Decrease the width of the ready queue (number of lanes) by 4
+void ready_queue_shrink(struct cluster * cltr);
+
+//-----------------------------------------------------------------------
+// Statics call at the end of each thread to register statistics
+#if !defined(__CFA_NO_STATISTICS__)
+void stats_tls_tally(struct cluster * cltr);
+#else
+static inline void stats_tls_tally(struct cluster * cltr) {}
+#endif
+
 // Local Variables: //
 // mode: c //

libcfa/src/concurrency/monitor.cfa

@@ -114 +114 @@
 
                 // Some one else has the monitor, wait in line for it
-                /* paranoid */ verify( thrd->next == 0p );
+                /* paranoid */ verify( thrd->link.next == 0p );
                 append( this->entry_queue, thrd );
-                /* paranoid */ verify( thrd->next == 1p );
+                /* paranoid */ verify( thrd->link.next == 1p );
 
                 unlock( this->lock );
@@ -199 +199 @@
 
                 // Some one else has the monitor, wait in line for it
-                /* paranoid */ verify( thrd->next == 0p );
+                /* paranoid */ verify( thrd->link.next == 0p );
                 append( this->entry_queue, thrd );
-                /* paranoid */ verify( thrd->next == 1p );
+                /* paranoid */ verify( thrd->link.next == 1p );
                 unlock( this->lock );
 
@@ -761 +761 @@
         $thread * new_owner = pop_head( this->entry_queue );
         /* paranoid */ verifyf( !this->owner || kernelTLS.this_thread == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", kernelTLS.this_thread, this->owner, this->recursion, this );
-        /* paranoid */ verify( !new_owner || new_owner->next == 0p );
+        /* paranoid */ verify( !new_owner || new_owner->link.next == 0p );
         __set_owner( this, new_owner );
 
@@ -883 +883 @@
         }
 
-        __cfaabi_dbg_print_safe( "Kernel :  Runing %i (%p)\n", ready2run, ready2run ? node->waiting_thread : 0p );
+        __cfaabi_dbg_print_safe( "Kernel :  Runing %i (%p)\n", ready2run, ready2run ? (thread*)node->waiting_thread : (thread*)0p );
         return ready2run ? node->waiting_thread : 0p;
 }
@@ -907 +907 @@
         // For each thread in the entry-queue
         for(    $thread ** thrd_it = &entry_queue.head;
-                *thrd_it != 1p;
-                thrd_it = &(*thrd_it)->next
+                *thrd_it;
+                thrd_it = &(*thrd_it)->link.next
         ) {
                 // For each acceptable check if it matches

libcfa/src/concurrency/preemption.cfa

@@ -121 +121 @@
         // If there are still alarms pending, reset the timer
         if( & (*alarms)`first ) {
-                __cfaabi_dbg_print_buffer_decl( " KERNEL: @%ju(%ju) resetting alarm to %ju.\n", currtime.tv, __kernel_get_time().tv, (alarms->head->alarm - currtime).tv);
+                __cfadbg_print_buffer_decl(preemption, " KERNEL: @%ju(%ju) resetting alarm to %ju.\n", currtime.tv, __kernel_get_time().tv, (alarms->head->alarm - currtime).tv);
                 Duration delta = (*alarms)`first.alarm - currtime;
                 Duration capped = max(delta, 50`us);

libcfa/src/concurrency/thread.cfa

@@ -35 +35 @@
         self_mon_p = &self_mon;
         curr_cluster = &cl;
-        next = 0p;
+        link.next = 0p;
+        link.prev = 0p;
 
         node.next = 0p;

libcfa/src/stdhdr/assert.h

@@ -33 +33 @@
         #define verify(x) assert(x)
         #define verifyf(x, ...) assertf(x, __VA_ARGS__)
+        #define verifyfail(...)
         #define __CFA_WITH_VERIFY__
 #else
         #define verify(x)
         #define verifyf(x, ...)
+        #define verifyfail(...)
 #endif
 

tests/concurrent/examples/datingService.cfa

@@ -35 +35 @@
                 signal_block( Boys[ccode] );                                    // restart boy to set phone number
         } // if
-        //sout | "Girl:" | PhoneNo | "is dating Boy at" | BoyPhoneNo | "with ccode" | ccode;
+        // sout | "Girl:" | PhoneNo | "is dating Boy at" | BoyPhoneNo | "with ccode" | ccode;
         return BoyPhoneNo;
 } // DatingService girl
@@ -47 +47 @@
                 signal_block( Girls[ccode] );                                   // restart girl to set phone number
         } // if
-        //sout | " Boy:" | PhoneNo | "is dating Girl" | GirlPhoneNo | "with ccode" | ccode;
+        // sout | " Boy:" | PhoneNo | "is dating Girl" | GirlPhoneNo | "with ccode" | ccode;
         return GirlPhoneNo;
 } // DatingService boy

tests/concurrent/waitfor/when.cfa

@@ -57 +57 @@
 
 void arbiter( global_t & mutex this ) {
+        // There is a race at start where callers can get in before the arbiter.
+        // It doesn't really matter here so just restart the loop correctly and move on
+        this.last_call = 6;
+
         for( int i = 0; i < N; i++ ) {
                    when( this.last_call == 6 ) waitfor( call1 : this ) { if( this.last_call != 1) { serr | "Expected last_call to be 1 got" | this.last_call; } }