Changeset 1eb239e4 for libcfa/src/concurrency/kernel.cfa

Timestamp:

Aug 10, 2020, 2:43:02 PM (4 years ago)

Author:

Thierry Delisle <tdelisle@…>

Branches:

ADT, arm-eh, ast-experimental, enum, forall-pointer-decay, jacob/cs343-translation, master, new-ast, new-ast-unique-expr, pthread-emulation, qualifiedEnum

Children:

8465b4d

Parents:

6c144d8

Message:

Removed snzi and replaced it with a fast/slow path

File:

• libcfa/src/concurrency/kernel.cfa (modified) (8 diffs)

libcfa/src/concurrency/kernel.cfa

@@ -86 +86 @@
 // Kernel Scheduling logic
 static $thread * __next_thread(cluster * this);
-static bool __has_next_thread(cluster * this);
+static $thread * __next_thread_slow(cluster * this);
 static void __run_thread(processor * this, $thread * dst);
-static bool __wake_one(struct __processor_id_t * id, cluster * cltr);
-static void __halt(processor * this);
-bool __wake_proc(processor *);
+static void __wake_one(struct __processor_id_t * id, cluster * cltr);
+
+static void push  (__cluster_idles & idles, processor & proc);
+static void remove(__cluster_idles & idles, processor & proc);
+static [unsigned idle, unsigned total, * processor] query( & __cluster_idles idles );
+
 
 //=============================================================================================
@@ -118 +121 @@
 
                 $thread * readyThread = 0p;
-                for( unsigned int spin_count = 0;; spin_count++ ) {
+                MAIN_LOOP:
+                for() {
                         // Try to get the next thread
                         readyThread = __next_thread( this->cltr );
 
-                        // Check if we actually found a thread
-                        if( readyThread ) {
-                                /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
-                                /* paranoid */ verifyf( readyThread->state == Ready || readyThread->preempted != __NO_PREEMPTION, "state : %d, preempted %d\n", readyThread->state, readyThread->preempted);
-                                /* paranoid */ verifyf( readyThread->link.next == 0p, "Expected null got %p", readyThread->link.next );
-                                __builtin_prefetch( readyThread->context.SP );
-
-                                // We found a thread run it
-                                __run_thread(this, readyThread);
-
-                                /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
+                        if( !readyThread ) {
+                                readyThread = __next_thread_slow( this->cltr );
                         }
 
-                        if(__atomic_load_n(&this->do_terminate, __ATOMIC_SEQ_CST)) break;
-
+                        HALT:
                         if( !readyThread ) {
-                                // Block until a thread is ready
-                                __halt(this);
+                                // Don't block if we are done
+                                if( __atomic_load_n(&this->do_terminate, __ATOMIC_SEQ_CST) ) break MAIN_LOOP;
+
+                                #if !defined(__CFA_NO_STATISTICS__)
+                                        __tls_stats()->ready.sleep.halts++;
+                                #endif
+
+                                // Push self to idle stack
+                                push(this->cltr->idles, * this);
+
+                                // Confirm the ready-queue is empty
+                                readyThread = __next_thread_slow( this->cltr );
+                                if( readyThread ) {
+                                        // A thread was found, cancel the halt
+                                        remove(this->cltr->idles, * this);
+
+                                        #if !defined(__CFA_NO_STATISTICS__)
+                                                __tls_stats()->ready.sleep.cancels++;
+                                        #endif
+
+                                        // continue the mai loop
+                                        break HALT;
+                                }
+
+                                #if !defined(__CFA_NO_STATISTICS__)
+                                        if(this->print_halts) {
+                                                __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 0\n", this->id, rdtscl());
+                                        }
+                                #endif
+
+                                wait( this->idle );
+
+                                #if !defined(__CFA_NO_STATISTICS__)
+                                        if(this->print_halts) {
+                                                __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 1\n", this->id, rdtscl());
+                                        }
+                                #endif
+
+                                // We were woken up, remove self from idle
+                                remove(this->cltr->idles, * this);
+
+                                // DON'T just proceed, start looking again
+                                continue MAIN_LOOP;
                         }
+
+                        /* paranoid */ verify( readyThread );
+
+                        // We found a thread run it
+                        __run_thread(this, readyThread);
+
+                        // Are we done?
+                        if( __atomic_load_n(&this->do_terminate, __ATOMIC_SEQ_CST) ) break MAIN_LOOP;
                 }
 
@@ -165 +208 @@
 // from the processor coroutine to the target thread
 static void __run_thread(processor * this, $thread * thrd_dst) {
+        /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
+        /* paranoid */ verifyf( thrd_dst->state == Ready || thrd_dst->preempted != __NO_PREEMPTION, "state : %d, preempted %d\n", thrd_dst->state, thrd_dst->preempted);
+        /* paranoid */ verifyf( thrd_dst->link.next == 0p, "Expected null got %p", thrd_dst->link.next );
+        __builtin_prefetch( thrd_dst->context.SP );
+
         $coroutine * proc_cor = get_coroutine(this->runner);
 
@@ -244 +292 @@
         proc_cor->state = Active;
         kernelTLS.this_thread = 0p;
+
+        /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
 }
 
@@ -300 +350 @@
         ready_schedule_lock  ( id );
                 push( thrd->curr_cluster, thrd );
-
-                #if !defined(__CFA_NO_STATISTICS__)
-                        bool woke =
-                #endif
-                        __wake_one(id, thrd->curr_cluster);
-
-                #if !defined(__CFA_NO_STATISTICS__)
-                        if(woke) __tls_stats()->ready.sleep.wakes++;
-                #endif
+                __wake_one(id, thrd->curr_cluster);
         ready_schedule_unlock( id );
 
@@ -315 +357 @@
 
 // KERNEL ONLY
-static $thread * __next_thread(cluster * this) with( *this ) {
+static inline $thread * __next_thread(cluster * this) with( *this ) {
         /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
 
         ready_schedule_lock  ( (__processor_id_t*)kernelTLS.this_processor );
-                $thread * head = pop( this );
+                $thread * thrd = pop( this );
         ready_schedule_unlock( (__processor_id_t*)kernelTLS.this_processor );
 
         /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
-        return head;
+        return thrd;
 }
 
 // KERNEL ONLY
-static bool __has_next_thread(cluster * this) with( *this ) {
+static inline $thread * __next_thread_slow(cluster * this) with( *this ) {
         /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
 
         ready_schedule_lock  ( (__processor_id_t*)kernelTLS.this_processor );
-                bool not_empty = query( this );
+                $thread * thrd = pop_slow( this );
         ready_schedule_unlock( (__processor_id_t*)kernelTLS.this_processor );
 
         /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
-        return not_empty;
+        return thrd;
 }
 
@@ -425 +467 @@
 //=============================================================================================
 // Wake a thread from the front if there are any
-static bool __wake_one(struct __processor_id_t * id, cluster * this) {
+static void __wake_one(struct __processor_id_t * id, cluster * this) {
+        /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
         /* paranoid */ verify( ready_schedule_islocked( id ) );
 
         // Check if there is a sleeping processor
-        processor * p = pop(this->idles);
+        processor * p;
+        unsigned idle;
+        unsigned total;
+        [idle, total, p] = query(this->idles);
 
         // If no one is sleeping, we are done
-        if( 0p == p ) return false;
+        if( idle == 0 ) return;
 
         // We found a processor, wake it up
         post( p->idle );
 
-        return true;
+        #if !defined(__CFA_NO_STATISTICS__)
+                __tls_stats()->ready.sleep.wakes++;
+        #endif
+
+        /* paranoid */ verify( ready_schedule_islocked( id ) );
+        /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
+
+        return;
 }
 
 // Unconditionnaly wake a thread
-bool __wake_proc(processor * this) {
+void __wake_proc(processor * this) {
         __cfadbg_print_safe(runtime_core, "Kernel : waking Processor %p\n", this);
 
@@ -448 +501 @@
                 bool ret = post( this->idle );
         enable_interrupts( __cfaabi_dbg_ctx );
-
-        return ret;
-}
-
-static void __halt(processor * this) with( *this ) {
-        if( do_terminate ) return;
-
-        #if !defined(__CFA_NO_STATISTICS__)
-                __tls_stats()->ready.sleep.halts++;
-        #endif
-        // Push self to queue
-        push(cltr->idles, *this);
-
-        // Makre sure we don't miss a thread
-        if( __has_next_thread(cltr) ) {
-                // A thread was posted, make sure a processor is woken up
-                struct __processor_id_t *id = (struct __processor_id_t *) this;
-                ready_schedule_lock  ( id );
-                        __wake_one( id, cltr );
-                ready_schedule_unlock( id );
-                #if !defined(__CFA_NO_STATISTICS__)
-                        __tls_stats()->ready.sleep.cancels++;
-                #endif
-        }
-
-        #if !defined(__CFA_NO_STATISTICS__)
-                if(this->print_halts) {
-                        __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 0\n", this->id, rdtscl());
-                }
-        #endif
-
-        wait( idle );
-
-        #if !defined(__CFA_NO_STATISTICS__)
-                if(this->print_halts) {
-                        __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 1\n", this->id, rdtscl());
-                }
-        #endif
+}
+
+static void push  (__cluster_idles & this, processor & proc) {
+        /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
+        lock( this );
+                this.idle++;
+                /* paranoid */ verify( this.idle <= this.total );
+
+                insert_first(this.list, proc);
+        unlock( this );
+        /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
+}
+
+static void remove(__cluster_idles & this, processor & proc) {
+        /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
+        lock( this );
+                this.idle--;
+                /* paranoid */ verify( this.idle >= 0 );
+
+                remove(proc);
+        unlock( this );
+        /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
+}
+
+static [unsigned idle, unsigned total, * processor] query( & __cluster_idles this ) {
+        for() {
+                uint64_t l = __atomic_load_n(&this.lock, __ATOMIC_SEQ_CST);
+                if( 1 == (l % 2) ) { Pause(); continue; }
+                unsigned idle    = this.idle;
+                unsigned total   = this.total;
+                processor * proc = &this.list`first;
+                if(l != __atomic_load_n(&this.lock, __ATOMIC_SEQ_CST)) { Pause(); continue; }
+                return [idle, total, proc];
+        }
 }