Changeset b067d9b for libcfa/src/concurrency/kernel.cfa

Timestamp:

Oct 29, 2019, 4:01:24 PM (6 years ago)

Author:

Thierry Delisle <tdelisle@…>

Branches:

ADT, arm-eh, ast-experimental, enum, forall-pointer-decay, jacob/cs343-translation, jenkins-sandbox, master, new-ast, new-ast-unique-expr, pthread-emulation, qualifiedEnum

Children:

773db65, 9421f3d8

Parents:

7951100 (diff), 8364209 (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the (diff) links above to see all the changes relative to each parent.

Message:

Merge branch 'master' of plg.uwaterloo.ca:software/cfa/cfa-cc

File:

• libcfa/src/concurrency/kernel.cfa (moved) (moved from src/libcfa/concurrency/kernel.c ) (33 diffs)

libcfa/src/concurrency/kernel.cfa

@@ -10 +10 @@
 // Created On       : Tue Jan 17 12:27:26 2017
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Mon Apr  9 16:11:46 2018
-// Update Count     : 24
+// Last Modified On : Thu Jun 20 17:21:23 2019
+// Update Count     : 25
 //
+
+#define __cforall_thread__
 
 //C Includes
@@ -27 +29 @@
 
 //CFA Includes
-#include "time"
-#include "kernel_private.h"
-#include "preemption.h"
-#include "startup.h"
+#include "time.hfa"
+#include "kernel_private.hfa"
+#include "preemption.hfa"
+#include "startup.hfa"
 
 //Private includes
@@ -36 +38 @@
 #include "invoke.h"
 
+//-----------------------------------------------------------------------------
+// Some assembly required
+#if   defined( __i386 )
+        #define CtxGet( ctx )        \
+                __asm__ volatile (     \
+                        "movl %%esp,%0\n"\
+                        "movl %%ebp,%1\n"\
+                        : "=rm" (ctx.SP),\
+                                "=rm" (ctx.FP) \
+                )
+
+        // mxcr : SSE Status and Control bits (control bits are preserved across function calls)
+        // fcw  : X87 FPU control word (preserved across function calls)
+        #define __x87_store         \
+                uint32_t __mxcr;      \
+                uint16_t __fcw;       \
+                __asm__ volatile (    \
+                        "stmxcsr %0\n"  \
+                        "fnstcw  %1\n"  \
+                        : "=m" (__mxcr),\
+                                "=m" (__fcw)  \
+                )
+
+        #define __x87_load         \
+                __asm__ volatile (   \
+                        "fldcw  %1\n"  \
+                        "ldmxcsr %0\n" \
+                        ::"m" (__mxcr),\
+                                "m" (__fcw)  \
+                )
+
+#elif defined( __x86_64 )
+        #define CtxGet( ctx )        \
+                __asm__ volatile (     \
+                        "movq %%rsp,%0\n"\
+                        "movq %%rbp,%1\n"\
+                        : "=rm" (ctx.SP),\
+                                "=rm" (ctx.FP) \
+                )
+
+        #define __x87_store         \
+                uint32_t __mxcr;      \
+                uint16_t __fcw;       \
+                __asm__ volatile (    \
+                        "stmxcsr %0\n"  \
+                        "fnstcw  %1\n"  \
+                        : "=m" (__mxcr),\
+                                "=m" (__fcw)  \
+                )
+
+        #define __x87_load          \
+                __asm__ volatile (    \
+                        "fldcw  %1\n"   \
+                        "ldmxcsr %0\n"  \
+                        :: "m" (__mxcr),\
+                                "m" (__fcw)  \
+                )
+
+
+#elif defined( __ARM_ARCH )
+#define CtxGet( ctx ) __asm__ ( \
+                "mov %0,%%sp\n"   \
+                "mov %1,%%r11\n"   \
+        : "=rm" (ctx.SP), "=rm" (ctx.FP) )
+#else
+        #error unknown hardware architecture
+#endif
+
+//-----------------------------------------------------------------------------
 //Start and stop routine for the kernel, declared first to make sure they run first
-void kernel_startup(void)  __attribute__(( constructor( STARTUP_PRIORITY_KERNEL ) ));
-void kernel_shutdown(void) __attribute__(( destructor ( STARTUP_PRIORITY_KERNEL ) ));
+static void kernel_startup(void)  __attribute__(( constructor( STARTUP_PRIORITY_KERNEL ) ));
+static void kernel_shutdown(void) __attribute__(( destructor ( STARTUP_PRIORITY_KERNEL ) ));
 
 //-----------------------------------------------------------------------------
 // Kernel storage
-KERNEL_STORAGE(cluster,           mainCluster);
-KERNEL_STORAGE(processor,         mainProcessor);
-KERNEL_STORAGE(thread_desc,       mainThread);
-KERNEL_STORAGE(machine_context_t, mainThreadCtx);
+KERNEL_STORAGE(cluster,         mainCluster);
+KERNEL_STORAGE(processor,       mainProcessor);
+KERNEL_STORAGE(thread_desc,     mainThread);
+KERNEL_STORAGE(__stack_t,       mainThreadCtx);
 
 cluster     * mainCluster;
@@ -55 +126 @@
 }
 
+size_t __page_size = 0;
+
 //-----------------------------------------------------------------------------
 // Global state
-thread_local struct KernelThreadData kernelTLS = {
-        NULL,
+thread_local struct KernelThreadData kernelTLS __attribute__ ((tls_model ( "initial-exec" ))) = {
         NULL,
         NULL,
@@ -67 +139 @@
 // Struct to steal stack
 struct current_stack_info_t {
-        machine_context_t ctx;
-        unsigned int size;              // size of stack
+        __stack_t * storage;            // pointer to stack object
         void *base;                             // base of stack
-        void *storage;                  // pointer to stack
         void *limit;                    // stack grows towards stack limit
         void *context;                  // address of cfa_context_t
-        void *top;                              // address of top of storage
 };
 
 void ?{}( current_stack_info_t & this ) {
-        CtxGet( this.ctx );
-        this.base = this.ctx.FP;
-        this.storage = this.ctx.SP;
+        __stack_context_t ctx;
+        CtxGet( ctx );
+        this.base = ctx.FP;
 
         rlimit r;
         getrlimit( RLIMIT_STACK, &r);
-        this.size = r.rlim_cur;
-
-        this.limit = (void *)(((intptr_t)this.base) - this.size);
+        size_t size = r.rlim_cur;
+
+        this.limit = (void *)(((intptr_t)this.base) - size);
         this.context = &storage_mainThreadCtx;
-        this.top = this.base;
 }
 
 //-----------------------------------------------------------------------------
 // Main thread construction
-void ?{}( coStack_t & this, current_stack_info_t * info) with( this ) {
-        size      = info->size;
-        storage   = info->storage;
-        limit     = info->limit;
-        base      = info->base;
-        context   = info->context;
-        top       = info->top;
-        userStack = true;
-}
 
 void ?{}( coroutine_desc & this, current_stack_info_t * info) with( this ) {
-        stack{ info };
+        stack.storage = info->storage;
+        with(*stack.storage) {
+                limit     = info->limit;
+                base      = info->base;
+        }
+        __attribute__((may_alias)) intptr_t * istorage = (intptr_t*) &stack.storage;
+        *istorage |= 0x1;
         name = "Main Thread";
-        errno_ = 0;
         state = Start;
         starter = NULL;
+        last = NULL;
+        cancellation = NULL;
 }
 
 void ?{}( thread_desc & this, current_stack_info_t * info) with( this ) {
+        state = Start;
         self_cor{ info };
         curr_cor = &self_cor;
@@ -133 +200 @@
 
 // Construct the processor context of non-main processors
-void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info) {
+static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info) {
         (this.__cor){ info };
         this.proc = proc;
 }
 
+static void start(processor * this);
 void ?{}(processor & this, const char * name, cluster & cltr) with( this ) {
         this.name = name;
@@ -147 +215 @@
         runner.proc = &this;
 
-        sem_init(&idleLock, 0, 0);
+        idleLock{};
 
         start( &this );
@@ -155 +223 @@
         if( ! __atomic_load_n(&do_terminate, __ATOMIC_ACQUIRE) ) {
                 __cfaabi_dbg_print_safe("Kernel : core %p signaling termination\n", &this);
-                terminate(&this);
-                verify( __atomic_load_n(&do_terminate, __ATOMIC_SEQ_CST) );
-                verify( kernelTLS.this_processor != &this);
+
+                __atomic_store_n(&do_terminate, true, __ATOMIC_RELAXED);
+                wake( &this );
+
                 P( terminated );
                 verify( kernelTLS.this_processor != &this);
-                pthread_join( kernel_thread, NULL );
-        }
-
-        sem_destroy(&idleLock);
+        }
+
+        pthread_join( kernel_thread, NULL );
 }
 
@@ -186 +254 @@
 // Kernel Scheduling logic
 //=============================================================================================
+static void runThread(processor * this, thread_desc * dst);
+static void finishRunning(processor * this);
+static void halt(processor * this);
+
 //Main of the processor contexts
 void main(processorCtx_t & runner) {
@@ -236 +308 @@
 }
 
+static int * __volatile_errno() __attribute__((noinline));
+static int * __volatile_errno() { asm(""); return &errno; }
+
 // KERNEL ONLY
 // runThread runs a thread by context switching
 // from the processor coroutine to the target thread
-void runThread(processor * this, thread_desc * dst) {
-        assert(dst->curr_cor);
+static void runThread(processor * this, thread_desc * thrd_dst) {
         coroutine_desc * proc_cor = get_coroutine(this->runner);
-        coroutine_desc * thrd_cor = dst->curr_cor;
 
         // Reset the terminating actions here
@@ -248 +321 @@
 
         // Update global state
-        kernelTLS.this_thread = dst;
-
-        // Context Switch to the thread
-        ThreadCtxSwitch(proc_cor, thrd_cor);
-        // when ThreadCtxSwitch returns we are back in the processor coroutine
+        kernelTLS.this_thread = thrd_dst;
+
+        // set state of processor coroutine to inactive and the thread to active
+        proc_cor->state = proc_cor->state == Halted ? Halted : Inactive;
+        thrd_dst->state = Active;
+
+        // set context switch to the thread that the processor is executing
+        verify( thrd_dst->context.SP );
+        CtxSwitch( &proc_cor->context, &thrd_dst->context );
+        // when CtxSwitch returns we are back in the processor coroutine
+
+        // set state of processor coroutine to active and the thread to inactive
+        thrd_dst->state = thrd_dst->state == Halted ? Halted : Inactive;
+        proc_cor->state = Active;
 }
 
 // KERNEL_ONLY
-void returnToKernel() {
+static void returnToKernel() {
         coroutine_desc * proc_cor = get_coroutine(kernelTLS.this_processor->runner);
-        coroutine_desc * thrd_cor = kernelTLS.this_thread->curr_cor = kernelTLS.this_coroutine;
-        ThreadCtxSwitch(thrd_cor, proc_cor);
+        thread_desc * thrd_src = kernelTLS.this_thread;
+
+        // set state of current coroutine to inactive
+        thrd_src->state = thrd_src->state == Halted ? Halted : Inactive;
+        proc_cor->state = Active;
+        int local_errno = *__volatile_errno();
+        #if defined( __i386 ) || defined( __x86_64 )
+                __x87_store;
+        #endif
+
+        // set new coroutine that the processor is executing
+        // and context switch to it
+        verify( proc_cor->context.SP );
+        CtxSwitch( &thrd_src->context, &proc_cor->context );
+
+        // set state of new coroutine to active
+        proc_cor->state = proc_cor->state == Halted ? Halted : Inactive;
+        thrd_src->state = Active;
+
+        #if defined( __i386 ) || defined( __x86_64 )
+                __x87_load;
+        #endif
+        *__volatile_errno() = local_errno;
 }
 
@@ -265 +368 @@
 // Once a thread has finished running, some of
 // its final actions must be executed from the kernel
-void finishRunning(processor * this) with( this->finish ) {
+static void finishRunning(processor * this) with( this->finish ) {
         verify( ! kernelTLS.preemption_state.enabled );
         choose( action_code ) {
@@ -295 +398 @@
 }
 
-// Handles spinning logic
-// TODO : find some strategy to put cores to sleep after some time
-void spin(processor * this, unsigned int * spin_count) {
-        // (*spin_count)++;
-        halt(this);
-}
-
 // KERNEL_ONLY
 // Context invoker for processors
 // This is the entry point for processors (kernel threads)
 // It effectively constructs a coroutine by stealing the pthread stack
-void * CtxInvokeProcessor(void * arg) {
+static void * CtxInvokeProcessor(void * arg) {
         processor * proc = (processor *) arg;
         kernelTLS.this_processor = proc;
-        kernelTLS.this_coroutine = NULL;
         kernelTLS.this_thread    = NULL;
         kernelTLS.preemption_state.[enabled, disable_count] = [false, 1];
@@ -316 +411 @@
         // to waste the perfectly valid stack create by pthread.
         current_stack_info_t info;
-        machine_context_t ctx;
-        info.context = &ctx;
+        __stack_t ctx;
+        info.storage = &ctx;
         (proc->runner){ proc, &info };
 
-        __cfaabi_dbg_print_safe("Coroutine : created stack %p\n", get_coroutine(proc->runner)->stack.base);
+        __cfaabi_dbg_print_safe("Coroutine : created stack %p\n", get_coroutine(proc->runner)->stack.storage);
 
         //Set global state
-        kernelTLS.this_coroutine = get_coroutine(proc->runner);
         kernelTLS.this_thread    = NULL;
 
@@ -343 +437 @@
 }
 
-void start(processor * this) {
+static void start(processor * this) {
         __cfaabi_dbg_print_safe("Kernel : Starting core %p\n", this);
 
@@ -352 +446 @@
 
 // KERNEL_ONLY
-void kernel_first_resume(processor * this) {
-        coroutine_desc * src = kernelTLS.this_coroutine;
+void kernel_first_resume( processor * this ) {
+        thread_desc * src = mainThread;
         coroutine_desc * dst = get_coroutine(this->runner);
 
         verify( ! kernelTLS.preemption_state.enabled );
 
-        create_stack(&dst->stack, dst->stack.size);
+        __stack_prepare( &dst->stack, 65000 );
         CtxStart(&this->runner, CtxInvokeCoroutine);
 
         verify( ! kernelTLS.preemption_state.enabled );
 
-        dst->last = src;
-        dst->starter = dst->starter ? dst->starter : src;
+        dst->last = &src->self_cor;
+        dst->starter = dst->starter ? dst->starter : &src->self_cor;
 
         // set state of current coroutine to inactive
         src->state = src->state == Halted ? Halted : Inactive;
 
-        // set new coroutine that task is executing
-        kernelTLS.this_coroutine = dst;
-
-        // SKULLDUGGERY normally interrupts are enable before leaving a coroutine ctxswitch.
-        // Therefore, when first creating a coroutine, interrupts are enable before calling the main.
-        // This is consistent with thread creation. However, when creating the main processor coroutine,
-        // we wan't interrupts to be disabled. Therefore, we double-disable interrupts here so they will
-        // stay disabled.
-        disable_interrupts();
-
         // context switch to specified coroutine
-        assert( src->stack.context );
-        CtxSwitch( src->stack.context, dst->stack.context );
+        verify( dst->context.SP );
+        CtxSwitch( &src->context, &dst->context );
         // when CtxSwitch returns we are back in the src coroutine
 
@@ -388 +472 @@
 
         verify( ! kernelTLS.preemption_state.enabled );
+}
+
+// KERNEL_ONLY
+void kernel_last_resume( processor * this ) {
+        coroutine_desc * src = &mainThread->self_cor;
+        coroutine_desc * dst = get_coroutine(this->runner);
+
+        verify( ! kernelTLS.preemption_state.enabled );
+        verify( dst->starter == src );
+        verify( dst->context.SP );
+
+        // context switch to the processor
+        CtxSwitch( &src->context, &dst->context );
 }
 
@@ -396 +493 @@
 void ScheduleThread( thread_desc * thrd ) {
         verify( thrd );
-        verify( thrd->self_cor.state != Halted );
+        verify( thrd->state != Halted );
 
         verify( ! kernelTLS.preemption_state.enabled );
@@ -408 +505 @@
                 unlock( ready_queue_lock );
 
-                if( was_empty ) {
+                if(was_empty) {
                         lock      (proc_list_lock __cfaabi_dbg_ctx2);
                         if(idles) {
-                                wake(idles.head);
+                                wake_fast(idles.head);
                         }
                         unlock    (proc_list_lock);
                 }
+                else if( struct processor * idle = idles.head ) {
+                        wake_fast(idle);
+                }
+
         }
 
@@ -545 +646 @@
 //-----------------------------------------------------------------------------
 // Kernel boot procedures
-void kernel_startup(void) {
+static void kernel_startup(void) {
         verify( ! kernelTLS.preemption_state.enabled );
         __cfaabi_dbg_print_safe("Kernel : Starting\n");
+
+        __page_size = sysconf( _SC_PAGESIZE );
 
         __cfa_dbg_global_clusters.list{ __get };
@@ -563 +666 @@
         mainThread = (thread_desc *)&storage_mainThread;
         current_stack_info_t info;
+        info.storage = (__stack_t*)&storage_mainThreadCtx;
         (*mainThread){ &info };
 
@@ -597 +701 @@
         kernelTLS.this_processor = mainProcessor;
         kernelTLS.this_thread    = mainThread;
-        kernelTLS.this_coroutine = &mainThread->self_cor;
 
         // Enable preemption
@@ -621 +724 @@
 }
 
-void kernel_shutdown(void) {
+static void kernel_shutdown(void) {
         __cfaabi_dbg_print_safe("\n--------------------------------------------------\nKernel : Shutting down\n");
 
@@ -632 +735 @@
         // which is currently here
         __atomic_store_n(&mainProcessor->do_terminate, true, __ATOMIC_RELEASE);
-        returnToKernel();
+        kernel_last_resume( kernelTLS.this_processor );
         mainThread->self_cor.state = Halted;
 
@@ -658 +761 @@
 // Kernel Quiescing
 //=============================================================================================
-
-void halt(processor * this) with( *this ) {
-        verify( ! __atomic_load_n(&do_terminate, __ATOMIC_SEQ_CST) );
+static void halt(processor * this) with( *this ) {
+        // verify( ! __atomic_load_n(&do_terminate, __ATOMIC_SEQ_CST) );
 
         with( *cltr ) {
@@ -671 +773 @@
         __cfaabi_dbg_print_safe("Kernel : Processor %p ready to sleep\n", this);
 
-        // #ifdef __CFA_WITH_VERIFY__
-        //      int sval = 0;
-        //      sem_getvalue(&this->idleLock, &sval);
-        //      verifyf(sval < 200, "Binary semaphore reached value %d : \n", sval);
-        // #endif
-
-        verify( ! __atomic_load_n(&do_terminate, __ATOMIC_SEQ_CST) );
-        int __attribute__((unused)) ret = sem_wait(&idleLock);
-        // verifyf(ret >= 0 || errno == EINTR, "Sem_wait returned %d (errno %d : %s\n", ret, errno, strerror(errno));
-
-        // wait( idleLock );
+        wait( idleLock );
 
         __cfaabi_dbg_print_safe("Kernel : Processor %p woke up and ready to run\n", this);
@@ -693 +785 @@
 }
 
-void wake(processor * this) {
-        __cfaabi_dbg_print_safe("Kernel : Waking up processor %p\n", this);
-        int __attribute__((unused)) ret = sem_post(&this->idleLock);
-        // verifyf(ret >= 0 || errno == EINTR, "Sem_post returned %d (errno %d : %s\n", ret, errno, strerror(errno));
-
-        // #ifdef __CFA_WITH_VERIFY__
-        //      int sval = 0;
-        //      sem_getvalue(&this->idleLock, &sval);
-        //      verifyf(sval < 200, "Binary semaphore reached value %d\n", sval);
-        // #endif
-
-        // post( this->idleLock );
-}
-
 //=============================================================================================
 // Unexpected Terminating logic
 //=============================================================================================
-
-
 static __spinlock_t kernel_abort_lock;
 static bool kernel_abort_called = false;
@@ -745 +821 @@
                 __cfaabi_dbg_bits_write( abort_text, len );
 
-                if ( get_coroutine(thrd) != kernelTLS.this_coroutine ) {
-                        len = snprintf( abort_text, abort_text_size, " in coroutine %.256s (%p).\n", kernelTLS.this_coroutine->name, kernelTLS.this_coroutine );
+                if ( &thrd->self_cor != thrd->curr_cor ) {
+                        len = snprintf( abort_text, abort_text_size, " in coroutine %.256s (%p).\n", thrd->curr_cor->name, thrd->curr_cor );
                         __cfaabi_dbg_bits_write( abort_text, len );
                 }
@@ -833 +909 @@
 void doregister( cluster * cltr, thread_desc & thrd ) {
         lock      (cltr->thread_list_lock __cfaabi_dbg_ctx2);
+        cltr->nthreads += 1;
         push_front(cltr->threads, thrd);
         unlock    (cltr->thread_list_lock);
@@ -840 +917 @@
         lock  (cltr->thread_list_lock __cfaabi_dbg_ctx2);
         remove(cltr->threads, thrd );
+        cltr->nthreads -= 1;
         unlock(cltr->thread_list_lock);
 }
@@ -845 +923 @@
 void doregister( cluster * cltr, processor * proc ) {
         lock      (cltr->proc_list_lock __cfaabi_dbg_ctx2);
+        cltr->nprocessors += 1;
         push_front(cltr->procs, *proc);
         unlock    (cltr->proc_list_lock);
@@ -852 +931 @@
         lock  (cltr->proc_list_lock __cfaabi_dbg_ctx2);
         remove(cltr->procs, *proc );
+        cltr->nprocessors -= 1;
         unlock(cltr->proc_list_lock);
 }
@@ -858 +938 @@
 // Debug
 __cfaabi_dbg_debug_do(
-        void __cfaabi_dbg_record(__spinlock_t & this, const char * prev_name) {
-                this.prev_name = prev_name;
-                this.prev_thrd = kernelTLS.this_thread;
+        extern "C" {
+                void __cfaabi_dbg_record(__spinlock_t & this, const char * prev_name) {
+                        this.prev_name = prev_name;
+                        this.prev_thrd = kernelTLS.this_thread;
+                }
         }
 )
+
+//-----------------------------------------------------------------------------
+// Debug
+bool threading_enabled(void) {
+        return true;
+}
 // Local Variables: //
 // mode: c //