source: src/libcfa/concurrency/kernel.c @ 1997b4e

ADTaaron-thesisarm-ehast-experimentalcleanup-dtorsdeferred_resndemanglerenumforall-pointer-decayjacob/cs343-translationjenkins-sandboxnew-astnew-ast-unique-exprno_listpersistent-indexerpthread-emulationqualifiedEnum
Last change on this file since 1997b4e was 1997b4e, checked in by Thierry Delisle <tdelisle@…>, 6 years ago

tentative fix of the linking ordering problem

  • Property mode set to 100644
File size: 23.9 KB
RevLine 
[8118303]1//
2// Cforall Version 1.0.0 Copyright (C) 2016 University of Waterloo
3//
4// The contents of this file are covered under the licence agreement in the
5// file "LICENCE" distributed with Cforall.
6//
7// kernel.c --
8//
9// Author           : Thierry Delisle
[75f3522]10// Created On       : Tue Jan 17 12:27:26 2017
[6b0b624]11// Last Modified By : Peter A. Buhr
[0f56058]12// Last Modified On : Mon Apr  9 16:11:46 2018
13// Update Count     : 24
[8118303]14//
15
16//C Includes
[c84e80a]17#include <stddef.h>
[214e8da]18#include <errno.h>
[ea8b2f7]19#include <string.h>
[eb2e723]20extern "C" {
[9d944b2]21#include <stdio.h>
[8fcbb4c]22#include <fenv.h>
[eb2e723]23#include <sys/resource.h>
[9d944b2]24#include <signal.h>
25#include <unistd.h>
[eb2e723]26}
[8118303]27
28//CFA Includes
[0f56058]29#include "time"
[2ac095d]30#include "kernel_private.h"
[c81ebf9]31#include "preemption.h"
[2ac095d]32#include "startup.h"
[8118303]33
34//Private includes
35#define __CFA_INVOKE_PRIVATE__
36#include "invoke.h"
37
[2ac095d]38//Start and stop routine for the kernel, declared first to make sure they run first
[c29c342]39static void kernel_startup(void)  __attribute__(( constructor( STARTUP_PRIORITY_KERNEL ) ));
40static void kernel_shutdown(void) __attribute__(( destructor ( STARTUP_PRIORITY_KERNEL ) ));
[2ac095d]41
[8def349]42//-----------------------------------------------------------------------------
43// Kernel storage
[969b3fe]44KERNEL_STORAGE(cluster,           mainCluster);
45KERNEL_STORAGE(processor,         mainProcessor);
46KERNEL_STORAGE(thread_desc,       mainThread);
[f2b12406]47KERNEL_STORAGE(machine_context_t, mainThreadCtx);
[8def349]48
[de6319f]49cluster     * mainCluster;
50processor   * mainProcessor;
[348006f]51thread_desc * mainThread;
[eb2e723]52
[ea8b2f7]53extern "C" {
54struct { __dllist_t(cluster) list; __spinlock_t lock; } __cfa_dbg_global_clusters;
55}
[de94a60]56
[bd98b58]57//-----------------------------------------------------------------------------
58// Global state
[14a61b5]59thread_local struct KernelThreadData kernelTLS = {
[b10affd]60        NULL,
61        NULL,
62        NULL,
63        { 1, false, false }
64};
[c84e80a]65
66//-----------------------------------------------------------------------------
[de6319f]67// Struct to steal stack
[8def349]68struct current_stack_info_t {
[1c273d0]69        machine_context_t ctx;
[8def349]70        unsigned int size;              // size of stack
71        void *base;                             // base of stack
72        void *storage;                  // pointer to stack
73        void *limit;                    // stack grows towards stack limit
74        void *context;                  // address of cfa_context_t
75        void *top;                              // address of top of storage
[c84e80a]76};
77
[242a902]78void ?{}( current_stack_info_t & this ) {
79        CtxGet( this.ctx );
80        this.base = this.ctx.FP;
81        this.storage = this.ctx.SP;
[8def349]82
83        rlimit r;
[132fad4]84        getrlimit( RLIMIT_STACK, &r);
[242a902]85        this.size = r.rlim_cur;
[8def349]86
[242a902]87        this.limit = (void *)(((intptr_t)this.base) - this.size);
[9236060]88        this.context = &storage_mainThreadCtx;
[242a902]89        this.top = this.base;
[8def349]90}
91
[de6319f]92//-----------------------------------------------------------------------------
93// Main thread construction
[65deb18]94void ?{}( coStack_t & this, current_stack_info_t * info) with( this ) {
95        size      = info->size;
96        storage   = info->storage;
97        limit     = info->limit;
98        base      = info->base;
99        context   = info->context;
100        top       = info->top;
101        userStack = true;
[8def349]102}
103
[65deb18]104void ?{}( coroutine_desc & this, current_stack_info_t * info) with( this ) {
105        stack{ info };
106        name = "Main Thread";
107        errno_ = 0;
108        state = Start;
109        starter = NULL;
[8def349]110}
111
[65deb18]112void ?{}( thread_desc & this, current_stack_info_t * info) with( this ) {
113        self_cor{ info };
[82c948c]114        curr_cor = &self_cor;
[de6319f]115        curr_cluster = mainCluster;
[82c948c]116        self_mon.owner = &this;
117        self_mon.recursion = 1;
118        self_mon_p = &self_mon;
119        next = NULL;
[de94a60]120
121        node.next = NULL;
122        node.prev = NULL;
[a1a17a7]123        doregister(curr_cluster, this);
[82c948c]124
125        monitors{ &self_mon_p, 1, (fptr_t)0 };
[8def349]126}
[c84e80a]127
[8def349]128//-----------------------------------------------------------------------------
129// Processor coroutine
[de6319f]130void ?{}(processorCtx_t & this) {
[39fea2f]131
[8def349]132}
133
[39fea2f]134// Construct the processor context of non-main processors
[c29c342]135static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info) {
[242a902]136        (this.__cor){ info };
137        this.proc = proc;
[8def349]138}
139
[c29c342]140static void start(processor * this);
[de6319f]141void ?{}(processor & this, const char * name, cluster & cltr) with( this ) {
142        this.name = name;
143        this.cltr = &cltr;
[c40e7c5]144        terminated{ 0 };
145        do_terminate = false;
146        preemption_alarm = NULL;
147        pending_preemption = false;
[094476d]148        runner.proc = &this;
[8def349]149
[85b1deb]150        idleLock{};
[6b4cdd3]151
[242a902]152        start( &this );
[c84e80a]153}
154
[65deb18]155void ^?{}(processor & this) with( this ){
[ea8b2f7]156        if( ! __atomic_load_n(&do_terminate, __ATOMIC_ACQUIRE) ) {
[36982fc]157                __cfaabi_dbg_print_safe("Kernel : core %p signaling termination\n", &this);
[85b1deb]158
159                __atomic_store_n(&do_terminate, true, __ATOMIC_RELAXED);
160                wake( &this );
161
[65deb18]162                P( terminated );
[14a61b5]163                verify( kernelTLS.this_processor != &this);
[8def349]164        }
[6b4cdd3]165
[85b1deb]166        pthread_join( kernel_thread, NULL );
[8def349]167}
168
[de6319f]169void ?{}(cluster & this, const char * name, Duration preemption_rate) with( this ) {
170        this.name = name;
171        this.preemption_rate = preemption_rate;
[65deb18]172        ready_queue{};
173        ready_queue_lock{};
[de94a60]174
175        procs{ __get };
176        idles{ __get };
[a1a17a7]177        threads{ __get };
[de94a60]178
179        doregister(this);
[8def349]180}
181
[242a902]182void ^?{}(cluster & this) {
[de94a60]183        unregister(this);
[c84e80a]184}
185
[75f3522]186//=============================================================================================
187// Kernel Scheduling logic
188//=============================================================================================
[c29c342]189static void runThread(processor * this, thread_desc * dst);
190static void finishRunning(processor * this);
191static void halt(processor * this);
192
[8fcbb4c]193//Main of the processor contexts
[83a071f9]194void main(processorCtx_t & runner) {
195        processor * this = runner.proc;
[094476d]196        verify(this);
[c81ebf9]197
[36982fc]198        __cfaabi_dbg_print_safe("Kernel : core %p starting\n", this);
[8118303]199
[de94a60]200        doregister(this->cltr, this);
201
[75f3522]202        {
[c81ebf9]203                // Setup preemption data
204                preemption_scope scope = { this };
205
[36982fc]206                __cfaabi_dbg_print_safe("Kernel : core %p started\n", this);
[8118303]207
[c81ebf9]208                thread_desc * readyThread = NULL;
[ea8b2f7]209                for( unsigned int spin_count = 0; ! __atomic_load_n(&this->do_terminate, __ATOMIC_SEQ_CST); spin_count++ )
[75f3522]210                {
[c81ebf9]211                        readyThread = nextThread( this->cltr );
[75f3522]212
[c81ebf9]213                        if(readyThread)
214                        {
[14a61b5]215                                verify( ! kernelTLS.preemption_state.enabled );
[4e6fb8e]216
[c81ebf9]217                                runThread(this, readyThread);
[75f3522]218
[14a61b5]219                                verify( ! kernelTLS.preemption_state.enabled );
[4e6fb8e]220
[c81ebf9]221                                //Some actions need to be taken from the kernel
222                                finishRunning(this);
223
224                                spin_count = 0;
225                        }
226                        else
227                        {
[ea8b2f7]228                                // spin(this, &spin_count);
229                                halt(this);
[c81ebf9]230                        }
231                }
232
[36982fc]233                __cfaabi_dbg_print_safe("Kernel : core %p stopping\n", this);
[c84e80a]234        }
[8118303]235
[de94a60]236        unregister(this->cltr, this);
237
[4cedd9f]238        V( this->terminated );
[bdeba0b]239
[36982fc]240        __cfaabi_dbg_print_safe("Kernel : core %p terminated\n", this);
[c84e80a]241}
242
[14a61b5]243// KERNEL ONLY
[1c273d0]244// runThread runs a thread by context switching
245// from the processor coroutine to the target thread
[c29c342]246static void runThread(processor * this, thread_desc * dst) {
[82c948c]247        assert(dst->curr_cor);
[094476d]248        coroutine_desc * proc_cor = get_coroutine(this->runner);
[82c948c]249        coroutine_desc * thrd_cor = dst->curr_cor;
[1c273d0]250
[14a61b5]251        // Reset the terminating actions here
[db6f06a]252        this->finish.action_code = No_Action;
[8fcbb4c]253
[14a61b5]254        // Update global state
255        kernelTLS.this_thread = dst;
[75f3522]256
257        // Context Switch to the thread
258        ThreadCtxSwitch(proc_cor, thrd_cor);
259        // when ThreadCtxSwitch returns we are back in the processor coroutine
260}
261
[14a61b5]262// KERNEL_ONLY
[c29c342]263static void returnToKernel() {
[14a61b5]264        coroutine_desc * proc_cor = get_coroutine(kernelTLS.this_processor->runner);
265        coroutine_desc * thrd_cor = kernelTLS.this_thread->curr_cor = kernelTLS.this_coroutine;
[82c948c]266        ThreadCtxSwitch(thrd_cor, proc_cor);
267}
268
[14a61b5]269// KERNEL_ONLY
[1c273d0]270// Once a thread has finished running, some of
[75f3522]271// its final actions must be executed from the kernel
[c29c342]272static void finishRunning(processor * this) with( this->finish ) {
[09800e9]273        verify( ! kernelTLS.preemption_state.enabled );
274        choose( action_code ) {
275        case No_Action:
276                break;
277        case Release:
[65deb18]278                unlock( *lock );
[09800e9]279        case Schedule:
[65deb18]280                ScheduleThread( thrd );
[09800e9]281        case Release_Schedule:
[65deb18]282                unlock( *lock );
283                ScheduleThread( thrd );
[09800e9]284        case Release_Multi:
[65deb18]285                for(int i = 0; i < lock_count; i++) {
286                        unlock( *locks[i] );
[0c78741]287                }
[09800e9]288        case Release_Multi_Schedule:
[65deb18]289                for(int i = 0; i < lock_count; i++) {
290                        unlock( *locks[i] );
[0c78741]291                }
[65deb18]292                for(int i = 0; i < thrd_count; i++) {
293                        ScheduleThread( thrds[i] );
[0c78741]294                }
[09800e9]295        case Callback:
296                callback();
297        default:
298                abort("KERNEL ERROR: Unexpected action to run after thread");
[8fcbb4c]299        }
[c84e80a]300}
301
[14a61b5]302// KERNEL_ONLY
[0c92c9f]303// Context invoker for processors
304// This is the entry point for processors (kernel threads)
305// It effectively constructs a coroutine by stealing the pthread stack
[c29c342]306static void * CtxInvokeProcessor(void * arg) {
[8def349]307        processor * proc = (processor *) arg;
[14a61b5]308        kernelTLS.this_processor = proc;
309        kernelTLS.this_coroutine = NULL;
310        kernelTLS.this_thread    = NULL;
311        kernelTLS.preemption_state.[enabled, disable_count] = [false, 1];
[8def349]312        // SKULLDUGGERY: We want to create a context for the processor coroutine
313        // which is needed for the 2-step context switch. However, there is no reason
[1c273d0]314        // to waste the perfectly valid stack create by pthread.
[8def349]315        current_stack_info_t info;
316        machine_context_t ctx;
317        info.context = &ctx;
[094476d]318        (proc->runner){ proc, &info };
[8def349]319
[094476d]320        __cfaabi_dbg_print_safe("Coroutine : created stack %p\n", get_coroutine(proc->runner)->stack.base);
[8fcbb4c]321
[0c92c9f]322        //Set global state
[14a61b5]323        kernelTLS.this_coroutine = get_coroutine(proc->runner);
324        kernelTLS.this_thread    = NULL;
[8def349]325
326        //We now have a proper context from which to schedule threads
[094476d]327        __cfaabi_dbg_print_safe("Kernel : core %p created (%p, %p)\n", proc, &proc->runner, &ctx);
[8def349]328
[1c273d0]329        // SKULLDUGGERY: Since the coroutine doesn't have its own stack, we can't
330        // resume it to start it like it normally would, it will just context switch
331        // back to here. Instead directly call the main since we already are on the
[8def349]332        // appropriate stack.
[094476d]333        get_coroutine(proc->runner)->state = Active;
334        main( proc->runner );
335        get_coroutine(proc->runner)->state = Halted;
[8def349]336
[0c92c9f]337        // Main routine of the core returned, the core is now fully terminated
[094476d]338        __cfaabi_dbg_print_safe("Kernel : core %p main ended (%p)\n", proc, &proc->runner);
[8def349]339
340        return NULL;
[c84e80a]341}
342
[c29c342]343static void start(processor * this) {
[36982fc]344        __cfaabi_dbg_print_safe("Kernel : Starting core %p\n", this);
[82ff5845]345
[8fcbb4c]346        pthread_create( &this->kernel_thread, NULL, CtxInvokeProcessor, (void*)this );
[eb2e723]347
[36982fc]348        __cfaabi_dbg_print_safe("Kernel : core %p started\n", this);
[eb2e723]349}
350
[14a61b5]351// KERNEL_ONLY
[b69ea6b]352void kernel_first_resume(processor * this) {
[14a61b5]353        coroutine_desc * src = kernelTLS.this_coroutine;
[094476d]354        coroutine_desc * dst = get_coroutine(this->runner);
[b69ea6b]355
[14a61b5]356        verify( ! kernelTLS.preemption_state.enabled );
[b69ea6b]357
358        create_stack(&dst->stack, dst->stack.size);
[094476d]359        CtxStart(&this->runner, CtxInvokeCoroutine);
[b69ea6b]360
[14a61b5]361        verify( ! kernelTLS.preemption_state.enabled );
[b69ea6b]362
363        dst->last = src;
364        dst->starter = dst->starter ? dst->starter : src;
365
366        // set state of current coroutine to inactive
367        src->state = src->state == Halted ? Halted : Inactive;
368
369        // set new coroutine that task is executing
[14a61b5]370        kernelTLS.this_coroutine = dst;
[b69ea6b]371
372        // SKULLDUGGERY normally interrupts are enable before leaving a coroutine ctxswitch.
373        // Therefore, when first creating a coroutine, interrupts are enable before calling the main.
374        // This is consistent with thread creation. However, when creating the main processor coroutine,
375        // we wan't interrupts to be disabled. Therefore, we double-disable interrupts here so they will
376        // stay disabled.
377        disable_interrupts();
378
379        // context switch to specified coroutine
380        assert( src->stack.context );
381        CtxSwitch( src->stack.context, dst->stack.context );
382        // when CtxSwitch returns we are back in the src coroutine
383
384        // set state of new coroutine to active
385        src->state = Active;
386
[14a61b5]387        verify( ! kernelTLS.preemption_state.enabled );
[b69ea6b]388}
389
[8def349]390//-----------------------------------------------------------------------------
391// Scheduler routines
[14a61b5]392
393// KERNEL ONLY
[348006f]394void ScheduleThread( thread_desc * thrd ) {
[135b431]395        verify( thrd );
[b18830e]396        verify( thrd->self_cor.state != Halted );
[1c273d0]397
[14a61b5]398        verify( ! kernelTLS.preemption_state.enabled );
[690f13c]399
[4aa2fb2]400        verifyf( thrd->next == NULL, "Expected null got %p", thrd->next );
[1c273d0]401
[de6319f]402        with( *thrd->curr_cluster ) {
[65deb18]403                lock  ( ready_queue_lock __cfaabi_dbg_ctx2 );
[6b4cdd3]404                bool was_empty = !(ready_queue != 0);
[65deb18]405                append( ready_queue, thrd );
406                unlock( ready_queue_lock );
[6b4cdd3]407
[85b1deb]408                if(was_empty) {
[6b4cdd3]409                        lock      (proc_list_lock __cfaabi_dbg_ctx2);
410                        if(idles) {
[85b1deb]411                                wake_fast(idles.head);
[6b4cdd3]412                        }
413                        unlock    (proc_list_lock);
414                }
[85b1deb]415                else if( struct processor * idle = idles.head ) {
416                        wake_fast(idle);
417                }
418
[65deb18]419        }
[1c273d0]420
[14a61b5]421        verify( ! kernelTLS.preemption_state.enabled );
[db6f06a]422}
423
[14a61b5]424// KERNEL ONLY
[65deb18]425thread_desc * nextThread(cluster * this) with( *this ) {
[14a61b5]426        verify( ! kernelTLS.preemption_state.enabled );
[65deb18]427        lock( ready_queue_lock __cfaabi_dbg_ctx2 );
428        thread_desc * head = pop_head( ready_queue );
429        unlock( ready_queue_lock );
[14a61b5]430        verify( ! kernelTLS.preemption_state.enabled );
[db6f06a]431        return head;
[eb2e723]432}
433
[82ff5845]434void BlockInternal() {
435        disable_interrupts();
[14a61b5]436        verify( ! kernelTLS.preemption_state.enabled );
[82c948c]437        returnToKernel();
[14a61b5]438        verify( ! kernelTLS.preemption_state.enabled );
[36982fc]439        enable_interrupts( __cfaabi_dbg_ctx );
[75f3522]440}
441
[ea7d2b0]442void BlockInternal( __spinlock_t * lock ) {
[82ff5845]443        disable_interrupts();
[14a61b5]444        with( *kernelTLS.this_processor ) {
[de6319f]445                finish.action_code = Release;
446                finish.lock        = lock;
447        }
[0b33412]448
[afd550c]449        verify( ! kernelTLS.preemption_state.enabled );
[82c948c]450        returnToKernel();
[afd550c]451        verify( ! kernelTLS.preemption_state.enabled );
[0b33412]452
[36982fc]453        enable_interrupts( __cfaabi_dbg_ctx );
[db6f06a]454}
455
[82ff5845]456void BlockInternal( thread_desc * thrd ) {
457        disable_interrupts();
[14a61b5]458        with( * kernelTLS.this_processor ) {
[de6319f]459                finish.action_code = Schedule;
460                finish.thrd        = thrd;
461        }
[0b33412]462
[14a61b5]463        verify( ! kernelTLS.preemption_state.enabled );
[82c948c]464        returnToKernel();
[14a61b5]465        verify( ! kernelTLS.preemption_state.enabled );
[0b33412]466
[36982fc]467        enable_interrupts( __cfaabi_dbg_ctx );
[db6f06a]468}
469
[ea7d2b0]470void BlockInternal( __spinlock_t * lock, thread_desc * thrd ) {
[97e3296]471        assert(thrd);
[82ff5845]472        disable_interrupts();
[14a61b5]473        with( * kernelTLS.this_processor ) {
[de6319f]474                finish.action_code = Release_Schedule;
475                finish.lock        = lock;
476                finish.thrd        = thrd;
477        }
[0b33412]478
[14a61b5]479        verify( ! kernelTLS.preemption_state.enabled );
[82c948c]480        returnToKernel();
[14a61b5]481        verify( ! kernelTLS.preemption_state.enabled );
[0b33412]482
[36982fc]483        enable_interrupts( __cfaabi_dbg_ctx );
[eb2e723]484}
485
[ea7d2b0]486void BlockInternal(__spinlock_t * locks [], unsigned short count) {
[82ff5845]487        disable_interrupts();
[14a61b5]488        with( * kernelTLS.this_processor ) {
[de6319f]489                finish.action_code = Release_Multi;
490                finish.locks       = locks;
491                finish.lock_count  = count;
492        }
[0b33412]493
[14a61b5]494        verify( ! kernelTLS.preemption_state.enabled );
[82c948c]495        returnToKernel();
[14a61b5]496        verify( ! kernelTLS.preemption_state.enabled );
[0b33412]497
[36982fc]498        enable_interrupts( __cfaabi_dbg_ctx );
[0c78741]499}
500
[ea7d2b0]501void BlockInternal(__spinlock_t * locks [], unsigned short lock_count, thread_desc * thrds [], unsigned short thrd_count) {
[82ff5845]502        disable_interrupts();
[14a61b5]503        with( *kernelTLS.this_processor ) {
[de6319f]504                finish.action_code = Release_Multi_Schedule;
505                finish.locks       = locks;
506                finish.lock_count  = lock_count;
507                finish.thrds       = thrds;
508                finish.thrd_count  = thrd_count;
509        }
[0b33412]510
[14a61b5]511        verify( ! kernelTLS.preemption_state.enabled );
[82c948c]512        returnToKernel();
[14a61b5]513        verify( ! kernelTLS.preemption_state.enabled );
[0b33412]514
[36982fc]515        enable_interrupts( __cfaabi_dbg_ctx );
[0c78741]516}
517
[09800e9]518void BlockInternal(__finish_callback_fptr_t callback) {
519        disable_interrupts();
520        with( *kernelTLS.this_processor ) {
521                finish.action_code = Callback;
522                finish.callback    = callback;
523        }
524
525        verify( ! kernelTLS.preemption_state.enabled );
526        returnToKernel();
527        verify( ! kernelTLS.preemption_state.enabled );
528
529        enable_interrupts( __cfaabi_dbg_ctx );
530}
531
[14a61b5]532// KERNEL ONLY
[ea7d2b0]533void LeaveThread(__spinlock_t * lock, thread_desc * thrd) {
[14a61b5]534        verify( ! kernelTLS.preemption_state.enabled );
535        with( * kernelTLS.this_processor ) {
[de6319f]536                finish.action_code = thrd ? Release_Schedule : Release;
537                finish.lock        = lock;
538                finish.thrd        = thrd;
539        }
[f2b12406]540
[82c948c]541        returnToKernel();
[f2b12406]542}
543
[fa21ac9]544//=============================================================================================
545// Kernel Setup logic
546//=============================================================================================
[eb2e723]547//-----------------------------------------------------------------------------
548// Kernel boot procedures
[c29c342]549static void kernel_startup(void) {
[14a61b5]550        verify( ! kernelTLS.preemption_state.enabled );
[36982fc]551        __cfaabi_dbg_print_safe("Kernel : Starting\n");
[eb2e723]552
[ea8b2f7]553        __cfa_dbg_global_clusters.list{ __get };
554        __cfa_dbg_global_clusters.lock{};
[de94a60]555
[de6319f]556        // Initialize the main cluster
557        mainCluster = (cluster *)&storage_mainCluster;
558        (*mainCluster){"Main Cluster"};
559
560        __cfaabi_dbg_print_safe("Kernel : Main cluster ready\n");
561
[eb2e723]562        // Start by initializing the main thread
[1c273d0]563        // SKULLDUGGERY: the mainThread steals the process main thread
[969b3fe]564        // which will then be scheduled by the mainProcessor normally
565        mainThread = (thread_desc *)&storage_mainThread;
[8fcbb4c]566        current_stack_info_t info;
[83a071f9]567        (*mainThread){ &info };
[eb2e723]568
[36982fc]569        __cfaabi_dbg_print_safe("Kernel : Main thread ready\n");
[fa21ac9]570
[bd98b58]571
[de6319f]572
573        // Construct the processor context of the main processor
574        void ?{}(processorCtx_t & this, processor * proc) {
575                (this.__cor){ "Processor" };
576                this.__cor.starter = NULL;
577                this.proc = proc;
578        }
579
580        void ?{}(processor & this) with( this ) {
581                name = "Main Processor";
582                cltr = mainCluster;
583                terminated{ 0 };
584                do_terminate = false;
585                preemption_alarm = NULL;
586                pending_preemption = false;
587                kernel_thread = pthread_self();
588
589                runner{ &this };
590                __cfaabi_dbg_print_safe("Kernel : constructed main processor context %p\n", &runner);
591        }
[fa21ac9]592
[969b3fe]593        // Initialize the main processor and the main processor ctx
[eb2e723]594        // (the coroutine that contains the processing control flow)
[969b3fe]595        mainProcessor = (processor *)&storage_mainProcessor;
[de6319f]596        (*mainProcessor){};
[eb2e723]597
[dcb42b8]598        //initialize the global state variables
[14a61b5]599        kernelTLS.this_processor = mainProcessor;
600        kernelTLS.this_thread    = mainThread;
601        kernelTLS.this_coroutine = &mainThread->self_cor;
[eb2e723]602
[82ff5845]603        // Enable preemption
604        kernel_start_preemption();
605
[969b3fe]606        // Add the main thread to the ready queue
607        // once resume is called on mainProcessor->runner the mainThread needs to be scheduled like any normal thread
608        ScheduleThread(mainThread);
609
610        // SKULLDUGGERY: Force a context switch to the main processor to set the main thread's context to the current UNIX
[dcb42b8]611        // context. Hence, the main thread does not begin through CtxInvokeThread, like all other threads. The trick here is that
[1c273d0]612        // mainThread is on the ready queue when this call is made.
[14a61b5]613        kernel_first_resume( kernelTLS.this_processor );
[eb2e723]614
[dcb42b8]615
616
617        // THE SYSTEM IS NOW COMPLETELY RUNNING
[36982fc]618        __cfaabi_dbg_print_safe("Kernel : Started\n--------------------------------------------------\n\n");
[82ff5845]619
[14a61b5]620        verify( ! kernelTLS.preemption_state.enabled );
[36982fc]621        enable_interrupts( __cfaabi_dbg_ctx );
[afd550c]622        verify( TL_GET( preemption_state.enabled ) );
[eb2e723]623}
624
[c29c342]625static void kernel_shutdown(void) {
[36982fc]626        __cfaabi_dbg_print_safe("\n--------------------------------------------------\nKernel : Shutting down\n");
[eb2e723]627
[afd550c]628        verify( TL_GET( preemption_state.enabled ) );
[4e6fb8e]629        disable_interrupts();
[14a61b5]630        verify( ! kernelTLS.preemption_state.enabled );
[4e6fb8e]631
[969b3fe]632        // SKULLDUGGERY: Notify the mainProcessor it needs to terminates.
[dcb42b8]633        // When its coroutine terminates, it return control to the mainThread
634        // which is currently here
[ea8b2f7]635        __atomic_store_n(&mainProcessor->do_terminate, true, __ATOMIC_RELEASE);
[82c948c]636        returnToKernel();
[ea8b2f7]637        mainThread->self_cor.state = Halted;
[eb2e723]638
[dcb42b8]639        // THE SYSTEM IS NOW COMPLETELY STOPPED
[eb2e723]640
[82ff5845]641        // Disable preemption
642        kernel_stop_preemption();
643
[969b3fe]644        // Destroy the main processor and its context in reverse order of construction
[dcb42b8]645        // These were manually constructed so we need manually destroy them
[094476d]646        ^(mainProcessor->runner){};
[969b3fe]647        ^(mainProcessor){};
[eb2e723]648
[dcb42b8]649        // Final step, destroy the main thread since it is no longer needed
650        // Since we provided a stack to this taxk it will not destroy anything
[eb2e723]651        ^(mainThread){};
652
[ea8b2f7]653        ^(__cfa_dbg_global_clusters.list){};
654        ^(__cfa_dbg_global_clusters.lock){};
[a1a17a7]655
[36982fc]656        __cfaabi_dbg_print_safe("Kernel : Shutdown complete\n");
[9d944b2]657}
658
[14a61b5]659//=============================================================================================
660// Kernel Quiescing
661//=============================================================================================
[c29c342]662static void halt(processor * this) with( *this ) {
[85b1deb]663        // verify( ! __atomic_load_n(&do_terminate, __ATOMIC_SEQ_CST) );
[ea8b2f7]664
[6b4cdd3]665        with( *cltr ) {
666                lock      (proc_list_lock __cfaabi_dbg_ctx2);
667                remove    (procs, *this);
668                push_front(idles, *this);
669                unlock    (proc_list_lock);
670        }
[14a61b5]671
[6b4cdd3]672        __cfaabi_dbg_print_safe("Kernel : Processor %p ready to sleep\n", this);
[14a61b5]673
[85b1deb]674        wait( idleLock );
[14a61b5]675
[6b4cdd3]676        __cfaabi_dbg_print_safe("Kernel : Processor %p woke up and ready to run\n", this);
[14a61b5]677
[6b4cdd3]678        with( *cltr ) {
679                lock      (proc_list_lock __cfaabi_dbg_ctx2);
680                remove    (idles, *this);
681                push_front(procs, *this);
682                unlock    (proc_list_lock);
683        }
684}
685
[dbe9b08]686//=============================================================================================
687// Unexpected Terminating logic
688//=============================================================================================
[ea7d2b0]689static __spinlock_t kernel_abort_lock;
[9d944b2]690static bool kernel_abort_called = false;
691
[afd550c]692void * kernel_abort(void) __attribute__ ((__nothrow__)) {
[9d944b2]693        // abort cannot be recursively entered by the same or different processors because all signal handlers return when
694        // the globalAbort flag is true.
[36982fc]695        lock( kernel_abort_lock __cfaabi_dbg_ctx2 );
[9d944b2]696
697        // first task to abort ?
[de94a60]698        if ( kernel_abort_called ) {                    // not first task to abort ?
[ea7d2b0]699                unlock( kernel_abort_lock );
[1c273d0]700
[9d944b2]701                sigset_t mask;
702                sigemptyset( &mask );
[de94a60]703                sigaddset( &mask, SIGALRM );            // block SIGALRM signals
704                sigsuspend( &mask );                    // block the processor to prevent further damage during abort
705                _exit( EXIT_FAILURE );                  // if processor unblocks before it is killed, terminate it
706        }
707        else {
708                kernel_abort_called = true;
709                unlock( kernel_abort_lock );
[9d944b2]710        }
711
[14a61b5]712        return kernelTLS.this_thread;
[9d944b2]713}
714
715void kernel_abort_msg( void * kernel_data, char * abort_text, int abort_text_size ) {
716        thread_desc * thrd = kernel_data;
717
[de94a60]718        if(thrd) {
719                int len = snprintf( abort_text, abort_text_size, "Error occurred while executing thread %.256s (%p)", thrd->self_cor.name, thrd );
[36982fc]720                __cfaabi_dbg_bits_write( abort_text, len );
[de94a60]721
722                if ( get_coroutine(thrd) != kernelTLS.this_coroutine ) {
723                        len = snprintf( abort_text, abort_text_size, " in coroutine %.256s (%p).\n", kernelTLS.this_coroutine->name, kernelTLS.this_coroutine );
724                        __cfaabi_dbg_bits_write( abort_text, len );
725                }
726                else {
727                        __cfaabi_dbg_bits_write( ".\n", 2 );
728                }
[1c273d0]729        }
[9d944b2]730        else {
[de94a60]731                int len = snprintf( abort_text, abort_text_size, "Error occurred outside of any thread.\n" );
[639991a]732                __cfaabi_dbg_bits_write( abort_text, len );
[9d944b2]733        }
734}
735
[2b8bc41]736int kernel_abort_lastframe( void ) __attribute__ ((__nothrow__)) {
[14a61b5]737        return get_coroutine(kernelTLS.this_thread) == get_coroutine(mainThread) ? 4 : 2;
[2b8bc41]738}
739
[de94a60]740static __spinlock_t kernel_debug_lock;
741
[9d944b2]742extern "C" {
[36982fc]743        void __cfaabi_dbg_bits_acquire() {
744                lock( kernel_debug_lock __cfaabi_dbg_ctx2 );
[9d944b2]745        }
746
[36982fc]747        void __cfaabi_dbg_bits_release() {
[ea7d2b0]748                unlock( kernel_debug_lock );
[9d944b2]749        }
[8118303]750}
751
[fa21ac9]752//=============================================================================================
753// Kernel Utilities
754//=============================================================================================
[bd98b58]755//-----------------------------------------------------------------------------
756// Locks
[242a902]757void  ?{}( semaphore & this, int count = 1 ) {
758        (this.lock){};
759        this.count = count;
760        (this.waiting){};
[db6f06a]761}
[242a902]762void ^?{}(semaphore & this) {}
[db6f06a]763
[65deb18]764void P(semaphore & this) with( this ){
765        lock( lock __cfaabi_dbg_ctx2 );
766        count -= 1;
767        if ( count < 0 ) {
[bdeba0b]768                // queue current task
[14a61b5]769                append( waiting, kernelTLS.this_thread );
[bdeba0b]770
771                // atomically release spin lock and block
[65deb18]772                BlockInternal( &lock );
[8def349]773        }
[4e6fb8e]774        else {
[65deb18]775            unlock( lock );
[4e6fb8e]776        }
[bd98b58]777}
778
[65deb18]779void V(semaphore & this) with( this ) {
[bdeba0b]780        thread_desc * thrd = NULL;
[65deb18]781        lock( lock __cfaabi_dbg_ctx2 );
782        count += 1;
783        if ( count <= 0 ) {
[bdeba0b]784                // remove task at head of waiting list
[65deb18]785                thrd = pop_head( waiting );
[bd98b58]786        }
[bdeba0b]787
[65deb18]788        unlock( lock );
[bdeba0b]789
790        // make new owner
791        WakeThread( thrd );
[bd98b58]792}
793
[f7d6bb0]794//-----------------------------------------------------------------------------
[de94a60]795// Global Queues
796void doregister( cluster     & cltr ) {
[ea8b2f7]797        lock      ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
798        push_front( __cfa_dbg_global_clusters.list, cltr );
799        unlock    ( __cfa_dbg_global_clusters.lock );
[de94a60]800}
[f7d6bb0]801
[de94a60]802void unregister( cluster     & cltr ) {
[ea8b2f7]803        lock  ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
804        remove( __cfa_dbg_global_clusters.list, cltr );
805        unlock( __cfa_dbg_global_clusters.lock );
[de94a60]806}
[f7d6bb0]807
[a1a17a7]808void doregister( cluster * cltr, thread_desc & thrd ) {
809        lock      (cltr->thread_list_lock __cfaabi_dbg_ctx2);
810        push_front(cltr->threads, thrd);
811        unlock    (cltr->thread_list_lock);
812}
813
814void unregister( cluster * cltr, thread_desc & thrd ) {
815        lock  (cltr->thread_list_lock __cfaabi_dbg_ctx2);
816        remove(cltr->threads, thrd );
817        unlock(cltr->thread_list_lock);
818}
[9181f1d]819
[de94a60]820void doregister( cluster * cltr, processor * proc ) {
[639991a]821        lock      (cltr->proc_list_lock __cfaabi_dbg_ctx2);
822        push_front(cltr->procs, *proc);
823        unlock    (cltr->proc_list_lock);
[de94a60]824}
825
826void unregister( cluster * cltr, processor * proc ) {
[639991a]827        lock  (cltr->proc_list_lock __cfaabi_dbg_ctx2);
828        remove(cltr->procs, *proc );
829        unlock(cltr->proc_list_lock);
[de94a60]830}
831
832//-----------------------------------------------------------------------------
833// Debug
834__cfaabi_dbg_debug_do(
[1997b4e]835        extern "C" {
836                void __cfaabi_dbg_record(__spinlock_t & this, const char * prev_name) {
837                        this.prev_name = prev_name;
838                        this.prev_thrd = kernelTLS.this_thread;
839                }
[9181f1d]840        }
[f7d6bb0]841)
[8118303]842// Local Variables: //
843// mode: c //
844// tab-width: 4 //
845// End: //
Note: See TracBrowser for help on using the repository browser.