Ignore:
File:
1 edited

Legend:

Unmodified
Added
Removed

  • src/libcfa/concurrency/kernel.c

    rde6319f rde94a60  
    4949thread_desc * mainThread;
    5050
     51struct { __dllist_t(thread_desc) list; __spinlock_t lock; } global_threads ;
     52struct { __dllist_t(cluster    ) list; __spinlock_t lock; } global_clusters;
     53
    5154//-----------------------------------------------------------------------------
    5255// Global state
    … …  
    5659// volatile thread_local unsigned short disable_preempt_count = 1;
    5760
    58 thread_local struct KernelThreadData kernelThreadData = {
     61thread_local struct KernelThreadData kernelTLS = {
    5962        NULL,
    6063        NULL,
    … …  
    117120        self_mon_p = &self_mon;
    118121        next = NULL;
    119         __cfaabi_dbg_debug_do(
    120                 dbg_next = NULL;
    121                 dbg_prev = NULL;
    122                 __cfaabi_dbg_thread_register(&this);
    123         )
     122
     123        node.next = NULL;
     124        node.prev = NULL;
     125        doregister(this);
    124126
    125127        monitors{ &self_mon_p, 1, (fptr_t)0 };
    … …  
    155157                terminate(&this);
    156158                verify(this.do_terminate);
    157                 verify(TL_GET( this_processor ) != &this);
     159                verify( kernelTLS.this_processor != &this);
    158160                P( terminated );
    159                 verify(TL_GET( this_processor ) != &this);
     161                verify( kernelTLS.this_processor != &this);
    160162                pthread_join( kernel_thread, NULL );
    161163        }
    … …  
    167169        ready_queue{};
    168170        ready_queue_lock{};
     171
     172        procs{ __get };
     173        idles{ __get };
     174
     175        doregister(this);
    169176}
    170177
    171178void ^?{}(cluster & this) {
    172 
     179        unregister(this);
    173180}
    174181
    … …  
    183190        __cfaabi_dbg_print_safe("Kernel : core %p starting\n", this);
    184191
     192        doregister(this->cltr, this);
     193
    185194        {
    186195                // Setup preemption data
    … …  
    196205                        if(readyThread)
    197206                        {
    198                                 verify( ! TL_GET( preemption_state ).enabled );
     207                                verify( ! kernelTLS.preemption_state.enabled );
    199208
    200209                                runThread(this, readyThread);
    201210
    202                                 verify( ! TL_GET( preemption_state ).enabled );
     211                                verify( ! kernelTLS.preemption_state.enabled );
    203212
    204213                                //Some actions need to be taken from the kernel
    … …  
    216225        }
    217226
     227        unregister(this->cltr, this);
     228
    218229        V( this->terminated );
    219230
    … …  
    221232}
    222233
     234// KERNEL ONLY
    223235// runThread runs a thread by context switching
    224236// from the processor coroutine to the target thread
    … …  
    228240        coroutine_desc * thrd_cor = dst->curr_cor;
    229241
    230         //Reset the terminating actions here
     242        // Reset the terminating actions here
    231243        this->finish.action_code = No_Action;
    232244
    233         //Update global state
    234         TL_SET( this_thread, dst );
     245        // Update global state
     246        kernelTLS.this_thread = dst;
    235247
    236248        // Context Switch to the thread
    … …  
    239251}
    240252
     253// KERNEL_ONLY
    241254void returnToKernel() {
    242         coroutine_desc * proc_cor = get_coroutine(TL_GET( this_processor )->runner);
    243         coroutine_desc * thrd_cor = TL_GET( this_thread )->curr_cor = TL_GET( this_coroutine );
     255        coroutine_desc * proc_cor = get_coroutine(kernelTLS.this_processor->runner);
     256        coroutine_desc * thrd_cor = kernelTLS.this_thread->curr_cor = kernelTLS.this_coroutine;
    244257        ThreadCtxSwitch(thrd_cor, proc_cor);
    245258}
    246259
     260// KERNEL_ONLY
    247261// Once a thread has finished running, some of
    248262// its final actions must be executed from the kernel
    249263void finishRunning(processor * this) with( this->finish ) {
    250264        if( action_code == Release ) {
    251                 verify( ! TL_GET( preemption_state ).enabled );
     265                verify( ! kernelTLS.preemption_state.enabled );
    252266                unlock( *lock );
    253267        }
    … …  
    256270        }
    257271        else if( action_code == Release_Schedule ) {
    258                 verify( ! TL_GET( preemption_state ).enabled );
     272                verify( ! kernelTLS.preemption_state.enabled );
    259273                unlock( *lock );
    260274                ScheduleThread( thrd );
    261275        }
    262276        else if( action_code == Release_Multi ) {
    263                 verify( ! TL_GET( preemption_state ).enabled );
     277                verify( ! kernelTLS.preemption_state.enabled );
    264278                for(int i = 0; i < lock_count; i++) {
    265279                        unlock( *locks[i] );
    … …  
    285299}
    286300
     301// KERNEL_ONLY
    287302// Context invoker for processors
    288303// This is the entry point for processors (kernel threads)
    … …  
    290305void * CtxInvokeProcessor(void * arg) {
    291306        processor * proc = (processor *) arg;
    292         TL_SET( this_processor, proc );
    293         TL_SET( this_coroutine, NULL );
    294         TL_SET( this_thread, NULL );
    295         TL_GET( preemption_state ).[enabled, disable_count] = [false, 1];
     307        kernelTLS.this_processor = proc;
     308        kernelTLS.this_coroutine = NULL;
     309        kernelTLS.this_thread    = NULL;
     310        kernelTLS.preemption_state.[enabled, disable_count] = [false, 1];
    296311        // SKULLDUGGERY: We want to create a context for the processor coroutine
    297312        // which is needed for the 2-step context switch. However, there is no reason
    … …  
    305320
    306321        //Set global state
    307         TL_SET( this_coroutine, get_coroutine(proc->runner) );
    308         TL_SET( this_thread, NULL );
     322        kernelTLS.this_coroutine = get_coroutine(proc->runner);
     323        kernelTLS.this_thread    = NULL;
    309324
    310325        //We now have a proper context from which to schedule threads
    … …  
    333348}
    334349
     350// KERNEL_ONLY
    335351void kernel_first_resume(processor * this) {
    336         coroutine_desc * src = TL_GET( this_coroutine );
     352        coroutine_desc * src = kernelTLS.this_coroutine;
    337353        coroutine_desc * dst = get_coroutine(this->runner);
    338354
    339         verify( ! TL_GET( preemption_state ).enabled );
     355        verify( ! kernelTLS.preemption_state.enabled );
    340356
    341357        create_stack(&dst->stack, dst->stack.size);
    342358        CtxStart(&this->runner, CtxInvokeCoroutine);
    343359
    344         verify( ! TL_GET( preemption_state ).enabled );
     360        verify( ! kernelTLS.preemption_state.enabled );
    345361
    346362        dst->last = src;
    … …  
    351367
    352368        // set new coroutine that task is executing
    353         TL_SET( this_coroutine, dst );
     369        kernelTLS.this_coroutine = dst;
    354370
    355371        // SKULLDUGGERY normally interrupts are enable before leaving a coroutine ctxswitch.
    … …  
    368384        src->state = Active;
    369385
    370         verify( ! TL_GET( preemption_state ).enabled );
     386        verify( ! kernelTLS.preemption_state.enabled );
    371387}
    372388
    373389//-----------------------------------------------------------------------------
    374390// Scheduler routines
     391
     392// KERNEL ONLY
    375393void ScheduleThread( thread_desc * thrd ) {
    376         // if( ! thrd ) return;
    377394        verify( thrd );
    378395        verify( thrd->self_cor.state != Halted );
    379396
    380         verify( ! TL_GET( preemption_state ).enabled );
     397        verify( ! kernelTLS.preemption_state.enabled );
    381398
    382399        verifyf( thrd->next == NULL, "Expected null got %p", thrd->next );
    … …  
    388405        }
    389406
    390         verify( ! TL_GET( preemption_state ).enabled );
    391 }
    392 
     407        verify( ! kernelTLS.preemption_state.enabled );
     408}
     409
     410// KERNEL ONLY
    393411thread_desc * nextThread(cluster * this) with( *this ) {
    394         verify( ! TL_GET( preemption_state ).enabled );
     412        verify( ! kernelTLS.preemption_state.enabled );
    395413        lock( ready_queue_lock __cfaabi_dbg_ctx2 );
    396414        thread_desc * head = pop_head( ready_queue );
    397415        unlock( ready_queue_lock );
    398         verify( ! TL_GET( preemption_state ).enabled );
     416        verify( ! kernelTLS.preemption_state.enabled );
    399417        return head;
    400418}
    … …  
    402420void BlockInternal() {
    403421        disable_interrupts();
    404         verify( ! TL_GET( preemption_state ).enabled );
     422        verify( ! kernelTLS.preemption_state.enabled );
    405423        returnToKernel();
    406         verify( ! TL_GET( preemption_state ).enabled );
     424        verify( ! kernelTLS.preemption_state.enabled );
    407425        enable_interrupts( __cfaabi_dbg_ctx );
    408426}
    … …  
    410428void BlockInternal( __spinlock_t * lock ) {
    411429        disable_interrupts();
    412         with( *TL_GET( this_processor ) ) {
     430        with( *kernelTLS.this_processor ) {
    413431                finish.action_code = Release;
    414432                finish.lock        = lock;
    415433        }
    416434
    417         verify( ! TL_GET( preemption_state ).enabled );
     435        verify( ! kernelTLS.preemption_state.enabled );
    418436        returnToKernel();
    419         verify( ! TL_GET( preemption_state ).enabled );
     437        verify( ! kernelTLS.preemption_state.enabled );
    420438
    421439        enable_interrupts( __cfaabi_dbg_ctx );
    … …  
    424442void BlockInternal( thread_desc * thrd ) {
    425443        disable_interrupts();
    426         with( *TL_GET( this_processor ) ) {
     444        with( * kernelTLS.this_processor ) {
    427445                finish.action_code = Schedule;
    428446                finish.thrd        = thrd;
    429447        }
    430448
    431         verify( ! TL_GET( preemption_state ).enabled );
     449        verify( ! kernelTLS.preemption_state.enabled );
    432450        returnToKernel();
    433         verify( ! TL_GET( preemption_state ).enabled );
     451        verify( ! kernelTLS.preemption_state.enabled );
    434452
    435453        enable_interrupts( __cfaabi_dbg_ctx );
    … …  
    439457        assert(thrd);
    440458        disable_interrupts();
    441         with( *TL_GET( this_processor ) ) {
     459        with( * kernelTLS.this_processor ) {
    442460                finish.action_code = Release_Schedule;
    443461                finish.lock        = lock;
    … …  
    445463        }
    446464
    447         verify( ! TL_GET( preemption_state ).enabled );
     465        verify( ! kernelTLS.preemption_state.enabled );
    448466        returnToKernel();
    449         verify( ! TL_GET( preemption_state ).enabled );
     467        verify( ! kernelTLS.preemption_state.enabled );
    450468
    451469        enable_interrupts( __cfaabi_dbg_ctx );
    … …  
    454472void BlockInternal(__spinlock_t * locks [], unsigned short count) {
    455473        disable_interrupts();
    456         with( *TL_GET( this_processor ) ) {
     474        with( * kernelTLS.this_processor ) {
    457475                finish.action_code = Release_Multi;
    458476                finish.locks       = locks;
    … …  
    460478        }
    461479
    462         verify( ! TL_GET( preemption_state ).enabled );
     480        verify( ! kernelTLS.preemption_state.enabled );
    463481        returnToKernel();
    464         verify( ! TL_GET( preemption_state ).enabled );
     482        verify( ! kernelTLS.preemption_state.enabled );
    465483
    466484        enable_interrupts( __cfaabi_dbg_ctx );
    … …  
    469487void BlockInternal(__spinlock_t * locks [], unsigned short lock_count, thread_desc * thrds [], unsigned short thrd_count) {
    470488        disable_interrupts();
    471         with( *TL_GET( this_processor ) ) {
     489        with( *kernelTLS.this_processor ) {
    472490                finish.action_code = Release_Multi_Schedule;
    473491                finish.locks       = locks;
    … …  
    477495        }
    478496
    479         verify( ! TL_GET( preemption_state ).enabled );
     497        verify( ! kernelTLS.preemption_state.enabled );
    480498        returnToKernel();
    481         verify( ! TL_GET( preemption_state ).enabled );
     499        verify( ! kernelTLS.preemption_state.enabled );
    482500
    483501        enable_interrupts( __cfaabi_dbg_ctx );
    484502}
    485503
     504// KERNEL ONLY
    486505void LeaveThread(__spinlock_t * lock, thread_desc * thrd) {
    487         verify( ! TL_GET( preemption_state ).enabled );
    488         with( *TL_GET( this_processor ) ) {
     506        verify( ! kernelTLS.preemption_state.enabled );
     507        with( * kernelTLS.this_processor ) {
    489508                finish.action_code = thrd ? Release_Schedule : Release;
    490509                finish.lock        = lock;
    … …  
    501520// Kernel boot procedures
    502521void kernel_startup(void) {
    503         verify( ! TL_GET( preemption_state ).enabled );
     522        verify( ! kernelTLS.preemption_state.enabled );
    504523        __cfaabi_dbg_print_safe("Kernel : Starting\n");
     524
     525        global_threads. list{ __get };
     526        global_threads. lock{};
     527        global_clusters.list{ __get };
     528        global_clusters.lock{};
    505529
    506530        // Initialize the main cluster
    … …  
    547571
    548572        //initialize the global state variables
    549         TL_SET( this_processor, mainProcessor );
    550         TL_SET( this_thread, mainThread );
    551         TL_SET( this_coroutine, &mainThread->self_cor );
     573        kernelTLS.this_processor = mainProcessor;
     574        kernelTLS.this_thread    = mainThread;
     575        kernelTLS.this_coroutine = &mainThread->self_cor;
    552576
    553577        // Enable preemption
    … …  
    561585        // context. Hence, the main thread does not begin through CtxInvokeThread, like all other threads. The trick here is that
    562586        // mainThread is on the ready queue when this call is made.
    563         kernel_first_resume( TL_GET( this_processor ) );
     587        kernel_first_resume( kernelTLS.this_processor );
    564588
    565589
    … …  
    568592        __cfaabi_dbg_print_safe("Kernel : Started\n--------------------------------------------------\n\n");
    569593
    570         verify( ! TL_GET( preemption_state ).enabled );
     594        verify( ! kernelTLS.preemption_state.enabled );
    571595        enable_interrupts( __cfaabi_dbg_ctx );
    572         verify( TL_GET( preemption_state ).enabled );
     596        verify( TL_GET( preemption_state.enabled ) );
    573597}
    574598
    … …  
    576600        __cfaabi_dbg_print_safe("\n--------------------------------------------------\nKernel : Shutting down\n");
    577601
    578         verify( TL_GET( preemption_state ).enabled );
     602        verify( TL_GET( preemption_state.enabled ) );
    579603        disable_interrupts();
    580         verify( ! TL_GET( preemption_state ).enabled );
     604        verify( ! kernelTLS.preemption_state.enabled );
    581605
    582606        // SKULLDUGGERY: Notify the mainProcessor it needs to terminates.
    … …  
    604628
    605629//=============================================================================================
     630// Kernel Quiescing
     631//=============================================================================================
     632
     633// void halt(processor * this) with( this ) {
     634//      pthread_mutex_lock( &idle.lock );
     635
     636
     637
     638//      // SKULLDUGGERY: Even if spurious wake-up is a thing
     639//      // spuriously waking up a kernel thread is not a big deal
     640//      // if it is very rare.
     641//      pthread_cond_wait( &idle.cond, &idle.lock);
     642//      pthread_mutex_unlock( &idle.lock );
     643// }
     644
     645// void wake(processor * this) with( this ) {
     646//      pthread_mutex_lock  (&idle.lock);
     647//      pthread_cond_signal (&idle.cond);
     648//      pthread_mutex_unlock(&idle.lock);
     649// }
     650
     651//=============================================================================================
    606652// Unexpected Terminating logic
    607653//=============================================================================================
    … …  
    609655
    610656static __spinlock_t kernel_abort_lock;
    611 static __spinlock_t kernel_debug_lock;
    612657static bool kernel_abort_called = false;
    613658
    614 void * kernel_abort    (void) __attribute__ ((__nothrow__)) {
     659void * kernel_abort(void) __attribute__ ((__nothrow__)) {
    615660        // abort cannot be recursively entered by the same or different processors because all signal handlers return when
    616661        // the globalAbort flag is true.
    … …  
    618663
    619664        // first task to abort ?
    620         if ( ! kernel_abort_called ) {                  // not first task to abort ?
     665        if ( kernel_abort_called ) {                    // not first task to abort ?
     666                unlock( kernel_abort_lock );
     667
     668                sigset_t mask;
     669                sigemptyset( &mask );
     670                sigaddset( &mask, SIGALRM );            // block SIGALRM signals
     671                sigsuspend( &mask );                    // block the processor to prevent further damage during abort
     672                _exit( EXIT_FAILURE );                  // if processor unblocks before it is killed, terminate it
     673        }
     674        else {
    621675                kernel_abort_called = true;
    622676                unlock( kernel_abort_lock );
    623677        }
    624         else {
    625                 unlock( kernel_abort_lock );
    626 
    627                 sigset_t mask;
    628                 sigemptyset( &mask );
    629                 sigaddset( &mask, SIGALRM );                    // block SIGALRM signals
    630                 sigaddset( &mask, SIGUSR1 );                    // block SIGUSR1 signals
    631                 sigsuspend( &mask );                            // block the processor to prevent further damage during abort
    632                 _exit( EXIT_FAILURE );                          // if processor unblocks before it is killed, terminate it
    633         }
    634 
    635         return TL_GET( this_thread );
     678
     679        return kernelTLS.this_thread;
    636680}
    637681
    … …  
    639683        thread_desc * thrd = kernel_data;
    640684
    641         int len = snprintf( abort_text, abort_text_size, "Error occurred while executing task %.256s (%p)", thrd->self_cor.name, thrd );
    642         __cfaabi_dbg_bits_write( abort_text, len );
    643 
    644         if ( get_coroutine(thrd) != TL_GET( this_coroutine ) ) {
    645                 len = snprintf( abort_text, abort_text_size, " in coroutine %.256s (%p).\n", TL_GET( this_coroutine )->name, TL_GET( this_coroutine ) );
     685        if(thrd) {
     686                int len = snprintf( abort_text, abort_text_size, "Error occurred while executing thread %.256s (%p)", thrd->self_cor.name, thrd );
    646687                __cfaabi_dbg_bits_write( abort_text, len );
     688
     689                if ( get_coroutine(thrd) != kernelTLS.this_coroutine ) {
     690                        len = snprintf( abort_text, abort_text_size, " in coroutine %.256s (%p).\n", kernelTLS.this_coroutine->name, kernelTLS.this_coroutine );
     691                        __cfaabi_dbg_bits_write( abort_text, len );
     692                }
     693                else {
     694                        __cfaabi_dbg_bits_write( ".\n", 2 );
     695                }
    647696        }
    648697        else {
    649                 __cfaabi_dbg_bits_write( ".\n", 2 );
     698                int len = snprintf( abort_text, abort_text_size, "Error occurred outside of any thread.\n" );
    650699        }
    651700}
    652701
    653702int kernel_abort_lastframe( void ) __attribute__ ((__nothrow__)) {
    654         return get_coroutine(TL_GET( this_thread )) == get_coroutine(mainThread) ? 4 : 2;
    655 }
     703        return get_coroutine(kernelTLS.this_thread) == get_coroutine(mainThread) ? 4 : 2;
     704}
     705
     706static __spinlock_t kernel_debug_lock;
    656707
    657708extern "C" {
    … …  
    682733        if ( count < 0 ) {
    683734                // queue current task
    684                 append( waiting, (thread_desc *)TL_GET( this_thread ) );
     735                append( waiting, kernelTLS.this_thread );
    685736
    686737                // atomically release spin lock and block
    … …  
    708759
    709760//-----------------------------------------------------------------------------
     761// Global Queues
     762void doregister( thread_desc & thrd ) {
     763        // lock      ( global_thread.lock );
     764        // push_front( global_thread.list, thrd );
     765        // unlock    ( global_thread.lock );
     766}
     767
     768void unregister( thread_desc & thrd ) {
     769        // lock  ( global_thread.lock );
     770        // remove( global_thread.list, thrd );
     771        // unlock( global_thread.lock );
     772}
     773
     774void doregister( cluster     & cltr ) {
     775        // lock      ( global_cluster.lock );
     776        // push_front( global_cluster.list, cltr );
     777        // unlock    ( global_cluster.lock );
     778}
     779
     780void unregister( cluster     & cltr ) {
     781        // lock  ( global_cluster.lock );
     782        // remove( global_cluster.list, cltr );
     783        // unlock( global_cluster.lock );
     784}
     785
     786
     787void doregister( cluster * cltr, processor * proc ) {
     788        // lock      (cltr->proc_list_lock __cfaabi_dbg_ctx2);
     789        // push_front(cltr->procs, *proc);
     790        // unlock    (cltr->proc_list_lock);
     791}
     792
     793void unregister( cluster * cltr, processor * proc ) {
     794        // lock  (cltr->proc_list_lock __cfaabi_dbg_ctx2);
     795        // remove(cltr->procs, *proc );
     796        // unlock(cltr->proc_list_lock);
     797}
     798
     799//-----------------------------------------------------------------------------
    710800// Debug
    711801__cfaabi_dbg_debug_do(
    712         struct {
    713                 thread_desc * tail;
    714         } __cfaabi_dbg_thread_list = { NULL };
    715 
    716         void __cfaabi_dbg_thread_register( thread_desc * thrd ) {
    717                 if( !__cfaabi_dbg_thread_list.tail ) {
    718                         __cfaabi_dbg_thread_list.tail = thrd;
    719                         return;
    720                 }
    721                 __cfaabi_dbg_thread_list.tail->dbg_next = thrd;
    722                 thrd->dbg_prev = __cfaabi_dbg_thread_list.tail;
    723                 __cfaabi_dbg_thread_list.tail = thrd;
    724         }
    725 
    726         void __cfaabi_dbg_thread_unregister( thread_desc * thrd ) {
    727                 thread_desc * prev = thrd->dbg_prev;
    728                 thread_desc * next = thrd->dbg_next;
    729 
    730                 if( next ) { next->dbg_prev = prev; }
    731                 else       {
    732                         assert( __cfaabi_dbg_thread_list.tail == thrd );
    733                         __cfaabi_dbg_thread_list.tail = prev;
    734                 }
    735 
    736                 if( prev ) { prev->dbg_next = next; }
    737 
    738                 thrd->dbg_prev = NULL;
    739                 thrd->dbg_next = NULL;
    740         }
    741 
    742802        void __cfaabi_dbg_record(__spinlock_t & this, const char * prev_name) {
    743803                this.prev_name = prev_name;
    744                 this.prev_thrd = TL_GET( this_thread );
     804                this.prev_thrd = kernelTLS.this_thread;
    745805        }
    746806)
Note: See TracChangeset for help on using the changeset viewer.