Ignore:
File:
1 edited

Legend:

Unmodified
Added
Removed

  • libcfa/src/concurrency/kernel.cfa

    r24e321c r1f45c7d  
    2222#include <errno.h>
    2323#include <stdio.h>
    24 #include <string.h>
    2524#include <signal.h>
    2625#include <unistd.h>
    … …  
    3231#include "kernel_private.hfa"
    3332#include "preemption.hfa"
    34 #include "strstream.hfa"
    35 #include "device/cpu.hfa"
    3633
    3734//Private includes
    … …  
    113110#endif
    114111
    115 extern thread$ * mainThread;
     112extern $thread * mainThread;
    116113extern processor * mainProcessor;
    117114
    118115//-----------------------------------------------------------------------------
    119116// Kernel Scheduling logic
    120 static thread$ * __next_thread(cluster * this);
    121 static thread$ * __next_thread_slow(cluster * this);
    122 static inline bool __must_unpark( thread$ * thrd ) __attribute((nonnull(1)));
    123 static void __run_thread(processor * this, thread$ * dst);
     117static $thread * __next_thread(cluster * this);
     118static $thread * __next_thread_slow(cluster * this);
     119static inline bool __must_unpark( $thread * thrd ) __attribute((nonnull(1)));
     120static void __run_thread(processor * this, $thread * dst);
    124121static void __wake_one(cluster * cltr);
    125122
    … …  
    184181                __cfadbg_print_safe(runtime_core, "Kernel : core %p started\n", this);
    185182
    186                 thread$ * readyThread = 0p;
     183                $thread * readyThread = 0p;
    187184                MAIN_LOOP:
    188185                for() {
    … …  
    234231                                __cfadbg_print_safe(runtime_core, "Kernel : core %p waiting on eventfd %d\n", this, this->idle);
    235232
    236                                 {
    237                                         eventfd_t val;
    238                                         ssize_t ret = read( this->idle, &val, sizeof(val) );
    239                                         if(ret < 0) {
    240                                                 switch((int)errno) {
    241                                                 case EAGAIN:
    242                                                 #if EAGAIN != EWOULDBLOCK
    243                                                         case EWOULDBLOCK:
    244                                                 #endif
    245                                                 case EINTR:
    246                                                         // No need to do anything special here, just assume it's a legitimate wake-up
    247                                                         break;
    248                                                 default:
    249                                                         abort( "KERNEL : internal error, read failure on idle eventfd, error(%d) %s.", (int)errno, strerror( (int)errno ) );
    250                                                 }
    251                                         }
    252                                 }
     233                                __disable_interrupts_hard();
     234                                eventfd_t val;
     235                                eventfd_read( this->idle, &val );
     236                                __enable_interrupts_hard();
    253237
    254238                                #if !defined(__CFA_NO_STATISTICS__)
    … …  
    341325                                }
    342326
    343                                 __STATS( if(this->print_halts) __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 0\n", this->unique_id, rdtscl()); )
     327                                        __STATS( if(this->print_halts) __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 0\n", this->unique_id, rdtscl()); )
    344328                                __cfadbg_print_safe(runtime_core, "Kernel : core %p waiting on eventfd %d\n", this, this->idle);
    345329
    346                                 {
    347                                         eventfd_t val;
    348                                         ssize_t ret = read( this->idle, &val, sizeof(val) );
    349                                         if(ret < 0) {
    350                                                 switch((int)errno) {
    351                                                 case EAGAIN:
    352                                                 #if EAGAIN != EWOULDBLOCK
    353                                                         case EWOULDBLOCK:
    354                                                 #endif
    355                                                 case EINTR:
    356                                                         // No need to do anything special here, just assume it's a legitimate wake-up
    357                                                         break;
    358                                                 default:
    359                                                         abort( "KERNEL : internal error, read failure on idle eventfd, error(%d) %s.", (int)errno, strerror( (int)errno ) );
    360                                                 }
    361                                         }
    362                                 }
     330                                // __disable_interrupts_hard();
     331                                eventfd_t val;
     332                                eventfd_read( this->idle, &val );
     333                                // __enable_interrupts_hard();
    363334
    364335                                        __STATS( if(this->print_halts) __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 1\n", this->unique_id, rdtscl()); )
    … …  
    417388// runThread runs a thread by context switching
    418389// from the processor coroutine to the target thread
    419 static void __run_thread(processor * this, thread$ * thrd_dst) {
     390static void __run_thread(processor * this, $thread * thrd_dst) {
    420391        /* paranoid */ verify( ! __preemption_enabled() );
    421392        /* paranoid */ verifyf( thrd_dst->state == Ready || thrd_dst->preempted != __NO_PREEMPTION, "state : %d, preempted %d\n", thrd_dst->state, thrd_dst->preempted);
    … …  
    423394        __builtin_prefetch( thrd_dst->context.SP );
    424395
     396        int curr = __kernel_getcpu();
     397        if(thrd_dst->last_cpu != curr) {
     398                int64_t l = thrd_dst->last_cpu;
     399                int64_t c = curr;
     400                int64_t v = (l << 32) | c;
     401                __push_stat( __tls_stats(), v, false, "Processor", this );
     402        }
     403
     404        thrd_dst->last_cpu = curr;
     405
    425406        __cfadbg_print_safe(runtime_core, "Kernel : core %p running thread %p (%s)\n", this, thrd_dst, thrd_dst->self_cor.name);
    426407
    427         coroutine$ * proc_cor = get_coroutine(this->runner);
     408        $coroutine * proc_cor = get_coroutine(this->runner);
    428409
    429410        // set state of processor coroutine to inactive
    … …  
    444425                /* paranoid */ verify( thrd_dst->context.SP );
    445426                /* paranoid */ verify( thrd_dst->state != Halted );
    446                 /* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) < ((uintptr_t)__get_stack(thrd_dst->curr_cor)->base ) || thrd_dst->curr_cor == proc_cor || thrd_dst->corctx_flag, "ERROR : Destination thread$ %p has been corrupted.\n StackPointer too small.\n", thrd_dst ); // add escape condition if we are setting up the processor
    447                 /* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) > ((uintptr_t)__get_stack(thrd_dst->curr_cor)->limit) || thrd_dst->curr_cor == proc_cor || thrd_dst->corctx_flag, "ERROR : Destination thread$ %p has been corrupted.\n StackPointer too large.\n", thrd_dst ); // add escape condition if we are setting up the processor
     427                /* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) < ((uintptr_t)__get_stack(thrd_dst->curr_cor)->base ) || thrd_dst->curr_cor == proc_cor || thrd_dst->corctx_flag, "ERROR : Destination $thread %p has been corrupted.\n StackPointer too small.\n", thrd_dst ); // add escape condition if we are setting up the processor
     428                /* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) > ((uintptr_t)__get_stack(thrd_dst->curr_cor)->limit) || thrd_dst->curr_cor == proc_cor || thrd_dst->corctx_flag, "ERROR : Destination $thread %p has been corrupted.\n StackPointer too large.\n", thrd_dst ); // add escape condition if we are setting up the processor
    448429                /* paranoid */ verify( 0x0D15EA5E0D15EA5Ep == thrd_dst->canary );
    449430
    … …  
    457438
    458439                /* paranoid */ verify( 0x0D15EA5E0D15EA5Ep == thrd_dst->canary );
    459                 /* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) > ((uintptr_t)__get_stack(thrd_dst->curr_cor)->limit) || thrd_dst->corctx_flag, "ERROR : Destination thread$ %p has been corrupted.\n StackPointer too large.\n", thrd_dst );
    460                 /* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) < ((uintptr_t)__get_stack(thrd_dst->curr_cor)->base ) || thrd_dst->corctx_flag, "ERROR : Destination thread$ %p has been corrupted.\n StackPointer too small.\n", thrd_dst );
     440                /* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) > ((uintptr_t)__get_stack(thrd_dst->curr_cor)->limit) || thrd_dst->corctx_flag, "ERROR : Destination $thread %p has been corrupted.\n StackPointer too large.\n", thrd_dst );
     441                /* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) < ((uintptr_t)__get_stack(thrd_dst->curr_cor)->base ) || thrd_dst->corctx_flag, "ERROR : Destination $thread %p has been corrupted.\n StackPointer too small.\n", thrd_dst );
    461442                /* paranoid */ verify( thrd_dst->context.SP );
    462443                /* paranoid */ verify( thrd_dst->curr_cluster == this->cltr );
    … …  
    476457                if(unlikely(thrd_dst->preempted != __NO_PREEMPTION)) {
    477458                        // The thread was preempted, reschedule it and reset the flag
    478                         schedule_thread$( thrd_dst, UNPARK_LOCAL );
     459                        schedule_thread$( thrd_dst );
    479460                        break RUNNING;
    480461                }
    … …  
    524505void returnToKernel() {
    525506        /* paranoid */ verify( ! __preemption_enabled() );
    526         coroutine$ * proc_cor = get_coroutine(kernelTLS().this_processor->runner);
    527         thread$ * thrd_src = kernelTLS().this_thread;
     507        $coroutine * proc_cor = get_coroutine(kernelTLS().this_processor->runner);
     508        $thread * thrd_src = kernelTLS().this_thread;
    528509
    529510        __STATS( thrd_src->last_proc = kernelTLS().this_processor; )
    … …  
    553534
    554535        /* paranoid */ verify( ! __preemption_enabled() );
    555         /* paranoid */ verifyf( ((uintptr_t)thrd_src->context.SP) < ((uintptr_t)__get_stack(thrd_src->curr_cor)->base ) || thrd_src->corctx_flag, "ERROR : Returning thread$ %p has been corrupted.\n StackPointer too small.\n", thrd_src );
    556         /* paranoid */ verifyf( ((uintptr_t)thrd_src->context.SP) > ((uintptr_t)__get_stack(thrd_src->curr_cor)->limit) || thrd_src->corctx_flag, "ERROR : Returning thread$ %p has been corrupted.\n StackPointer too large.\n", thrd_src );
     536        /* paranoid */ verifyf( ((uintptr_t)thrd_src->context.SP) < ((uintptr_t)__get_stack(thrd_src->curr_cor)->base ) || thrd_src->corctx_flag, "ERROR : Returning $thread %p has been corrupted.\n StackPointer too small.\n", thrd_src );
     537        /* paranoid */ verifyf( ((uintptr_t)thrd_src->context.SP) > ((uintptr_t)__get_stack(thrd_src->curr_cor)->limit) || thrd_src->corctx_flag, "ERROR : Returning $thread %p has been corrupted.\n StackPointer too large.\n", thrd_src );
    557538}
    558539
    … …  
    560541// Scheduler routines
    561542// KERNEL ONLY
    562 static void __schedule_thread( thread$ * thrd, unpark_hint hint ) {
     543static void __schedule_thread( $thread * thrd ) {
    563544        /* paranoid */ verify( ! __preemption_enabled() );
    564545        /* paranoid */ verify( ready_schedule_islocked());
    … …  
    580561        // Dereference the thread now because once we push it, there is not guaranteed it's still valid.
    581562        struct cluster * cl = thrd->curr_cluster;
    582         __STATS(bool outside = hint == UNPARK_LOCAL && thrd->last_proc && thrd->last_proc != kernelTLS().this_processor; )
     563        __STATS(bool outside = thrd->last_proc && thrd->last_proc != kernelTLS().this_processor; )
    583564
    584565        // push the thread to the cluster ready-queue
    585         push( cl, thrd, hint );
     566        push( cl, thrd, local );
    586567
    587568        // variable thrd is no longer safe to use
    … …  
    608589}
    609590
    610 void schedule_thread$( thread$ * thrd, unpark_hint hint ) {
     591void schedule_thread$( $thread * thrd ) {
    611592        ready_schedule_lock();
    612                 __schedule_thread( thrd, hint );
     593                __schedule_thread( thrd );
    613594        ready_schedule_unlock();
    614595}
    615596
    616597// KERNEL ONLY
    617 static inline thread$ * __next_thread(cluster * this) with( *this ) {
     598static inline $thread * __next_thread(cluster * this) with( *this ) {
    618599        /* paranoid */ verify( ! __preemption_enabled() );
    619600
    620601        ready_schedule_lock();
    621                 thread$ * thrd = pop_fast( this );
     602                $thread * thrd = pop_fast( this );
    622603        ready_schedule_unlock();
    623604
    … …  
    627608
    628609// KERNEL ONLY
    629 static inline thread$ * __next_thread_slow(cluster * this) with( *this ) {
     610static inline $thread * __next_thread_slow(cluster * this) with( *this ) {
    630611        /* paranoid */ verify( ! __preemption_enabled() );
    631612
    632613        ready_schedule_lock();
    633                 thread$ * thrd;
     614                $thread * thrd;
    634615                for(25) {
    635616                        thrd = pop_slow( this );
    … …  
    645626}
    646627
    647 static inline bool __must_unpark( thread$ * thrd ) {
     628static inline bool __must_unpark( $thread * thrd ) {
    648629        int old_ticket = __atomic_fetch_add(&thrd->ticket, 1, __ATOMIC_SEQ_CST);
    649630        switch(old_ticket) {
    … …  
    661642}
    662643
    663 void __kernel_unpark( thread$ * thrd, unpark_hint hint ) {
     644void __kernel_unpark( $thread * thrd ) {
    664645        /* paranoid */ verify( ! __preemption_enabled() );
    665646        /* paranoid */ verify( ready_schedule_islocked());
    … …  
    669650        if(__must_unpark(thrd)) {
    670651                // Wake lost the race,
    671                 __schedule_thread( thrd, hint );
     652                __schedule_thread( thrd );
    672653        }
    673654
    … …  
    676657}
    677658
    678 void unpark( thread$ * thrd, unpark_hint hint ) {
     659void unpark( $thread * thrd ) {
    679660        if( !thrd ) return;
    680661
    … …  
    682663                disable_interrupts();
    683664                        // Wake lost the race,
    684                         schedule_thread$( thrd, hint );
     665                        schedule_thread$( thrd );
    685666                enable_interrupts(false);
    686667        }
    … …  
    700681        // Should never return
    701682        void __cfactx_thrd_leave() {
    702                 thread$ * thrd = active_thread();
    703                 monitor$ * this = &thrd->self_mon;
     683                $thread * thrd = active_thread();
     684                $monitor * this = &thrd->self_mon;
    704685
    705686                // Lock the monitor now
    … …  
    713694                /* paranoid */ verify( kernelTLS().this_thread == thrd );
    714695                /* paranoid */ verify( thrd->context.SP );
    715                 /* paranoid */ verifyf( ((uintptr_t)thrd->context.SP) > ((uintptr_t)__get_stack(thrd->curr_cor)->limit), "ERROR : thread$ %p has been corrupted.\n StackPointer too large.\n", thrd );
    716                 /* paranoid */ verifyf( ((uintptr_t)thrd->context.SP) < ((uintptr_t)__get_stack(thrd->curr_cor)->base ), "ERROR : thread$ %p has been corrupted.\n StackPointer too small.\n", thrd );
     696                /* paranoid */ verifyf( ((uintptr_t)thrd->context.SP) > ((uintptr_t)__get_stack(thrd->curr_cor)->limit), "ERROR : $thread %p has been corrupted.\n StackPointer too large.\n", thrd );
     697                /* paranoid */ verifyf( ((uintptr_t)thrd->context.SP) < ((uintptr_t)__get_stack(thrd->curr_cor)->base ), "ERROR : $thread %p has been corrupted.\n StackPointer too small.\n", thrd );
    717698
    718699                thrd->state = Halting;
    … …  
    732713bool force_yield( __Preemption_Reason reason ) {
    733714        __disable_interrupts_checked();
    734                 thread$ * thrd = kernelTLS().this_thread;
     715                $thread * thrd = kernelTLS().this_thread;
    735716                /* paranoid */ verify(thrd->state == Active);
    736717
    … …  
    844825//=============================================================================================
    845826void __kernel_abort_msg( char * abort_text, int abort_text_size ) {
    846         thread$ * thrd = __cfaabi_tls.this_thread;
     827        $thread * thrd = __cfaabi_tls.this_thread;
    847828
    848829        if(thrd) {
Note: See TracChangeset for help on using the changeset viewer.