Ignore:
File:
1 edited

Legend:

Unmodified
Added
Removed

  • libcfa/src/concurrency/kernel.cfa

    r1f45c7d r24e321c  
    2222#include <errno.h>
    2323#include <stdio.h>
     24#include <string.h>
    2425#include <signal.h>
    2526#include <unistd.h>
    … …  
    3132#include "kernel_private.hfa"
    3233#include "preemption.hfa"
     34#include "strstream.hfa"
     35#include "device/cpu.hfa"
    3336
    3437//Private includes
    … …  
    110113#endif
    111114
    112 extern $thread * mainThread;
     115extern thread$ * mainThread;
    113116extern processor * mainProcessor;
    114117
    115118//-----------------------------------------------------------------------------
    116119// Kernel Scheduling logic
    117 static $thread * __next_thread(cluster * this);
    118 static $thread * __next_thread_slow(cluster * this);
    119 static inline bool __must_unpark( $thread * thrd ) __attribute((nonnull(1)));
    120 static void __run_thread(processor * this, $thread * dst);
     120static thread$ * __next_thread(cluster * this);
     121static thread$ * __next_thread_slow(cluster * this);
     122static inline bool __must_unpark( thread$ * thrd ) __attribute((nonnull(1)));
     123static void __run_thread(processor * this, thread$ * dst);
    121124static void __wake_one(cluster * cltr);
    122125
    … …  
    181184                __cfadbg_print_safe(runtime_core, "Kernel : core %p started\n", this);
    182185
    183                 $thread * readyThread = 0p;
     186                thread$ * readyThread = 0p;
    184187                MAIN_LOOP:
    185188                for() {
    … …  
    231234                                __cfadbg_print_safe(runtime_core, "Kernel : core %p waiting on eventfd %d\n", this, this->idle);
    232235
    233                                 __disable_interrupts_hard();
    234                                 eventfd_t val;
    235                                 eventfd_read( this->idle, &val );
    236                                 __enable_interrupts_hard();
     236                                {
     237                                        eventfd_t val;
     238                                        ssize_t ret = read( this->idle, &val, sizeof(val) );
     239                                        if(ret < 0) {
     240                                                switch((int)errno) {
     241                                                case EAGAIN:
     242                                                #if EAGAIN != EWOULDBLOCK
     243                                                        case EWOULDBLOCK:
     244                                                #endif
     245                                                case EINTR:
     246                                                        // No need to do anything special here, just assume it's a legitimate wake-up
     247                                                        break;
     248                                                default:
     249                                                        abort( "KERNEL : internal error, read failure on idle eventfd, error(%d) %s.", (int)errno, strerror( (int)errno ) );
     250                                                }
     251                                        }
     252                                }
    237253
    238254                                #if !defined(__CFA_NO_STATISTICS__)
    … …  
    325341                                }
    326342
    327                                         __STATS( if(this->print_halts) __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 0\n", this->unique_id, rdtscl()); )
     343                                __STATS( if(this->print_halts) __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 0\n", this->unique_id, rdtscl()); )
    328344                                __cfadbg_print_safe(runtime_core, "Kernel : core %p waiting on eventfd %d\n", this, this->idle);
    329345
    330                                 // __disable_interrupts_hard();
    331                                 eventfd_t val;
    332                                 eventfd_read( this->idle, &val );
    333                                 // __enable_interrupts_hard();
     346                                {
     347                                        eventfd_t val;
     348                                        ssize_t ret = read( this->idle, &val, sizeof(val) );
     349                                        if(ret < 0) {
     350                                                switch((int)errno) {
     351                                                case EAGAIN:
     352                                                #if EAGAIN != EWOULDBLOCK
     353                                                        case EWOULDBLOCK:
     354                                                #endif
     355                                                case EINTR:
     356                                                        // No need to do anything special here, just assume it's a legitimate wake-up
     357                                                        break;
     358                                                default:
     359                                                        abort( "KERNEL : internal error, read failure on idle eventfd, error(%d) %s.", (int)errno, strerror( (int)errno ) );
     360                                                }
     361                                        }
     362                                }
    334363
    335364                                        __STATS( if(this->print_halts) __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 1\n", this->unique_id, rdtscl()); )
    … …  
    388417// runThread runs a thread by context switching
    389418// from the processor coroutine to the target thread
    390 static void __run_thread(processor * this, $thread * thrd_dst) {
     419static void __run_thread(processor * this, thread$ * thrd_dst) {
    391420        /* paranoid */ verify( ! __preemption_enabled() );
    392421        /* paranoid */ verifyf( thrd_dst->state == Ready || thrd_dst->preempted != __NO_PREEMPTION, "state : %d, preempted %d\n", thrd_dst->state, thrd_dst->preempted);
    … …  
    394423        __builtin_prefetch( thrd_dst->context.SP );
    395424
    396         int curr = __kernel_getcpu();
    397         if(thrd_dst->last_cpu != curr) {
    398                 int64_t l = thrd_dst->last_cpu;
    399                 int64_t c = curr;
    400                 int64_t v = (l << 32) | c;
    401                 __push_stat( __tls_stats(), v, false, "Processor", this );
    402         }
    403 
    404         thrd_dst->last_cpu = curr;
    405 
    406425        __cfadbg_print_safe(runtime_core, "Kernel : core %p running thread %p (%s)\n", this, thrd_dst, thrd_dst->self_cor.name);
    407426
    408         $coroutine * proc_cor = get_coroutine(this->runner);
     427        coroutine$ * proc_cor = get_coroutine(this->runner);
    409428
    410429        // set state of processor coroutine to inactive
    … …  
    425444                /* paranoid */ verify( thrd_dst->context.SP );
    426445                /* paranoid */ verify( thrd_dst->state != Halted );
    427                 /* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) < ((uintptr_t)__get_stack(thrd_dst->curr_cor)->base ) || thrd_dst->curr_cor == proc_cor || thrd_dst->corctx_flag, "ERROR : Destination $thread %p has been corrupted.\n StackPointer too small.\n", thrd_dst ); // add escape condition if we are setting up the processor
    428                 /* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) > ((uintptr_t)__get_stack(thrd_dst->curr_cor)->limit) || thrd_dst->curr_cor == proc_cor || thrd_dst->corctx_flag, "ERROR : Destination $thread %p has been corrupted.\n StackPointer too large.\n", thrd_dst ); // add escape condition if we are setting up the processor
     446                /* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) < ((uintptr_t)__get_stack(thrd_dst->curr_cor)->base ) || thrd_dst->curr_cor == proc_cor || thrd_dst->corctx_flag, "ERROR : Destination thread$ %p has been corrupted.\n StackPointer too small.\n", thrd_dst ); // add escape condition if we are setting up the processor
     447                /* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) > ((uintptr_t)__get_stack(thrd_dst->curr_cor)->limit) || thrd_dst->curr_cor == proc_cor || thrd_dst->corctx_flag, "ERROR : Destination thread$ %p has been corrupted.\n StackPointer too large.\n", thrd_dst ); // add escape condition if we are setting up the processor
    429448                /* paranoid */ verify( 0x0D15EA5E0D15EA5Ep == thrd_dst->canary );
    430449
    … …  
    438457
    439458                /* paranoid */ verify( 0x0D15EA5E0D15EA5Ep == thrd_dst->canary );
    440                 /* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) > ((uintptr_t)__get_stack(thrd_dst->curr_cor)->limit) || thrd_dst->corctx_flag, "ERROR : Destination $thread %p has been corrupted.\n StackPointer too large.\n", thrd_dst );
    441                 /* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) < ((uintptr_t)__get_stack(thrd_dst->curr_cor)->base ) || thrd_dst->corctx_flag, "ERROR : Destination $thread %p has been corrupted.\n StackPointer too small.\n", thrd_dst );
     459                /* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) > ((uintptr_t)__get_stack(thrd_dst->curr_cor)->limit) || thrd_dst->corctx_flag, "ERROR : Destination thread$ %p has been corrupted.\n StackPointer too large.\n", thrd_dst );
     460                /* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) < ((uintptr_t)__get_stack(thrd_dst->curr_cor)->base ) || thrd_dst->corctx_flag, "ERROR : Destination thread$ %p has been corrupted.\n StackPointer too small.\n", thrd_dst );
    442461                /* paranoid */ verify( thrd_dst->context.SP );
    443462                /* paranoid */ verify( thrd_dst->curr_cluster == this->cltr );
    … …  
    457476                if(unlikely(thrd_dst->preempted != __NO_PREEMPTION)) {
    458477                        // The thread was preempted, reschedule it and reset the flag
    459                         schedule_thread$( thrd_dst );
     478                        schedule_thread$( thrd_dst, UNPARK_LOCAL );
    460479                        break RUNNING;
    461480                }
    … …  
    505524void returnToKernel() {
    506525        /* paranoid */ verify( ! __preemption_enabled() );
    507         $coroutine * proc_cor = get_coroutine(kernelTLS().this_processor->runner);
    508         $thread * thrd_src = kernelTLS().this_thread;
     526        coroutine$ * proc_cor = get_coroutine(kernelTLS().this_processor->runner);
     527        thread$ * thrd_src = kernelTLS().this_thread;
    509528
    510529        __STATS( thrd_src->last_proc = kernelTLS().this_processor; )
    … …  
    534553
    535554        /* paranoid */ verify( ! __preemption_enabled() );
    536         /* paranoid */ verifyf( ((uintptr_t)thrd_src->context.SP) < ((uintptr_t)__get_stack(thrd_src->curr_cor)->base ) || thrd_src->corctx_flag, "ERROR : Returning $thread %p has been corrupted.\n StackPointer too small.\n", thrd_src );
    537         /* paranoid */ verifyf( ((uintptr_t)thrd_src->context.SP) > ((uintptr_t)__get_stack(thrd_src->curr_cor)->limit) || thrd_src->corctx_flag, "ERROR : Returning $thread %p has been corrupted.\n StackPointer too large.\n", thrd_src );
     555        /* paranoid */ verifyf( ((uintptr_t)thrd_src->context.SP) < ((uintptr_t)__get_stack(thrd_src->curr_cor)->base ) || thrd_src->corctx_flag, "ERROR : Returning thread$ %p has been corrupted.\n StackPointer too small.\n", thrd_src );
     556        /* paranoid */ verifyf( ((uintptr_t)thrd_src->context.SP) > ((uintptr_t)__get_stack(thrd_src->curr_cor)->limit) || thrd_src->corctx_flag, "ERROR : Returning thread$ %p has been corrupted.\n StackPointer too large.\n", thrd_src );
    538557}
    539558
    … …  
    541560// Scheduler routines
    542561// KERNEL ONLY
    543 static void __schedule_thread( $thread * thrd ) {
     562static void __schedule_thread( thread$ * thrd, unpark_hint hint ) {
    544563        /* paranoid */ verify( ! __preemption_enabled() );
    545564        /* paranoid */ verify( ready_schedule_islocked());
    … …  
    561580        // Dereference the thread now because once we push it, there is not guaranteed it's still valid.
    562581        struct cluster * cl = thrd->curr_cluster;
    563         __STATS(bool outside = thrd->last_proc && thrd->last_proc != kernelTLS().this_processor; )
     582        __STATS(bool outside = hint == UNPARK_LOCAL && thrd->last_proc && thrd->last_proc != kernelTLS().this_processor; )
    564583
    565584        // push the thread to the cluster ready-queue
    566         push( cl, thrd, local );
     585        push( cl, thrd, hint );
    567586
    568587        // variable thrd is no longer safe to use
    … …  
    589608}
    590609
    591 void schedule_thread$( $thread * thrd ) {
     610void schedule_thread$( thread$ * thrd, unpark_hint hint ) {
    592611        ready_schedule_lock();
    593                 __schedule_thread( thrd );
     612                __schedule_thread( thrd, hint );
    594613        ready_schedule_unlock();
    595614}
    596615
    597616// KERNEL ONLY
    598 static inline $thread * __next_thread(cluster * this) with( *this ) {
     617static inline thread$ * __next_thread(cluster * this) with( *this ) {
    599618        /* paranoid */ verify( ! __preemption_enabled() );
    600619
    601620        ready_schedule_lock();
    602                 $thread * thrd = pop_fast( this );
     621                thread$ * thrd = pop_fast( this );
    603622        ready_schedule_unlock();
    604623
    … …  
    608627
    609628// KERNEL ONLY
    610 static inline $thread * __next_thread_slow(cluster * this) with( *this ) {
     629static inline thread$ * __next_thread_slow(cluster * this) with( *this ) {
    611630        /* paranoid */ verify( ! __preemption_enabled() );
    612631
    613632        ready_schedule_lock();
    614                 $thread * thrd;
     633                thread$ * thrd;
    615634                for(25) {
    616635                        thrd = pop_slow( this );
    … …  
    626645}
    627646
    628 static inline bool __must_unpark( $thread * thrd ) {
     647static inline bool __must_unpark( thread$ * thrd ) {
    629648        int old_ticket = __atomic_fetch_add(&thrd->ticket, 1, __ATOMIC_SEQ_CST);
    630649        switch(old_ticket) {
    … …  
    642661}
    643662
    644 void __kernel_unpark( $thread * thrd ) {
     663void __kernel_unpark( thread$ * thrd, unpark_hint hint ) {
    645664        /* paranoid */ verify( ! __preemption_enabled() );
    646665        /* paranoid */ verify( ready_schedule_islocked());
    … …  
    650669        if(__must_unpark(thrd)) {
    651670                // Wake lost the race,
    652                 __schedule_thread( thrd );
     671                __schedule_thread( thrd, hint );
    653672        }
    654673
    … …  
    657676}
    658677
    659 void unpark( $thread * thrd ) {
     678void unpark( thread$ * thrd, unpark_hint hint ) {
    660679        if( !thrd ) return;
    661680
    … …  
    663682                disable_interrupts();
    664683                        // Wake lost the race,
    665                         schedule_thread$( thrd );
     684                        schedule_thread$( thrd, hint );
    666685                enable_interrupts(false);
    667686        }
    … …  
    681700        // Should never return
    682701        void __cfactx_thrd_leave() {
    683                 $thread * thrd = active_thread();
    684                 $monitor * this = &thrd->self_mon;
     702                thread$ * thrd = active_thread();
     703                monitor$ * this = &thrd->self_mon;
    685704
    686705                // Lock the monitor now
    … …  
    694713                /* paranoid */ verify( kernelTLS().this_thread == thrd );
    695714                /* paranoid */ verify( thrd->context.SP );
    696                 /* paranoid */ verifyf( ((uintptr_t)thrd->context.SP) > ((uintptr_t)__get_stack(thrd->curr_cor)->limit), "ERROR : $thread %p has been corrupted.\n StackPointer too large.\n", thrd );
    697                 /* paranoid */ verifyf( ((uintptr_t)thrd->context.SP) < ((uintptr_t)__get_stack(thrd->curr_cor)->base ), "ERROR : $thread %p has been corrupted.\n StackPointer too small.\n", thrd );
     715                /* paranoid */ verifyf( ((uintptr_t)thrd->context.SP) > ((uintptr_t)__get_stack(thrd->curr_cor)->limit), "ERROR : thread$ %p has been corrupted.\n StackPointer too large.\n", thrd );
     716                /* paranoid */ verifyf( ((uintptr_t)thrd->context.SP) < ((uintptr_t)__get_stack(thrd->curr_cor)->base ), "ERROR : thread$ %p has been corrupted.\n StackPointer too small.\n", thrd );
    698717
    699718                thrd->state = Halting;
    … …  
    713732bool force_yield( __Preemption_Reason reason ) {
    714733        __disable_interrupts_checked();
    715                 $thread * thrd = kernelTLS().this_thread;
     734                thread$ * thrd = kernelTLS().this_thread;
    716735                /* paranoid */ verify(thrd->state == Active);
    717736
    … …  
    825844//=============================================================================================
    826845void __kernel_abort_msg( char * abort_text, int abort_text_size ) {
    827         $thread * thrd = __cfaabi_tls.this_thread;
     846        thread$ * thrd = __cfaabi_tls.this_thread;
    828847
    829848        if(thrd) {
Note: See TracChangeset for help on using the changeset viewer.