Index: libcfa/src/concurrency/kernel.cfa =================================================================== --- libcfa/src/concurrency/kernel.cfa (revision c993b1521e96d36bed6b395f8eec5284bc85f5f5) +++ libcfa/src/concurrency/kernel.cfa (revision 34b27967130121fdf7c2603b7cac89f172507c65) @@ -184,4 +184,5 @@ MAIN_LOOP: for() { + #if 1 // Check if there is pending io __maybe_io_drain( this ); @@ -270,116 +271,113 @@ } - // SEARCH: { - // /* paranoid */ verify( ! __preemption_enabled() ); - // /* paranoid */ verify( kernelTLS().this_proc_id ); - - // // First, lock the scheduler since we are searching for a thread - - // // Try to get the next thread - // ready_schedule_lock(); - // readyThread = pop_fast( this->cltr ); - // ready_schedule_unlock(); - // if(readyThread) { break SEARCH; } - - // // If we can't find a thread, might as well flush any outstanding I/O - // if(this->io.pending) { __cfa_io_flush( this ); } - - // // Spin a little on I/O, just in case - // for(25) { - // __maybe_io_drain( this ); - // ready_schedule_lock(); - // readyThread = pop_fast( this->cltr ); - // ready_schedule_unlock(); - // if(readyThread) { break SEARCH; } - // } - - // // no luck, try stealing a few times - // for(25) { - // if( __maybe_io_drain( this ) ) { - // ready_schedule_lock(); - // readyThread = pop_fast( this->cltr ); - // } else { - // ready_schedule_lock(); - // readyThread = pop_slow( this->cltr ); - // } - // ready_schedule_unlock(); - // if(readyThread) { break SEARCH; } - // } - - // // still no luck, search for a thread - // ready_schedule_lock(); - // readyThread = pop_search( this->cltr ); - // ready_schedule_unlock(); - // if(readyThread) { break SEARCH; } - - // // Don't block if we are done - // if( __atomic_load_n(&this->do_terminate, __ATOMIC_SEQ_CST) ) break MAIN_LOOP; - - // __STATS( __tls_stats()->ready.sleep.halts++; ) - - // // Push self to idle stack - // mark_idle(this->cltr->procs, * this); - - // // Confirm the ready-queue is empty - // __maybe_io_drain( this ); - // ready_schedule_lock(); - // readyThread = pop_search( this->cltr ); - // ready_schedule_unlock(); - - // if( readyThread ) { - // // A thread was found, cancel the halt - // mark_awake(this->cltr->procs, * this); - - // __STATS( __tls_stats()->ready.sleep.cancels++; ) - - // // continue the main loop - // break SEARCH; - // } - - // __STATS( if(this->print_halts) __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 0\n", this->id, rdtscl()); ) - // __cfadbg_print_safe(runtime_core, "Kernel : core %p waiting on eventfd %d\n", this, this->idle); - - // // __disable_interrupts_hard(); - // eventfd_t val; - // eventfd_read( this->idle, &val ); - // // __enable_interrupts_hard(); - - // __STATS( if(this->print_halts) __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 1\n", this->id, rdtscl()); ) - - // // We were woken up, remove self from idle - // mark_awake(this->cltr->procs, * this); - - // // DON'T just proceed, start looking again - // continue MAIN_LOOP; - // } - - // RUN_THREAD: - // /* paranoid */ verify( kernelTLS().this_proc_id ); - // /* paranoid */ verify( ! __preemption_enabled() ); - // /* paranoid */ verify( readyThread ); - - // // Reset io dirty bit - // this->io.dirty = false; - - // // We found a thread run it - // __run_thread(this, readyThread); - - // // Are we done? - // if( __atomic_load_n(&this->do_terminate, __ATOMIC_SEQ_CST) ) break MAIN_LOOP; - - // #if !defined(__CFA_NO_STATISTICS__) - // unsigned long long curr = rdtscl(); - // if(curr > (last_tally + 500000000)) { - // __tally_stats(this->cltr->stats, __cfaabi_tls.this_stats); - // last_tally = curr; - // } - // #endif - - // if(this->io.pending && !this->io.dirty) { - // __cfa_io_flush( this ); - // } - - // // Check if there is pending io - // __maybe_io_drain( this ); + #else + + SEARCH: { + /* paranoid */ verify( ! __preemption_enabled() ); + /* paranoid */ verify( kernelTLS().this_proc_id ); + + // First, lock the scheduler since we are searching for a thread + ready_schedule_lock(); + + // Try to get the next thread + readyThread = pop_fast( this->cltr ); + if(readyThread) { ready_schedule_unlock(); break SEARCH; } + + // If we can't find a thread, might as well flush any outstanding I/O + if(this->io.pending) { __cfa_io_flush( this ); } + + // Spin a little on I/O, just in case + for(25) { + __maybe_io_drain( this ); + readyThread = pop_fast( this->cltr ); + if(readyThread) { ready_schedule_unlock(); break SEARCH; } + } + + // no luck, try stealing a few times + for(25) { + if( __maybe_io_drain( this ) ) { + readyThread = pop_fast( this->cltr ); + } else { + readyThread = pop_slow( this->cltr ); + } + if(readyThread) { ready_schedule_unlock(); break SEARCH; } + } + + // still no luck, search for a thread + readyThread = pop_search( this->cltr ); + if(readyThread) { ready_schedule_unlock(); break SEARCH; } + + // Don't block if we are done + if( __atomic_load_n(&this->do_terminate, __ATOMIC_SEQ_CST) ) break MAIN_LOOP; + + __STATS( __tls_stats()->ready.sleep.halts++; ) + + // Push self to idle stack + ready_schedule_unlock(); + mark_idle(this->cltr->procs, * this); + ready_schedule_lock(); + + // Confirm the ready-queue is empty + __maybe_io_drain( this ); + readyThread = pop_search( this->cltr ); + ready_schedule_unlock(); + + if( readyThread ) { + // A thread was found, cancel the halt + mark_awake(this->cltr->procs, * this); + + __STATS( __tls_stats()->ready.sleep.cancels++; ) + + // continue the main loop + break SEARCH; + } + + __STATS( if(this->print_halts) __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 0\n", this->id, rdtscl()); ) + __cfadbg_print_safe(runtime_core, "Kernel : core %p waiting on eventfd %d\n", this, this->idle); + + // __disable_interrupts_hard(); + eventfd_t val; + eventfd_read( this->idle, &val ); + // __enable_interrupts_hard(); + + __STATS( if(this->print_halts) __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 1\n", this->id, rdtscl()); ) + + // We were woken up, remove self from idle + mark_awake(this->cltr->procs, * this); + + // DON'T just proceed, start looking again + continue MAIN_LOOP; + } + + RUN_THREAD: + /* paranoid */ verify( kernelTLS().this_proc_id ); + /* paranoid */ verify( ! __preemption_enabled() ); + /* paranoid */ verify( readyThread ); + + // Reset io dirty bit + this->io.dirty = false; + + // We found a thread run it + __run_thread(this, readyThread); + + // Are we done? + if( __atomic_load_n(&this->do_terminate, __ATOMIC_SEQ_CST) ) break MAIN_LOOP; + + #if !defined(__CFA_NO_STATISTICS__) + unsigned long long curr = rdtscl(); + if(curr > (last_tally + 500000000)) { + __tally_stats(this->cltr->stats, __cfaabi_tls.this_stats); + last_tally = curr; + } + #endif + + if(this->io.pending && !this->io.dirty) { + __cfa_io_flush( this ); + } + + ready_schedule_lock(); + __maybe_io_drain( this ); + ready_schedule_unlock(); + #endif }