Changes in / [cfff639:254ad1b]
- Location:
- libcfa/src/concurrency
- Files:
-
- 9 edited
-
clib/cfathread.cfa (modified) (2 diffs)
-
invoke.h (modified) (2 diffs)
-
kernel.cfa (modified) (8 diffs)
-
kernel/startup.cfa (modified) (1 diff)
-
kernel_private.hfa (modified) (2 diffs)
-
ready_queue.cfa (modified) (2 diffs)
-
stats.cfa (modified) (3 diffs)
-
stats.hfa (modified) (1 diff)
-
thread.cfa (modified) (1 diff)
Legend:
- Unmodified
- Added
- Removed
-
libcfa/src/concurrency/clib/cfathread.cfa
rcfff639 r254ad1b 243 243 // Mutex 244 244 struct cfathread_mutex { 245 fast_lock impl;245 single_acquisition_lock impl; 246 246 }; 247 247 int cfathread_mutex_init(cfathread_mutex_t *restrict mut, const cfathread_mutexattr_t *restrict) __attribute__((nonnull (1))) { *mut = new(); return 0; } … … 258 258 // Condition 259 259 struct cfathread_condition { 260 condition_variable( fast_lock) impl;260 condition_variable(single_acquisition_lock) impl; 261 261 }; 262 262 int cfathread_cond_init(cfathread_cond_t *restrict cond, const cfathread_condattr_t *restrict) __attribute__((nonnull (1))) { *cond = new(); return 0; } -
libcfa/src/concurrency/invoke.h
rcfff639 r254ad1b 148 148 struct $thread * prev; 149 149 volatile unsigned long long ts; 150 unsigned preferred;151 150 }; 152 151 … … 200 199 } node; 201 200 202 struct processor * last_proc;203 204 201 #if defined( __CFA_WITH_VERIFY__ ) 205 202 void * canary; -
libcfa/src/concurrency/kernel.cfa
rcfff639 r254ad1b 34 34 #include "invoke.h" 35 35 36 #if !defined(__CFA_NO_STATISTICS__)37 #define __STATS( ...) __VA_ARGS__38 #else39 #define __STATS( ...)40 #endif41 36 42 37 //----------------------------------------------------------------------------- … … 171 166 preemption_scope scope = { this }; 172 167 173 __STATS( unsigned long long last_tally = rdtscl(); ) 168 #if !defined(__CFA_NO_STATISTICS__) 169 unsigned long long last_tally = rdtscl(); 170 #endif 174 171 175 172 // if we need to run some special setup, now is the time to do it. … … 269 266 __cfa_io_flush( this ); 270 267 } 271 272 // SEARCH: {273 // /* paranoid */ verify( ! __preemption_enabled() );274 // /* paranoid */ verify( kernelTLS().this_proc_id );275 276 // // First, lock the scheduler since we are searching for a thread277 278 // // Try to get the next thread279 // ready_schedule_lock();280 // readyThread = pop_fast( this->cltr );281 // ready_schedule_unlock();282 // if(readyThread) { break SEARCH; }283 284 // // If we can't find a thread, might as well flush any outstanding I/O285 // if(this->io.pending) { __cfa_io_flush( this ); }286 287 // // Spin a little on I/O, just in case288 // for(25) {289 // __maybe_io_drain( this );290 // ready_schedule_lock();291 // readyThread = pop_fast( this->cltr );292 // ready_schedule_unlock();293 // if(readyThread) { break SEARCH; }294 // }295 296 // // no luck, try stealing a few times297 // for(25) {298 // if( __maybe_io_drain( this ) ) {299 // ready_schedule_lock();300 // readyThread = pop_fast( this->cltr );301 // } else {302 // ready_schedule_lock();303 // readyThread = pop_slow( this->cltr );304 // }305 // ready_schedule_unlock();306 // if(readyThread) { break SEARCH; }307 // }308 309 // // still no luck, search for a thread310 // ready_schedule_lock();311 // readyThread = pop_search( this->cltr );312 // ready_schedule_unlock();313 // if(readyThread) { break SEARCH; }314 315 // // Don't block if we are done316 // if( __atomic_load_n(&this->do_terminate, __ATOMIC_SEQ_CST) ) break MAIN_LOOP;317 318 // __STATS( __tls_stats()->ready.sleep.halts++; )319 320 // // Push self to idle stack321 // mark_idle(this->cltr->procs, * this);322 323 // // Confirm the ready-queue is empty324 // __maybe_io_drain( this );325 // ready_schedule_lock();326 // readyThread = pop_search( this->cltr );327 // ready_schedule_unlock();328 329 // if( readyThread ) {330 // // A thread was found, cancel the halt331 // mark_awake(this->cltr->procs, * this);332 333 // __STATS( __tls_stats()->ready.sleep.cancels++; )334 335 // // continue the main loop336 // break SEARCH;337 // }338 339 // __STATS( if(this->print_halts) __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 0\n", this->id, rdtscl()); )340 // __cfadbg_print_safe(runtime_core, "Kernel : core %p waiting on eventfd %d\n", this, this->idle);341 342 // // __disable_interrupts_hard();343 // eventfd_t val;344 // eventfd_read( this->idle, &val );345 // // __enable_interrupts_hard();346 347 // __STATS( if(this->print_halts) __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 1\n", this->id, rdtscl()); )348 349 // // We were woken up, remove self from idle350 // mark_awake(this->cltr->procs, * this);351 352 // // DON'T just proceed, start looking again353 // continue MAIN_LOOP;354 // }355 356 // RUN_THREAD:357 // /* paranoid */ verify( kernelTLS().this_proc_id );358 // /* paranoid */ verify( ! __preemption_enabled() );359 // /* paranoid */ verify( readyThread );360 361 // // Reset io dirty bit362 // this->io.dirty = false;363 364 // // We found a thread run it365 // __run_thread(this, readyThread);366 367 // // Are we done?368 // if( __atomic_load_n(&this->do_terminate, __ATOMIC_SEQ_CST) ) break MAIN_LOOP;369 370 // #if !defined(__CFA_NO_STATISTICS__)371 // unsigned long long curr = rdtscl();372 // if(curr > (last_tally + 500000000)) {373 // __tally_stats(this->cltr->stats, __cfaabi_tls.this_stats);374 // last_tally = curr;375 // }376 // #endif377 378 // if(this->io.pending && !this->io.dirty) {379 // __cfa_io_flush( this );380 // }381 382 // // Check if there is pending io383 // __maybe_io_drain( this );384 268 } 385 269 … … 518 402 $thread * thrd_src = kernelTLS().this_thread; 519 403 520 __STATS( thrd_src->last_proc = kernelTLS().this_processor; ) 404 #if !defined(__CFA_NO_STATISTICS__) 405 struct processor * last_proc = kernelTLS().this_processor; 406 #endif 521 407 522 408 // Run the thread on this processor … … 537 423 538 424 #if !defined(__CFA_NO_STATISTICS__) 539 /* paranoid */ verify( thrd_src->last_proc != 0p ); 540 if(thrd_src->last_proc != kernelTLS().this_processor) { 425 if(last_proc != kernelTLS().this_processor) { 541 426 __tls_stats()->ready.threads.migration++; 542 427 } … … 572 457 // Dereference the thread now because once we push it, there is not guaranteed it's still valid. 573 458 struct cluster * cl = thrd->curr_cluster; 574 __STATS(bool outside = thrd->last_proc && thrd->last_proc != kernelTLS().this_processor; )575 459 576 460 // push the thread to the cluster ready-queue … … 586 470 if( kernelTLS().this_stats ) { 587 471 __tls_stats()->ready.threads.threads++; 588 if(outside) {589 __tls_stats()->ready.threads.extunpark++;590 }591 472 __push_stat( __tls_stats(), __tls_stats()->ready.threads.threads, false, "Processor", kernelTLS().this_processor ); 592 473 } 593 474 else { 594 475 __atomic_fetch_add(&cl->stats->ready.threads.threads, 1, __ATOMIC_RELAXED); 595 __atomic_fetch_add(&cl->stats->ready.threads.extunpark, 1, __ATOMIC_RELAXED);596 476 __push_stat( cl->stats, cl->stats->ready.threads.threads, true, "Cluster", cl ); 597 477 } … … 628 508 629 509 ready_schedule_lock(); 630 $thread * thrd; 631 for(25) { 632 thrd = pop_slow( this ); 633 if(thrd) goto RET; 634 } 635 thrd = pop_search( this ); 636 637 RET: 510 $thread * thrd = pop_slow( this ); 638 511 ready_schedule_unlock(); 639 512 -
libcfa/src/concurrency/kernel/startup.cfa
rcfff639 r254ad1b 447 447 link.next = 0p; 448 448 link.prev = 0p; 449 link.preferred = -1u;450 last_proc = 0p;451 449 #if defined( __CFA_WITH_VERIFY__ ) 452 450 canary = 0x0D15EA5E0D15EA5Ep; -
libcfa/src/concurrency/kernel_private.hfa
rcfff639 r254ad1b 284 284 285 285 //----------------------------------------------------------------------- 286 // pop thread from the local queuesof a cluster286 // pop thread from the ready queue of a cluster 287 287 // returns 0p if empty 288 288 // May return 0p spuriously … … 290 290 291 291 //----------------------------------------------------------------------- 292 // pop thread from any ready queue of a cluster 293 // returns 0p if empty 294 // May return 0p spuriously 295 __attribute__((hot)) struct $thread * pop_slow(struct cluster * cltr); 296 297 //----------------------------------------------------------------------- 298 // search all ready queues of a cluster for any thread 292 // pop thread from the ready queue of a cluster 299 293 // returns 0p if empty 300 294 // guaranteed to find any threads added before this call 301 __attribute__((hot)) struct $thread * pop_s earch(struct cluster * cltr);295 __attribute__((hot)) struct $thread * pop_slow(struct cluster * cltr); 302 296 303 297 //----------------------------------------------------------------------- -
libcfa/src/concurrency/ready_queue.cfa
rcfff639 r254ad1b 344 344 } 345 345 346 __attribute__((hot)) struct $thread * pop_slow(struct cluster * cltr) { return pop_fast(cltr); } 347 __attribute__((hot)) struct $thread * pop_search(struct cluster * cltr) { 346 __attribute__((hot)) struct $thread * pop_slow(struct cluster * cltr) { 348 347 return search(cltr); 349 348 } … … 437 436 438 437 __attribute__((hot)) struct $thread * pop_slow(struct cluster * cltr) with (cltr->ready_queue) { 439 unsigned i = __tls_rand() % lanes.count; 440 return try_pop(cltr, i __STATS(, __tls_stats()->ready.pop.steal)); 441 } 442 443 __attribute__((hot)) struct $thread * pop_search(struct cluster * cltr) with (cltr->ready_queue) { 438 for(25) { 439 unsigned i = __tls_rand() % lanes.count; 440 $thread * t = try_pop(cltr, i __STATS(, __tls_stats()->ready.pop.steal)); 441 if(t) return t; 442 } 443 444 444 return search(cltr); 445 445 } -
libcfa/src/concurrency/stats.cfa
rcfff639 r254ad1b 38 38 stats->ready.pop.search.espec = 0; 39 39 stats->ready.threads.migration = 0; 40 stats->ready.threads.extunpark = 0;41 40 stats->ready.threads.threads = 0; 42 41 stats->ready.sleep.halts = 0; … … 96 95 __atomic_fetch_add( &cltr->ready.pop.search.espec , proc->ready.pop.search.espec , __ATOMIC_SEQ_CST ); proc->ready.pop.search.espec = 0; 97 96 __atomic_fetch_add( &cltr->ready.threads.migration , proc->ready.threads.migration , __ATOMIC_SEQ_CST ); proc->ready.threads.migration = 0; 98 __atomic_fetch_add( &cltr->ready.threads.extunpark , proc->ready.threads.extunpark , __ATOMIC_SEQ_CST ); proc->ready.threads.extunpark = 0;99 97 __atomic_fetch_add( &cltr->ready.threads.threads , proc->ready.threads.threads , __ATOMIC_SEQ_CST ); proc->ready.threads.threads = 0; 100 98 __atomic_fetch_add( &cltr->ready.sleep.halts , proc->ready.sleep.halts , __ATOMIC_SEQ_CST ); proc->ready.sleep.halts = 0; … … 134 132 uint64_t totalR = ready.pop.local.success + ready.pop.help.success + ready.pop.steal.success + ready.pop.search.success; 135 133 uint64_t totalS = ready.push.local.success + ready.push.share.success + ready.push.extrn.success; 136 sstr | "- totals : " | eng3(totalR) | "run," | eng3(totalS) | "schd (" | eng3(ready.push.extrn.success) | "ext," | eng3(ready.threads.migration) | "mig ," | eng3(ready.threads.extunpark) | " eupk)";134 sstr | "- totals : " | eng3(totalR) | "run," | eng3(totalS) | "schd (" | eng3(ready.push.extrn.success) | "ext," | eng3(ready.threads.migration) | "mig)"; 137 135 138 136 double push_len = ((double)ready.push.local.attempt + ready.push.share.attempt + ready.push.extrn.attempt) / totalS; -
libcfa/src/concurrency/stats.hfa
rcfff639 r254ad1b 70 70 struct { 71 71 volatile uint64_t migration; 72 volatile uint64_t extunpark;73 72 volatile int64_t threads; // number of threads in the system, includes only local change 74 73 } threads; -
libcfa/src/concurrency/thread.cfa
rcfff639 r254ad1b 39 39 link.next = 0p; 40 40 link.prev = 0p; 41 link.preferred = -1u;42 last_proc = 0p;43 41 #if defined( __CFA_WITH_VERIFY__ ) 44 42 canary = 0x0D15EA5E0D15EA5Ep;
Note:
See TracChangeset
for help on using the changeset viewer.
