- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
libcfa/src/concurrency/ready_queue.cfa
re84ab3d r1f45c7d 67 67 #endif 68 68 69 static inline struct thread$* try_pop(struct cluster * cltr, unsigned w __STATS(, __stats_readyQ_pop_t & stats));70 static inline struct thread$* try_pop(struct cluster * cltr, unsigned i, unsigned j __STATS(, __stats_readyQ_pop_t & stats));71 static inline struct thread$* search(struct cluster * cltr);69 static inline struct $thread * try_pop(struct cluster * cltr, unsigned w __STATS(, __stats_readyQ_pop_t & stats)); 70 static inline struct $thread * try_pop(struct cluster * cltr, unsigned i, unsigned j __STATS(, __stats_readyQ_pop_t & stats)); 71 static inline struct $thread * search(struct cluster * cltr); 72 72 static inline [unsigned, bool] idx_from_r(unsigned r, unsigned preferred); 73 73 … … 274 274 //----------------------------------------------------------------------- 275 275 #if defined(USE_CPU_WORK_STEALING) 276 __attribute__((hot)) void push(struct cluster * cltr, struct thread$* thrd, bool push_local) with (cltr->ready_queue) {276 __attribute__((hot)) void push(struct cluster * cltr, struct $thread * thrd, bool push_local) with (cltr->ready_queue) { 277 277 __cfadbg_print_safe(ready_queue, "Kernel : Pushing %p on cluster %p\n", thrd, cltr); 278 278 … … 316 316 317 317 // Pop from the ready queue from a given cluster 318 __attribute__((hot)) thread$* pop_fast(struct cluster * cltr) with (cltr->ready_queue) {318 __attribute__((hot)) $thread * pop_fast(struct cluster * cltr) with (cltr->ready_queue) { 319 319 /* paranoid */ verify( lanes.count > 0 ); 320 320 /* paranoid */ verify( kernelTLS().this_processor ); … … 345 345 /* paranoid */ verify(lanes.count < 65536); // The following code assumes max 65536 cores. 346 346 /* paranoid */ verify(map.count < 65536); // The following code assumes max 65536 cores. 347 uint64_t chaos = __tls_rand(); 348 uint64_t high_chaos = (chaos >> 32); 349 uint64_t mid_chaos = (chaos >> 16) & 0xffff; 350 uint64_t low_chaos = chaos & 0xffff; 351 352 unsigned me = map.self; 353 unsigned cpu_chaos = map.start + (mid_chaos % map.count); 354 bool global = cpu_chaos == me; 355 356 if(global) { 357 proc->rdq.target = high_chaos % lanes.count; 347 348 if(0 == (__tls_rand() % 10_000)) { 349 proc->rdq.target = __tls_rand() % lanes.count; 358 350 } else { 359 proc->rdq.target = (cpu_chaos * READYQ_SHARD_FACTOR) + (low_chaos % READYQ_SHARD_FACTOR); 351 unsigned cpu_chaos = map.start + (__tls_rand() % map.count); 352 proc->rdq.target = (cpu_chaos * READYQ_SHARD_FACTOR) + (__tls_rand() % READYQ_SHARD_FACTOR); 360 353 /* paranoid */ verify(proc->rdq.target >= (map.start * READYQ_SHARD_FACTOR)); 361 354 /* paranoid */ verify(proc->rdq.target < ((map.start + map.count) * READYQ_SHARD_FACTOR)); … … 371 364 proc->rdq.target = -1u; 372 365 if(lanes.tscs[target].tv < cutoff && ts(lanes.data[target]) < cutoff) { 373 thread$* t = try_pop(cltr, target __STATS(, __tls_stats()->ready.pop.help));366 $thread * t = try_pop(cltr, target __STATS(, __tls_stats()->ready.pop.help)); 374 367 proc->rdq.last = target; 375 368 if(t) return t; … … 379 372 unsigned last = proc->rdq.last; 380 373 if(last != -1u && lanes.tscs[last].tv < cutoff && ts(lanes.data[last]) < cutoff) { 381 thread$* t = try_pop(cltr, last __STATS(, __tls_stats()->ready.pop.help));374 $thread * t = try_pop(cltr, last __STATS(, __tls_stats()->ready.pop.help)); 382 375 if(t) return t; 383 376 } … … 389 382 for(READYQ_SHARD_FACTOR) { 390 383 unsigned i = start + (proc->rdq.itr++ % READYQ_SHARD_FACTOR); 391 if( thread$* t = try_pop(cltr, i __STATS(, __tls_stats()->ready.pop.local))) return t;384 if($thread * t = try_pop(cltr, i __STATS(, __tls_stats()->ready.pop.local))) return t; 392 385 } 393 386 … … 396 389 } 397 390 398 __attribute__((hot)) struct thread$* pop_slow(struct cluster * cltr) with (cltr->ready_queue) {391 __attribute__((hot)) struct $thread * pop_slow(struct cluster * cltr) with (cltr->ready_queue) { 399 392 processor * const proc = kernelTLS().this_processor; 400 393 unsigned last = proc->rdq.last; 401 394 if(last != -1u) { 402 struct thread$* t = try_pop(cltr, last __STATS(, __tls_stats()->ready.pop.steal));395 struct $thread * t = try_pop(cltr, last __STATS(, __tls_stats()->ready.pop.steal)); 403 396 if(t) return t; 404 397 proc->rdq.last = -1u; … … 408 401 return try_pop(cltr, i __STATS(, __tls_stats()->ready.pop.steal)); 409 402 } 410 __attribute__((hot)) struct thread$* pop_search(struct cluster * cltr) {403 __attribute__((hot)) struct $thread * pop_search(struct cluster * cltr) { 411 404 return search(cltr); 412 405 } … … 435 428 } 436 429 437 __attribute__((hot)) void push(struct cluster * cltr, struct thread$* thrd, bool push_local) with (cltr->ready_queue) {430 __attribute__((hot)) void push(struct cluster * cltr, struct $thread * thrd, bool push_local) with (cltr->ready_queue) { 438 431 __cfadbg_print_safe(ready_queue, "Kernel : Pushing %p on cluster %p\n", thrd, cltr); 439 432 … … 482 475 483 476 // Pop from the ready queue from a given cluster 484 __attribute__((hot)) thread$* pop_fast(struct cluster * cltr) with (cltr->ready_queue) {477 __attribute__((hot)) $thread * pop_fast(struct cluster * cltr) with (cltr->ready_queue) { 485 478 /* paranoid */ verify( lanes.count > 0 ); 486 479 /* paranoid */ verify( kernelTLS().this_processor ); … … 506 499 507 500 // try popping from the 2 picked lists 508 struct thread$* thrd = try_pop(cltr, i, j __STATS(, *(locali || localj ? &__tls_stats()->ready.pop.local : &__tls_stats()->ready.pop.help)));501 struct $thread * thrd = try_pop(cltr, i, j __STATS(, *(locali || localj ? &__tls_stats()->ready.pop.local : &__tls_stats()->ready.pop.help))); 509 502 if(thrd) { 510 503 return thrd; … … 516 509 } 517 510 518 __attribute__((hot)) struct thread$* pop_slow(struct cluster * cltr) { return pop_fast(cltr); }519 __attribute__((hot)) struct thread$* pop_search(struct cluster * cltr) {511 __attribute__((hot)) struct $thread * pop_slow(struct cluster * cltr) { return pop_fast(cltr); } 512 __attribute__((hot)) struct $thread * pop_search(struct cluster * cltr) { 520 513 return search(cltr); 521 514 } 522 515 #endif 523 516 #if defined(USE_WORK_STEALING) 524 __attribute__((hot)) void push(struct cluster * cltr, struct thread$* thrd, bool push_local) with (cltr->ready_queue) {517 __attribute__((hot)) void push(struct cluster * cltr, struct $thread * thrd, bool push_local) with (cltr->ready_queue) { 525 518 __cfadbg_print_safe(ready_queue, "Kernel : Pushing %p on cluster %p\n", thrd, cltr); 526 519 … … 576 569 577 570 // Pop from the ready queue from a given cluster 578 __attribute__((hot)) thread$* pop_fast(struct cluster * cltr) with (cltr->ready_queue) {571 __attribute__((hot)) $thread * pop_fast(struct cluster * cltr) with (cltr->ready_queue) { 579 572 /* paranoid */ verify( lanes.count > 0 ); 580 573 /* paranoid */ verify( kernelTLS().this_processor ); … … 598 591 const unsigned long long cutoff = proc->rdq.cutoff > bias ? proc->rdq.cutoff - bias : proc->rdq.cutoff; 599 592 if(lanes.tscs[target].tv < cutoff && ts(lanes.data[target]) < cutoff) { 600 thread$* t = try_pop(cltr, target __STATS(, __tls_stats()->ready.pop.help));593 $thread * t = try_pop(cltr, target __STATS(, __tls_stats()->ready.pop.help)); 601 594 if(t) return t; 602 595 } … … 605 598 for(READYQ_SHARD_FACTOR) { 606 599 unsigned i = proc->rdq.id + (proc->rdq.itr++ % READYQ_SHARD_FACTOR); 607 if( thread$* t = try_pop(cltr, i __STATS(, __tls_stats()->ready.pop.local))) return t;600 if($thread * t = try_pop(cltr, i __STATS(, __tls_stats()->ready.pop.local))) return t; 608 601 } 609 602 return 0p; 610 603 } 611 604 612 __attribute__((hot)) struct thread$* pop_slow(struct cluster * cltr) with (cltr->ready_queue) {605 __attribute__((hot)) struct $thread * pop_slow(struct cluster * cltr) with (cltr->ready_queue) { 613 606 unsigned i = __tls_rand() % lanes.count; 614 607 return try_pop(cltr, i __STATS(, __tls_stats()->ready.pop.steal)); 615 608 } 616 609 617 __attribute__((hot)) struct thread$* pop_search(struct cluster * cltr) with (cltr->ready_queue) {610 __attribute__((hot)) struct $thread * pop_search(struct cluster * cltr) with (cltr->ready_queue) { 618 611 return search(cltr); 619 612 } … … 628 621 //----------------------------------------------------------------------- 629 622 // try to pop from a lane given by index w 630 static inline struct thread$* try_pop(struct cluster * cltr, unsigned w __STATS(, __stats_readyQ_pop_t & stats)) with (cltr->ready_queue) {623 static inline struct $thread * try_pop(struct cluster * cltr, unsigned w __STATS(, __stats_readyQ_pop_t & stats)) with (cltr->ready_queue) { 631 624 __STATS( stats.attempt++; ) 632 625 … … 651 644 652 645 // Actually pop the list 653 struct thread$* thrd;646 struct $thread * thrd; 654 647 unsigned long long tsv; 655 648 [thrd, tsv] = pop(lane); … … 678 671 // try to pop from any lanes making sure you don't miss any threads push 679 672 // before the start of the function 680 static inline struct thread$* search(struct cluster * cltr) with (cltr->ready_queue) {673 static inline struct $thread * search(struct cluster * cltr) with (cltr->ready_queue) { 681 674 /* paranoid */ verify( lanes.count > 0 ); 682 675 unsigned count = __atomic_load_n( &lanes.count, __ATOMIC_RELAXED ); … … 684 677 for(i; count) { 685 678 unsigned idx = (offset + i) % count; 686 struct thread$* thrd = try_pop(cltr, idx __STATS(, __tls_stats()->ready.pop.search));679 struct $thread * thrd = try_pop(cltr, idx __STATS(, __tls_stats()->ready.pop.search)); 687 680 if(thrd) { 688 681 return thrd; … … 719 712 //----------------------------------------------------------------------- 720 713 // Given 2 indexes, pick the list with the oldest push an try to pop from it 721 static inline struct thread$* try_pop(struct cluster * cltr, unsigned i, unsigned j __STATS(, __stats_readyQ_pop_t & stats)) with (cltr->ready_queue) {714 static inline struct $thread * try_pop(struct cluster * cltr, unsigned i, unsigned j __STATS(, __stats_readyQ_pop_t & stats)) with (cltr->ready_queue) { 722 715 // Pick the bet list 723 716 int w = i; … … 854 847 // As long as we can pop from this lane to push the threads somewhere else in the queue 855 848 while(!is_empty(lanes.data[idx])) { 856 struct thread$* thrd;849 struct $thread * thrd; 857 850 unsigned long long _; 858 851 [thrd, _] = pop(lanes.data[idx]);
Note: See TracChangeset
for help on using the changeset viewer.