- File:
-
- 1 edited
-
libcfa/src/concurrency/ready_queue.cfa (modified) (18 diffs)
Legend:
- Unmodified
- Added
- Removed
-
libcfa/src/concurrency/ready_queue.cfa
r5f6a172 r5cb51502 17 17 // #define __CFA_DEBUG_PRINT_READY_QUEUE__ 18 18 19 // #define USE_SNZI 19 20 // #define USE_MPSC 20 21 #define USE_RELAXED_FIFO22 // #define USE_WORK_STEALING23 21 24 22 #include "bits/defs.hfa" … … 31 29 #include <unistd.h> 32 30 31 #include "snzi.hfa" 33 32 #include "ready_subqueue.hfa" 34 33 … … 41 40 #endif 42 41 43 #if defined(USE_RELAXED_FIFO) 44 #define BIAS 4 45 #define READYQ_SHARD_FACTOR 4 46 #define SEQUENTIAL_SHARD 1 47 #elif defined(USE_WORK_STEALING) 48 #define READYQ_SHARD_FACTOR 2 49 #define SEQUENTIAL_SHARD 2 50 #else 51 #error no scheduling strategy selected 52 #endif 53 54 static inline [unsigned, bool] idx_from_r(unsigned r, unsigned preferred); 55 static inline struct $thread * try_pop(struct cluster * cltr, unsigned w); 56 static inline struct $thread * try_pop(struct cluster * cltr, unsigned i, unsigned j); 57 static inline struct $thread * search(struct cluster * cltr); 58 42 #define BIAS 4 59 43 60 44 // returns the maximum number of processors the RWLock support … … 110 94 //======================================================================= 111 95 // Lock-Free registering/unregistering of threads 112 void register_proc_id( struct __processor_id_t * proc ) with(*__scheduler_lock) {96 unsigned doregister( struct __processor_id_t * proc ) with(*__scheduler_lock) { 113 97 __cfadbg_print_safe(ready_queue, "Kernel : Registering proc %p for RW-Lock\n", proc); 114 98 … … 124 108 /*paranoid*/ verify(0 == (__alignof__(data[i]) % cache_line_size)); 125 109 /*paranoid*/ verify((((uintptr_t)&data[i]) % cache_line_size) == 0); 126 proc->id =i;110 return i; 127 111 } 128 112 } … … 151 135 /*paranoid*/ verify(__alignof__(data[n]) == (2 * cache_line_size)); 152 136 /*paranoid*/ verify((((uintptr_t)&data[n]) % cache_line_size) == 0); 153 proc->id =n;154 } 155 156 void unregister _proc_id( struct __processor_id_t * proc ) with(*__scheduler_lock) {137 return n; 138 } 139 140 void unregister( struct __processor_id_t * proc ) with(*__scheduler_lock) { 157 141 unsigned id = proc->id; 158 142 /*paranoid*/ verify(id < ready); … … 209 193 210 194 //======================================================================= 211 // Cforall Re ady Queue used for scheduling195 // Cforall Reqdy Queue used for scheduling 212 196 //======================================================================= 213 197 void ?{}(__ready_queue_t & this) with (this) { 214 198 lanes.data = 0p; 215 lanes.tscs = 0p;216 199 lanes.count = 0; 217 200 } 218 201 219 202 void ^?{}(__ready_queue_t & this) with (this) { 220 verify( SEQUENTIAL_SHARD == lanes.count ); 203 verify( 1 == lanes.count ); 204 #ifdef USE_SNZI 205 verify( !query( snzi ) ); 206 #endif 221 207 free(lanes.data); 222 free(lanes.tscs);223 208 } 224 209 225 210 //----------------------------------------------------------------------- 226 #if defined(USE_RELAXED_FIFO) 227 //----------------------------------------------------------------------- 228 // get index from random number with or without bias towards queues 229 static inline [unsigned, bool] idx_from_r(unsigned r, unsigned preferred) { 230 unsigned i; 231 bool local; 211 __attribute__((hot)) bool query(struct cluster * cltr) { 212 #ifdef USE_SNZI 213 return query(cltr->ready_queue.snzi); 214 #endif 215 return true; 216 } 217 218 static inline [unsigned, bool] idx_from_r(unsigned r, unsigned preferred) { 219 unsigned i; 220 bool local; 221 #if defined(BIAS) 232 222 unsigned rlow = r % BIAS; 233 223 unsigned rhigh = r / BIAS; … … 235 225 // (BIAS - 1) out of BIAS chances 236 226 // Use perferred queues 237 i = preferred + (rhigh % READYQ_SHARD_FACTOR);227 i = preferred + (rhigh % 4); 238 228 local = true; 239 229 } … … 244 234 local = false; 245 235 } 246 return [i, local]; 247 } 248 249 __attribute__((hot)) void push(struct cluster * cltr, struct $thread * thrd) with (cltr->ready_queue) { 250 __cfadbg_print_safe(ready_queue, "Kernel : Pushing %p on cluster %p\n", thrd, cltr); 251 252 const bool external = (!kernelTLS().this_processor) || (cltr != kernelTLS().this_processor->cltr); 253 /* paranoid */ verify(external || kernelTLS().this_processor->rdq.id < lanes.count ); 254 255 // write timestamp 256 thrd->link.ts = rdtscl(); 257 258 bool local; 259 int preferred = external ? -1 : kernelTLS().this_processor->rdq.id; 260 261 // Try to pick a lane and lock it 262 unsigned i; 263 do { 264 // Pick the index of a lane 265 unsigned r = __tls_rand_fwd(); 266 [i, local] = idx_from_r(r, preferred); 267 268 i %= __atomic_load_n( &lanes.count, __ATOMIC_RELAXED ); 269 270 #if !defined(__CFA_NO_STATISTICS__) 271 if(external) { 272 if(local) __atomic_fetch_add(&cltr->stats->ready.pick.ext.local, 1, __ATOMIC_RELAXED); 273 __atomic_fetch_add(&cltr->stats->ready.pick.ext.attempt, 1, __ATOMIC_RELAXED); 274 } 275 else { 276 if(local) __tls_stats()->ready.pick.push.local++; 277 __tls_stats()->ready.pick.push.attempt++; 278 } 279 #endif 280 281 #if defined(USE_MPSC) 282 // mpsc always succeeds 283 } while( false ); 284 #else 285 // If we can't lock it retry 286 } while( !__atomic_try_acquire( &lanes.data[i].lock ) ); 287 #endif 288 289 // Actually push it 290 push(lanes.data[i], thrd); 291 292 #if !defined(USE_MPSC) 293 // Unlock and return 294 __atomic_unlock( &lanes.data[i].lock ); 295 #endif 296 297 // Mark the current index in the tls rng instance as having an item 298 __tls_rand_advance_bck(); 299 300 __cfadbg_print_safe(ready_queue, "Kernel : Pushed %p on cluster %p (idx: %u, mask %llu, first %d)\n", thrd, cltr, i, used.mask[0], lane_first); 301 302 // Update statistics 236 #else 237 i = r; 238 local = false; 239 #endif 240 return [i, local]; 241 } 242 243 //----------------------------------------------------------------------- 244 __attribute__((hot)) bool push(struct cluster * cltr, struct $thread * thrd) with (cltr->ready_queue) { 245 __cfadbg_print_safe(ready_queue, "Kernel : Pushing %p on cluster %p\n", thrd, cltr); 246 247 const bool external = (!kernelTLS().this_processor) || (cltr != kernelTLS().this_processor->cltr); 248 249 // write timestamp 250 thrd->link.ts = rdtscl(); 251 252 bool first = false; 253 __attribute__((unused)) bool local; 254 __attribute__((unused)) int preferred; 255 #if defined(BIAS) 256 preferred = 257 //* 258 external ? -1 : kernelTLS().this_processor->cltr_id; 259 /*/ 260 thrd->link.preferred * 4; 261 //*/ 262 #endif 263 264 // Try to pick a lane and lock it 265 unsigned i; 266 do { 267 // Pick the index of a lane 268 // unsigned r = __tls_rand(); 269 unsigned r = __tls_rand_fwd(); 270 [i, local] = idx_from_r(r, preferred); 271 272 i %= __atomic_load_n( &lanes.count, __ATOMIC_RELAXED ); 273 303 274 #if !defined(__CFA_NO_STATISTICS__) 304 275 if(external) { 305 if(local) __atomic_fetch_add(&cltr->stats->ready.pick.ext.l success, 1, __ATOMIC_RELAXED);306 __atomic_fetch_add(&cltr->stats->ready.pick.ext. success, 1, __ATOMIC_RELAXED);276 if(local) __atomic_fetch_add(&cltr->stats->ready.pick.ext.local, 1, __ATOMIC_RELAXED); 277 __atomic_fetch_add(&cltr->stats->ready.pick.ext.attempt, 1, __ATOMIC_RELAXED); 307 278 } 308 279 else { 309 if(local) __tls_stats()->ready.pick.push.l success++;310 __tls_stats()->ready.pick.push. success++;280 if(local) __tls_stats()->ready.pick.push.local++; 281 __tls_stats()->ready.pick.push.attempt++; 311 282 } 312 283 #endif 313 } 314 315 // Pop from the ready queue from a given cluster 316 __attribute__((hot)) $thread * pop_fast(struct cluster * cltr) with (cltr->ready_queue) { 317 /* paranoid */ verify( lanes.count > 0 ); 318 /* paranoid */ verify( kernelTLS().this_processor ); 319 /* paranoid */ verify( kernelTLS().this_processor->rdq.id < lanes.count ); 320 321 unsigned count = __atomic_load_n( &lanes.count, __ATOMIC_RELAXED ); 322 int preferred = kernelTLS().this_processor->rdq.id; 323 324 325 // As long as the list is not empty, try finding a lane that isn't empty and pop from it 326 for(25) { 327 // Pick two lists at random 328 unsigned ri = __tls_rand_bck(); 329 unsigned rj = __tls_rand_bck(); 330 331 unsigned i, j; 332 __attribute__((unused)) bool locali, localj; 333 [i, locali] = idx_from_r(ri, preferred); 334 [j, localj] = idx_from_r(rj, preferred); 335 336 #if !defined(__CFA_NO_STATISTICS__) 337 if(locali && localj) { 338 __tls_stats()->ready.pick.pop.local++; 339 } 340 #endif 341 342 i %= count; 343 j %= count; 344 345 // try popping from the 2 picked lists 346 struct $thread * thrd = try_pop(cltr, i, j); 347 if(thrd) { 348 #if !defined(__CFA_NO_STATISTICS__) 349 if( locali || localj ) __tls_stats()->ready.pick.pop.lsuccess++; 350 #endif 351 return thrd; 352 } 353 } 354 355 // All lanes where empty return 0p 356 return 0p; 357 } 358 359 __attribute__((hot)) struct $thread * pop_slow(struct cluster * cltr) { 360 return search(cltr); 361 } 362 #endif 363 #if defined(USE_WORK_STEALING) 364 __attribute__((hot)) void push(struct cluster * cltr, struct $thread * thrd) with (cltr->ready_queue) { 365 __cfadbg_print_safe(ready_queue, "Kernel : Pushing %p on cluster %p\n", thrd, cltr); 366 367 const bool external = (!kernelTLS().this_processor) || (cltr != kernelTLS().this_processor->cltr); 368 /* paranoid */ verify(external || kernelTLS().this_processor->rdq.id < lanes.count ); 369 370 // write timestamp 371 thrd->link.ts = rdtscl(); 372 373 // Try to pick a lane and lock it 374 unsigned i; 375 do { 376 if(unlikely(external)) { 377 i = __tls_rand() % lanes.count; 378 } 379 else { 380 processor * proc = kernelTLS().this_processor; 381 unsigned r = proc->rdq.its++; 382 i = proc->rdq.id + (r % READYQ_SHARD_FACTOR); 383 } 384 385 386 #if defined(USE_MPSC) 387 // mpsc always succeeds 388 } while( false ); 389 #else 390 // If we can't lock it retry 391 } while( !__atomic_try_acquire( &lanes.data[i].lock ) ); 392 #endif 393 394 // Actually push it 395 push(lanes.data[i], thrd); 396 397 #if !defined(USE_MPSC) 398 // Unlock and return 399 __atomic_unlock( &lanes.data[i].lock ); 400 #endif 401 402 __cfadbg_print_safe(ready_queue, "Kernel : Pushed %p on cluster %p (idx: %u, mask %llu, first %d)\n", thrd, cltr, i, used.mask[0], lane_first); 403 } 404 405 // Pop from the ready queue from a given cluster 406 __attribute__((hot)) $thread * pop_fast(struct cluster * cltr) with (cltr->ready_queue) { 407 /* paranoid */ verify( lanes.count > 0 ); 408 /* paranoid */ verify( kernelTLS().this_processor ); 409 /* paranoid */ verify( kernelTLS().this_processor->rdq.id < lanes.count ); 410 411 processor * proc = kernelTLS().this_processor; 412 413 if(proc->rdq.target == -1u) { 414 proc->rdq.target = __tls_rand() % lanes.count; 415 unsigned it1 = proc->rdq.itr; 416 unsigned it2 = proc->rdq.itr + 1; 417 unsigned idx1 = proc->rdq.id + (it1 % READYQ_SHARD_FACTOR); 418 unsigned idx2 = proc->rdq.id + (it1 % READYQ_SHARD_FACTOR); 419 unsigned long long tsc1 = ts(lanes.data[idx1]); 420 unsigned long long tsc2 = ts(lanes.data[idx2]); 421 proc->rdq.cutoff = min(tsc1, tsc2); 422 } 423 else if(lanes.tscs[proc->rdq.target].tv < proc->rdq.cutoff) { 424 $thread * t = try_pop(cltr, proc->rdq.target); 425 proc->rdq.target = -1u; 426 if(t) return t; 427 } 428 429 for(READYQ_SHARD_FACTOR) { 430 unsigned i = proc->rdq.id + (--proc->rdq.itr % READYQ_SHARD_FACTOR); 431 if($thread * t = try_pop(cltr, i)) return t; 432 } 433 return 0p; 434 } 435 436 __attribute__((hot)) struct $thread * pop_slow(struct cluster * cltr) with (cltr->ready_queue) { 437 for(25) { 438 unsigned i = __tls_rand() % lanes.count; 439 $thread * t = try_pop(cltr, i); 440 if(t) return t; 441 } 442 443 return search(cltr); 444 } 445 #endif 446 447 //======================================================================= 448 // Various Ready Queue utilities 449 //======================================================================= 450 // these function work the same or almost the same 451 // whether they are using work-stealing or relaxed fifo scheduling 452 453 //----------------------------------------------------------------------- 454 // try to pop from a lane given by index w 455 static inline struct $thread * try_pop(struct cluster * cltr, unsigned w) with (cltr->ready_queue) { 456 // Get relevant elements locally 457 __intrusive_lane_t & lane = lanes.data[w]; 458 459 // If list looks empty retry 460 if( is_empty(lane) ) return 0p; 461 462 // If we can't get the lock retry 463 if( !__atomic_try_acquire(&lane.lock) ) return 0p; 464 465 // If list is empty, unlock and retry 466 if( is_empty(lane) ) { 467 __atomic_unlock(&lane.lock); 468 return 0p; 469 } 470 471 // Actually pop the list 472 struct $thread * thrd; 473 thrd = pop(lane); 474 475 /* paranoid */ verify(thrd); 476 /* paranoid */ verify(lane.lock); 477 478 // Unlock and return 479 __atomic_unlock(&lane.lock); 284 285 #if defined(USE_MPSC) 286 // mpsc always succeeds 287 } while( false ); 288 #else 289 // If we can't lock it retry 290 } while( !__atomic_try_acquire( &lanes.data[i].lock ) ); 291 #endif 292 293 // Actually push it 294 #ifdef USE_SNZI 295 bool lane_first = 296 #endif 297 298 push(lanes.data[i], thrd); 299 300 #ifdef USE_SNZI 301 // If this lane used to be empty we need to do more 302 if(lane_first) { 303 // Check if the entire queue used to be empty 304 first = !query(snzi); 305 306 // Update the snzi 307 arrive( snzi, i ); 308 } 309 #endif 310 311 #if !defined(USE_MPSC) 312 // Unlock and return 313 __atomic_unlock( &lanes.data[i].lock ); 314 #endif 315 316 // Mark the current index in the tls rng instance as having an item 317 __tls_rand_advance_bck(); 318 319 __cfadbg_print_safe(ready_queue, "Kernel : Pushed %p on cluster %p (idx: %u, mask %llu, first %d)\n", thrd, cltr, i, used.mask[0], lane_first); 480 320 481 321 // Update statistics 482 322 #if !defined(__CFA_NO_STATISTICS__) 483 __tls_stats()->ready.pick.pop.success++; 484 #endif 485 486 #if defined(USE_WORK_STEALING) 487 lanes.tscs[w].tv = thrd->link.ts; 488 #endif 489 490 // return the popped thread 491 return thrd; 492 } 493 494 //----------------------------------------------------------------------- 495 // try to pop from any lanes making sure you don't miss any threads push 496 // before the start of the function 497 static inline struct $thread * search(struct cluster * cltr) with (cltr->ready_queue) { 323 if(external) { 324 if(local) __atomic_fetch_add(&cltr->stats->ready.pick.ext.lsuccess, 1, __ATOMIC_RELAXED); 325 __atomic_fetch_add(&cltr->stats->ready.pick.ext.success, 1, __ATOMIC_RELAXED); 326 } 327 else { 328 if(local) __tls_stats()->ready.pick.push.lsuccess++; 329 __tls_stats()->ready.pick.push.success++; 330 } 331 #endif 332 333 // return whether or not the list was empty before this push 334 return first; 335 } 336 337 static struct $thread * try_pop(struct cluster * cltr, unsigned i, unsigned j); 338 static struct $thread * try_pop(struct cluster * cltr, unsigned i); 339 340 // Pop from the ready queue from a given cluster 341 __attribute__((hot)) $thread * pop(struct cluster * cltr) with (cltr->ready_queue) { 342 /* paranoid */ verify( lanes.count > 0 ); 343 unsigned count = __atomic_load_n( &lanes.count, __ATOMIC_RELAXED ); 344 int preferred; 345 #if defined(BIAS) 346 // Don't bother trying locally too much 347 preferred = kernelTLS().this_processor->cltr_id; 348 #endif 349 350 351 // As long as the list is not empty, try finding a lane that isn't empty and pop from it 352 #ifdef USE_SNZI 353 while( query(snzi) ) { 354 #else 355 for(25) { 356 #endif 357 // Pick two lists at random 358 // unsigned ri = __tls_rand(); 359 // unsigned rj = __tls_rand(); 360 unsigned ri = __tls_rand_bck(); 361 unsigned rj = __tls_rand_bck(); 362 363 unsigned i, j; 364 __attribute__((unused)) bool locali, localj; 365 [i, locali] = idx_from_r(ri, preferred); 366 [j, localj] = idx_from_r(rj, preferred); 367 368 #if !defined(__CFA_NO_STATISTICS__) 369 if(locali && localj) { 370 __tls_stats()->ready.pick.pop.local++; 371 } 372 #endif 373 374 i %= count; 375 j %= count; 376 377 // try popping from the 2 picked lists 378 struct $thread * thrd = try_pop(cltr, i, j); 379 if(thrd) { 380 #if defined(BIAS) && !defined(__CFA_NO_STATISTICS__) 381 if( locali || localj ) __tls_stats()->ready.pick.pop.lsuccess++; 382 #endif 383 return thrd; 384 } 385 } 386 387 // All lanes where empty return 0p 388 return 0p; 389 } 390 391 __attribute__((hot)) struct $thread * pop_slow(struct cluster * cltr) with (cltr->ready_queue) { 498 392 /* paranoid */ verify( lanes.count > 0 ); 499 393 unsigned count = __atomic_load_n( &lanes.count, __ATOMIC_RELAXED ); … … 511 405 } 512 406 407 513 408 //----------------------------------------------------------------------- 514 // Check that all the intrusive queues in the data structure are still consistent 409 // Given 2 indexes, pick the list with the oldest push an try to pop from it 410 static inline struct $thread * try_pop(struct cluster * cltr, unsigned i, unsigned j) with (cltr->ready_queue) { 411 #if !defined(__CFA_NO_STATISTICS__) 412 __tls_stats()->ready.pick.pop.attempt++; 413 #endif 414 415 // Pick the bet list 416 int w = i; 417 if( __builtin_expect(!is_empty(lanes.data[j]), true) ) { 418 w = (ts(lanes.data[i]) < ts(lanes.data[j])) ? i : j; 419 } 420 421 return try_pop(cltr, w); 422 } 423 424 static inline struct $thread * try_pop(struct cluster * cltr, unsigned w) with (cltr->ready_queue) { 425 // Get relevant elements locally 426 __intrusive_lane_t & lane = lanes.data[w]; 427 428 // If list looks empty retry 429 if( is_empty(lane) ) return 0p; 430 431 // If we can't get the lock retry 432 if( !__atomic_try_acquire(&lane.lock) ) return 0p; 433 434 435 // If list is empty, unlock and retry 436 if( is_empty(lane) ) { 437 __atomic_unlock(&lane.lock); 438 return 0p; 439 } 440 441 // Actually pop the list 442 struct $thread * thrd; 443 thrd = pop(lane); 444 445 /* paranoid */ verify(thrd); 446 /* paranoid */ verify(lane.lock); 447 448 #ifdef USE_SNZI 449 // If this was the last element in the lane 450 if(emptied) { 451 depart( snzi, w ); 452 } 453 #endif 454 455 // Unlock and return 456 __atomic_unlock(&lane.lock); 457 458 // Update statistics 459 #if !defined(__CFA_NO_STATISTICS__) 460 __tls_stats()->ready.pick.pop.success++; 461 #endif 462 463 // Update the thread bias 464 thrd->link.preferred = w / 4; 465 466 // return the popped thread 467 return thrd; 468 } 469 //----------------------------------------------------------------------- 470 471 bool remove_head(struct cluster * cltr, struct $thread * thrd) with (cltr->ready_queue) { 472 for(i; lanes.count) { 473 __intrusive_lane_t & lane = lanes.data[i]; 474 475 bool removed = false; 476 477 __atomic_acquire(&lane.lock); 478 if(head(lane)->link.next == thrd) { 479 $thread * pthrd; 480 pthrd = pop(lane); 481 482 /* paranoid */ verify( pthrd == thrd ); 483 484 removed = true; 485 #ifdef USE_SNZI 486 if(emptied) { 487 depart( snzi, i ); 488 } 489 #endif 490 } 491 __atomic_unlock(&lane.lock); 492 493 if( removed ) return true; 494 } 495 return false; 496 } 497 498 //----------------------------------------------------------------------- 499 515 500 static void check( __ready_queue_t & q ) with (q) { 516 501 #if defined(__CFA_WITH_VERIFY__) && !defined(USE_MPSC) … … 535 520 } 536 521 #endif 537 }538 539 //-----------------------------------------------------------------------540 // Given 2 indexes, pick the list with the oldest push an try to pop from it541 static inline struct $thread * try_pop(struct cluster * cltr, unsigned i, unsigned j) with (cltr->ready_queue) {542 #if !defined(__CFA_NO_STATISTICS__)543 __tls_stats()->ready.pick.pop.attempt++;544 #endif545 546 // Pick the bet list547 int w = i;548 if( __builtin_expect(!is_empty(lanes.data[j]), true) ) {549 w = (ts(lanes.data[i]) < ts(lanes.data[j])) ? i : j;550 }551 552 return try_pop(cltr, w);553 522 } 554 523 … … 572 541 } 573 542 574 static void assign_list(unsigned & value, dlist(processor, processor) & list, unsigned count) {575 processor * it = &list`first;576 for(unsigned i = 0; i < count; i++) {577 /* paranoid */ verifyf( it, "Unexpected null iterator, at index %u of %u\n", i, count);578 it->rdq.id = value;579 it->rdq.target = -1u;580 value += READYQ_SHARD_FACTOR;581 it = &(*it)`next;582 }583 }584 585 static void reassign_cltr_id(struct cluster * cltr) {586 unsigned preferred = 0;587 assign_list(preferred, cltr->procs.actives, cltr->procs.total - cltr->procs.idle);588 assign_list(preferred, cltr->procs.idles , cltr->procs.idle );589 }590 591 static void fix_times( struct cluster * cltr ) with( cltr->ready_queue ) {592 #if defined(USE_WORK_STEALING)593 lanes.tscs = alloc(lanes.count, lanes.tscs`realloc);594 for(i; lanes.count) {595 lanes.tscs[i].tv = ts(lanes.data[i]);596 }597 #endif598 }599 600 543 // Grow the ready queue 601 void ready_queue_grow(struct cluster * cltr) { 544 unsigned ready_queue_grow(struct cluster * cltr, int target) { 545 unsigned preferred; 602 546 size_t ncount; 603 int target = cltr->procs.total;604 547 605 548 /* paranoid */ verify( ready_mutate_islocked() ); … … 611 554 // grow the ready queue 612 555 with( cltr->ready_queue ) { 556 #ifdef USE_SNZI 557 ^(snzi){}; 558 #endif 559 613 560 // Find new count 614 561 // Make sure we always have atleast 1 list 615 562 if(target >= 2) { 616 ncount = target * READYQ_SHARD_FACTOR; 563 ncount = target * 4; 564 preferred = ncount - 4; 617 565 } else { 618 ncount = SEQUENTIAL_SHARD; 566 ncount = 1; 567 preferred = 0; 619 568 } 620 569 … … 634 583 // Update original 635 584 lanes.count = ncount; 636 } 637 638 fix_times(cltr); 639 640 reassign_cltr_id(cltr); 585 586 #ifdef USE_SNZI 587 // Re-create the snzi 588 snzi{ log2( lanes.count / 8 ) }; 589 for( idx; (size_t)lanes.count ) { 590 if( !is_empty(lanes.data[idx]) ) { 591 arrive(snzi, idx); 592 } 593 } 594 #endif 595 } 641 596 642 597 // Make sure that everything is consistent … … 646 601 647 602 /* paranoid */ verify( ready_mutate_islocked() ); 603 return preferred; 648 604 } 649 605 650 606 // Shrink the ready queue 651 void ready_queue_shrink(struct cluster * cltr ) {607 void ready_queue_shrink(struct cluster * cltr, int target) { 652 608 /* paranoid */ verify( ready_mutate_islocked() ); 653 609 __cfadbg_print_safe(ready_queue, "Kernel : Shrinking ready queue\n"); … … 656 612 /* paranoid */ check( cltr->ready_queue ); 657 613 658 int target = cltr->procs.total;659 660 614 with( cltr->ready_queue ) { 615 #ifdef USE_SNZI 616 ^(snzi){}; 617 #endif 618 661 619 // Remember old count 662 620 size_t ocount = lanes.count; … … 664 622 // Find new count 665 623 // Make sure we always have atleast 1 list 666 lanes.count = target >= 2 ? target * READYQ_SHARD_FACTOR: SEQUENTIAL_SHARD;624 lanes.count = target >= 2 ? target * 4: 1; 667 625 /* paranoid */ verify( ocount >= lanes.count ); 668 /* paranoid */ verify( lanes.count == target * READYQ_SHARD_FACTOR|| target < 2 );626 /* paranoid */ verify( lanes.count == target * 4 || target < 2 ); 669 627 670 628 // for printing count the number of displaced threads … … 709 667 fix(lanes.data[idx]); 710 668 } 711 } 712 713 fix_times(cltr); 714 715 reassign_cltr_id(cltr); 669 670 #ifdef USE_SNZI 671 // Re-create the snzi 672 snzi{ log2( lanes.count / 8 ) }; 673 for( idx; (size_t)lanes.count ) { 674 if( !is_empty(lanes.data[idx]) ) { 675 arrive(snzi, idx); 676 } 677 } 678 #endif 679 } 716 680 717 681 // Make sure that everything is consistent
Note:
See TracChangeset
for help on using the changeset viewer.
