Changeset d2fadeb for libcfa/src/concurrency/ready_queue.cfa
 Timestamp:
 Apr 19, 2021, 5:07:16 PM (2 years ago)
 Branches:
 armeh, enum, forallpointerdecay, jacob/cs343translation, master, newastuniqueexpr, pthreademulation, qualifiedEnum
 Children:
 59f3f61, 665edf40, ddd473f
 Parents:
 4aa495f
 File:

 1 edited
Legend:
 Unmodified
 Added
 Removed

libcfa/src/concurrency/ready_queue.cfa
r4aa495f rd2fadeb 35 35 static const size_t cache_line_size = 64; 36 36 37 #if !defined(__CFA_NO_STATISTICS__) 38 #define __STATS(...) __VA_ARGS__ 39 #else 40 #define __STATS(...) 41 #endif 42 37 43 // No overriden function, no environment variable, no define 38 44 // fall back to a magic number … … 52 58 #endif 53 59 60 static inline struct $thread * try_pop(struct cluster * cltr, unsigned w __STATS(, __stats_readyQ_pop_t & stats)); 61 static inline struct $thread * try_pop(struct cluster * cltr, unsigned i, unsigned j __STATS(, __stats_readyQ_pop_t & stats)); 62 static inline struct $thread * search(struct cluster * cltr); 54 63 static inline [unsigned, bool] idx_from_r(unsigned r, unsigned preferred); 55 static inline struct $thread * try_pop(struct cluster * cltr, unsigned w);56 static inline struct $thread * try_pop(struct cluster * cltr, unsigned i, unsigned j);57 static inline struct $thread * search(struct cluster * cltr);58 64 59 65 … … 269 275 270 276 #if !defined(__CFA_NO_STATISTICS__) 271 if(external) { 272 if(local) __atomic_fetch_add(&cltr>stats>ready.pick.ext.local, 1, __ATOMIC_RELAXED); 273 __atomic_fetch_add(&cltr>stats>ready.pick.ext.attempt, 1, __ATOMIC_RELAXED); 274 } 275 else { 276 if(local) __tls_stats()>ready.pick.push.local++; 277 __tls_stats()>ready.pick.push.attempt++; 278 } 277 if(unlikely(external)) __atomic_fetch_add(&cltr>stats>ready.push.extrn.attempt, 1, __ATOMIC_RELAXED); 278 else if(local) __tls_stats()>ready.push.local.attempt++; 279 else __tls_stats()>ready.push.share.attempt++; 279 280 #endif 280 281 … … 302 303 // Update statistics 303 304 #if !defined(__CFA_NO_STATISTICS__) 304 if(external) { 305 if(local) __atomic_fetch_add(&cltr>stats>ready.pick.ext.lsuccess, 1, __ATOMIC_RELAXED); 306 __atomic_fetch_add(&cltr>stats>ready.pick.ext.success, 1, __ATOMIC_RELAXED); 307 } 308 else { 309 if(local) __tls_stats()>ready.pick.push.lsuccess++; 310 __tls_stats()>ready.pick.push.success++; 311 } 305 if(unlikely(external)) __atomic_fetch_add(&cltr>stats>ready.push.extrn.success, 1, __ATOMIC_RELAXED); 306 else if(local) __tls_stats()>ready.push.local.success++; 307 else __tls_stats()>ready.push.share.success++; 312 308 #endif 313 309 } … … 334 330 [j, localj] = idx_from_r(rj, preferred); 335 331 336 #if !defined(__CFA_NO_STATISTICS__)337 if(locali && localj) {338 __tls_stats()>ready.pick.pop.local++;339 }340 #endif341 342 332 i %= count; 343 333 j %= count; 344 334 345 335 // try popping from the 2 picked lists 346 struct $thread * thrd = try_pop(cltr, i, j );336 struct $thread * thrd = try_pop(cltr, i, j __STATS(, *(locali  localj ? &__tls_stats()>ready.pop.local : &__tls_stats()>ready.pop.help))); 347 337 if(thrd) { 348 #if !defined(__CFA_NO_STATISTICS__)349 if( locali  localj ) __tls_stats()>ready.pick.pop.lsuccess++;350 #endif351 338 return thrd; 352 339 } … … 374 361 unsigned i; 375 362 do { 363 #if !defined(__CFA_NO_STATISTICS__) 364 if(unlikely(external)) __atomic_fetch_add(&cltr>stats>ready.push.extrn.attempt, 1, __ATOMIC_RELAXED); 365 else __tls_stats()>ready.push.local.attempt++; 366 #endif 367 376 368 if(unlikely(external)) { 377 369 i = __tls_rand() % lanes.count; … … 398 390 // Unlock and return 399 391 __atomic_unlock( &lanes.data[i].lock ); 392 #endif 393 394 #if !defined(__CFA_NO_STATISTICS__) 395 if(unlikely(external)) __atomic_fetch_add(&cltr>stats>ready.push.extrn.success, 1, __ATOMIC_RELAXED); 396 else __tls_stats()>ready.push.local.success++; 400 397 #endif 401 398 … … 422 419 } 423 420 else if(lanes.tscs[proc>rdq.target].tv < proc>rdq.cutoff) { 424 $thread * t = try_pop(cltr, proc>rdq.target );421 $thread * t = try_pop(cltr, proc>rdq.target __STATS(, __tls_stats()>ready.pop.help)); 425 422 proc>rdq.target = 1u; 426 423 if(t) return t; … … 429 426 for(READYQ_SHARD_FACTOR) { 430 427 unsigned i = proc>rdq.id + (proc>rdq.itr % READYQ_SHARD_FACTOR); 431 if($thread * t = try_pop(cltr, i )) return t;428 if($thread * t = try_pop(cltr, i __STATS(, __tls_stats()>ready.pop.local))) return t; 432 429 } 433 430 return 0p; … … 437 434 for(25) { 438 435 unsigned i = __tls_rand() % lanes.count; 439 $thread * t = try_pop(cltr, i );436 $thread * t = try_pop(cltr, i __STATS(, __tls_stats()>ready.pop.steal)); 440 437 if(t) return t; 441 438 } … … 453 450 // 454 451 // try to pop from a lane given by index w 455 static inline struct $thread * try_pop(struct cluster * cltr, unsigned w) with (cltr>ready_queue) { 452 static inline struct $thread * try_pop(struct cluster * cltr, unsigned w __STATS(, __stats_readyQ_pop_t & stats)) with (cltr>ready_queue) { 453 __STATS( stats.attempt++; ) 454 456 455 // Get relevant elements locally 457 456 __intrusive_lane_t & lane = lanes.data[w]; 458 457 459 458 // If list looks empty retry 460 if( is_empty(lane) ) return 0p; 459 if( is_empty(lane) ) { 460 __STATS( stats.espec++; ) 461 return 0p; 462 } 461 463 462 464 // If we can't get the lock retry 463 if( !__atomic_try_acquire(&lane.lock) ) return 0p; 465 if( !__atomic_try_acquire(&lane.lock) ) { 466 __STATS( stats.elock++; ) 467 return 0p; 468 } 464 469 465 470 // If list is empty, unlock and retry 466 471 if( is_empty(lane) ) { 467 472 __atomic_unlock(&lane.lock); 473 __STATS( stats.eempty++; ) 468 474 return 0p; 469 475 } … … 480 486 481 487 // Update statistics 482 #if !defined(__CFA_NO_STATISTICS__) 483 __tls_stats()>ready.pick.pop.success++; 484 #endif 488 __STATS( stats.success++; ) 485 489 486 490 #if defined(USE_WORK_STEALING) … … 501 505 for(i; count) { 502 506 unsigned idx = (offset + i) % count; 503 struct $thread * thrd = try_pop(cltr, idx );507 struct $thread * thrd = try_pop(cltr, idx __STATS(, __tls_stats()>ready.pop.search)); 504 508 if(thrd) { 505 509 return thrd; … … 539 543 // 540 544 // Given 2 indexes, pick the list with the oldest push an try to pop from it 541 static inline struct $thread * try_pop(struct cluster * cltr, unsigned i, unsigned j) with (cltr>ready_queue) { 542 #if !defined(__CFA_NO_STATISTICS__) 543 __tls_stats()>ready.pick.pop.attempt++; 544 #endif 545 545 static inline struct $thread * try_pop(struct cluster * cltr, unsigned i, unsigned j __STATS(, __stats_readyQ_pop_t & stats)) with (cltr>ready_queue) { 546 546 // Pick the bet list 547 547 int w = i; … … 550 550 } 551 551 552 return try_pop(cltr, w );552 return try_pop(cltr, w __STATS(, stats)); 553 553 } 554 554
Note: See TracChangeset
for help on using the changeset viewer.