Changes in libcfa/src/concurrency/io.cfa [6f121b8:7bfc849]
- File:
-
- 1 edited
-
libcfa/src/concurrency/io.cfa (modified) (21 diffs)
Legend:
- Unmodified
- Added
- Removed
-
libcfa/src/concurrency/io.cfa
r6f121b8 r7bfc849 135 135 void * ring_ptr; 136 136 size_t ring_sz; 137 138 // Statistics139 #if !defined(__CFA_NO_STATISTICS__)140 struct {141 struct {142 volatile unsigned long long int rdy;143 volatile unsigned long long int csm;144 volatile unsigned long long int avl;145 volatile unsigned long long int cnt;146 } submit_avg;147 struct {148 volatile unsigned long long int val;149 volatile unsigned long long int cnt;150 volatile unsigned long long int block;151 } look_avg;152 struct {153 volatile unsigned long long int val;154 volatile unsigned long long int cnt;155 volatile unsigned long long int block;156 } alloc_avg;157 } stats;158 #endif159 137 }; 160 138 … … 177 155 void * ring_ptr; 178 156 size_t ring_sz; 179 180 // Statistics181 #if !defined(__CFA_NO_STATISTICS__)182 struct {183 struct {184 unsigned long long int val;185 unsigned long long int slow_cnt;186 unsigned long long int fast_cnt;187 } completed_avg;188 } stats;189 #endif190 157 }; 191 158 … … 200 167 struct { 201 168 struct { 169 __processor_id_t id; 202 170 void * stack; 203 171 pthread_t kthrd; … … 331 299 (this.io->submit){ min(*sq.num, *cq.num) }; 332 300 333 // Initialize statistics334 #if !defined(__CFA_NO_STATISTICS__)335 this.io->submit_q.stats.submit_avg.rdy = 0;336 this.io->submit_q.stats.submit_avg.csm = 0;337 this.io->submit_q.stats.submit_avg.avl = 0;338 this.io->submit_q.stats.submit_avg.cnt = 0;339 this.io->submit_q.stats.look_avg.val = 0;340 this.io->submit_q.stats.look_avg.cnt = 0;341 this.io->submit_q.stats.look_avg.block = 0;342 this.io->submit_q.stats.alloc_avg.val = 0;343 this.io->submit_q.stats.alloc_avg.cnt = 0;344 this.io->submit_q.stats.alloc_avg.block = 0;345 this.io->completion_q.stats.completed_avg.val = 0;346 this.io->completion_q.stats.completed_avg.slow_cnt = 0;347 this.io->completion_q.stats.completed_avg.fast_cnt = 0;348 #endif349 350 301 if(!main_cluster) { 351 302 __kernel_io_finish_start( this ); … … 384 335 if( this.io->cltr_flags & CFA_CLUSTER_IO_POLLER_USER_THREAD ) { 385 336 with( this.io->poller.fast ) { 386 /* paranoid */ verify( this. procs.head == 0p|| &this == mainCluster );387 /* paranoid */ verify( this.idles.head == 0p || &this == mainCluster);337 /* paranoid */ verify( this.nprocessors == 0 || &this == mainCluster ); 338 /* paranoid */ verify( !ready_mutate_islocked() ); 388 339 389 340 // We need to adjust the clean-up based on where the thread is 390 341 if( thrd.state == Ready || thrd.preempted != __NO_PREEMPTION ) { 391 342 392 // This is the tricky case 393 // The thread was preempted and now it is on the ready queue 394 /* paranoid */ verify( thrd.next == 1p ); // The thread should be the last on the list 395 /* paranoid */ verify( this.ready_queue.head == &thrd ); // The thread should be the only thing on the list 396 397 // Remove the thread from the ready queue of this cluster 398 this.ready_queue.head = 1p; 399 thrd.next = 0p; 400 __cfaabi_dbg_debug_do( thrd.unpark_stale = true ); 401 402 // Fixup the thread state 403 thrd.state = Blocked; 404 thrd.preempted = __NO_PREEMPTION; 343 ready_schedule_lock( (struct __processor_id_t *)active_processor() ); 344 345 // This is the tricky case 346 // The thread was preempted and now it is on the ready queue 347 // The thread should be the last on the list 348 /* paranoid */ verify( thrd.link.next != 0p ); 349 350 // Remove the thread from the ready queue of this cluster 351 __attribute__((unused)) bool removed = remove_head( &this, &thrd ); 352 /* paranoid */ verify( removed ); 353 thrd.link.next = 0p; 354 thrd.link.prev = 0p; 355 __cfaabi_dbg_debug_do( thrd.unpark_stale = true ); 356 357 // Fixup the thread state 358 thrd.state = Blocked; 359 thrd.ticket = 0; 360 thrd.preempted = __NO_PREEMPTION; 361 362 ready_schedule_unlock( (struct __processor_id_t *)active_processor() ); 405 363 406 364 // Pretend like the thread was blocked all along … … 414 372 thrd.curr_cluster = active_cluster(); 415 373 416 // unpark the fast io_poller374 // unpark the fast io_poller 417 375 unpark( &thrd __cfaabi_dbg_ctx2 ); 418 376 } … … 436 394 __kernel_io_prepare_stop( this ); 437 395 } 438 439 // print statistics440 #if !defined(__CFA_NO_STATISTICS__)441 if(this.print_stats) {442 with(this.io->submit_q.stats, this.io->completion_q.stats) {443 double avgrdy = ((double)submit_avg.rdy) / submit_avg.cnt;444 double avgcsm = ((double)submit_avg.csm) / submit_avg.cnt;445 double avgavl = ((double)submit_avg.avl) / submit_avg.cnt;446 447 double lavgv = 0;448 double lavgb = 0;449 if(look_avg.cnt != 0) {450 lavgv = ((double)look_avg.val ) / look_avg.cnt;451 lavgb = ((double)look_avg.block) / look_avg.cnt;452 }453 454 double aavgv = 0;455 double aavgb = 0;456 if(alloc_avg.cnt != 0) {457 aavgv = ((double)alloc_avg.val ) / alloc_avg.cnt;458 aavgb = ((double)alloc_avg.block) / alloc_avg.cnt;459 }460 461 __cfaabi_bits_print_safe( STDOUT_FILENO,462 "----- I/O uRing Stats -----\n"463 "- total submit calls : %'15llu\n"464 "- avg ready entries : %'18.2lf\n"465 "- avg submitted entries : %'18.2lf\n"466 "- avg available entries : %'18.2lf\n"467 "- total ready search : %'15llu\n"468 "- avg ready search len : %'18.2lf\n"469 "- avg ready search block : %'18.2lf\n"470 "- total alloc search : %'15llu\n"471 "- avg alloc search len : %'18.2lf\n"472 "- avg alloc search block : %'18.2lf\n"473 "- total wait calls : %'15llu (%'llu slow, %'llu fast)\n"474 "- avg completion/wait : %'18.2lf\n",475 submit_avg.cnt,476 avgrdy,477 avgcsm,478 avgavl,479 look_avg.cnt,480 lavgv,481 lavgb,482 alloc_avg.cnt,483 aavgv,484 aavgb,485 completed_avg.slow_cnt + completed_avg.fast_cnt,486 completed_avg.slow_cnt, completed_avg.fast_cnt,487 ((double)completed_avg.val) / (completed_avg.slow_cnt + completed_avg.fast_cnt)488 );489 }490 }491 #endif492 396 493 397 // Shutdown the io rings … … 561 465 } 562 466 563 verify( (shead + ret) == *ring.submit_q.head );564 565 467 // Release the consumed SQEs 566 468 for( i; ret ) { … … 577 479 // update statistics 578 480 #if !defined(__CFA_NO_STATISTICS__) 579 ring.submit_q.stats.submit_avg.rdy += to_submit;580 ring.submit_q.stats.submit_avg.csm += ret;581 ring.submit_q.stats.submit_avg.avl += avail;582 ring.submit_q.stats.submit_avg.cnt += 1;481 __tls_stats()->io.submit_q.submit_avg.rdy += to_submit; 482 __tls_stats()->io.submit_q.submit_avg.csm += ret; 483 __tls_stats()->io.submit_q.submit_avg.avl += avail; 484 __tls_stats()->io.submit_q.submit_avg.cnt += 1; 583 485 #endif 584 486 … … 608 510 data->result = cqe.res; 609 511 if(!in_kernel) { unpark( data->thrd __cfaabi_dbg_ctx2 ); } 610 else { __unpark( data->thrd __cfaabi_dbg_ctx2 ); }512 else { __unpark( &ring.poller.slow.id, data->thrd __cfaabi_dbg_ctx2 ); } 611 513 } 612 514 … … 623 525 624 526 static void * __io_poller_slow( void * arg ) { 527 #if !defined( __CFA_NO_STATISTICS__ ) 528 __stats_t local_stats; 529 __init_stats( &local_stats ); 530 kernelTLS.this_stats = &local_stats; 531 #endif 532 625 533 cluster * cltr = (cluster *)arg; 626 534 struct __io_data & ring = *cltr->io; 535 536 ring.poller.slow.id.id = doregister( &ring.poller.slow.id ); 627 537 628 538 sigset_t mask; … … 654 564 // Update statistics 655 565 #if !defined(__CFA_NO_STATISTICS__) 656 ring.completion_q.stats.completed_avg.val += count;657 ring.completion_q.stats.completed_avg.slow_cnt += 1;566 __tls_stats()->io.complete_q.completed_avg.val += count; 567 __tls_stats()->io.complete_q.completed_avg.slow_cnt += 1; 658 568 #endif 659 569 660 570 if(again) { 661 571 __cfadbg_print_safe(io_core, "Kernel I/O : Moving to ring %p to fast poller\n", &ring); 662 __unpark( &ring.poller. fast.thrd __cfaabi_dbg_ctx2 );572 __unpark( &ring.poller.slow.id, &ring.poller.fast.thrd __cfaabi_dbg_ctx2 ); 663 573 wait( ring.poller.sem ); 664 574 } … … 674 584 // Update statistics 675 585 #if !defined(__CFA_NO_STATISTICS__) 676 ring.completion_q.stats.completed_avg.val += count;677 ring.completion_q.stats.completed_avg.slow_cnt += 1;586 __tls_stats()->io.complete_q.completed_avg.val += count; 587 __tls_stats()->io.complete_q.completed_avg.slow_cnt += 1; 678 588 #endif 679 589 } … … 681 591 682 592 __cfadbg_print_safe(io_core, "Kernel I/O : Slow poller for ring %p stopping\n", &ring); 593 594 unregister( &ring.poller.slow.id ); 683 595 684 596 return 0p; … … 701 613 int count; 702 614 bool again; 703 [count, again] = __drain_io( *this.ring, 0p, 0, false ); 704 705 if(!again) reset++; 706 707 // Update statistics 708 #if !defined(__CFA_NO_STATISTICS__) 709 this.ring->completion_q.stats.completed_avg.val += count; 710 this.ring->completion_q.stats.completed_avg.fast_cnt += 1; 711 #endif 615 disable_interrupts(); 616 [count, again] = __drain_io( *this.ring, 0p, 0, false ); 617 618 if(!again) reset++; 619 620 // Update statistics 621 #if !defined(__CFA_NO_STATISTICS__) 622 __tls_stats()->io.complete_q.completed_avg.val += count; 623 __tls_stats()->io.complete_q.completed_avg.fast_cnt += 1; 624 #endif 625 enable_interrupts( __cfaabi_dbg_ctx ); 712 626 713 627 // If we got something, just yield and check again … … 770 684 verify( data != 0 ); 771 685 686 772 687 // Prepare the data we need 773 688 __attribute((unused)) int len = 0; … … 775 690 uint32_t cnt = *ring.submit_q.num; 776 691 uint32_t mask = *ring.submit_q.mask; 777 uint32_t off = __tls_rand(); 692 693 disable_interrupts(); 694 uint32_t off = __tls_rand(); 695 enable_interrupts( __cfaabi_dbg_ctx ); 778 696 779 697 // Loop around looking for an available spot 780 LOOKING:for() {698 for() { 781 699 // Look through the list starting at some offset 782 700 for(i; cnt) { … … 791 709 // update statistics 792 710 #if !defined(__CFA_NO_STATISTICS__) 793 __atomic_fetch_add( &ring.submit_q.stats.alloc_avg.val, len, __ATOMIC_RELAXED ); 794 __atomic_fetch_add( &ring.submit_q.stats.alloc_avg.block, block, __ATOMIC_RELAXED ); 795 __atomic_fetch_add( &ring.submit_q.stats.alloc_avg.cnt, 1, __ATOMIC_RELAXED ); 711 disable_interrupts(); 712 __tls_stats()->io.submit_q.alloc_avg.val += len; 713 __tls_stats()->io.submit_q.alloc_avg.block += block; 714 __tls_stats()->io.submit_q.alloc_avg.cnt += 1; 715 enable_interrupts( __cfaabi_dbg_ctx ); 796 716 #endif 717 797 718 798 719 // Success return the data … … 813 734 uint32_t * const tail = ring.submit_q.tail; 814 735 const uint32_t mask = *ring.submit_q.mask; 736 737 disable_interrupts(); 815 738 816 739 // There are 2 submission schemes, check which one we are using … … 846 769 // update statistics 847 770 #if !defined(__CFA_NO_STATISTICS__) 848 __ atomic_fetch_add( &ring.submit_q.stats.look_avg.val, len, __ATOMIC_RELAXED );849 __ atomic_fetch_add( &ring.submit_q.stats.look_avg.block, block, __ATOMIC_RELAXED );850 __ atomic_fetch_add( &ring.submit_q.stats.look_avg.cnt, 1, __ATOMIC_RELAXED );771 __tls_stats()->io.submit_q.look_avg.val += len; 772 __tls_stats()->io.submit_q.look_avg.block += block; 773 __tls_stats()->io.submit_q.look_avg.cnt += 1; 851 774 #endif 852 775 … … 875 798 // update statistics 876 799 #if !defined(__CFA_NO_STATISTICS__) 877 ring.submit_q.stats.submit_avg.csm += 1;878 ring.submit_q.stats.submit_avg.cnt += 1;800 __tls_stats()->io.submit_q.submit_avg.csm += 1; 801 __tls_stats()->io.submit_q.submit_avg.cnt += 1; 879 802 #endif 880 803 804 ring.submit_q.sqes[ idx & mask ].user_data = 0; 805 881 806 unlock(ring.submit_q.lock); 882 807 883 808 __cfadbg_print_safe( io, "Kernel I/O : Performed io_submit for %p, returned %d\n", active_thread(), ret ); 884 809 } 810 811 enable_interrupts( __cfaabi_dbg_ctx ); 885 812 } 886 813
Note:
See TracChangeset
for help on using the changeset viewer.
