Changes in libcfa/src/concurrency/io.cfa [7bfc849:6f121b8]
- File:
-
- 1 edited
-
libcfa/src/concurrency/io.cfa (modified) (21 diffs)
Legend:
- Unmodified
- Added
- Removed
-
libcfa/src/concurrency/io.cfa
r7bfc849 r6f121b8 135 135 void * ring_ptr; 136 136 size_t ring_sz; 137 138 // Statistics 139 #if !defined(__CFA_NO_STATISTICS__) 140 struct { 141 struct { 142 volatile unsigned long long int rdy; 143 volatile unsigned long long int csm; 144 volatile unsigned long long int avl; 145 volatile unsigned long long int cnt; 146 } submit_avg; 147 struct { 148 volatile unsigned long long int val; 149 volatile unsigned long long int cnt; 150 volatile unsigned long long int block; 151 } look_avg; 152 struct { 153 volatile unsigned long long int val; 154 volatile unsigned long long int cnt; 155 volatile unsigned long long int block; 156 } alloc_avg; 157 } stats; 158 #endif 137 159 }; 138 160 … … 155 177 void * ring_ptr; 156 178 size_t ring_sz; 179 180 // Statistics 181 #if !defined(__CFA_NO_STATISTICS__) 182 struct { 183 struct { 184 unsigned long long int val; 185 unsigned long long int slow_cnt; 186 unsigned long long int fast_cnt; 187 } completed_avg; 188 } stats; 189 #endif 157 190 }; 158 191 … … 167 200 struct { 168 201 struct { 169 __processor_id_t id;170 202 void * stack; 171 203 pthread_t kthrd; … … 299 331 (this.io->submit){ min(*sq.num, *cq.num) }; 300 332 333 // Initialize statistics 334 #if !defined(__CFA_NO_STATISTICS__) 335 this.io->submit_q.stats.submit_avg.rdy = 0; 336 this.io->submit_q.stats.submit_avg.csm = 0; 337 this.io->submit_q.stats.submit_avg.avl = 0; 338 this.io->submit_q.stats.submit_avg.cnt = 0; 339 this.io->submit_q.stats.look_avg.val = 0; 340 this.io->submit_q.stats.look_avg.cnt = 0; 341 this.io->submit_q.stats.look_avg.block = 0; 342 this.io->submit_q.stats.alloc_avg.val = 0; 343 this.io->submit_q.stats.alloc_avg.cnt = 0; 344 this.io->submit_q.stats.alloc_avg.block = 0; 345 this.io->completion_q.stats.completed_avg.val = 0; 346 this.io->completion_q.stats.completed_avg.slow_cnt = 0; 347 this.io->completion_q.stats.completed_avg.fast_cnt = 0; 348 #endif 349 301 350 if(!main_cluster) { 302 351 __kernel_io_finish_start( this ); … … 335 384 if( this.io->cltr_flags & CFA_CLUSTER_IO_POLLER_USER_THREAD ) { 336 385 with( this.io->poller.fast ) { 337 /* paranoid */ verify( this. nprocessors == 0|| &this == mainCluster );338 /* paranoid */ verify( !ready_mutate_islocked());386 /* paranoid */ verify( this.procs.head == 0p || &this == mainCluster ); 387 /* paranoid */ verify( this.idles.head == 0p || &this == mainCluster ); 339 388 340 389 // We need to adjust the clean-up based on where the thread is 341 390 if( thrd.state == Ready || thrd.preempted != __NO_PREEMPTION ) { 342 391 343 ready_schedule_lock( (struct __processor_id_t *)active_processor() ); 344 345 // This is the tricky case 346 // The thread was preempted and now it is on the ready queue 347 // The thread should be the last on the list 348 /* paranoid */ verify( thrd.link.next != 0p ); 349 350 // Remove the thread from the ready queue of this cluster 351 __attribute__((unused)) bool removed = remove_head( &this, &thrd ); 352 /* paranoid */ verify( removed ); 353 thrd.link.next = 0p; 354 thrd.link.prev = 0p; 355 __cfaabi_dbg_debug_do( thrd.unpark_stale = true ); 356 357 // Fixup the thread state 358 thrd.state = Blocked; 359 thrd.ticket = 0; 360 thrd.preempted = __NO_PREEMPTION; 361 362 ready_schedule_unlock( (struct __processor_id_t *)active_processor() ); 392 // This is the tricky case 393 // The thread was preempted and now it is on the ready queue 394 /* paranoid */ verify( thrd.next == 1p ); // The thread should be the last on the list 395 /* paranoid */ verify( this.ready_queue.head == &thrd ); // The thread should be the only thing on the list 396 397 // Remove the thread from the ready queue of this cluster 398 this.ready_queue.head = 1p; 399 thrd.next = 0p; 400 __cfaabi_dbg_debug_do( thrd.unpark_stale = true ); 401 402 // Fixup the thread state 403 thrd.state = Blocked; 404 thrd.preempted = __NO_PREEMPTION; 363 405 364 406 // Pretend like the thread was blocked all along … … 372 414 thrd.curr_cluster = active_cluster(); 373 415 374 416 // unpark the fast io_poller 375 417 unpark( &thrd __cfaabi_dbg_ctx2 ); 376 418 } … … 394 436 __kernel_io_prepare_stop( this ); 395 437 } 438 439 // print statistics 440 #if !defined(__CFA_NO_STATISTICS__) 441 if(this.print_stats) { 442 with(this.io->submit_q.stats, this.io->completion_q.stats) { 443 double avgrdy = ((double)submit_avg.rdy) / submit_avg.cnt; 444 double avgcsm = ((double)submit_avg.csm) / submit_avg.cnt; 445 double avgavl = ((double)submit_avg.avl) / submit_avg.cnt; 446 447 double lavgv = 0; 448 double lavgb = 0; 449 if(look_avg.cnt != 0) { 450 lavgv = ((double)look_avg.val ) / look_avg.cnt; 451 lavgb = ((double)look_avg.block) / look_avg.cnt; 452 } 453 454 double aavgv = 0; 455 double aavgb = 0; 456 if(alloc_avg.cnt != 0) { 457 aavgv = ((double)alloc_avg.val ) / alloc_avg.cnt; 458 aavgb = ((double)alloc_avg.block) / alloc_avg.cnt; 459 } 460 461 __cfaabi_bits_print_safe( STDOUT_FILENO, 462 "----- I/O uRing Stats -----\n" 463 "- total submit calls : %'15llu\n" 464 "- avg ready entries : %'18.2lf\n" 465 "- avg submitted entries : %'18.2lf\n" 466 "- avg available entries : %'18.2lf\n" 467 "- total ready search : %'15llu\n" 468 "- avg ready search len : %'18.2lf\n" 469 "- avg ready search block : %'18.2lf\n" 470 "- total alloc search : %'15llu\n" 471 "- avg alloc search len : %'18.2lf\n" 472 "- avg alloc search block : %'18.2lf\n" 473 "- total wait calls : %'15llu (%'llu slow, %'llu fast)\n" 474 "- avg completion/wait : %'18.2lf\n", 475 submit_avg.cnt, 476 avgrdy, 477 avgcsm, 478 avgavl, 479 look_avg.cnt, 480 lavgv, 481 lavgb, 482 alloc_avg.cnt, 483 aavgv, 484 aavgb, 485 completed_avg.slow_cnt + completed_avg.fast_cnt, 486 completed_avg.slow_cnt, completed_avg.fast_cnt, 487 ((double)completed_avg.val) / (completed_avg.slow_cnt + completed_avg.fast_cnt) 488 ); 489 } 490 } 491 #endif 396 492 397 493 // Shutdown the io rings … … 465 561 } 466 562 563 verify( (shead + ret) == *ring.submit_q.head ); 564 467 565 // Release the consumed SQEs 468 566 for( i; ret ) { … … 479 577 // update statistics 480 578 #if !defined(__CFA_NO_STATISTICS__) 481 __tls_stats()->io.submit_q.submit_avg.rdy += to_submit;482 __tls_stats()->io.submit_q.submit_avg.csm += ret;483 __tls_stats()->io.submit_q.submit_avg.avl += avail;484 __tls_stats()->io.submit_q.submit_avg.cnt += 1;579 ring.submit_q.stats.submit_avg.rdy += to_submit; 580 ring.submit_q.stats.submit_avg.csm += ret; 581 ring.submit_q.stats.submit_avg.avl += avail; 582 ring.submit_q.stats.submit_avg.cnt += 1; 485 583 #endif 486 584 … … 510 608 data->result = cqe.res; 511 609 if(!in_kernel) { unpark( data->thrd __cfaabi_dbg_ctx2 ); } 512 else { __unpark( &ring.poller.slow.id,data->thrd __cfaabi_dbg_ctx2 ); }610 else { __unpark( data->thrd __cfaabi_dbg_ctx2 ); } 513 611 } 514 612 … … 525 623 526 624 static void * __io_poller_slow( void * arg ) { 527 #if !defined( __CFA_NO_STATISTICS__ )528 __stats_t local_stats;529 __init_stats( &local_stats );530 kernelTLS.this_stats = &local_stats;531 #endif532 533 625 cluster * cltr = (cluster *)arg; 534 626 struct __io_data & ring = *cltr->io; 535 536 ring.poller.slow.id.id = doregister( &ring.poller.slow.id );537 627 538 628 sigset_t mask; … … 564 654 // Update statistics 565 655 #if !defined(__CFA_NO_STATISTICS__) 566 __tls_stats()->io.complete_q.completed_avg.val += count;567 __tls_stats()->io.complete_q.completed_avg.slow_cnt += 1;656 ring.completion_q.stats.completed_avg.val += count; 657 ring.completion_q.stats.completed_avg.slow_cnt += 1; 568 658 #endif 569 659 570 660 if(again) { 571 661 __cfadbg_print_safe(io_core, "Kernel I/O : Moving to ring %p to fast poller\n", &ring); 572 __unpark( &ring.poller. slow.id, &ring.poller.fast.thrd __cfaabi_dbg_ctx2 );662 __unpark( &ring.poller.fast.thrd __cfaabi_dbg_ctx2 ); 573 663 wait( ring.poller.sem ); 574 664 } … … 584 674 // Update statistics 585 675 #if !defined(__CFA_NO_STATISTICS__) 586 __tls_stats()->io.complete_q.completed_avg.val += count;587 __tls_stats()->io.complete_q.completed_avg.slow_cnt += 1;676 ring.completion_q.stats.completed_avg.val += count; 677 ring.completion_q.stats.completed_avg.slow_cnt += 1; 588 678 #endif 589 679 } … … 591 681 592 682 __cfadbg_print_safe(io_core, "Kernel I/O : Slow poller for ring %p stopping\n", &ring); 593 594 unregister( &ring.poller.slow.id );595 683 596 684 return 0p; … … 613 701 int count; 614 702 bool again; 615 disable_interrupts(); 616 [count, again] = __drain_io( *this.ring, 0p, 0, false ); 617 618 if(!again) reset++; 619 620 // Update statistics 621 #if !defined(__CFA_NO_STATISTICS__) 622 __tls_stats()->io.complete_q.completed_avg.val += count; 623 __tls_stats()->io.complete_q.completed_avg.fast_cnt += 1; 624 #endif 625 enable_interrupts( __cfaabi_dbg_ctx ); 703 [count, again] = __drain_io( *this.ring, 0p, 0, false ); 704 705 if(!again) reset++; 706 707 // Update statistics 708 #if !defined(__CFA_NO_STATISTICS__) 709 this.ring->completion_q.stats.completed_avg.val += count; 710 this.ring->completion_q.stats.completed_avg.fast_cnt += 1; 711 #endif 626 712 627 713 // If we got something, just yield and check again … … 684 770 verify( data != 0 ); 685 771 686 687 772 // Prepare the data we need 688 773 __attribute((unused)) int len = 0; … … 690 775 uint32_t cnt = *ring.submit_q.num; 691 776 uint32_t mask = *ring.submit_q.mask; 692 693 disable_interrupts(); 694 uint32_t off = __tls_rand(); 695 enable_interrupts( __cfaabi_dbg_ctx ); 777 uint32_t off = __tls_rand(); 696 778 697 779 // Loop around looking for an available spot 698 for() {780 LOOKING: for() { 699 781 // Look through the list starting at some offset 700 782 for(i; cnt) { … … 709 791 // update statistics 710 792 #if !defined(__CFA_NO_STATISTICS__) 711 disable_interrupts(); 712 __tls_stats()->io.submit_q.alloc_avg.val += len; 713 __tls_stats()->io.submit_q.alloc_avg.block += block; 714 __tls_stats()->io.submit_q.alloc_avg.cnt += 1; 715 enable_interrupts( __cfaabi_dbg_ctx ); 793 __atomic_fetch_add( &ring.submit_q.stats.alloc_avg.val, len, __ATOMIC_RELAXED ); 794 __atomic_fetch_add( &ring.submit_q.stats.alloc_avg.block, block, __ATOMIC_RELAXED ); 795 __atomic_fetch_add( &ring.submit_q.stats.alloc_avg.cnt, 1, __ATOMIC_RELAXED ); 716 796 #endif 717 718 797 719 798 // Success return the data … … 734 813 uint32_t * const tail = ring.submit_q.tail; 735 814 const uint32_t mask = *ring.submit_q.mask; 736 737 disable_interrupts();738 815 739 816 // There are 2 submission schemes, check which one we are using … … 769 846 // update statistics 770 847 #if !defined(__CFA_NO_STATISTICS__) 771 __ tls_stats()->io.submit_q.look_avg.val += len;772 __ tls_stats()->io.submit_q.look_avg.block += block;773 __ tls_stats()->io.submit_q.look_avg.cnt += 1;848 __atomic_fetch_add( &ring.submit_q.stats.look_avg.val, len, __ATOMIC_RELAXED ); 849 __atomic_fetch_add( &ring.submit_q.stats.look_avg.block, block, __ATOMIC_RELAXED ); 850 __atomic_fetch_add( &ring.submit_q.stats.look_avg.cnt, 1, __ATOMIC_RELAXED ); 774 851 #endif 775 852 … … 798 875 // update statistics 799 876 #if !defined(__CFA_NO_STATISTICS__) 800 __tls_stats()->io.submit_q.submit_avg.csm += 1;801 __tls_stats()->io.submit_q.submit_avg.cnt += 1;877 ring.submit_q.stats.submit_avg.csm += 1; 878 ring.submit_q.stats.submit_avg.cnt += 1; 802 879 #endif 803 880 804 ring.submit_q.sqes[ idx & mask ].user_data = 0;805 806 881 unlock(ring.submit_q.lock); 807 882 808 883 __cfadbg_print_safe( io, "Kernel I/O : Performed io_submit for %p, returned %d\n", active_thread(), ret ); 809 884 } 810 811 enable_interrupts( __cfaabi_dbg_ctx );812 885 } 813 886
Note:
See TracChangeset
for help on using the changeset viewer.
