Changeset 13c5e19 for libcfa/src/concurrency/io.cfa

Timestamp:

Jun 23, 2020, 4:42:58 PM (4 years ago)

Author:

Thierry Delisle <tdelisle@…>

Branches:

ADT, arm-eh, ast-experimental, enum, forall-pointer-decay, jacob/cs343-translation, master, new-ast, new-ast-unique-expr, pthread-emulation, qualifiedEnum

Children:

de917da3

Parents:

b232745

Message:

• Moved snzi and subqueues outside of ready_queue.cfa.
• Added random.hfa with multiple prng.
• Minor optimizations to ready-queue
• Stats now track number of local pops( bias pops )
• Fixed stats for io
• Fixed calculaton of nprocessors
• Fixed IO to work with new ready-queue

File:

• libcfa/src/concurrency/io.cfa (modified) (18 diffs)

libcfa/src/concurrency/io.cfa

@@ -167 +167 @@
                 struct {
                         struct {
+                                __processor_id_t id;
                                 void * stack;
                                 pthread_t kthrd;
@@ -334 +335 @@
                 if( this.io->cltr_flags & CFA_CLUSTER_IO_POLLER_USER_THREAD ) {
                         with( this.io->poller.fast ) {
-                                /* paranoid */ verify( this.procs.head == 0p || &this == mainCluster );
-                                /* paranoid */ verify( this.idles.head == 0p || &this == mainCluster );
+                                /* paranoid */ verify( this.nprocessors == 0 || &this == mainCluster );
+                                /* paranoid */ verify( !ready_mutate_islocked() );
 
                                 // We need to adjust the clean-up based on where the thread is
                                 if( thrd.state == Ready || thrd.preempted != __NO_PREEMPTION ) {
 
-                                        // This is the tricky case
-                                        // The thread was preempted and now it is on the ready queue
-
-                                        /* paranoid */ verify( thrd.next != 0p );                // The thread should be the last on the list
-                                        /* paranoid */ verify( this.ready_queue.head == &thrd ); // The thread should be the only thing on the list
-
-                                        // Remove the thread from the ready queue of this cluster
-                                        this.ready_queue.head = 1p;
-                                        thrd.next = 0p;
-                                        __cfaabi_dbg_debug_do( thrd.unpark_stale = true );
-
-                                        // Fixup the thread state
-                                        thrd.state = Blocked;
-                                        thrd.preempted = __NO_PREEMPTION;
+                                        ready_schedule_lock( (struct __processor_id_t *)active_processor() );
+
+                                                // This is the tricky case
+                                                // The thread was preempted and now it is on the ready queue
+                                                // The thread should be the last on the list
+                                                /* paranoid */ verify( thrd.link.next != 0p );
+
+                                                // Remove the thread from the ready queue of this cluster
+                                                __attribute__((unused)) bool removed = remove_head( &this, &thrd );
+                                                /* paranoid */ verify( removed );
+                                                thrd.link.next = 0p;
+                                                thrd.link.prev = 0p;
+                                                __cfaabi_dbg_debug_do( thrd.unpark_stale = true );
+
+                                                // Fixup the thread state
+                                                thrd.state = Blocked;
+                                                thrd.ticket = 0;
+                                                thrd.preempted = __NO_PREEMPTION;
+
+                                        ready_schedule_unlock( (struct __processor_id_t *)active_processor() );
 
                                         // Pretend like the thread was blocked all along
@@ -365 +372 @@
                                         thrd.curr_cluster = active_cluster();
 
-                        // unpark the fast io_poller
+                                        // unpark the fast io_poller
                                         unpark( &thrd __cfaabi_dbg_ctx2 );
                                 }
@@ -458 +465 @@
                 }
 
-                verify( (shead + ret) == *ring.submit_q.head );
+                uint32_t nhead = *ring.submit_q.head;
+                verifyf( (shead + ret) == nhead, "Expected %u got %u\n", (shead + ret), nhead );
 
                 // Release the consumed SQEs
@@ -474 +482 @@
                 // update statistics
                 #if !defined(__CFA_NO_STATISTICS__)
-                        __tls_stats()->io.submit_q.stats.submit_avg.rdy += to_submit;
-                        __tls_stats()->io.submit_q.stats.submit_avg.csm += ret;
-                        __tls_stats()->io.submit_q.stats.submit_avg.avl += avail;
-                        __tls_stats()->io.submit_q.stats.submit_avg.cnt += 1;
+                        __tls_stats()->io.submit_q.submit_avg.rdy += to_submit;
+                        __tls_stats()->io.submit_q.submit_avg.csm += ret;
+                        __tls_stats()->io.submit_q.submit_avg.avl += avail;
+                        __tls_stats()->io.submit_q.submit_avg.cnt += 1;
                 #endif
 
@@ -505 +513 @@
                         data->result = cqe.res;
                         if(!in_kernel) { unpark( data->thrd __cfaabi_dbg_ctx2 ); }
-                        else         { __unpark( data->thrd __cfaabi_dbg_ctx2 ); }
+                        else         { __unpark( &ring.poller.slow.id, data->thrd __cfaabi_dbg_ctx2 ); }
                 }
 
@@ -520 +528 @@
 
         static void * __io_poller_slow( void * arg ) {
+                #if !defined( __CFA_NO_STATISTICS__ )
+                        __stats_t local_stats;
+                        __init_stats( &local_stats );
+                        kernelTLS.this_stats = &local_stats;
+                #endif
+
                 cluster * cltr = (cluster *)arg;
                 struct __io_data & ring = *cltr->io;
+
+                ring.poller.slow.id.id = doregister( &ring.poller.slow.id );
 
                 sigset_t mask;
@@ -551 +567 @@
                                 // Update statistics
                                 #if !defined(__CFA_NO_STATISTICS__)
-                                        __tls_stats()->io.complete_q.stats.completed_avg.val += count;
-                                        __tls_stats()->io.complete_q.stats.completed_avg.slow_cnt += 1;
+                                        __tls_stats()->io.complete_q.completed_avg.val += count;
+                                        __tls_stats()->io.complete_q.completed_avg.slow_cnt += 1;
                                 #endif
 
                                 if(again) {
                                         __cfadbg_print_safe(io_core, "Kernel I/O : Moving to ring %p to fast poller\n", &ring);
-                                        __unpark( &ring.poller.fast.thrd __cfaabi_dbg_ctx2 );
+                                        __unpark( &ring.poller.slow.id, &ring.poller.fast.thrd __cfaabi_dbg_ctx2 );
                                         wait( ring.poller.sem );
                                 }
@@ -571 +587 @@
                                 // Update statistics
                                 #if !defined(__CFA_NO_STATISTICS__)
-                                        __tls_stats()->io.complete_q.stats.completed_avg.val += count;
-                                        __tls_stats()->io.complete_q.stats.completed_avg.slow_cnt += 1;
+                                        __tls_stats()->io.complete_q.completed_avg.val += count;
+                                        __tls_stats()->io.complete_q.completed_avg.slow_cnt += 1;
                                 #endif
                         }
@@ -578 +594 @@
 
                 __cfadbg_print_safe(io_core, "Kernel I/O : Slow poller for ring %p stopping\n", &ring);
+
+                unregister( &ring.poller.slow.id );
 
                 return 0p;
@@ -598 +616 @@
                         int count;
                         bool again;
-                        [count, again] = __drain_io( *this.ring, 0p, 0, false );
-
-                        if(!again) reset++;
-
-                        // Update statistics
-                        #if !defined(__CFA_NO_STATISTICS__)
-                                __tls_stats()->io.complete_q.stats.completed_avg.val += count;
-                                __tls_stats()->io.complete_q.stats.completed_avg.fast_cnt += 1;
-                        #endif
+                        disable_interrupts();
+                                [count, again] = __drain_io( *this.ring, 0p, 0, false );
+
+                                if(!again) reset++;
+
+                                // Update statistics
+                                #if !defined(__CFA_NO_STATISTICS__)
+                                        __tls_stats()->io.complete_q.completed_avg.val += count;
+                                        __tls_stats()->io.complete_q.completed_avg.fast_cnt += 1;
+                                #endif
+                        enable_interrupts( __cfaabi_dbg_ctx );
 
                         // If we got something, just yield and check again
@@ -667 +687 @@
                 verify( data != 0 );
 
+                disable_interrupts();
+
                 // Prepare the data we need
                 __attribute((unused)) int len   = 0;
@@ -675 +697 @@
 
                 // Loop around looking for an available spot
-                LOOKING: for() {
+                for() {
                         // Look through the list starting at some offset
                         for(i; cnt) {
@@ -688 +710 @@
                                         // update statistics
                                         #if !defined(__CFA_NO_STATISTICS__)
-                                                __tls_stats()->io.submit_q.stats.alloc_avg.val   += len;
-                                                __tls_stats()->io.submit_q.stats.alloc_avg.block += block;
-                                                __tls_stats()->io.submit_q.stats.alloc_avg.cnt   += 1;
+                                                __tls_stats()->io.submit_q.alloc_avg.val   += len;
+                                                __tls_stats()->io.submit_q.alloc_avg.block += block;
+                                                __tls_stats()->io.submit_q.alloc_avg.cnt   += 1;
                                         #endif
+
+                                        enable_interrupts( __cfaabi_dbg_ctx );
 
                                         // Success return the data
@@ -710 +734 @@
                 uint32_t * const tail = ring.submit_q.tail;
                 const uint32_t mask = *ring.submit_q.mask;
+
+                disable_interrupts();
 
                 // There are 2 submission schemes, check which one we are using
@@ -743 +769 @@
                         // update statistics
                         #if !defined(__CFA_NO_STATISTICS__)
-                                __tls_stats()->io.submit_q.stats.look_avg.val   += len;
-                                __tls_stats()->io.submit_q.stats.look_avg.block += block;
-                                __tls_stats()->io.submit_q.stats.look_avg.cnt   += 1;
+                                __tls_stats()->io.submit_q.look_avg.val   += len;
+                                __tls_stats()->io.submit_q.look_avg.block += block;
+                                __tls_stats()->io.submit_q.look_avg.cnt   += 1;
                         #endif
 
@@ -772 +798 @@
                         // update statistics
                         #if !defined(__CFA_NO_STATISTICS__)
-                                __tls_stats()->io.submit_q.stats.submit_avg.csm += 1;
-                                __tls_stats()->io.submit_q.stats.submit_avg.cnt += 1;
+                                __tls_stats()->io.submit_q.submit_avg.csm += 1;
+                                __tls_stats()->io.submit_q.submit_avg.cnt += 1;
                         #endif
 
@@ -780 +806 @@
                         __cfadbg_print_safe( io, "Kernel I/O : Performed io_submit for %p, returned %d\n", active_thread(), ret );
                 }
+
+                enable_interrupts( __cfaabi_dbg_ctx );
         }