Changes in / [31bb2e1:4ec028d]

File:

• libcfa/src/concurrency/io.cfa (modified) (5 diffs)

libcfa/src/concurrency/io.cfa

@@ -576 +576 @@
                                 int count;
                                 bool again;
-                                [count, again] = __drain_io( ring, &mask, 1, true );
+                                [count, again] = __drain_io( ring, &mask, 0, true );
 
                                 // Update statistics
@@ -589 +589 @@
 
                 unregister( &ring.poller.slow.id );
+
+                #if !defined(__CFA_NO_STATISTICS__)
+                        __tally_stats(cltr->stats, &local_stats);
+                #endif
 
                 return 0p;
@@ -726 +730 @@
         }
 
+        static inline uint32_t __submit_to_ready_array( struct __io_data & ring, uint32_t idx, const uint32_t mask ) {
+                /* paranoid */ verify( idx <= mask   );
+                /* paranoid */ verify( idx != -1ul32 );
+
+                // We need to find a spot in the ready array
+                __attribute((unused)) int len   = 0;
+                __attribute((unused)) int block = 0;
+                uint32_t ready_mask = ring.submit_q.ready_cnt - 1;
+
+                disable_interrupts();
+                        uint32_t off = __tls_rand();
+                enable_interrupts( __cfaabi_dbg_ctx );
+
+                uint32_t picked;
+                LOOKING: for() {
+                        for(i; ring.submit_q.ready_cnt) {
+                                picked = (i + off) & ready_mask;
+                                uint32_t expected = -1ul32;
+                                if( __atomic_compare_exchange_n( &ring.submit_q.ready[picked], &expected, idx, true, __ATOMIC_SEQ_CST, __ATOMIC_RELAXED ) ) {
+                                        break LOOKING;
+                                }
+                                verify(expected != idx);
+
+                                len ++;
+                        }
+
+                        block++;
+                        yield();
+                }
+
+                // update statistics
+                #if !defined(__CFA_NO_STATISTICS__)
+                disable_interrupts();
+                        __tls_stats()->io.submit_q.look_avg.val   += len;
+                        __tls_stats()->io.submit_q.look_avg.block += block;
+                        __tls_stats()->io.submit_q.look_avg.cnt   += 1;
+                enable_interrupts( __cfaabi_dbg_ctx );
+                #endif
+
+                return picked;
+        }
+
         void __submit( struct __io_data & ring, uint32_t idx ) {
                 // Get now the data we definetely need
@@ -731 +777 @@
                 const uint32_t mask = *ring.submit_q.mask;
 
-                disable_interrupts();
-
                 // There are 2 submission schemes, check which one we are using
                 if( ring.cltr_flags & CFA_CLUSTER_IO_POLLER_THREAD_SUBMITS ) {
                         // If the poller thread submits, then we just need to add this to the ready array
 
-                        /* paranoid */ verify( idx <= mask   );
-                        /* paranoid */ verify( idx != -1ul32 );
-
-                        // We need to find a spot in the ready array
-                        __attribute((unused)) int len   = 0;
-                        __attribute((unused)) int block = 0;
-                        uint32_t ready_mask = ring.submit_q.ready_cnt - 1;
-                        uint32_t off = __tls_rand();
-                        LOOKING: for() {
-                                for(i; ring.submit_q.ready_cnt) {
-                                        uint32_t ii = (i + off) & ready_mask;
-                                        uint32_t expected = -1ul32;
-                                        if( __atomic_compare_exchange_n( &ring.submit_q.ready[ii], &expected, idx, true, __ATOMIC_SEQ_CST, __ATOMIC_RELAXED ) ) {
-                                                break LOOKING;
-                                        }
-                                        verify(expected != idx);
-
-                                        len ++;
-                                }
-
-                                block++;
-                                yield();
-                        }
+                        __submit_to_ready_array( ring, idx, mask );
 
                         __wake_poller( ring );
-
-                        // update statistics
-                        #if !defined(__CFA_NO_STATISTICS__)
-                                __tls_stats()->io.submit_q.look_avg.val   += len;
-                                __tls_stats()->io.submit_q.look_avg.block += block;
-                                __tls_stats()->io.submit_q.look_avg.cnt   += 1;
-                        #endif
 
                         __cfadbg_print_safe( io, "Kernel I/O : Added %u to ready for %p\n", idx, active_thread() );
@@ -804 +819 @@
                         __cfadbg_print_safe( io, "Kernel I/O : Performed io_submit for %p, returned %d\n", active_thread(), ret );
                 }
-
-                enable_interrupts( __cfaabi_dbg_ctx );
-        }
-
-        static inline void ?{}(struct io_uring_sqe & this, uint8_t opcode, int fd) {
-                this.opcode = opcode;
-                #if !defined(IOSQE_ASYNC)
-                        this.flags = 0;
-                #else
-                        this.flags = IOSQE_ASYNC;
-                #endif
-                this.ioprio = 0;
-                this.fd = fd;
-                this.off = 0;
-                this.addr = 0;
-                this.len = 0;
-                this.rw_flags = 0;
-                this.__pad2[0] = this.__pad2[1] = this.__pad2[2] = 0;
-        }
-
-        static inline void ?{}(struct io_uring_sqe & this, uint8_t opcode, int fd, void * addr, uint32_t len, uint64_t off ) {
-                (this){ opcode, fd };
-                this.off = off;
-                this.addr = (uint64_t)(uintptr_t)addr;
-                this.len = len;
-        }
+        }
+#endif