Changes in libcfa/src/concurrency/io.cfa [e8ac228:d3605f8]

File:

• libcfa/src/concurrency/io.cfa (modified) (8 diffs)

libcfa/src/concurrency/io.cfa

@@ -33 +33 @@
                 #include <sys/syscall.h>
                 #include <sys/eventfd.h>
+                #include <sys/uio.h>
 
                 #include <linux/io_uring.h>
@@ -133 +134 @@
         }
 
-        void __cfa_io_flush( processor * proc ) {
+        bool __cfa_io_flush( processor * proc, int min_comp ) {
                 /* paranoid */ verify( ! __preemption_enabled() );
                 /* paranoid */ verify( proc );
@@ -141 +142 @@
                 $io_context & ctx = *proc->io.ctx;
 
-                // for(i; 2) {
-                //      unsigned idx = proc->rdq.id + i;
-                //      cltr->ready_queue.lanes.tscs[idx].tv = -1ull;
-                // }
-
                 __ioarbiter_flush( ctx );
 
                 __STATS__( true, io.calls.flush++; )
-                int ret = syscall( __NR_io_uring_enter, ctx.fd, ctx.sq.to_submit, 0, 0, (sigset_t *)0p, _NSIG / 8);
+                int ret = syscall( __NR_io_uring_enter, ctx.fd, ctx.sq.to_submit, min_comp, min_comp > 0 ? IORING_ENTER_GETEVENTS : 0, (sigset_t *)0p, _NSIG / 8);
                 if( ret < 0 ) {
                         switch((int)errno) {
@@ -157 +153 @@
                                 // Update statistics
                                 __STATS__( false, io.calls.errors.busy ++; )
-                                // for(i; 2) {
-                                //      unsigned idx = proc->rdq.id + i;
-                                //      cltr->ready_queue.lanes.tscs[idx].tv = rdtscl();
-                                // }
-                                return;
+                                return false;
                         default:
                                 abort( "KERNEL ERROR: IO_URING SYSCALL - (%d) %s\n", (int)errno, strerror(errno) );
@@ -182 +174 @@
 
                 ctx.proc->io.pending = false;
-
-                __cfa_io_drain( proc );
-                // for(i; 2) {
-                //      unsigned idx = proc->rdq.id + i;
-                //      cltr->ready_queue.lanes.tscs[idx].tv = rdtscl();
-                // }
+                ready_schedule_lock();
+                bool ret = __cfa_io_drain( proc );
+                ready_schedule_unlock();
+                return ret;
         }
 
@@ -291 +281 @@
         }
 
-
         //=============================================================================================
         // submission
@@ -314 +303 @@
                 ctx->proc->io.dirty   = true;
                 if(sq.to_submit > 30 || !lazy) {
-                        ready_schedule_lock();
-                        __cfa_io_flush( ctx->proc );
-                        ready_schedule_unlock();
+                        __cfa_io_flush( ctx->proc, 0 );
                 }
         }
@@ -515 +502 @@
                 }
         }
+
+        #if defined(CFA_WITH_IO_URING_IDLE)
+                bool __kernel_read(processor * proc, io_future_t & future, iovec & iov, int fd) {
+                        $io_context * ctx = proc->io.ctx;
+                        /* paranoid */ verify( ! __preemption_enabled() );
+                        /* paranoid */ verify( proc == __cfaabi_tls.this_processor );
+                        /* paranoid */ verify( ctx );
+
+                        __u32 idx;
+                        struct io_uring_sqe * sqe;
+
+                        // We can proceed to the fast path
+                        if( !__alloc(ctx, &idx, 1) ) return false;
+
+                        // Allocation was successful
+                        __fill( &sqe, 1, &idx, ctx );
+
+                        sqe->user_data = (uintptr_t)&future;
+                        sqe->flags = 0;
+                        sqe->fd = fd;
+                        sqe->off = 0;
+                        sqe->ioprio = 0;
+                        sqe->fsync_flags = 0;
+                        sqe->__pad2[0] = 0;
+                        sqe->__pad2[1] = 0;
+                        sqe->__pad2[2] = 0;
+
+                        #if defined(CFA_HAVE_IORING_OP_READ)
+                                sqe->opcode = IORING_OP_READ;
+                                sqe->addr = (uint64_t)iov.iov_base;
+                                sqe->len = iov.iov_len;
+                        #elif defined(CFA_HAVE_READV) && defined(CFA_HAVE_IORING_OP_READV)
+                                sqe->opcode = IORING_OP_READV;
+                                sqe->addr = (uintptr_t)&iov;
+                                sqe->len = 1;
+                        #else
+                                #error CFA_WITH_IO_URING_IDLE but none of CFA_HAVE_READV, CFA_HAVE_IORING_OP_READV or CFA_HAVE_IORING_OP_READ defined
+                        #endif
+
+                        asm volatile("": : :"memory");
+
+                        /* paranoid */ verify( sqe->user_data == (uintptr_t)&future );
+                        __submit( ctx, &idx, 1, true );
+
+                        /* paranoid */ verify( proc == __cfaabi_tls.this_processor );
+                        /* paranoid */ verify( ! __preemption_enabled() );
+                }
+        #endif
 #endif