Changeset 13cdc8c for libcfa/src/concurrency

Timestamp:

Mar 30, 2022, 10:44:10 PM (3 years ago)

Author:

Peter A. Buhr <pabuhr@…>

Branches:

ADT, ast-experimental, enum, master, pthread-emulation, qualifiedEnum

Children:

f870e257

Parents:

ee3da78 (diff), bdfd0bd (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the (diff) links above to see all the changes relative to each parent.

Message:

Merge branch 'master' of plg.uwaterloo.ca:software/cfa/cfa-cc

Location:

libcfa/src/concurrency

Files:

• io.cfa (modified) (7 diffs)
• kernel.cfa (modified) (3 diffs)
• kernel.hfa (modified) (1 diff)
• kernel/startup.cfa (modified) (1 diff)

libcfa/src/concurrency/io.cfa

@@ -96 +96 @@
         static void ioring_syscsll( struct $io_context & ctx, unsigned int min_comp, unsigned int flags ) {
                 __STATS__( true, io.calls.flush++; )
-                int ret = syscall( __NR_io_uring_enter, ctx.fd, ctx.sq.to_submit, min_comp, flags, (sigset_t *)0p, _NSIG / 8);
-                if( ret < 0 ) {
-                        switch((int)errno) {
-                        case EAGAIN:
-                        case EINTR:
-                        case EBUSY:
-                                // Update statistics
-                                __STATS__( false, io.calls.errors.busy ++; )
-                                return false;
-                        default:
-                                abort( "KERNEL ERROR: IO_URING SYSCALL - (%d) %s\n", (int)errno, strerror(errno) );
+                int ret;
+                for() {
+                        ret = syscall( __NR_io_uring_enter, ctx.fd, ctx.sq.to_submit, min_comp, flags, (sigset_t *)0p, _NSIG / 8);
+                        if( ret < 0 ) {
+                                switch((int)errno) {
+                                case EINTR:
+                                        continue;
+                                case EAGAIN:
+                                case EBUSY:
+                                        // Update statistics
+                                        __STATS__( false, io.calls.errors.busy ++; )
+                                        return false;
+                                default:
+                                        abort( "KERNEL ERROR: IO_URING SYSCALL - (%d) %s\n", (int)errno, strerror(errno) );
+                                }
                         }
+                        break;
                 }
 
@@ -179 +184 @@
                 // Ensure that the kernel only sees the new value of the head index after the CQEs have been read.
                 __atomic_store_n( ctx->cq.head, head + count, __ATOMIC_SEQ_CST );
+                ctx->proc->idle_wctx.drain_time = ts_next;
 
                 __cfadbg_print_safe(io, "Kernel I/O : %u completed age %llu\n", count, ts_next);
@@ -230 +236 @@
                                         const unsigned long long cutoff = calc_cutoff(ctsc, ctx->cq.id, ctxs_count, io.data, io.tscs, __shard_factor.io);
                                         const unsigned long long age = moving_average(ctsc, io.tscs[target].tv, io.tscs[target].ma);
-                                        __cfadbg_print_safe(io, "Kernel I/O: Help attempt on %u from %u, age %'llu vs cutoff %'llu, %s\n", target, this, age, cutoff, age > cutoff ? "yes" : "no");
+                                        __cfadbg_print_safe(io, "Kernel I/O: Help attempt on %u from %u, age %'llu vs cutoff %'llu, %s\n", target, ctx->cq.id, age, cutoff, age > cutoff ? "yes" : "no");
                                         if(age <= cutoff) break HELP;
 
@@ -635 +641 @@
 
                         // We can proceed to the fast path
-                        if( !__alloc(ctx, &idx, 1) ) return false;
+                        if( !__alloc(ctx, &idx, 1) ) {
+                                /* paranoid */ verify( false ); // for now check if this happens, next time just abort the sleep.
+                                return false;
+                        }
 
                         // Allocation was successful
@@ -665 +674 @@
 
                         /* paranoid */ verify( sqe->user_data == (uintptr_t)&future );
-                        __submit( ctx, &idx, 1, true );
+                        __submit_only( ctx, &idx, 1 );
 
                         /* paranoid */ verify( proc == __cfaabi_tls.this_processor );
@@ -676 +685 @@
                         iovec iov;
                         __atomic_acquire( &proc->io.ctx->cq.lock );
+
+                        __attribute__((used)) volatile bool was_reset = false;
 
                         with( proc->idle_wctx) {
@@ -687 +698 @@
                                         iov.iov_len  = sizeof(eventfd_t);
                                         __kernel_read(proc, *ftr, iov, evfd );
+                                        ftr->result = 0xDEADDEAD;
+                                        *((eventfd_t *)rdbuf) = 0xDEADDEADDEADDEAD;
+                                        was_reset = true;
                                 }
                         }
 
-                        __ioarbiter_flush( *proc->io.ctx );
-                        ioring_syscsll( *proc->io.ctx, 1, IORING_ENTER_GETEVENTS);
+                        if( !__atomic_load_n( &proc->do_terminate, __ATOMIC_SEQ_CST ) ) {
+                                __ioarbiter_flush( *proc->io.ctx );
+                                proc->idle_wctx.sleep_time = rdtscl();
+                                ioring_syscsll( *proc->io.ctx, 1, IORING_ENTER_GETEVENTS);
+                        }
 
                         ready_schedule_lock();
                         __cfa_do_drain( proc->io.ctx, proc->cltr );
                         ready_schedule_unlock();
+
+                        asm volatile ("" :: "m" (was_reset));
                 }
         #endif

libcfa/src/concurrency/kernel.cfa

@@ -642 +642 @@
 
         switch(fd) {
+                __attribute__((unused)) int ret;
         case 0:
                 // If the processor isn't ready to sleep then the exchange will already wake it up
@@ -659 +660 @@
                 // If the processor was ready to sleep, we need to wake it up with an actual write
                 val = 1;
-                eventfd_write( fd, val );
+                ret = eventfd_write( fd, val );
+                /* paranoid */ verifyf( ret == 0, "Expected return to be 0, was %d\n", ret );
 
                 #if !defined(__CFA_NO_STATISTICS__)
@@ -682 +684 @@
         this->idle_wctx.sem = 1;
 
+        this->idle_wctx.wake__time = rdtscl();
+
         eventfd_t val;
         val = 1;
-        eventfd_write( this->idle_wctx.evfd, val );
-
+        __attribute__((unused)) int ret = eventfd_write( this->idle_wctx.evfd, val );
+
+        /* paranoid */ verifyf( ret == 0, "Expected return to be 0, was %d\n", ret );
         /* paranoid */ verify( ! __preemption_enabled() );
 }

libcfa/src/concurrency/kernel.hfa

@@ -74 +74 @@
         // unused if not using io_uring for idle sleep
         io_future_t * ftr;
+
+        volatile unsigned long long wake__time;
+        volatile unsigned long long sleep_time;
+        volatile unsigned long long drain_time;
 };
 

libcfa/src/concurrency/kernel/startup.cfa

@@ -558 +558 @@
 
         idle_wctx.sem = 0;
+        idle_wctx.wake__time = 0;
 
         // I'm assuming these two are reserved for standard input and output