Changeset 376c632a for libcfa/src

Timestamp:

Feb 1, 2022, 10:10:46 AM (4 years ago)

Author:

Thierry Delisle <tdelisle@…>

Branches:

ADT, ast-experimental, enum, forall-pointer-decay, master, pthread-emulation, qualifiedEnum, stuck-waitfor-destruct

Children:

7b2c8c3c

Parents:

f681823 (diff), 89a5a1f (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the (diff) links above to see all the changes relative to each parent.

Message:

Merge branch 'master' of plg.uwaterloo.ca:software/cfa/cfa-cc

Location:

libcfa/src/concurrency

Files:

• io.cfa (modified) (1 diff)
• kernel.cfa (modified) (13 diffs)
• kernel/fwd.hfa (modified) (1 diff)
• stats.cfa (modified) (3 diffs)
• stats.hfa (modified) (1 diff)

libcfa/src/concurrency/io.cfa

@@ -306 +306 @@
                 ctx->proc->io.pending = true;
                 ctx->proc->io.dirty   = true;
-                if(sq.to_submit > 30 || !lazy) {
+                if(sq.to_submit > 30) {
+                        __tls_stats()->io.flush.full++;
+                        __cfa_io_flush( ctx->proc, 0 );
+                }
+                if(!lazy) {
+                        __tls_stats()->io.flush.eager++;
                         __cfa_io_flush( ctx->proc, 0 );
                 }

libcfa/src/concurrency/kernel.cfa

@@ -42 +42 @@
 
 #if !defined(__CFA_NO_STATISTICS__)
-        #define __STATS( ...) __VA_ARGS__
+        #define __STATS_DEF( ...) __VA_ARGS__
 #else
-        #define __STATS( ...)
+        #define __STATS_DEF( ...)
 #endif
 
@@ -122 +122 @@
 static thread$ * __next_thread(cluster * this);
 static thread$ * __next_thread_slow(cluster * this);
+static thread$ * __next_thread_search(cluster * this);
 static inline bool __must_unpark( thread$ * thrd ) __attribute((nonnull(1)));
 static void __run_thread(processor * this, thread$ * dst);
@@ -187 +188 @@
                 MAIN_LOOP:
                 for() {
-                        #define OLD_MAIN 1
-                        #if OLD_MAIN
                         // Check if there is pending io
                         __maybe_io_drain( this );
@@ -196 +195 @@
 
                         if( !readyThread ) {
+                                __IO_STATS__(true, io.flush.idle++; )
                                 __cfa_io_flush( this, 0 );
 
+                                readyThread = __next_thread( this->cltr );
+                        }
+
+                        if( !readyThread ) for(5) {
+                                __IO_STATS__(true, io.flush.idle++; )
+
                                 readyThread = __next_thread_slow( this->cltr );
+
+                                if( readyThread ) break;
+
+                                __cfa_io_flush( this, 0 );
                         }
 
@@ -210 +220 @@
 
                                 // Confirm the ready-queue is empty
-                                readyThread = __next_thread_slow( this->cltr );
+                                readyThread = __next_thread_search( this->cltr );
                                 if( readyThread ) {
                                         // A thread was found, cancel the halt
                                         mark_awake(this->cltr->procs, * this);
 
-                                        #if !defined(__CFA_NO_STATISTICS__)
-                                                __tls_stats()->ready.sleep.cancels++;
-                                        #endif
+                                        __STATS__(true, ready.sleep.cancels++; )
 
                                         // continue the mai loop
@@ -244 +252 @@
 
                         if(this->io.pending && !this->io.dirty) {
+                                __IO_STATS__(true, io.flush.dirty++; )
                                 __cfa_io_flush( this, 0 );
                         }
-
-                        #else
-                                #warning new kernel loop
-                        SEARCH: {
-                                /* paranoid */ verify( ! __preemption_enabled() );
-
-                                // First, lock the scheduler since we are searching for a thread
-                                ready_schedule_lock();
-
-                                // Try to get the next thread
-                                readyThread = pop_fast( this->cltr );
-                                if(readyThread) { ready_schedule_unlock(); break SEARCH; }
-
-                                // If we can't find a thread, might as well flush any outstanding I/O
-                                if(this->io.pending) { __cfa_io_flush( this, 0 ); }
-
-                                // Spin a little on I/O, just in case
-                                for(5) {
-                                        __maybe_io_drain( this );
-                                        readyThread = pop_fast( this->cltr );
-                                        if(readyThread) { ready_schedule_unlock(); break SEARCH; }
-                                }
-
-                                // no luck, try stealing a few times
-                                for(5) {
-                                        if( __maybe_io_drain( this ) ) {
-                                                readyThread = pop_fast( this->cltr );
-                                        } else {
-                                                readyThread = pop_slow( this->cltr );
-                                        }
-                                        if(readyThread) { ready_schedule_unlock(); break SEARCH; }
-                                }
-
-                                // still no luck, search for a thread
-                                readyThread = pop_search( this->cltr );
-                                if(readyThread) { ready_schedule_unlock(); break SEARCH; }
-
-                                // Don't block if we are done
-                                if( __atomic_load_n(&this->do_terminate, __ATOMIC_SEQ_CST) ) {
-                                        ready_schedule_unlock();
-                                        break MAIN_LOOP;
-                                }
-
-                                __STATS( __tls_stats()->ready.sleep.halts++; )
-
-                                // Push self to idle stack
-                                ready_schedule_unlock();
-                                if(!mark_idle(this->cltr->procs, * this)) goto SEARCH;
-                                ready_schedule_lock();
-
-                                // Confirm the ready-queue is empty
-                                __maybe_io_drain( this );
-                                readyThread = pop_search( this->cltr );
-                                ready_schedule_unlock();
-
-                                if( readyThread ) {
-                                        // A thread was found, cancel the halt
-                                        mark_awake(this->cltr->procs, * this);
-
-                                        __STATS( __tls_stats()->ready.sleep.cancels++; )
-
-                                        // continue the main loop
-                                        break SEARCH;
-                                }
-
-                                __STATS( if(this->print_halts) __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 0\n", this->unique_id, rdtscl()); )
-                                __cfadbg_print_safe(runtime_core, "Kernel : core %p waiting on eventfd %d\n", this, this->idle_fd);
-
-                                {
-                                        eventfd_t val;
-                                        ssize_t ret = read( this->idle_fd, &val, sizeof(val) );
-                                        if(ret < 0) {
-                                                switch((int)errno) {
-                                                case EAGAIN:
-                                                #if EAGAIN != EWOULDBLOCK
-                                                        case EWOULDBLOCK:
-                                                #endif
-                                                case EINTR:
-                                                        // No need to do anything special here, just assume it's a legitimate wake-up
-                                                        break;
-                                                default:
-                                                        abort( "KERNEL : internal error, read failure on idle eventfd, error(%d) %s.", (int)errno, strerror( (int)errno ) );
-                                                }
-                                        }
-                                }
-
-                                        __STATS( if(this->print_halts) __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 1\n", this->unique_id, rdtscl()); )
-
-                                // We were woken up, remove self from idle
-                                mark_awake(this->cltr->procs, * this);
-
-                                // DON'T just proceed, start looking again
-                                continue MAIN_LOOP;
-                        }
-
-                RUN_THREAD:
-                        /* paranoid */ verify( ! __preemption_enabled() );
-                        /* paranoid */ verify( readyThread );
-
-                        // Reset io dirty bit
-                        this->io.dirty = false;
-
-                        // We found a thread run it
-                        __run_thread(this, readyThread);
-
-                        // Are we done?
-                        if( __atomic_load_n(&this->do_terminate, __ATOMIC_SEQ_CST) ) break MAIN_LOOP;
-
-                        if(this->io.pending && !this->io.dirty) {
-                                __cfa_io_flush( this, 0 );
-                        }
-
-                        ready_schedule_lock();
-                        __maybe_io_drain( this );
-                        ready_schedule_unlock();
-                        #endif
                 }
 
@@ -472 +365 @@
                                 break RUNNING;
                         case TICKET_UNBLOCK:
-                                #if !defined(__CFA_NO_STATISTICS__)
-                                        __tls_stats()->ready.threads.threads++;
-                                #endif
+                                __STATS__(true, ready.threads.threads++; )
                                 // This is case 2, the racy case, someone tried to run this thread before it finished blocking
                                 // In this case, just run it again.
@@ -489 +380 @@
         __cfadbg_print_safe(runtime_core, "Kernel : core %p finished running thread %p\n", this, thrd_dst);
 
-        #if !defined(__CFA_NO_STATISTICS__)
-                __tls_stats()->ready.threads.threads--;
-        #endif
+        __STATS__(true, ready.threads.threads--; )
 
         /* paranoid */ verify( ! __preemption_enabled() );
@@ -502 +391 @@
         thread$ * thrd_src = kernelTLS().this_thread;
 
-        __STATS( thrd_src->last_proc = kernelTLS().this_processor; )
+        __STATS_DEF( thrd_src->last_proc = kernelTLS().this_processor; )
 
         // Run the thread on this processor
@@ -554 +443 @@
         // Dereference the thread now because once we push it, there is not guaranteed it's still valid.
         struct cluster * cl = thrd->curr_cluster;
-        __STATS(bool outside = hint == UNPARK_LOCAL && thrd->last_proc && thrd->last_proc != kernelTLS().this_processor; )
+        __STATS_DEF(bool outside = hint == UNPARK_LOCAL && thrd->last_proc && thrd->last_proc != kernelTLS().this_processor; )
 
         // push the thread to the cluster ready-queue
@@ -605 +494 @@
 
         ready_schedule_lock();
-                thread$ * thrd;
-                for(25) {
-                        thrd = pop_slow( this );
-                        if(thrd) goto RET;
-                }
-                thrd = pop_search( this );
-
-                RET:
+                thread$ * thrd = pop_slow( this );
+        ready_schedule_unlock();
+
+        /* paranoid */ verify( ! __preemption_enabled() );
+        return thrd;
+}
+
+// KERNEL ONLY
+static inline thread$ * __next_thread_search(cluster * this) with( *this ) {
+        /* paranoid */ verify( ! __preemption_enabled() );
+
+        ready_schedule_lock();
+                thread$ * thrd = pop_search( this );
         ready_schedule_unlock();
 
@@ -854 +748 @@
 
 static bool mark_idle(__cluster_proc_list & this, processor & proc) {
-        #if !defined(__CFA_NO_STATISTICS__)
-                __tls_stats()->ready.sleep.halts++;
-        #endif
+        __STATS__(true, ready.sleep.halts++; )
 
         proc.idle_wctx.fd = 0;
@@ -949 +841 @@
                 unsigned tail = *ctx->cq.tail;
                 if(head == tail) return false;
-                #if OLD_MAIN
-                        ready_schedule_lock();
-                        ret = __cfa_io_drain( proc );
-                        ready_schedule_unlock();
-                #else
-                        ret = __cfa_io_drain( proc );
-                #endif
+                ready_schedule_lock();
+                ret = __cfa_io_drain( proc );
+                ready_schedule_unlock();
         #endif
         return ret;

libcfa/src/concurrency/kernel/fwd.hfa

@@ -396 +396 @@
                                 if( !(in_kernel) ) enable_interrupts(); \
                         }
+                        #if defined(CFA_HAVE_LINUX_IO_URING_H)
+                                #define __IO_STATS__(in_kernel, ...) { \
+                                        if( !(in_kernel) ) disable_interrupts(); \
+                                        with( *__tls_stats() ) { \
+                                                __VA_ARGS__ \
+                                        } \
+                                        if( !(in_kernel) ) enable_interrupts(); \
+                                }
+                        #else
+                                #define __IO_STATS__(in_kernel, ...)
+                        #endif
                 #else
                         #define __STATS__(in_kernel, ...)
+                        #define __IO_STATS__(in_kernel, ...)
                 #endif
         }

libcfa/src/concurrency/stats.cfa

@@ -45 +45 @@
                         stats->io.submit.slow       = 0;
                         stats->io.flush.external    = 0;
+                        stats->io.flush.dirty       = 0;
+                        stats->io.flush.full        = 0;
+                        stats->io.flush.idle        = 0;
+                        stats->io.flush.eager       = 0;
                         stats->io.calls.flush       = 0;
                         stats->io.calls.submitted   = 0;
@@ -107 +111 @@
                         tally_one( &cltr->io.submit.slow      , &proc->io.submit.slow       );
                         tally_one( &cltr->io.flush.external   , &proc->io.flush.external    );
+                        tally_one( &cltr->io.flush.dirty      , &proc->io.flush.dirty       );
+                        tally_one( &cltr->io.flush.full       , &proc->io.flush.full        );
+                        tally_one( &cltr->io.flush.idle       , &proc->io.flush.idle        );
+                        tally_one( &cltr->io.flush.eager      , &proc->io.flush.eager       );
                         tally_one( &cltr->io.calls.flush      , &proc->io.calls.flush       );
                         tally_one( &cltr->io.calls.submitted  , &proc->io.calls.submitted   );
@@ -184 +192 @@
                                 if(io.alloc.fail || io.alloc.revoke || io.alloc.block)
                                         sstr | "-     failures      : " | eng3(io.alloc.fail) | "oom, " | eng3(io.alloc.revoke) | "rvk, " | eng3(io.alloc.block) | "blk";
-                                if(io.flush.external)
-                                        sstr | "- flush external    : " | eng3(io.flush.external);
+                                // if(io.flush.external)
+                                //      sstr | "- flush external    : " | eng3(io.flush.external);
 
                                 double avgsubs = ((double)io.calls.submitted) / io.calls.flush;
                                 double avgcomp = ((double)io.calls.completed) / io.calls.drain;
                                 sstr | "- syscll : "
-                                     |   " sub " | eng3(io.calls.flush) | "/" | eng3(io.calls.submitted) | "(" | ws(3, 3, avgsubs) | "/flush)"
-                                     | " - cmp " | eng3(io.calls.drain) | "/" | eng3(io.calls.completed) | "(" | ws(3, 3, avgcomp) | "/drain)"
+                                     |   " sub " | eng3(io.calls.submitted) | "/" | eng3(io.calls.flush) | "(" | ws(3, 3, avgsubs) | "/flush)"
+                                     | " - cmp " | eng3(io.calls.completed) | "/" | eng3(io.calls.drain) | "(" | ws(3, 3, avgcomp) | "/drain)"
                                      | " - " | eng3(io.calls.errors.busy) | " EBUSY";
+                                sstr | " - sub: " | eng3(io.flush.full) | "full, " | eng3(io.flush.dirty) | "drty, " | eng3(io.flush.idle) | "idle, " | eng3(io.flush.eager) | "eagr, " | eng3(io.flush.external) | "ext";
                                 sstr | "- ops blk: "
                                      |   " sk rd: " | eng3(io.ops.sockread)  | "epll: " | eng3(io.ops.epllread)

libcfa/src/concurrency/stats.hfa

@@ -91 +91 @@
                         struct {
                                 volatile uint64_t external;
+                                volatile uint64_t dirty;
+                                volatile uint64_t full;
+                                volatile uint64_t idle;
+                                volatile uint64_t eager;
                         } flush;
                         struct {