Changeset 4479890

Timestamp:

Mar 22, 2022, 4:59:53 PM (3 years ago)

Author:

Thierry Delisle <tdelisle@…>

Branches:

ADT, ast-experimental, enum, master, pthread-emulation, qualifiedEnum

Children:

1a567d0

Parents:

0521a1a

Message:

Implemented helping for io drain based on timestamps.

Location:

libcfa/src/concurrency

Files:

• io.cfa (modified) (3 diffs)
• io/types.hfa (modified) (1 diff)
• kernel.cfa (modified) (3 diffs)
• kernel/cluster.hfa (modified) (2 diffs)
• ready_queue.cfa (modified) (1 diff)

libcfa/src/concurrency/io.cfa

@@ -94 +94 @@
         extern void __kernel_unpark( thread$ * thrd, unpark_hint );
 
-        bool __cfa_io_drain( $io_context * ctx, cluster * cltr ) {
+        static bool __cfa_do_drain( $io_context * ctx, cluster * cltr ) {
                 /* paranoid */ verify( ! __preemption_enabled() );
                 /* paranoid */ verify( ready_schedule_islocked() );
@@ -142 +142 @@
 
                 return true;
+        }
+
+        bool __cfa_io_drain( processor * proc ) {
+                bool local = false;
+                bool remote = false;
+
+                cluster * const cltr = proc->cltr;
+                $io_context * const ctx = proc->io.ctx;
+                /* paranoid */ verify( cltr );
+                /* paranoid */ verify( ctx );
+
+                with(cltr->sched) {
+                        const size_t ctxs_count = io.count;
+
+                        /* paranoid */ verify( ready_schedule_islocked() );
+                        /* paranoid */ verify( ! __preemption_enabled() );
+                        /* paranoid */ verify( active_processor() == proc );
+                        /* paranoid */ verify( __shard_factor.io > 0 );
+                        /* paranoid */ verify( ctxs_count > 0 );
+                        /* paranoid */ verify( ctx->cq.id < ctxs_count );
+
+                        const unsigned this_cache = cache_id(cltr, ctx->cq.id / __shard_factor.io);
+                        const unsigned long long ctsc = rdtscl();
+
+                        if(proc->io.target == MAX) {
+                                uint64_t chaos = __tls_rand();
+                                unsigned ext = chaos & 0xff;
+                                unsigned other  = (chaos >> 8) % (ctxs_count);
+
+                                if(ext < 3 || __atomic_load_n(&caches[other / __shard_factor.io].id, __ATOMIC_RELAXED) == this_cache) {
+                                        proc->io.target = other;
+                                }
+                        }
+                        else {
+                                const unsigned target = proc->io.target;
+                                /* paranoid */ verify( io.tscs[target].tv != MAX );
+                                if(target < ctxs_count) {
+                                        const unsigned long long cutoff = calc_cutoff(ctsc, ctx->cq.id, ctxs_count, io.data, io.tscs, __shard_factor.io);
+                                        const unsigned long long age = moving_average(ctsc, io.tscs[target].tv, io.tscs[target].ma);
+                                        // __cfadbg_print_safe(ready_queue, "Kernel : Help attempt on %u from %u, age %'llu vs cutoff %'llu, %s\n", target, this, age, cutoff, age > cutoff ? "yes" : "no");
+                                        if(age > cutoff) {
+                                                remote = __cfa_do_drain( io.data[target], cltr );
+                                        }
+                                }
+                                proc->io.target = MAX;
+                        }
+                }
+
+
+                // Drain the local queue
+                local = __cfa_do_drain( proc->io.ctx, cltr );
+
+                /* paranoid */ verify( ready_schedule_islocked() );
+                /* paranoid */ verify( ! __preemption_enabled() );
+                /* paranoid */ verify( active_processor() == proc );
+                return local || remote;
         }
 
@@ -189 +245 @@
 
                 ready_schedule_lock();
-                bool ret = __cfa_io_drain( &ctx, cltr );
+                bool ret = __cfa_io_drain( proc );
                 ready_schedule_unlock();
                 return ret;

libcfa/src/concurrency/io/types.hfa

@@ -135 +135 @@
         };
 
+        static inline unsigned long long ts($io_context *& this) {
+                return this->cq.ts;
+        }
+
         struct __pending_alloc {
                 inline __outstanding_io;

libcfa/src/concurrency/kernel.cfa

@@ -136 +136 @@
 static void mark_awake(__cluster_proc_list & idles, processor & proc);
 
-extern bool __cfa_io_drain( $io_context *, cluster * cltr );
+extern bool __cfa_io_drain( processor * proc ) __attribute__((nonnull (1)));
 extern bool __cfa_io_flush( processor *, int min_comp );
 static inline bool __maybe_io_drain( processor * );
@@ -829 +829 @@
 
 static inline bool __maybe_io_drain( processor * proc ) {
+        /* paranoid */ verify( proc );
         bool ret = false;
         #if defined(CFA_HAVE_LINUX_IO_URING_H)
@@ -839 +840 @@
                 if(head == tail) return false;
                 ready_schedule_lock();
-                ret = __cfa_io_drain( ctx, proc->cltr );
+                ret = __cfa_io_drain( proc );
                 ready_schedule_unlock();
         #endif

libcfa/src/concurrency/kernel/cluster.hfa

@@ -49 +49 @@
 static inline unsigned long long calc_cutoff(
         const unsigned long long ctsc,
-        const processor * proc,
+        unsigned procid,
         size_t count,
         Data_t * data,
@@ -55 +55 @@
         const unsigned shard_factor
 ) {
-        unsigned start = proc->rdq.id;
+        unsigned start = procid;
         unsigned long long max = 0;
         for(i; shard_factor) {

libcfa/src/concurrency/ready_queue.cfa

@@ -139 +139 @@
                 /* paranoid */ verify( readyQ.tscs[target].tv != MAX );
                 if(target < lanes_count) {
-                        const unsigned long long cutoff = calc_cutoff(ctsc, proc, lanes_count, cltr->sched.readyQ.data, cltr->sched.readyQ.tscs, __shard_factor.readyq);
+                        const unsigned long long cutoff = calc_cutoff(ctsc, proc->rdq.id, lanes_count, cltr->sched.readyQ.data, cltr->sched.readyQ.tscs, __shard_factor.readyq);
                         const unsigned long long age = moving_average(ctsc, readyQ.tscs[target].tv, readyQ.tscs[target].ma);
                         __cfadbg_print_safe(ready_queue, "Kernel : Help attempt on %u from %u, age %'llu vs cutoff %'llu, %s\n", target, this, age, cutoff, age > cutoff ? "yes" : "no");