Changes in libcfa/src/concurrency/kernel.cfa [431cd4f:254ad1b]

File:

• libcfa/src/concurrency/kernel.cfa (modified) (12 diffs)

libcfa/src/concurrency/kernel.cfa

@@ -110 +110 @@
 static $thread * __next_thread(cluster * this);
 static $thread * __next_thread_slow(cluster * this);
+static inline bool __must_unpark( $thread * thrd ) __attribute((nonnull(1)));
 static void __run_thread(processor * this, $thread * dst);
 static void __wake_one(cluster * cltr);
@@ -118 +119 @@
 
 extern void __cfa_io_start( processor * );
-extern void __cfa_io_drain( processor * );
+extern bool __cfa_io_drain( processor * );
 extern void __cfa_io_flush( processor * );
 extern void __cfa_io_stop ( processor * );
-static inline void __maybe_io_drain( processor * );
+static inline bool __maybe_io_drain( processor * );
 
 extern void __disable_interrupts_hard();
 extern void __enable_interrupts_hard();
+
+static inline void __disable_interrupts_checked() {
+        /* paranoid */ verify( __preemption_enabled() );
+        disable_interrupts();
+        /* paranoid */ verify( ! __preemption_enabled() );
+}
+
+static inline void __enable_interrupts_checked( bool poll = true ) {
+        /* paranoid */ verify( ! __preemption_enabled() );
+        enable_interrupts( poll );
+        /* paranoid */ verify( __preemption_enabled() );
+}
 
 //=============================================================================================
@@ -336 +349 @@
                 if(unlikely(thrd_dst->preempted != __NO_PREEMPTION)) {
                         // The thread was preempted, reschedule it and reset the flag
-                        __schedule_thread( thrd_dst );
+                        schedule_thread$( thrd_dst );
                         break RUNNING;
                 }
@@ -426 +439 @@
         /* paranoid */ verify( ! __preemption_enabled() );
         /* paranoid */ verify( kernelTLS().this_proc_id );
+        /* paranoid */ verify( ready_schedule_islocked());
         /* paranoid */ verify( thrd );
         /* paranoid */ verify( thrd->state != Halted );
@@ -444 +458 @@
         struct cluster * cl = thrd->curr_cluster;
 
-        ready_schedule_lock();
-                // push the thread to the cluster ready-queue
-                push( cl, thrd );
-
-                // variable thrd is no longer safe to use
-
-                // wake the cluster using the save variable.
-                __wake_one( cl );
-        ready_schedule_unlock();
+        // push the thread to the cluster ready-queue
+        push( cl, thrd );
+
+        // variable thrd is no longer safe to use
+        thrd = 0xdeaddeaddeaddeadp;
+
+        // wake the cluster using the save variable.
+        __wake_one( cl );
 
         #if !defined(__CFA_NO_STATISTICS__)
@@ -465 +478 @@
         #endif
 
-        /* paranoid */ verify( ! __preemption_enabled() );
+        /* paranoid */ verify( ready_schedule_islocked());
+        /* paranoid */ verify( ! __preemption_enabled() );
+}
+
+void schedule_thread$( $thread * thrd ) {
+        ready_schedule_lock();
+                __schedule_thread( thrd );
+        ready_schedule_unlock();
 }
 
@@ -496 +516 @@
 }
 
-void unpark( $thread * thrd ) {
-        if( !thrd ) return;
-
+static inline bool __must_unpark( $thread * thrd ) {
         int old_ticket = __atomic_fetch_add(&thrd->ticket, 1, __ATOMIC_SEQ_CST);
         switch(old_ticket) {
                 case TICKET_RUNNING:
                         // Wake won the race, the thread will reschedule/rerun itself
-                        break;
+                        return false;
                 case TICKET_BLOCKED:
                         /* paranoid */ verify( ! thrd->preempted != __NO_PREEMPTION );
                         /* paranoid */ verify( thrd->state == Blocked );
-
-                        {
-                                /* paranoid */ verify( publicTLS_get(this_proc_id) );
-                                disable_interrupts();
-
-                                /* paranoid */ verify( ! __preemption_enabled() );
-
-                                // Wake lost the race,
-                                __schedule_thread( thrd );
-
-                                /* paranoid */ verify( ! __preemption_enabled() );
-
-                                enable_interrupts_noPoll();
-                                /* paranoid */ verify( publicTLS_get(this_proc_id) );
-                        }
-
-                        break;
+                        return true;
                 default:
                         // This makes no sense, something is wrong abort
@@ -530 +532 @@
 }
 
+void unpark( $thread * thrd ) {
+        if( !thrd ) return;
+
+        if(__must_unpark(thrd)) {
+                disable_interrupts();
+                        // Wake lost the race,
+                        schedule_thread$( thrd );
+                enable_interrupts(false);
+        }
+}
+
 void park( void ) {
-        /* paranoid */ verify( __preemption_enabled() );
-        disable_interrupts();
-        /* paranoid */ verify( ! __preemption_enabled() );
-        /* paranoid */ verify( kernelTLS().this_thread->preempted == __NO_PREEMPTION );
-
-        returnToKernel();
-
-        /* paranoid */ verify( ! __preemption_enabled() );
-        enable_interrupts( __cfaabi_dbg_ctx );
-        /* paranoid */ verify( __preemption_enabled() );
+        __disable_interrupts_checked();
+                /* paranoid */ verify( kernelTLS().this_thread->preempted == __NO_PREEMPTION );
+                returnToKernel();
+        __enable_interrupts_checked();
 
 }
@@ -580 +587 @@
 // KERNEL ONLY
 bool force_yield( __Preemption_Reason reason ) {
-        /* paranoid */ verify( __preemption_enabled() );
-        disable_interrupts();
-        /* paranoid */ verify( ! __preemption_enabled() );
-
-        $thread * thrd = kernelTLS().this_thread;
-        /* paranoid */ verify(thrd->state == Active);
-
-        // SKULLDUGGERY: It is possible that we are preempting this thread just before
-        // it was going to park itself. If that is the case and it is already using the
-        // intrusive fields then we can't use them to preempt the thread
-        // If that is the case, abandon the preemption.
-        bool preempted = false;
-        if(thrd->link.next == 0p) {
-                preempted = true;
-                thrd->preempted = reason;
-                returnToKernel();
-        }
-
-        /* paranoid */ verify( ! __preemption_enabled() );
-        enable_interrupts_noPoll();
-        /* paranoid */ verify( __preemption_enabled() );
-
+        __disable_interrupts_checked();
+                $thread * thrd = kernelTLS().this_thread;
+                /* paranoid */ verify(thrd->state == Active);
+
+                // SKULLDUGGERY: It is possible that we are preempting this thread just before
+                // it was going to park itself. If that is the case and it is already using the
+                // intrusive fields then we can't use them to preempt the thread
+                // If that is the case, abandon the preemption.
+                bool preempted = false;
+                if(thrd->link.next == 0p) {
+                        preempted = true;
+                        thrd->preempted = reason;
+                        returnToKernel();
+                }
+        __enable_interrupts_checked( false );
         return preempted;
 }
@@ -646 +646 @@
         __cfadbg_print_safe(runtime_core, "Kernel : waking Processor %p\n", this);
 
-        disable_interrupts();
+        __disable_interrupts_checked();
                 /* paranoid */ verify( ! __preemption_enabled() );
                 eventfd_t val;
                 val = 1;
                 eventfd_write( this->idle, val );
-        enable_interrupts( __cfaabi_dbg_ctx );
+        __enable_interrupts_checked();
 }
 
@@ -743 +743 @@
 #endif
 
-static inline void __maybe_io_drain( processor * proc ) {
+static inline bool __maybe_io_drain( processor * proc ) {
         #if defined(CFA_HAVE_LINUX_IO_URING_H)
                 __cfadbg_print_safe(runtime_core, "Kernel : core %p checking io for ring %d\n", proc, proc->io.ctx->fd);
@@ -751 +751 @@
                 unsigned head = *ctx->cq.head;
                 unsigned tail = *ctx->cq.tail;
-                if(head != tail) __cfa_io_drain( proc );
+                if(head == tail) return false;
+                return __cfa_io_drain( proc );
         #endif
 }