Changes in libcfa/src/concurrency/kernel.cfa [a757ba1:8bee858]

File:

• libcfa/src/concurrency/kernel.cfa (modified) (11 diffs)

libcfa/src/concurrency/kernel.cfa

@@ -138 +138 @@
 extern bool __cfa_io_drain( processor * proc ) __attribute__((nonnull (1)));
 extern bool __cfa_io_flush( processor * ) __attribute__((nonnull (1)));
-
+extern void __cfa_io_idle( processor * ) __attribute__((nonnull (1)));
+
+#if defined(CFA_WITH_IO_URING_IDLE)
+        extern bool __kernel_read(processor * proc, io_future_t & future, iovec &, int fd);
+#endif
 
 extern void __disable_interrupts_hard();
@@ -158 +162 @@
         verify(this);
 
+        /* paranoid */ verify( this->idle_wctx.ftr   != 0p );
+        /* paranoid */ verify( this->idle_wctx.rdbuf != 0p );
+
+        // used for idle sleep when io_uring is present
+        // mark it as already fulfilled so we know if there is a pending request or not
+        this->idle_wctx.ftr->self.ptr = 1p;
+
         __cfadbg_print_safe(runtime_core, "Kernel : core %p starting\n", this);
         #if !defined(__CFA_NO_STATISTICS__)
@@ -280 +291 @@
         /* paranoid */ verify( ! __preemption_enabled() );
         /* paranoid */ verifyf( thrd_dst->state == Ready || thrd_dst->preempted != __NO_PREEMPTION, "state : %d, preempted %d\n", thrd_dst->state, thrd_dst->preempted);
-        /* paranoid */ verifyf( thrd_dst->rdy_link.next == 0p, "Expected null got %p", thrd_dst->rdy_link.next );
+        /* paranoid */ verifyf( thrd_dst->link.next == 0p, "Expected null got %p", thrd_dst->link.next );
         __builtin_prefetch( thrd_dst->context.SP );
 
@@ -310 +321 @@
                 /* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) < ((uintptr_t)__get_stack(thrd_dst->curr_cor)->base ) || thrd_dst->curr_cor == proc_cor || thrd_dst->corctx_flag, "ERROR : Destination thread$ %p has been corrupted.\n StackPointer too small.\n", thrd_dst ); // add escape condition if we are setting up the processor
                 /* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) > ((uintptr_t)__get_stack(thrd_dst->curr_cor)->limit) || thrd_dst->curr_cor == proc_cor || thrd_dst->corctx_flag, "ERROR : Destination thread$ %p has been corrupted.\n StackPointer too large.\n", thrd_dst ); // add escape condition if we are setting up the processor
-                /* paranoid */ verify( __atomic_exchange_n( &thrd_dst->executing, this, __ATOMIC_SEQ_CST) == 0p );
                 /* paranoid */ verify( 0x0D15EA5E0D15EA5Ep == thrd_dst->canary );
 
@@ -322 +332 @@
 
                 /* paranoid */ verify( 0x0D15EA5E0D15EA5Ep == thrd_dst->canary );
-                /* paranoid */ verify( __atomic_exchange_n( &thrd_dst->executing, 0p, __ATOMIC_SEQ_CST) == this );
                 /* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) > ((uintptr_t)__get_stack(thrd_dst->curr_cor)->limit) || thrd_dst->corctx_flag, "ERROR : Destination thread$ %p has been corrupted.\n StackPointer too large.\n", thrd_dst );
                 /* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) < ((uintptr_t)__get_stack(thrd_dst->curr_cor)->base ) || thrd_dst->corctx_flag, "ERROR : Destination thread$ %p has been corrupted.\n StackPointer too small.\n", thrd_dst );
-                /* paranoid */ verify( thrd_dst->state != Halted );
                 /* paranoid */ verify( thrd_dst->context.SP );
+                /* paranoid */ verify( thrd_dst->curr_cluster == this->cltr );
                 /* paranoid */ verify( kernelTLS().this_thread == thrd_dst );
                 /* paranoid */ verify( ! __preemption_enabled() );
@@ -443 +452 @@
                                         "Error preempted thread marked as not currently running, state %d, preemption %d\n", thrd->state, thrd->preempted );
         /* paranoid */ #endif
-        /* paranoid */ verifyf( thrd->rdy_link.next == 0p, "Expected null got %p", thrd->rdy_link.next );
+        /* paranoid */ verifyf( thrd->link.next == 0p, "Expected null got %p", thrd->link.next );
         /* paranoid */ verify( 0x0D15EA5E0D15EA5Ep == thrd->canary );
 
@@ -591 +600 @@
                 /* paranoid */ verifyf( ((uintptr_t)thrd->context.SP) < ((uintptr_t)__get_stack(thrd->curr_cor)->base ), "ERROR : thread$ %p has been corrupted.\n StackPointer too small.\n", thrd );
 
+                thrd->state = Halting;
                 if( TICKET_RUNNING != thrd->ticket ) { abort( "Thread terminated with pending unpark" ); }
                 if( thrd != this->owner ) { abort( "Thread internal monitor has incorrect owner" ); }
                 if( this->recursion != 1) { abort( "Thread internal monitor has unbalanced recursion" ); }
-
-                thrd->state = Halting;
-                thrd->ticket = TICKET_DEAD;
 
                 // Leave the thread
@@ -617 +624 @@
                 // If that is the case, abandon the preemption.
                 bool preempted = false;
-                if(thrd->rdy_link.next == 0p) {
+                if(thrd->link.next == 0p) {
                         preempted = true;
                         thrd->preempted = reason;
@@ -719 +726 @@
 
 
-        #if !defined(__CFA_NO_STATISTICS__)
-                if(this->print_halts) {
-                        __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 0\n", this->unique_id, rdtscl());
-                }
-        #endif
-
-        __cfadbg_print_safe(runtime_core, "Kernel : core %p waiting on eventfd %d\n", this, this->idle_fd);
-
-        {
-                eventfd_t val;
-                ssize_t ret = read( this->idle_wctx.evfd, &val, sizeof(val) );
-                if(ret < 0) {
-                        switch((int)errno) {
-                        case EAGAIN:
-                        #if EAGAIN != EWOULDBLOCK
-                                case EWOULDBLOCK:
-                        #endif
-                        case EINTR:
-                                // No need to do anything special here, just assume it's a legitimate wake-up
-                                break;
-                        default:
-                                abort( "KERNEL : internal error, read failure on idle eventfd, error(%d) %s.", (int)errno, strerror( (int)errno ) );
+        #if !defined(CFA_WITH_IO_URING_IDLE)
+                #if !defined(__CFA_NO_STATISTICS__)
+                        if(this->print_halts) {
+                                __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 0\n", this->unique_id, rdtscl());
                         }
-                }
-        }
-
-        #if !defined(__CFA_NO_STATISTICS__)
-                if(this->print_halts) {
-                        __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 1\n", this->unique_id, rdtscl());
-                }
+                #endif
+
+                __cfadbg_print_safe(runtime_core, "Kernel : core %p waiting on eventfd %d\n", this, this->idle_fd);
+
+                {
+                        eventfd_t val;
+                        ssize_t ret = read( this->idle_wctx.evfd, &val, sizeof(val) );
+                        if(ret < 0) {
+                                switch((int)errno) {
+                                case EAGAIN:
+                                #if EAGAIN != EWOULDBLOCK
+                                        case EWOULDBLOCK:
+                                #endif
+                                case EINTR:
+                                        // No need to do anything special here, just assume it's a legitimate wake-up
+                                        break;
+                                default:
+                                        abort( "KERNEL : internal error, read failure on idle eventfd, error(%d) %s.", (int)errno, strerror( (int)errno ) );
+                                }
+                        }
+                }
+
+                #if !defined(__CFA_NO_STATISTICS__)
+                        if(this->print_halts) {
+                                __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 1\n", this->unique_id, rdtscl());
+                        }
+                #endif
+        #else
+                __cfa_io_idle( this );
         #endif
 }
@@ -764 +775 @@
                 insert_first(this.idles, proc);
 
-                // update the pointer to the head wait context, which should now point to this proc.
                 __atomic_store_n(&this.fdw, &proc.idle_wctx, __ATOMIC_SEQ_CST);
         unlock( this );
@@ -781 +791 @@
 
                 {
-                        // update the pointer to the head wait context
                         struct __fd_waitctx * wctx = 0;
                         if(!this.idles`isEmpty) wctx = &this.idles`first.idle_wctx;