Changeset cfff639 for libcfa/src/concurrency

Timestamp:

Apr 24, 2021, 7:45:02 PM (4 years ago)

Author:

Thierry Delisle <tdelisle@…>

Branches:

ADT, arm-eh, ast-experimental, enum, forall-pointer-decay, jacob/cs343-translation, master, new-ast-unique-expr, pthread-emulation, qualifiedEnum

Children:

fb0be05

Parents:

89eff25 (diff), 254ad1b (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the (diff) links above to see all the changes relative to each parent.

Message:

Merge branch 'master' of plg.uwaterloo.ca:software/cfa/cfa-cc

Location:

libcfa/src/concurrency

Files:

• alarm.cfa (modified) (2 diffs)
• clib/cfathread.cfa (modified) (1 diff)
• invoke.c (modified) (2 diffs)
• io.cfa (modified) (4 diffs)
• kernel.cfa (modified) (10 diffs)
• kernel/fwd.hfa (modified) (2 diffs)
• kernel/startup.cfa (modified) (6 diffs)
• kernel_private.hfa (modified) (1 diff)
• preemption.cfa (modified) (2 diffs)
• thread.cfa (modified) (2 diffs)

libcfa/src/concurrency/alarm.cfa

@@ -116 +116 @@
         unlock( event_kernel->lock );
         this->set = true;
-        enable_interrupts( __cfaabi_dbg_ctx );
+        enable_interrupts();
 }
 
@@ -127 +127 @@
         }
         unlock( event_kernel->lock );
-        enable_interrupts( __cfaabi_dbg_ctx );
+        enable_interrupts();
         this->set = false;
 }

libcfa/src/concurrency/clib/cfathread.cfa

@@ -117 +117 @@
 
         this_thrd->state = Ready;
-        enable_interrupts( __cfaabi_dbg_ctx );
+        enable_interrupts();
 }
 

libcfa/src/concurrency/invoke.c

@@ -34 +34 @@
 
 extern void disable_interrupts() OPTIONAL_THREAD;
-extern void enable_interrupts( __cfaabi_dbg_ctx_param );
+extern void enable_interrupts( _Bool poll );
 
 void __cfactx_invoke_coroutine(
@@ -82 +82 @@
 ) {
         // Officially start the thread by enabling preemption
-        enable_interrupts( __cfaabi_dbg_ctx );
+        enable_interrupts( true );
 
         // Call the main of the thread

libcfa/src/concurrency/io.cfa

@@ -244 +244 @@
                         // Allocation was successful
                         __STATS__( true, io.alloc.fast += 1; )
-                        enable_interrupts( __cfaabi_dbg_ctx );
+                        enable_interrupts();
 
                         __cfadbg_print_safe(io, "Kernel I/O : fast allocation successful from ring %d\n", ctx->fd);
@@ -256 +256 @@
                 // Fast path failed, fallback on arbitration
                 __STATS__( true, io.alloc.slow += 1; )
-                enable_interrupts( __cfaabi_dbg_ctx );
+                enable_interrupts();
 
                 $io_arbiter * ioarb = proc->cltr->io.arbiter;
@@ -314 +314 @@
                         // Mark the instance as no longer in-use, re-enable interrupts and return
                         __STATS__( true, io.submit.fast += 1; )
-                        enable_interrupts( __cfaabi_dbg_ctx );
+                        enable_interrupts();
 
                         __cfadbg_print_safe(io, "Kernel I/O : submitted on fast path\n");
@@ -322 +322 @@
                 // Fast path failed, fallback on arbitration
                 __STATS__( true, io.submit.slow += 1; )
-                enable_interrupts( __cfaabi_dbg_ctx );
+                enable_interrupts();
 
                 __cfadbg_print_safe(io, "Kernel I/O : falling back on arbiter for submission\n");

libcfa/src/concurrency/kernel.cfa

@@ -115 +115 @@
 static $thread * __next_thread(cluster * this);
 static $thread * __next_thread_slow(cluster * this);
+static inline bool __must_unpark( $thread * thrd ) __attribute((nonnull(1)));
 static void __run_thread(processor * this, $thread * dst);
 static void __wake_one(cluster * cltr);
@@ -130 +131 @@
 extern void __disable_interrupts_hard();
 extern void __enable_interrupts_hard();
+
+static inline void __disable_interrupts_checked() {
+        /* paranoid */ verify( __preemption_enabled() );
+        disable_interrupts();
+        /* paranoid */ verify( ! __preemption_enabled() );
+}
+
+static inline void __enable_interrupts_checked( bool poll = true ) {
+        /* paranoid */ verify( ! __preemption_enabled() );
+        enable_interrupts( poll );
+        /* paranoid */ verify( __preemption_enabled() );
+}
 
 //=============================================================================================
@@ -452 +465 @@
                 if(unlikely(thrd_dst->preempted != __NO_PREEMPTION)) {
                         // The thread was preempted, reschedule it and reset the flag
-                        __schedule_thread( thrd_dst );
+                        schedule_thread$( thrd_dst );
                         break RUNNING;
                 }
@@ -541 +554 @@
         /* paranoid */ verify( ! __preemption_enabled() );
         /* paranoid */ verify( kernelTLS().this_proc_id );
+        /* paranoid */ verify( ready_schedule_islocked());
         /* paranoid */ verify( thrd );
         /* paranoid */ verify( thrd->state != Halted );
@@ -560 +574 @@
         __STATS(bool outside = thrd->last_proc && thrd->last_proc != kernelTLS().this_processor; )
 
-        ready_schedule_lock();
-                // push the thread to the cluster ready-queue
-                push( cl, thrd );
-
-                // variable thrd is no longer safe to use
-
-                // wake the cluster using the save variable.
-                __wake_one( cl );
-        ready_schedule_unlock();
+        // push the thread to the cluster ready-queue
+        push( cl, thrd );
+
+        // variable thrd is no longer safe to use
+        thrd = 0xdeaddeaddeaddeadp;
+
+        // wake the cluster using the save variable.
+        __wake_one( cl );
 
         #if !defined(__CFA_NO_STATISTICS__)
@@ -585 +598 @@
         #endif
 
-        /* paranoid */ verify( ! __preemption_enabled() );
+        /* paranoid */ verify( ready_schedule_islocked());
+        /* paranoid */ verify( ! __preemption_enabled() );
+}
+
+void schedule_thread$( $thread * thrd ) {
+        ready_schedule_lock();
+                __schedule_thread( thrd );
+        ready_schedule_unlock();
 }
 
@@ -623 +643 @@
 }
 
-void unpark( $thread * thrd ) {
-        if( !thrd ) return;
-
+static inline bool __must_unpark( $thread * thrd ) {
         int old_ticket = __atomic_fetch_add(&thrd->ticket, 1, __ATOMIC_SEQ_CST);
         switch(old_ticket) {
                 case TICKET_RUNNING:
                         // Wake won the race, the thread will reschedule/rerun itself
-                        break;
+                        return false;
                 case TICKET_BLOCKED:
                         /* paranoid */ verify( ! thrd->preempted != __NO_PREEMPTION );
                         /* paranoid */ verify( thrd->state == Blocked );
-
-                        {
-                                /* paranoid */ verify( publicTLS_get(this_proc_id) );
-                                disable_interrupts();
-
-                                /* paranoid */ verify( ! __preemption_enabled() );
-
-                                // Wake lost the race,
-                                __schedule_thread( thrd );
-
-                                /* paranoid */ verify( ! __preemption_enabled() );
-
-                                enable_interrupts_noPoll();
-                                /* paranoid */ verify( publicTLS_get(this_proc_id) );
-                        }
-
-                        break;
+                        return true;
                 default:
                         // This makes no sense, something is wrong abort
@@ -657 +659 @@
 }
 
+void unpark( $thread * thrd ) {
+        if( !thrd ) return;
+
+        if(__must_unpark(thrd)) {
+                disable_interrupts();
+                        // Wake lost the race,
+                        schedule_thread$( thrd );
+                enable_interrupts(false);
+        }
+}
+
 void park( void ) {
-        /* paranoid */ verify( __preemption_enabled() );
-        disable_interrupts();
-        /* paranoid */ verify( ! __preemption_enabled() );
-        /* paranoid */ verify( kernelTLS().this_thread->preempted == __NO_PREEMPTION );
-
-        returnToKernel();
-
-        /* paranoid */ verify( ! __preemption_enabled() );
-        enable_interrupts( __cfaabi_dbg_ctx );
-        /* paranoid */ verify( __preemption_enabled() );
+        __disable_interrupts_checked();
+                /* paranoid */ verify( kernelTLS().this_thread->preempted == __NO_PREEMPTION );
+                returnToKernel();
+        __enable_interrupts_checked();
 
 }
@@ -707 +714 @@
 // KERNEL ONLY
 bool force_yield( __Preemption_Reason reason ) {
-        /* paranoid */ verify( __preemption_enabled() );
-        disable_interrupts();
-        /* paranoid */ verify( ! __preemption_enabled() );
-
-        $thread * thrd = kernelTLS().this_thread;
-        /* paranoid */ verify(thrd->state == Active);
-
-        // SKULLDUGGERY: It is possible that we are preempting this thread just before
-        // it was going to park itself. If that is the case and it is already using the
-        // intrusive fields then we can't use them to preempt the thread
-        // If that is the case, abandon the preemption.
-        bool preempted = false;
-        if(thrd->link.next == 0p) {
-                preempted = true;
-                thrd->preempted = reason;
-                returnToKernel();
-        }
-
-        /* paranoid */ verify( ! __preemption_enabled() );
-        enable_interrupts_noPoll();
-        /* paranoid */ verify( __preemption_enabled() );
-
+        __disable_interrupts_checked();
+                $thread * thrd = kernelTLS().this_thread;
+                /* paranoid */ verify(thrd->state == Active);
+
+                // SKULLDUGGERY: It is possible that we are preempting this thread just before
+                // it was going to park itself. If that is the case and it is already using the
+                // intrusive fields then we can't use them to preempt the thread
+                // If that is the case, abandon the preemption.
+                bool preempted = false;
+                if(thrd->link.next == 0p) {
+                        preempted = true;
+                        thrd->preempted = reason;
+                        returnToKernel();
+                }
+        __enable_interrupts_checked( false );
         return preempted;
 }
@@ -773 +773 @@
         __cfadbg_print_safe(runtime_core, "Kernel : waking Processor %p\n", this);
 
-        disable_interrupts();
+        __disable_interrupts_checked();
                 /* paranoid */ verify( ! __preemption_enabled() );
                 eventfd_t val;
                 val = 1;
                 eventfd_write( this->idle, val );
-        enable_interrupts( __cfaabi_dbg_ctx );
+        __enable_interrupts_checked();
 }
 

libcfa/src/concurrency/kernel/fwd.hfa

@@ -108 +108 @@
 
         extern void disable_interrupts();
-        extern void enable_interrupts_noPoll();
-        extern void enable_interrupts( __cfaabi_dbg_ctx_param );
+        extern void enable_interrupts( bool poll = false );
 
         extern "Cforall" {
@@ -403 +402 @@
                                         __VA_ARGS__ \
                                 } \
-                                if( !(in_kernel) ) enable_interrupts( __cfaabi_dbg_ctx ); \
+                                if( !(in_kernel) ) enable_interrupts(); \
                         }
                 #else

libcfa/src/concurrency/kernel/startup.cfa

@@ -225 +225 @@
         // Add the main thread to the ready queue
         // once resume is called on mainProcessor->runner the mainThread needs to be scheduled like any normal thread
-        __schedule_thread(mainThread);
+        schedule_thread$(mainThread);
 
         // SKULLDUGGERY: Force a context switch to the main processor to set the main thread's context to the current UNIX
@@ -238 +238 @@
 
         /* paranoid */ verify( ! __preemption_enabled() );
-        enable_interrupts( __cfaabi_dbg_ctx );
+        enable_interrupts();
         /* paranoid */ verify( __preemption_enabled() );
 
@@ -532 +532 @@
         disable_interrupts();
                 init( this, name, _cltr, initT );
-        enable_interrupts( __cfaabi_dbg_ctx );
+        enable_interrupts();
 
         __cfadbg_print_safe(runtime_core, "Kernel : Starting core %p\n", &this);
@@ -559 +559 @@
         disable_interrupts();
                 deinit( this );
-        enable_interrupts( __cfaabi_dbg_ctx );
+        enable_interrupts();
 }
 
@@ -597 +597 @@
         // Unlock the RWlock
         ready_mutate_unlock( last_size );
-        enable_interrupts_noPoll(); // Don't poll, could be in main cluster
+        enable_interrupts( false ); // Don't poll, could be in main cluster
 }
 
@@ -612 +612 @@
         // Unlock the RWlock
         ready_mutate_unlock( last_size );
-        enable_interrupts_noPoll(); // Don't poll, could be in main cluster
+        enable_interrupts( false ); // Don't poll, could be in main cluster
 
         #if !defined(__CFA_NO_STATISTICS__)

libcfa/src/concurrency/kernel_private.hfa

@@ -29 +29 @@
 extern "C" {
         void disable_interrupts() OPTIONAL_THREAD;
-        void enable_interrupts_noPoll();
-        void enable_interrupts( __cfaabi_dbg_ctx_param );
-}
-
-void __schedule_thread( $thread * )
-#if defined(NDEBUG) || (!defined(__CFA_DEBUG__) && !defined(__CFA_VERIFY__))
-        __attribute__((nonnull (1)))
-#endif
-;
+        void enable_interrupts( bool poll = true );
+}
+
+void schedule_thread$( $thread * ) __attribute__((nonnull (1)));
 
 extern bool __preemption_enabled();

libcfa/src/concurrency/preemption.cfa

@@ -315 +315 @@
         // Enable interrupts by decrementing the counter
         // If counter reaches 0, execute any pending __cfactx_switch
-        void enable_interrupts( __cfaabi_dbg_ctx_param ) {
+        void enable_interrupts( bool poll ) {
                 // Cache the processor now since interrupts can start happening after the atomic store
                 processor   * proc = __cfaabi_tls.this_processor;
-                /* paranoid */ verify( proc );
+                /* paranoid */ verify( !poll || proc );
 
                 with( __cfaabi_tls.preemption_state ){
@@ -340 +340 @@
                                 // Signal the compiler that a fence is needed but only for signal handlers
                                 __atomic_signal_fence(__ATOMIC_RELEASE);
-                                if( proc->pending_preemption ) {
+                                if( poll && proc->pending_preemption ) {
                                         proc->pending_preemption = false;
                                         force_yield( __POLL_PREEMPTION );
                                 }
                         }
-                }
-
-                // For debugging purposes : keep track of the last person to enable the interrupts
-                __cfaabi_dbg_debug_do( proc->last_enable = caller; )
-        }
-
-        // Disable interrupts by incrementint the counter
-        // Don't execute any pending __cfactx_switch even if counter reaches 0
-        void enable_interrupts_noPoll() {
-                unsigned short prev = __cfaabi_tls.preemption_state.disable_count;
-                __cfaabi_tls.preemption_state.disable_count -= 1;
-                // If this triggers someone is enabled already enabled interrupts
-                /* paranoid */ verifyf( prev != 0u, "Incremented from %u\n", prev );
-                if( prev == 1 ) {
-                        #if GCC_VERSION > 50000
-                                static_assert(__atomic_always_lock_free(sizeof(__cfaabi_tls.preemption_state.enabled), &__cfaabi_tls.preemption_state.enabled), "Must be lock-free");
-                        #endif
-                        // Set enabled flag to true
-                        // should be atomic to avoid preemption in the middle of the operation.
-                        // use memory order RELAXED since there is no inter-thread on this variable requirements
-                        __atomic_store_n(&__cfaabi_tls.preemption_state.enabled, true, __ATOMIC_RELAXED);
-
-                        // Signal the compiler that a fence is needed but only for signal handlers
-                        __atomic_signal_fence(__ATOMIC_RELEASE);
                 }
         }

libcfa/src/concurrency/thread.cfa

@@ -136 +136 @@
         /* paranoid */ verify( this_thrd->context.SP );
 
-        __schedule_thread( this_thrd );
-        enable_interrupts( __cfaabi_dbg_ctx );
+        schedule_thread$( this_thrd );
+        enable_interrupts();
 }
 
@@ -170 +170 @@
         disable_interrupts();
         uint64_t ret = __tls_rand();
-        enable_interrupts( __cfaabi_dbg_ctx );
+        enable_interrupts();
         return ret;
 }