Changeset 8fc652e0 for libcfa/src/concurrency/preemption.cfa

Timestamp:

Nov 6, 2020, 11:22:57 AM (3 years ago)

Author:

Thierry Delisle <tdelisle@…>

Branches:

ADT, arm-eh, ast-experimental, enum, forall-pointer-decay, jacob/cs343-translation, master, new-ast-unique-expr, pthread-emulation, qualifiedEnum

Children:

54dcab1

Parents:

be5e34b

Message:

Change usage of TLS to more strongly segregate in kernel and out of kernel usage.

File:

• libcfa/src/concurrency/preemption.cfa (modified) (16 diffs)

libcfa/src/concurrency/preemption.cfa

@@ -164 +164 @@
 //=============================================================================================
 
+//----------
+// special case for preemption since used often
+bool __preemption_enabled() {
+        // create a assembler label before
+        // marked as clobber all to avoid movement
+        asm volatile("__cfaasm_check_before:":::"memory");
+
+        // access tls as normal
+        bool enabled = __cfaabi_tls.preemption_state.enabled;
+
+        // create a assembler label after
+        // marked as clobber all to avoid movement
+        asm volatile("__cfaasm_check_after:":::"memory");
+        return enabled;
+}
+
+//----------
+// Get data from the TLS block
+uintptr_t __cfatls_get( unsigned long int offset ) __attribute__((__noinline__)); //no inline to avoid problems
+uintptr_t __cfatls_get( unsigned long int offset ) {
+        // create a assembler label before
+        // marked as clobber all to avoid movement
+        asm volatile("__cfaasm_get_before:":::"memory");
+
+        // access tls as normal (except for pointer arithmetic)
+        uintptr_t val = *(uintptr_t*)((uintptr_t)&__cfaabi_tls + offset);
+
+        // create a assembler label after
+        // marked as clobber all to avoid movement
+        asm volatile("__cfaasm_get_after:":::"memory");
+        return val;
+}
+
+// //----------
+// // Write data to the TLS block
+// // sadly it looses the type information and can only write 1 word at a time
+// // use with __builtin_offsetof
+// void __cfatls_set(uintptr_t offset, void * value) __attribute__((__noinline__));
+// void __cfatls_set(uintptr_t offset, void * value) {
+//     // create a assembler label before
+//     // marked as clobber all to avoid movement
+//     asm volatile("__cfaasm_set_before:":::"memory");
+
+//     // access tls as normal (except for type information)
+//     *(void**)(offset + (uintptr_t)&my_tls) = value;
+
+//     // create a assembler label after
+//     // marked as clobber all to avoid movement
+//     asm volatile("__cfaasm_set_after:":::"memory");
+// }
+
+// //----------
+// #include <stdio.h>
+// int main() {
+//     // Get the information
+//     // Must use inline assembly to get access to label
+//     // C is annoying here because this could easily be a static const but "initializer element is not a compile-time constant"
+//     // The big advantage of this approach is that there is 0 overhead for the read and writes function
+//     void * __cfaasm_addr_get_before = ({ void * value; asm("movq $__cfaasm_get_before, %[v]\n\t" : [v]"=r"(value) ); value; });
+//     void * __cfaasm_addr_get_after  = ({ void * value; asm("movq $__cfaasm_get_after , %[v]\n\t" : [v]"=r"(value) ); value; });
+//     void * __cfaasm_addr_set_before = ({ void * value; asm("movq $__cfaasm_set_before, %[v]\n\t" : [v]"=r"(value) ); value; });
+//     void * __cfaasm_addr_set_after  = ({ void * value; asm("movq $__cfaasm_set_after , %[v]\n\t" : [v]"=r"(value) ); value; });
+
+//     printf("%p to %p\n", __cfaasm_addr_get_before, __cfaasm_addr_get_after);
+//     printf("%p to %p\n", __cfaasm_addr_set_before, __cfaasm_addr_set_after);
+//     return 0;
+// }
+
 __cfaabi_dbg_debug_do( static thread_local void * last_interrupt = 0; )
 
@@ -169 +237 @@
         // Disable interrupts by incrementing the counter
         void disable_interrupts() {
-                with( kernelTLS.preemption_state ) {
+                // create a assembler label before
+                // marked as clobber all to avoid movement
+                asm volatile("__cfaasm_disable_before:":::"memory");
+
+                with( __cfaabi_tls.preemption_state ) {
                         #if GCC_VERSION > 50000
                         static_assert(__atomic_always_lock_free(sizeof(enabled), &enabled), "Must be lock-free");
@@ -186 +258 @@
                         verify( new_val < 65_000u );              // If this triggers someone is disabling interrupts without enabling them
                 }
+
+                // create a assembler label after
+                // marked as clobber all to avoid movement
+                asm volatile("__cfaasm_disable_after:":::"memory");
         }
 
@@ -191 +267 @@
         // If counter reaches 0, execute any pending __cfactx_switch
         void enable_interrupts( __cfaabi_dbg_ctx_param ) {
-                processor   * proc = kernelTLS.this_processor; // Cache the processor now since interrupts can start happening after the atomic store
+                // create a assembler label before
+                // marked as clobber all to avoid movement
+                asm volatile("__cfaasm_enable_before:":::"memory");
+
+                processor   * proc = __cfaabi_tls.this_processor; // Cache the processor now since interrupts can start happening after the atomic store
                 /* paranoid */ verify( proc );
 
-                with( kernelTLS.preemption_state ){
+                with( __cfaabi_tls.preemption_state ){
                         unsigned short prev = disable_count;
                         disable_count -= 1;
@@ -221 +301 @@
                 // For debugging purposes : keep track of the last person to enable the interrupts
                 __cfaabi_dbg_debug_do( proc->last_enable = caller; )
+
+                // create a assembler label after
+                // marked as clobber all to avoid movement
+                asm volatile("__cfaasm_enable_after:":::"memory");
         }
 
@@ -226 +310 @@
         // Don't execute any pending __cfactx_switch even if counter reaches 0
         void enable_interrupts_noPoll() {
-                unsigned short prev = kernelTLS.preemption_state.disable_count;
-                kernelTLS.preemption_state.disable_count -= 1;
+                // create a assembler label before
+                // marked as clobber all to avoid movement
+                asm volatile("__cfaasm_nopoll_before:":::"memory");
+
+                unsigned short prev = __cfaabi_tls.preemption_state.disable_count;
+                __cfaabi_tls.preemption_state.disable_count -= 1;
                 verifyf( prev != 0u, "Incremented from %u\n", prev );                     // If this triggers someone is enabled already enabled interrupts
                 if( prev == 1 ) {
                         #if GCC_VERSION > 50000
-                        static_assert(__atomic_always_lock_free(sizeof(kernelTLS.preemption_state.enabled), &kernelTLS.preemption_state.enabled), "Must be lock-free");
+                        static_assert(__atomic_always_lock_free(sizeof(__cfaabi_tls.preemption_state.enabled), &__cfaabi_tls.preemption_state.enabled), "Must be lock-free");
                         #endif
                         // Set enabled flag to true
                         // should be atomic to avoid preemption in the middle of the operation.
                         // use memory order RELAXED since there is no inter-thread on this variable requirements
-                        __atomic_store_n(&kernelTLS.preemption_state.enabled, true, __ATOMIC_RELAXED);
+                        __atomic_store_n(&__cfaabi_tls.preemption_state.enabled, true, __ATOMIC_RELAXED);
 
                         // Signal the compiler that a fence is needed but only for signal handlers
                         __atomic_signal_fence(__ATOMIC_RELEASE);
                 }
+
+                // create a assembler label after
+                // marked as clobber all to avoid movement
+                asm volatile("__cfaasm_nopoll_after:":::"memory");
         }
 }
@@ -275 +367 @@
 static void timeout( $thread * this ) {
         #if !defined( __CFA_NO_STATISTICS__ )
-                kernelTLS.this_stats = this->curr_cluster->stats;
+                kernelTLS().this_stats = this->curr_cluster->stats;
         #endif
         unpark( this );
@@ -286 +378 @@
 static inline bool preemption_ready() {
         // Check if preemption is safe
-        bool ready = kernelTLS.preemption_state.enabled && ! kernelTLS.preemption_state.in_progress;
+        bool ready = __cfaabi_tls.preemption_state.enabled && ! __cfaabi_tls.preemption_state.in_progress;
 
         // Adjust the pending flag accordingly
-        kernelTLS.this_processor->pending_preemption = !ready;
+        __cfaabi_tls.this_processor->pending_preemption = !ready;
         return ready;
 }
@@ -303 +395 @@
 
         // Start with preemption disabled until ready
-        kernelTLS.preemption_state.enabled = false;
-        kernelTLS.preemption_state.disable_count = 1;
+        __cfaabi_tls.preemption_state.enabled = false;
+        __cfaabi_tls.preemption_state.disable_count = 1;
 
         // Initialize the event kernel
@@ -362 +454 @@
 // Kernel Signal Handlers
 //=============================================================================================
+struct asm_region {
+        void * before;
+        void * after;
+};
+
+//-----------------------------------------------------------------------------
+// Some assembly required
+#if defined( __i386 )
+        #define __cfaasm_label( label ) \
+                ({ \
+                        struct asm_region region; \
+                        asm( \
+                                "movl $__cfaasm_" #label "_before, %[vb]\n\t" \
+                                "movl $__cfaasm_" #label "_after , %[va]\n\t" \
+                                 : [vb]"=r"(region.before), [vb]"=r"(region.before) \
+                        ); \
+                        region; \
+                });
+#elif defined( __x86_64 )
+        #ifdef __PIC__
+                #define PLT "@PLT"
+        #else
+                #define PLT ""
+        #endif
+        #define __cfaasm_label( label ) \
+                ({ \
+                        struct asm_region region; \
+                        asm( \
+                                "movq $__cfaasm_" #label "_before" PLT ", %[vb]\n\t" \
+                                "movq $__cfaasm_" #label "_after"  PLT ", %[va]\n\t" \
+                                 : [vb]"=r"(region.before), [va]"=r"(region.after) \
+                        ); \
+                        region; \
+                });
+#elif defined( __aarch64__ )
+        #error __cfaasm_label undefined for arm
+#else
+        #error unknown hardware architecture
+#endif
 
 // Context switch signal handler
 // Receives SIGUSR1 signal and causes the current thread to yield
 static void sigHandler_ctxSwitch( __CFA_SIGPARMS__ ) {
-        __cfaabi_dbg_debug_do( last_interrupt = (void *)(cxt->uc_mcontext.CFA_REG_IP); )
+        void * ip = (void *)(cxt->uc_mcontext.CFA_REG_IP);
+        __cfaabi_dbg_debug_do( last_interrupt = ip; )
 
         // SKULLDUGGERY: if a thread creates a processor and the immediately deletes it,
@@ -372 +504 @@
         // before the kernel thread has even started running. When that happens, an interrupt
         // with a null 'this_processor' will be caught, just ignore it.
-        if(! kernelTLS.this_processor ) return;
+        if(! __cfaabi_tls.this_processor ) return;
 
         choose(sfp->si_value.sival_int) {
                 case PREEMPT_NORMAL   : ;// Normal case, nothing to do here
-                case PREEMPT_TERMINATE: verify( __atomic_load_n( &kernelTLS.this_processor->do_terminate, __ATOMIC_SEQ_CST ) );
+                case PREEMPT_TERMINATE: verify( __atomic_load_n( &__cfaabi_tls.this_processor->do_terminate, __ATOMIC_SEQ_CST ) );
                 default:
                         abort( "internal error, signal value is %d", sfp->si_value.sival_int );
@@ -384 +516 @@
         if( !preemption_ready() ) { return; }
 
-        __cfaabi_dbg_print_buffer_decl( " KERNEL: preempting core %p (%p @ %p).\n", kernelTLS.this_processor, kernelTLS.this_thread, (void *)(cxt->uc_mcontext.CFA_REG_IP) );
+        struct asm_region region;
+        region = __cfaasm_label( get     ); if( ip >= region.before && ip <= region.after ) return;
+        region = __cfaasm_label( check   ); if( ip >= region.before && ip <= region.after ) return;
+        region = __cfaasm_label( disable ); if( ip >= region.before && ip <= region.after ) return;
+        region = __cfaasm_label( enable  ); if( ip >= region.before && ip <= region.after ) return;
+        region = __cfaasm_label( nopoll  ); if( ip >= region.before && ip <= region.after ) return;
+
+        __cfaabi_dbg_print_buffer_decl( " KERNEL: preempting core %p (%p @ %p).\n", __cfaabi_tls.this_processor, __cfaabi_tls.this_thread, (void *)(cxt->uc_mcontext.CFA_REG_IP) );
 
         // Sync flag : prevent recursive calls to the signal handler
-        kernelTLS.preemption_state.in_progress = true;
+        __cfaabi_tls.preemption_state.in_progress = true;
 
         // Clear sighandler mask before context switching.
@@ -397 +536 @@
         }
 
-        // TODO: this should go in finish action
         // Clear the in progress flag
-        kernelTLS.preemption_state.in_progress = false;
+        __cfaabi_tls.preemption_state.in_progress = false;
 
         // Preemption can occur here
@@ -416 +554 @@
         id.full_proc = false;
         id.id = doregister(&id);
-        kernelTLS.this_proc_id = &id;
+        __cfaabi_tls.this_proc_id = &id;
 
         // Block sigalrms to control when they arrive
@@ -484 +622 @@
 
 void __cfaabi_check_preemption() {
-        bool ready = kernelTLS.preemption_state.enabled;
+        bool ready = __preemption_enabled();
         if(!ready) { abort("Preemption should be ready"); }
 
@@ -507 +645 @@
 #ifdef __CFA_WITH_VERIFY__
 bool __cfaabi_dbg_in_kernel() {
-        return !kernelTLS.preemption_state.enabled;
+        return !__preemption_enabled();
 }
 #endif