Changeset 58fe85a for libcfa/src/concurrency/preemption.cfa

Timestamp:

Jan 7, 2021, 3:27:00 PM (5 years ago)

Author:

Thierry Delisle <tdelisle@…>

Branches:

ADT, arm-eh, ast-experimental, enum, forall-pointer-decay, jacob/cs343-translation, master, new-ast-unique-expr, pthread-emulation, qualifiedEnum

Children:

2b4daf2, 64aeca0

Parents:

3c64c668 (diff), eef8dfb (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the (diff) links above to see all the changes relative to each parent.

Message:

Merge branch 'master' into park_unpark

File:

• libcfa/src/concurrency/preemption.cfa (modified) (25 diffs)

libcfa/src/concurrency/preemption.cfa

@@ -10 +10 @@
 // Created On       : Mon Jun 5 14:20:42 2017
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Thu Dec  5 16:34:05 2019
-// Update Count     : 43
+// Last Modified On : Fri Nov  6 07:42:13 2020
+// Update Count     : 54
 //
 
@@ -19 +19 @@
 #include <assert.h>
 
-extern "C" {
 #include <errno.h>
 #include <stdio.h>
@@ -25 +24 @@
 #include <unistd.h>
 #include <limits.h>                                                                             // PTHREAD_STACK_MIN
-}
 
 #include "bits/signal.hfa"
+#include "kernel_private.hfa"
 
 #if !defined(__CFA_DEFAULT_PREEMPTION__)
@@ -43 +42 @@
 // FwdDeclarations : Signal handlers
 static void sigHandler_ctxSwitch( __CFA_SIGPARMS__ );
+static void sigHandler_alarm    ( __CFA_SIGPARMS__ );
 static void sigHandler_segv     ( __CFA_SIGPARMS__ );
 static void sigHandler_ill      ( __CFA_SIGPARMS__ );
@@ -56 +56 @@
 #elif defined( __x86_64 )
 #define CFA_REG_IP gregs[REG_RIP]
-#elif defined( __ARM_ARCH )
+#elif defined( __arm__ )
 #define CFA_REG_IP arm_pc
+#elif defined( __aarch64__ )
+#define CFA_REG_IP pc
 #else
-#error unknown hardware architecture
+#error unsupported hardware architecture
 #endif
 
@@ -83 +85 @@
 // Get next expired node
 static inline alarm_node_t * get_expired( alarm_list_t * alarms, Time currtime ) {
-        if( !alarms->head ) return 0p;                                          // If no alarms return null
-        if( alarms->head->alarm >= currtime ) return 0p;        // If alarms head not expired return null
+        if( ! & (*alarms)`first ) return 0p;                                            // If no alarms return null
+        if( (*alarms)`first.alarm >= currtime ) return 0p;      // If alarms head not expired return null
         return pop(alarms);                                                                     // Otherwise just pop head
 }
 
 // Tick one frame of the Discrete Event Simulation for alarms
-static void tick_preemption() {
+static void tick_preemption(void) {
         alarm_node_t * node = 0p;                                                       // Used in the while loop but cannot be declared in the while condition
         alarm_list_t * alarms = &event_kernel->alarms;          // Local copy for ease of reading
@@ -97 +99 @@
         while( node = get_expired( alarms, currtime ) ) {
                 // __cfaabi_dbg_print_buffer_decl( " KERNEL: preemption tick.\n" );
+                Duration period = node->period;
+                if( period == 0) {
+                        node->set = false;                  // Node is one-shot, just mark it as not pending
+                }
 
                 // Check if this is a kernel
-                if( node->kernel_alarm ) {
+                if( node->type == Kernel ) {
                         preempt( node->proc );
                 }
+                else if( node->type == User ) {
+                        timeout( node->thrd );
+                }
                 else {
-                        timeout( node->thrd );
+                        node->callback(*node);
                 }
 
                 // Check if this is a periodic alarm
-                Duration period = node->period;
                 if( period > 0 ) {
                         // __cfaabi_dbg_print_buffer_local( " KERNEL: alarm period is %lu.\n", period.tv );
@@ -113 +121 @@
                         insert( alarms, node );             // Reinsert the node for the next time it triggers
                 }
-                else {
-                        node->set = false;                  // Node is one-shot, just mark it as not pending
-                }
         }
 
         // If there are still alarms pending, reset the timer
-        if( alarms->head ) {
-                __cfaabi_dbg_print_buffer_decl( " KERNEL: @%ju(%ju) resetting alarm to %ju.\n", currtime.tv, __kernel_get_time().tv, (alarms->head->alarm - currtime).tv);
-                Duration delta = alarms->head->alarm - currtime;
-                Duration caped = max(delta, 50`us);
+        if( & (*alarms)`first ) {
+                __cfadbg_print_buffer_decl(preemption, " KERNEL: @%ju(%ju) resetting alarm to %ju.\n", currtime.tv, __kernel_get_time().tv, (alarms->head->alarm - currtime).tv);
+                Duration delta = (*alarms)`first.alarm - currtime;
+                Duration capped = max(delta, 50`us);
                 // itimerval tim  = { caped };
                 // __cfaabi_dbg_print_buffer_local( "    Values are %lu, %lu, %lu %lu.\n", delta.tv, caped.tv, tim.it_value.tv_sec, tim.it_value.tv_usec);
 
-                __kernel_set_timer( caped );
+                __kernel_set_timer( capped );
         }
 }
@@ -158 +163 @@
 // Kernel Signal Tools
 //=============================================================================================
-
-__cfaabi_dbg_debug_do( static thread_local void * last_interrupt = 0; )
+// In a user-level threading system, there are handful of thread-local variables where this problem occurs on the ARM.
+//
+// For each kernel thread running user-level threads, there is a flag variable to indicate if interrupts are
+// enabled/disabled for that kernel thread. Therefore, this variable is made thread local.
+//
+// For example, this code fragment sets the state of the "interrupt" variable in thread-local memory.
+//
+// _Thread_local volatile int interrupts;
+// int main() {
+//     interrupts = 0; // disable interrupts }
+//
+// which generates the following code on the ARM
+//
+// (gdb) disassemble main
+// Dump of assembler code for function main:
+//    0x0000000000000610 <+0>:  mrs     x1, tpidr_el0
+//    0x0000000000000614 <+4>:  mov     w0, #0x0                        // #0
+//    0x0000000000000618 <+8>:  add     x1, x1, #0x0, lsl #12
+//    0x000000000000061c <+12>: add     x1, x1, #0x10
+//    0x0000000000000620 <+16>: str     wzr, [x1]
+//    0x0000000000000624 <+20>: ret
+//
+// The mrs moves a pointer from coprocessor register tpidr_el0 into register x1.  Register w0 is set to 0. The two adds
+// increase the TLS pointer with the displacement (offset) 0x10, which is the location in the TSL of variable
+// "interrupts".  Finally, 0 is stored into "interrupts" through the pointer in register x1 that points into the
+// TSL. Now once x1 has the pointer to the location of the TSL for kernel thread N, it can be be preempted at a
+// user-level and the user thread is put on the user-level ready-queue. When the preempted thread gets to the front of
+// the user-level ready-queue it is run on kernel thread M. It now stores 0 into "interrupts" back on kernel thread N,
+// turning off interrupt on the wrong kernel thread.
+//
+// On the x86, the following code is generated for the same code fragment.
+//
+// (gdb) disassemble main
+// Dump of assembler code for function main:
+//    0x0000000000400420 <+0>:  movl   $0x0,%fs:0xfffffffffffffffc
+//    0x000000000040042c <+12>: xor    %eax,%eax
+//    0x000000000040042e <+14>: retq
+//
+// and there is base-displacement addressing used to atomically reset variable "interrupts" off of the TSL pointer in
+// register "fs".
+//
+// Hence, the ARM has base-displacement address for the general purpose registers, BUT not to the coprocessor
+// registers. As a result, generating the address for the write into variable "interrupts" is no longer atomic.
+//
+// Note this problem does NOT occur when just using multiple kernel threads because the preemption ALWAYS restarts the
+// thread on the same kernel thread.
+//
+// The obvious question is why does ARM use a coprocessor register to store the TSL pointer given that coprocessor
+// registers are second-class registers with respect to the instruction set. One possible answer is that they did not
+// want to dedicate one of the general registers to hold the TLS pointer and there was a free coprocessor register
+// available.
+
+//-----------------------------------------------------------------------------
+// Some assembly required
+#define __cfaasm_label(label, when) when: asm volatile goto(".global __cfaasm_" #label "_" #when "\n" "__cfaasm_" #label "_" #when ":":::"memory":when)
+
+//----------
+// special case for preemption since used often
+bool __preemption_enabled() {
+        // create a assembler label before
+        // marked as clobber all to avoid movement
+        __cfaasm_label(check, before);
+
+        // access tls as normal
+        bool enabled = __cfaabi_tls.preemption_state.enabled;
+
+        // create a assembler label after
+        // marked as clobber all to avoid movement
+        __cfaasm_label(check, after);
+        return enabled;
+}
+
+struct asm_region {
+        void * before;
+        void * after;
+};
+
+static inline bool __cfaasm_in( void * ip, struct asm_region & region ) {
+        return ip >= region.before && ip <= region.after;
+}
+
+
+//----------
+// Get data from the TLS block
+// struct asm_region __cfaasm_get;
+uintptr_t __cfatls_get( unsigned long int offset ) __attribute__((__noinline__)); //no inline to avoid problems
+uintptr_t __cfatls_get( unsigned long int offset ) {
+        // create a assembler label before
+        // marked as clobber all to avoid movement
+        __cfaasm_label(get, before);
+
+        // access tls as normal (except for pointer arithmetic)
+        uintptr_t val = *(uintptr_t*)((uintptr_t)&__cfaabi_tls + offset);
+
+        // create a assembler label after
+        // marked as clobber all to avoid movement
+        __cfaasm_label(get, after);
+        return val;
+}
 
 extern "C" {
         // Disable interrupts by incrementing the counter
         void disable_interrupts() {
-                with( kernelTLS.preemption_state ) {
+                // create a assembler label before
+                // marked as clobber all to avoid movement
+                __cfaasm_label(dsable, before);
+
+                with( __cfaabi_tls.preemption_state ) {
                         #if GCC_VERSION > 50000
                         static_assert(__atomic_always_lock_free(sizeof(enabled), &enabled), "Must be lock-free");
@@ -181 +287 @@
                         verify( new_val < 65_000u );              // If this triggers someone is disabling interrupts without enabling them
                 }
+
+                // create a assembler label after
+                // marked as clobber all to avoid movement
+                __cfaasm_label(dsable, after);
+
         }
 
@@ -186 +297 @@
         // If counter reaches 0, execute any pending __cfactx_switch
         void enable_interrupts( __cfaabi_dbg_ctx_param ) {
-                processor   * proc = kernelTLS.this_processor; // Cache the processor now since interrupts can start happening after the atomic store
-
-                with( kernelTLS.preemption_state ){
+                // Cache the processor now since interrupts can start happening after the atomic store
+                processor   * proc = __cfaabi_tls.this_processor;
+                /* paranoid */ verify( proc );
+
+                with( __cfaabi_tls.preemption_state ){
                         unsigned short prev = disable_count;
                         disable_count -= 1;
-                        verify( prev != 0u );                     // If this triggers someone is enabled already enabled interruptsverify( prev != 0u );
+
+                        // If this triggers someone is enabled already enabled interruptsverify( prev != 0u );
+                        /* paranoid */ verify( prev != 0u );
 
                         // Check if we need to prempt the thread because an interrupt was missed
                         if( prev == 1 ) {
                                 #if GCC_VERSION > 50000
-                                static_assert(__atomic_always_lock_free(sizeof(enabled), &enabled), "Must be lock-free");
+                                        static_assert(__atomic_always_lock_free(sizeof(enabled), &enabled), "Must be lock-free");
                                 #endif
 
@@ -220 +335 @@
         // Don't execute any pending __cfactx_switch even if counter reaches 0
         void enable_interrupts_noPoll() {
-                unsigned short prev = kernelTLS.preemption_state.disable_count;
-                kernelTLS.preemption_state.disable_count -= 1;
-                verifyf( prev != 0u, "Incremented from %u\n", prev );                     // If this triggers someone is enabled already enabled interrupts
+                unsigned short prev = __cfaabi_tls.preemption_state.disable_count;
+                __cfaabi_tls.preemption_state.disable_count -= 1;
+                // If this triggers someone is enabled already enabled interrupts
+                /* paranoid */ verifyf( prev != 0u, "Incremented from %u\n", prev );
                 if( prev == 1 ) {
                         #if GCC_VERSION > 50000
-                        static_assert(__atomic_always_lock_free(sizeof(kernelTLS.preemption_state.enabled), &kernelTLS.preemption_state.enabled), "Must be lock-free");
+                                static_assert(__atomic_always_lock_free(sizeof(__cfaabi_tls.preemption_state.enabled), &__cfaabi_tls.preemption_state.enabled), "Must be lock-free");
                         #endif
                         // Set enabled flag to true
                         // should be atomic to avoid preemption in the middle of the operation.
                         // use memory order RELAXED since there is no inter-thread on this variable requirements
-                        __atomic_store_n(&kernelTLS.preemption_state.enabled, true, __ATOMIC_RELAXED);
+                        __atomic_store_n(&__cfaabi_tls.preemption_state.enabled, true, __ATOMIC_RELAXED);
 
                         // Signal the compiler that a fence is needed but only for signal handlers
@@ -237 +353 @@
         }
 }
+
+//-----------------------------------------------------------------------------
+// Kernel Signal Debug
+void __cfaabi_check_preemption() {
+        bool ready = __preemption_enabled();
+        if(!ready) { abort("Preemption should be ready"); }
+
+        __cfaasm_label(debug, before);
+
+                sigset_t oldset;
+                int ret;
+                ret = pthread_sigmask(0, ( const sigset_t * ) 0p, &oldset);  // workaround trac#208: cast should be unnecessary
+                if(ret != 0) { abort("ERROR sigprocmask returned %d", ret); }
+
+                ret = sigismember(&oldset, SIGUSR1);
+                if(ret <  0) { abort("ERROR sigismember returned %d", ret); }
+                if(ret == 1) { abort("ERROR SIGUSR1 is disabled"); }
+
+                ret = sigismember(&oldset, SIGALRM);
+                if(ret <  0) { abort("ERROR sigismember returned %d", ret); }
+                if(ret == 0) { abort("ERROR SIGALRM is enabled"); }
+
+                ret = sigismember(&oldset, SIGTERM);
+                if(ret <  0) { abort("ERROR sigismember returned %d", ret); }
+                if(ret == 1) { abort("ERROR SIGTERM is disabled"); }
+
+        __cfaasm_label(debug, after);
+}
+
+#ifdef __CFA_WITH_VERIFY__
+bool __cfaabi_dbg_in_kernel() {
+        return !__preemption_enabled();
+}
+#endif
+
+#undef __cfaasm_label
+
+//-----------------------------------------------------------------------------
+// Signal handling
 
 // sigprocmask wrapper : unblock a single signal
@@ -256 +411 @@
 
         if ( pthread_sigmask( SIG_BLOCK, &mask, 0p ) == -1 ) {
-            abort( "internal error, pthread_sigmask" );
+                abort( "internal error, pthread_sigmask" );
         }
 }
@@ -268 +423 @@
 // reserved for future use
 static void timeout( $thread * this ) {
-        //TODO : implement waking threads
-}
+        unpark( this );
+}
+
+//-----------------------------------------------------------------------------
+// Some assembly required
+#if defined( __i386 )
+        #ifdef __PIC__
+                #define RELOC_PRELUDE( label ) \
+                        "calll   .Lcfaasm_prelude_" #label "$pb\n\t" \
+                        ".Lcfaasm_prelude_" #label "$pb:\n\t" \
+                        "popl    %%eax\n\t" \
+                        ".Lcfaasm_prelude_" #label "_end:\n\t" \
+                        "addl    $_GLOBAL_OFFSET_TABLE_+(.Lcfaasm_prelude_" #label "_end-.Lcfaasm_prelude_" #label "$pb), %%eax\n\t"
+                #define RELOC_PREFIX ""
+                #define RELOC_SUFFIX "@GOT(%%eax)"
+        #else
+                #define RELOC_PREFIX "$"
+                #define RELOC_SUFFIX ""
+        #endif
+        #define __cfaasm_label( label ) struct asm_region label = \
+                ({ \
+                        struct asm_region region; \
+                        asm( \
+                                RELOC_PRELUDE( label ) \
+                                "movl " RELOC_PREFIX "__cfaasm_" #label "_before" RELOC_SUFFIX ", %[vb]\n\t" \
+                                "movl " RELOC_PREFIX "__cfaasm_" #label "_after"  RELOC_SUFFIX ", %[va]\n\t" \
+                                 : [vb]"=r"(region.before), [va]"=r"(region.after) \
+                        ); \
+                        region; \
+                });
+#elif defined( __x86_64 )
+        #ifdef __PIC__
+                #define RELOC_PREFIX ""
+                #define RELOC_SUFFIX "@GOTPCREL(%%rip)"
+        #else
+                #define RELOC_PREFIX "$"
+                #define RELOC_SUFFIX ""
+        #endif
+        #define __cfaasm_label( label ) struct asm_region label = \
+                ({ \
+                        struct asm_region region; \
+                        asm( \
+                                "movq " RELOC_PREFIX "__cfaasm_" #label "_before" RELOC_SUFFIX ", %[vb]\n\t" \
+                                "movq " RELOC_PREFIX "__cfaasm_" #label "_after"  RELOC_SUFFIX ", %[va]\n\t" \
+                                 : [vb]"=r"(region.before), [va]"=r"(region.after) \
+                        ); \
+                        region; \
+                });
+#elif defined( __aarch64__ )
+        #ifdef __PIC__
+                // Note that this works only for gcc
+                #define __cfaasm_label( label ) struct asm_region label = \
+                ({ \
+                        struct asm_region region; \
+                        asm( \
+                                "adrp %[vb], _GLOBAL_OFFSET_TABLE_"                              "\n\t" \
+                                "ldr  %[vb], [%[vb], #:gotpage_lo15:__cfaasm_" #label "_before]" "\n\t" \
+                                "adrp %[va], _GLOBAL_OFFSET_TABLE_"                              "\n\t" \
+                                "ldr  %[va], [%[va], #:gotpage_lo15:__cfaasm_" #label "_after]"  "\n\t" \
+                                 : [vb]"=r"(region.before), [va]"=r"(region.after) \
+                        ); \
+                        region; \
+                });
+        #else
+                #error this is not the right thing to do
+                /*
+                #define __cfaasm_label( label ) struct asm_region label = \
+                ({ \
+                        struct asm_region region; \
+                        asm( \
+                                "adrp %[vb], __cfaasm_" #label "_before"              "\n\t" \
+                                "add  %[vb], %[vb], :lo12:__cfaasm_" #label "_before" "\n\t" \
+                                "adrp %[va], :got:__cfaasm_" #label "_after"          "\n\t" \
+                                "add  %[va], %[va], :lo12:__cfaasm_" #label "_after"  "\n\t" \
+                                 : [vb]"=r"(region.before), [va]"=r"(region.after) \
+                        ); \
+                        region; \
+                });
+                */
+        #endif
+#else
+        #error unknown hardware architecture
+#endif
 
 // KERNEL ONLY
@@ -275 +511 @@
 // If true  : preemption is safe
 // If false : preemption is unsafe and marked as pending
-static inline bool preemption_ready() {
+static inline bool preemption_ready( void * ip ) {
+        // Get all the region for which it is not safe to preempt
+        __cfaasm_label( get    );
+        __cfaasm_label( check  );
+        __cfaasm_label( dsable );
+        __cfaasm_label( debug  );
+
         // Check if preemption is safe
-        bool ready = kernelTLS.preemption_state.enabled && ! kernelTLS.preemption_state.in_progress;
-
+        bool ready = true;
+        if( __cfaasm_in( ip, get    ) ) { ready = false; goto EXIT; };
+        if( __cfaasm_in( ip, check  ) ) { ready = false; goto EXIT; };
+        if( __cfaasm_in( ip, dsable ) ) { ready = false; goto EXIT; };
+        if( __cfaasm_in( ip, debug  ) ) { ready = false; goto EXIT; };
+        if( !__cfaabi_tls.preemption_state.enabled) { ready = false; goto EXIT; };
+        if( __cfaabi_tls.preemption_state.in_progress ) { ready = false; goto EXIT; };
+
+EXIT:
         // Adjust the pending flag accordingly
-        kernelTLS.this_processor->pending_preemption = !ready;
+        __cfaabi_tls.this_processor->pending_preemption = !ready;
         return ready;
 }
@@ -290 +539 @@
 // Startup routine to activate preemption
 // Called from kernel_startup
-void kernel_start_preemption() {
+void __kernel_alarm_startup() {
         __cfaabi_dbg_print_safe( "Kernel : Starting preemption\n" );
 
         // Start with preemption disabled until ready
-        kernelTLS.preemption_state.enabled = false;
-        kernelTLS.preemption_state.disable_count = 1;
+        __cfaabi_tls.preemption_state.enabled = false;
+        __cfaabi_tls.preemption_state.disable_count = 1;
 
         // Initialize the event kernel
@@ -303 +552 @@
         // Setup proper signal handlers
         __cfaabi_sigaction( SIGUSR1, sigHandler_ctxSwitch, SA_SIGINFO | SA_RESTART ); // __cfactx_switch handler
+        __cfaabi_sigaction( SIGALRM, sigHandler_alarm    , SA_SIGINFO | SA_RESTART ); // debug handler
 
         signal_block( SIGALRM );
@@ -311 +561 @@
 // Shutdown routine to deactivate preemption
 // Called from kernel_shutdown
-void kernel_stop_preemption() {
+void __kernel_alarm_shutdown() {
         __cfaabi_dbg_print_safe( "Kernel : Preemption stopping\n" );
 
@@ -325 +575 @@
         // Wait for the preemption thread to finish
 
-        pthread_join( alarm_thread, 0p );
-        free( alarm_stack );
+        __destroy_pthread( alarm_thread, alarm_stack, 0p );
 
         // Preemption is now fully stopped
@@ -352 +601 @@
 // Kernel Signal Handlers
 //=============================================================================================
+__cfaabi_dbg_debug_do( static thread_local void * last_interrupt = 0; )
 
 // Context switch signal handler
 // Receives SIGUSR1 signal and causes the current thread to yield
 static void sigHandler_ctxSwitch( __CFA_SIGPARMS__ ) {
-        __cfaabi_dbg_debug_do( last_interrupt = (void *)(cxt->uc_mcontext.CFA_REG_IP); )
+        void * ip = (void *)(cxt->uc_mcontext.CFA_REG_IP);
+        __cfaabi_dbg_debug_do( last_interrupt = ip; )
 
         // SKULLDUGGERY: if a thread creates a processor and the immediately deletes it,
         // the interrupt that is supposed to force the kernel thread to preempt might arrive
-        // before the kernel thread has even started running. When that happens an iterrupt
-        // we a null 'this_processor' will be caught, just ignore it.
-        if(! kernelTLS.this_processor ) return;
+        // before the kernel thread has even started running. When that happens, an interrupt
+        // with a null 'this_processor' will be caught, just ignore it.
+        if(! __cfaabi_tls.this_processor ) return;
 
         choose(sfp->si_value.sival_int) {
                 case PREEMPT_NORMAL   : ;// Normal case, nothing to do here
-                case PREEMPT_TERMINATE: verify( __atomic_load_n( &kernelTLS.this_processor->do_terminate, __ATOMIC_SEQ_CST ) );
+                case PREEMPT_TERMINATE: verify( __atomic_load_n( &__cfaabi_tls.this_processor->do_terminate, __ATOMIC_SEQ_CST ) );
                 default:
                         abort( "internal error, signal value is %d", sfp->si_value.sival_int );
@@ -372 +623 @@
 
         // Check if it is safe to preempt here
-        if( !preemption_ready() ) { return; }
-
-        __cfaabi_dbg_print_buffer_decl( " KERNEL: preempting core %p (%p @ %p).\n", kernelTLS.this_processor, kernelTLS.this_thread, (void *)(cxt->uc_mcontext.CFA_REG_IP) );
+        if( !preemption_ready( ip ) ) { return; }
+
+        __cfaabi_dbg_print_buffer_decl( " KERNEL: preempting core %p (%p @ %p).\n", __cfaabi_tls.this_processor, __cfaabi_tls.this_thread, (void *)(cxt->uc_mcontext.CFA_REG_IP) );
 
         // Sync flag : prevent recursive calls to the signal handler
-        kernelTLS.preemption_state.in_progress = true;
+        __cfaabi_tls.preemption_state.in_progress = true;
 
         // Clear sighandler mask before context switching.
@@ -387 +638 @@
         }
 
-        // TODO: this should go in finish action
         // Clear the in progress flag
-        kernelTLS.preemption_state.in_progress = false;
+        __cfaabi_tls.preemption_state.in_progress = false;
 
         // Preemption can occur here
@@ -395 +645 @@
         force_yield( __ALARM_PREEMPTION ); // Do the actual __cfactx_switch
 }
+
+static void sigHandler_alarm( __CFA_SIGPARMS__ ) {
+        abort("SIGALRM should never reach the signal handler");
+}
+
+#if !defined(__CFA_NO_STATISTICS__)
+        int __print_alarm_stats = 0;
+#endif
 
 // Main of the alarm thread
 // Waits on SIGALRM and send SIGUSR1 to whom ever needs it
 static void * alarm_loop( __attribute__((unused)) void * args ) {
+        __processor_id_t id;
+        id.full_proc = false;
+        id.id = doregister(&id);
+        __cfaabi_tls.this_proc_id = &id;
+
+        #if !defined(__CFA_NO_STATISTICS__)
+                struct __stats_t local_stats;
+                __cfaabi_tls.this_stats = &local_stats;
+                __init_stats( &local_stats );
+        #endif
+
         // Block sigalrms to control when they arrive
         sigset_t mask;
@@ -456 +725 @@
 EXIT:
         __cfaabi_dbg_print_safe( "Kernel : Preemption thread stopping\n" );
+        unregister(&id);
+
+        #if !defined(__CFA_NO_STATISTICS__)
+                if( 0 != __print_alarm_stats ) {
+                        __print_stats( &local_stats, __print_alarm_stats, "Alarm", "Thread", 0p );
+                }
+        #endif
         return 0p;
 }
-
-//=============================================================================================
-// Kernel Signal Debug
-//=============================================================================================
-
-void __cfaabi_check_preemption() {
-        bool ready = kernelTLS.preemption_state.enabled;
-        if(!ready) { abort("Preemption should be ready"); }
-
-        sigset_t oldset;
-        int ret;
-        ret = pthread_sigmask(0, 0p, &oldset);
-        if(ret != 0) { abort("ERROR sigprocmask returned %d", ret); }
-
-        ret = sigismember(&oldset, SIGUSR1);
-        if(ret <  0) { abort("ERROR sigismember returned %d", ret); }
-        if(ret == 1) { abort("ERROR SIGUSR1 is disabled"); }
-
-        ret = sigismember(&oldset, SIGALRM);
-        if(ret <  0) { abort("ERROR sigismember returned %d", ret); }
-        if(ret == 0) { abort("ERROR SIGALRM is enabled"); }
-
-        ret = sigismember(&oldset, SIGTERM);
-        if(ret <  0) { abort("ERROR sigismember returned %d", ret); }
-        if(ret == 1) { abort("ERROR SIGTERM is disabled"); }
-}
-
-#ifdef __CFA_WITH_VERIFY__
-bool __cfaabi_dbg_in_kernel() {
-        return !kernelTLS.preemption_state.enabled;
-}
-#endif
 
 // Local Variables: //