Ignore:
Timestamp:
May 11, 2020, 1:53:29 PM (5 years ago)
Author:
Thierry Delisle <tdelisle@…>
Branches:
ADT, arm-eh, ast-experimental, enum, forall-pointer-decay, jacob/cs343-translation, master, new-ast, new-ast-unique-expr, pthread-emulation, qualifiedEnum
Children:
504a7dc
Parents:
b7d6a36 (diff), a7b486b (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the (diff) links above to see all the changes relative to each parent.
Message:

Merge branch 'master' into relaxed_ready

Location:
libcfa/src/concurrency
Files:
3 added
19 edited

Legend:

Unmodified
Added
Removed

  • libcfa/src/concurrency/CtxSwitch-arm.S

    rb7d6a36 r6a490b2  
    1313        .text
    1414        .align  2
    15         .global CtxSwitch
    16         .type   CtxSwitch, %function
     15        .global __cfactx_switch
     16        .type   __cfactx_switch, %function
    1717
    18 CtxSwitch:
     18__cfactx_switch:
    1919        @ save callee-saved registers: r4-r8, r10, r11, r13(sp) (plus r9 depending on platform specification)
    2020        @ I've seen reference to 31 registers on 64-bit, if this is the case, more need to be saved
    … …  
    5252        mov r15, r14
    5353        #endif // R9_SPECIAL
    54        
     54
    5555        .text
    5656        .align  2
    57         .global CtxInvokeStub
    58         .type   CtxInvokeStub, %function
     57        .global __cfactx_invoke_stub
     58        .type   __cfactx_invoke_stub, %function
    5959
    60 CtxInvokeStub:
     60__cfactx_invoke_stub:
    6161        ldmfd r13!, {r0-r1}
    6262        mov r15, r1

  • libcfa/src/concurrency/CtxSwitch-i386.S

    rb7d6a36 r6a490b2  
    4343        .text
    4444        .align 2
    45         .globl CtxSwitch
    46         .type  CtxSwitch, @function
    47 CtxSwitch:
     45        .globl __cfactx_switch
     46        .type  __cfactx_switch, @function
     47__cfactx_switch:
    4848
    4949        // Copy the "from" context argument from the stack to register eax
    … …  
    8383
    8484        ret
    85         .size  CtxSwitch, .-CtxSwitch
     85        .size  __cfactx_switch, .-__cfactx_switch
    8686
    8787// Local Variables: //

  • libcfa/src/concurrency/CtxSwitch-x86_64.S

    rb7d6a36 r6a490b2  
    4444        .text
    4545        .align 2
    46         .globl CtxSwitch
    47         .type  CtxSwitch, @function
    48 CtxSwitch:
     46        .globl __cfactx_switch
     47        .type  __cfactx_switch, @function
     48__cfactx_switch:
    4949
    5050        // Save volatile registers on the stack.
    … …  
    7777
    7878        ret
    79         .size  CtxSwitch, .-CtxSwitch
     79        .size  __cfactx_switch, .-__cfactx_switch
    8080
    8181//-----------------------------------------------------------------------------
    … …  
    8383        .text
    8484        .align 2
    85         .globl CtxInvokeStub
    86         .type    CtxInvokeStub, @function
    87 CtxInvokeStub:
     85        .globl __cfactx_invoke_stub
     86        .type    __cfactx_invoke_stub, @function
     87__cfactx_invoke_stub:
    8888        movq %rbx, %rdi
    8989        movq %r12, %rsi
    9090        jmp *%r13
    91         .size  CtxInvokeStub, .-CtxInvokeStub
     91        .size  __cfactx_invoke_stub, .-__cfactx_invoke_stub
    9292
    9393// Local Variables: //

  • libcfa/src/concurrency/alarm.cfa

    rb7d6a36 r6a490b2  
    4747//=============================================================================================
    4848
    49 void ?{}( alarm_node_t & this, thread_desc * thrd, Time alarm, Duration period ) with( this ) {
     49void ?{}( alarm_node_t & this, $thread * thrd, Time alarm, Duration period ) with( this ) {
    5050        this.thrd = thrd;
    5151        this.alarm = alarm;
    5252        this.period = period;
    53         next = 0;
    5453        set = false;
    5554        kernel_alarm = false;
    … …  
    6059        this.alarm = alarm;
    6160        this.period = period;
    62         next = 0;
    6361        set = false;
    6462        kernel_alarm = true;
    … …  
    7169}
    7270
    73 #if !defined(NDEBUG) && (defined(__CFA_DEBUG__) || defined(__CFA_VERIFY__))
    74 bool validate( alarm_list_t * this ) {
    75         alarm_node_t ** it = &this->head;
    76         while( (*it) ) {
    77                 it = &(*it)->next;
     71void insert( alarm_list_t * this, alarm_node_t * n ) {
     72        alarm_node_t * it = & (*this)`first;
     73        while( it && (n->alarm > it->alarm) ) {
     74                it = & (*it)`next;
     75        }
     76        if ( it ) {
     77                insert_before( *it, *n );
     78        } else {
     79                insert_last(*this, *n);
    7880        }
    7981
    80         return it == this->tail;
    81 }
    82 #endif
    83 
    84 static inline void insert_at( alarm_list_t * this, alarm_node_t * n, __alarm_it_t p ) {
    85         verify( !n->next );
    86         if( p == this->tail ) {
    87                 this->tail = &n->next;
    88         }
    89         else {
    90                 n->next = *p;
    91         }
    92         *p = n;
    93 
    94         verify( validate( this ) );
    95 }
    96 
    97 void insert( alarm_list_t * this, alarm_node_t * n ) {
    98         alarm_node_t ** it = &this->head;
    99         while( (*it) && (n->alarm > (*it)->alarm) ) {
    100                 it = &(*it)->next;
    101         }
    102 
    103         insert_at( this, n, it );
    104 
    105         verify( validate( this ) );
     82        verify( validate( *this ) );
    10683}
    10784
    10885alarm_node_t * pop( alarm_list_t * this ) {
    109         alarm_node_t * head = this->head;
     86        verify( validate( *this ) );
     87        alarm_node_t * head = & (*this)`first;
    11088        if( head ) {
    111                 this->head = head->next;
    112                 if( !head->next ) {
    113                         this->tail = &this->head;
    114                 }
    115                 head->next = 0p;
     89                remove(*head);
    11690        }
    117         verify( validate( this ) );
     91        verify( validate( *this ) );
    11892        return head;
    11993}
    12094
    121 static inline void remove_at( alarm_list_t * this, alarm_node_t * n, __alarm_it_t it ) {
    122         verify( it );
    123         verify( (*it) == n );
    124 
    125         (*it) = n->next;
    126         if( !n-> next ) {
    127                 this->tail = it;
    128         }
    129         n->next = 0p;
    130 
    131         verify( validate( this ) );
    132 }
    133 
    134 static inline void remove( alarm_list_t * this, alarm_node_t * n ) {
    135         alarm_node_t ** it = &this->head;
    136         while( (*it) && (*it) != n ) {
    137                 it = &(*it)->next;
    138         }
    139 
    140         verify( validate( this ) );
    141 
    142         if( *it ) { remove_at( this, n, it ); }
    143 
    144         verify( validate( this ) );
    145 }
    146 
    14795void register_self( alarm_node_t * this ) {
    148         alarm_list_t * alarms = &event_kernel->alarms;
     96        alarm_list_t & alarms = event_kernel->alarms;
    14997
    15098        disable_interrupts();
    … …  
    152100        {
    153101                verify( validate( alarms ) );
    154                 bool first = !alarms->head;
     102                bool first = ! & alarms`first;
    155103
    156                 insert( alarms, this );
     104                insert( &alarms, this );
    157105                if( first ) {
    158                         __kernel_set_timer( alarms->head->alarm - __kernel_get_time() );
     106                        __kernel_set_timer( alarms`first.alarm - __kernel_get_time() );
    159107                }
    160108        }
    … …  
    168116        lock( event_kernel->lock __cfaabi_dbg_ctx2 );
    169117        {
    170                 verify( validate( &event_kernel->alarms ) );
    171                 remove( &event_kernel->alarms, this );
     118                verify( validate( event_kernel->alarms ) );
     119                remove( *this );
    172120        }
    173121        unlock( event_kernel->lock );
    … …  
    176124}
    177125
     126//=============================================================================================
     127// Utilities
     128//=============================================================================================
     129
     130void sleep( Duration duration ) {
     131        alarm_node_t node = { active_thread(), __kernel_get_time() + duration, 0`s };
     132
     133        register_self( &node );
     134        park( __cfaabi_dbg_ctx );
     135
     136        /* paranoid */ verify( !node.set );
     137        /* paranoid */ verify( & node`next == 0p );
     138        /* paranoid */ verify( & node`prev == 0p );
     139}
     140
    178141// Local Variables: //
    179142// mode: c //

  • libcfa/src/concurrency/alarm.hfa

    rb7d6a36 r6a490b2  
    2323#include "time.hfa"
    2424
    25 struct thread_desc;
     25#include <containers/list.hfa>
     26
     27struct $thread;
    2628struct processor;
    2729
    … …  
    4042        Time alarm;                             // time when alarm goes off
    4143        Duration period;                        // if > 0 => period of alarm
    42         alarm_node_t * next;            // intrusive link list field
     44
     45        DLISTED_MGD_IMPL_IN(alarm_node_t)
    4346
    4447        union {
    45                 thread_desc * thrd;     // thrd who created event
     48                $thread * thrd; // thrd who created event
    4649                processor * proc;               // proc who created event
    4750        };
    … …  
    5053        bool kernel_alarm       :1;             // true if this is not a user defined alarm
    5154};
     55DLISTED_MGD_IMPL_OUT(alarm_node_t)
    5256
    53 typedef alarm_node_t ** __alarm_it_t;
    54 
    55 void ?{}( alarm_node_t & this, thread_desc * thrd, Time alarm, Duration period );
     57void ?{}( alarm_node_t & this, $thread * thrd, Time alarm, Duration period );
    5658void ?{}( alarm_node_t & this, processor   * proc, Time alarm, Duration period );
    5759void ^?{}( alarm_node_t & this );
    5860
    59 struct alarm_list_t {
    60         alarm_node_t * head;
    61         __alarm_it_t tail;
    62 };
    63 
    64 static inline void ?{}( alarm_list_t & this ) with( this ) {
    65         head = 0;
    66         tail = &head;
    67 }
     61typedef dlist(alarm_node_t, alarm_node_t) alarm_list_t;
    6862
    6963void insert( alarm_list_t * this, alarm_node_t * n );

  • libcfa/src/concurrency/coroutine.cfa

    rb7d6a36 r6a490b2  
    3737
    3838extern "C" {
    39         void _CtxCoroutine_Unwind(struct _Unwind_Exception * storage, struct coroutine_desc *) __attribute__ ((__noreturn__));
     39        void _CtxCoroutine_Unwind(struct _Unwind_Exception * storage, struct $coroutine *) __attribute__ ((__noreturn__));
    4040        static void _CtxCoroutine_UnwindCleanup(_Unwind_Reason_Code, struct _Unwind_Exception *) __attribute__ ((__noreturn__));
    4141        static void _CtxCoroutine_UnwindCleanup(_Unwind_Reason_Code, struct _Unwind_Exception *) {
    … …  
    8989}
    9090
    91 void ?{}( coroutine_desc & this, const char name[], void * storage, size_t storageSize ) with( this ) {
     91void ?{}( $coroutine & this, const char name[], void * storage, size_t storageSize ) with( this ) {
    9292        (this.context){0p, 0p};
    9393        (this.stack){storage, storageSize};
    … …  
    9999}
    100100
    101 void ^?{}(coroutine_desc& this) {
     101void ^?{}($coroutine& this) {
    102102        if(this.state != Halted && this.state != Start && this.state != Primed) {
    103                 coroutine_desc * src = TL_GET( this_thread )->curr_cor;
    104                 coroutine_desc * dst = &this;
     103                $coroutine * src = TL_GET( this_thread )->curr_cor;
     104                $coroutine * dst = &this;
    105105
    106106                struct _Unwind_Exception storage;
    … …  
    115115                }
    116116
    117                 CoroutineCtxSwitch( src, dst );
     117                $ctx_switch( src, dst );
    118118        }
    119119}
    … …  
    123123forall(dtype T | is_coroutine(T))
    124124void prime(T& cor) {
    125         coroutine_desc* this = get_coroutine(cor);
     125        $coroutine* this = get_coroutine(cor);
    126126        assert(this->state == Start);
    127127
    … …  
    187187// is not inline (We can't inline Cforall in C)
    188188extern "C" {
    189         void __leave_coroutine( struct coroutine_desc * src ) {
    190                 coroutine_desc * starter = src->cancellation != 0 ? src->last : src->starter;
     189        void __cfactx_cor_leave( struct $coroutine * src ) {
     190                $coroutine * starter = src->cancellation != 0 ? src->last : src->starter;
    191191
    192192                src->state = Halted;
    … …  
    201201                        src->name, src, starter->name, starter );
    202202
    203                 CoroutineCtxSwitch( src, starter );
    204         }
    205 
    206         struct coroutine_desc * __finish_coroutine(void) {
    207                 struct coroutine_desc * cor = kernelTLS.this_thread->curr_cor;
     203                $ctx_switch( src, starter );
     204        }
     205
     206        struct $coroutine * __cfactx_cor_finish(void) {
     207                struct $coroutine * cor = kernelTLS.this_thread->curr_cor;
    208208
    209209                if(cor->state == Primed) {
    210                         suspend();
     210                        __cfactx_suspend();
    211211                }
    212212

  • libcfa/src/concurrency/coroutine.hfa

    rb7d6a36 r6a490b2  
    2525trait is_coroutine(dtype T) {
    2626      void main(T & this);
    27       coroutine_desc * get_coroutine(T & this);
     27      $coroutine * get_coroutine(T & this);
    2828};
    2929
    30 #define DECL_COROUTINE(X) static inline coroutine_desc* get_coroutine(X& this) { return &this.__cor; } void main(X& this)
     30#define DECL_COROUTINE(X) static inline $coroutine* get_coroutine(X& this) { return &this.__cor; } void main(X& this)
    3131
    3232//-----------------------------------------------------------------------------
    … …  
    3535// void ^?{}( coStack_t & this );
    3636
    37 void ?{}( coroutine_desc & this, const char name[], void * storage, size_t storageSize );
    38 void ^?{}( coroutine_desc & this );
     37void  ?{}( $coroutine & this, const char name[], void * storage, size_t storageSize );
     38void ^?{}( $coroutine & this );
    3939
    40 static inline void ?{}( coroutine_desc & this)                                       { this{ "Anonymous Coroutine", 0p, 0 }; }
    41 static inline void ?{}( coroutine_desc & this, size_t stackSize)                     { this{ "Anonymous Coroutine", 0p, stackSize }; }
    42 static inline void ?{}( coroutine_desc & this, void * storage, size_t storageSize )  { this{ "Anonymous Coroutine", storage, storageSize }; }
    43 static inline void ?{}( coroutine_desc & this, const char name[])                    { this{ name, 0p, 0 }; }
    44 static inline void ?{}( coroutine_desc & this, const char name[], size_t stackSize ) { this{ name, 0p, stackSize }; }
     40static inline void ?{}( $coroutine & this)                                       { this{ "Anonymous Coroutine", 0p, 0 }; }
     41static inline void ?{}( $coroutine & this, size_t stackSize)                     { this{ "Anonymous Coroutine", 0p, stackSize }; }
     42static inline void ?{}( $coroutine & this, void * storage, size_t storageSize )  { this{ "Anonymous Coroutine", storage, storageSize }; }
     43static inline void ?{}( $coroutine & this, const char name[])                    { this{ name, 0p, 0 }; }
     44static inline void ?{}( $coroutine & this, const char name[], size_t stackSize ) { this{ name, 0p, stackSize }; }
    4545
    4646//-----------------------------------------------------------------------------
    4747// Public coroutine API
    48 static inline void suspend(void);
    49 
    50 forall(dtype T | is_coroutine(T))
    51 static inline T & resume(T & cor);
    52 
    5348forall(dtype T | is_coroutine(T))
    5449void prime(T & cor);
    5550
    56 static inline struct coroutine_desc * active_coroutine() { return TL_GET( this_thread )->curr_cor; }
     51static inline struct $coroutine * active_coroutine() { return TL_GET( this_thread )->curr_cor; }
    5752
    5853//-----------------------------------------------------------------------------
    … …  
    6156// Start coroutine routines
    6257extern "C" {
    63         void CtxInvokeCoroutine(void (*main)(void *), void * this);
     58        void __cfactx_invoke_coroutine(void (*main)(void *), void * this);
    6459
    6560        forall(dtype T)
    66         void CtxStart(void (*main)(T &), struct coroutine_desc * cor, T & this, void (*invoke)(void (*main)(void *), void *));
     61        void __cfactx_start(void (*main)(T &), struct $coroutine * cor, T & this, void (*invoke)(void (*main)(void *), void *));
    6762
    68         extern void _CtxCoroutine_Unwind(struct _Unwind_Exception * storage, struct coroutine_desc *) __attribute__ ((__noreturn__));
     63        extern void __cfactx_coroutine_unwind(struct _Unwind_Exception * storage, struct $coroutine *) __attribute__ ((__noreturn__));
    6964
    70         extern void CtxSwitch( struct __stack_context_t * from, struct __stack_context_t * to ) asm ("CtxSwitch");
     65        extern void __cfactx_switch( struct __stack_context_t * from, struct __stack_context_t * to ) asm ("__cfactx_switch");
    7166}
    7267
    7368// Private wrappers for context switch and stack creation
    7469// Wrapper for co
    75 static inline void CoroutineCtxSwitch(coroutine_desc* src, coroutine_desc* dst) {
     70static inline void $ctx_switch( $coroutine * src, $coroutine * dst ) __attribute__((nonnull (1, 2))) {
    7671        // set state of current coroutine to inactive
    77         src->state = src->state == Halted ? Halted : Inactive;
     72        src->state = src->state == Halted ? Halted : Blocked;
    7873
    7974        // set new coroutine that task is executing
    … …  
    8277        // context switch to specified coroutine
    8378        verify( dst->context.SP );
    84         CtxSwitch( &src->context, &dst->context );
    85         // when CtxSwitch returns we are back in the src coroutine
     79        __cfactx_switch( &src->context, &dst->context );
     80        // when __cfactx_switch returns we are back in the src coroutine
    8681
    8782        // set state of new coroutine to active
    … …  
    8984
    9085        if( unlikely(src->cancellation != 0p) ) {
    91                 _CtxCoroutine_Unwind(src->cancellation, src);
     86                __cfactx_coroutine_unwind(src->cancellation, src);
    9287        }
    9388}
    … …  
    9691
    9792// Suspend implementation inlined for performance
    98 static inline void suspend(void) {
    99         // optimization : read TLS once and reuse it
    100         // Safety note: this is preemption safe since if
    101         // preemption occurs after this line, the pointer
    102         // will also migrate which means this value will
    103         // stay in syn with the TLS
    104         coroutine_desc * src = TL_GET( this_thread )->curr_cor;
     93extern "C" {
     94        static inline void __cfactx_suspend(void) {
     95                // optimization : read TLS once and reuse it
     96                // Safety note: this is preemption safe since if
     97                // preemption occurs after this line, the pointer
     98                // will also migrate which means this value will
     99                // stay in syn with the TLS
     100                $coroutine * src = TL_GET( this_thread )->curr_cor;
    105101
    106         assertf( src->last != 0,
    107                 "Attempt to suspend coroutine \"%.256s\" (%p) that has never been resumed.\n"
    108                 "Possible cause is a suspend executed in a member called by a coroutine user rather than by the coroutine main.",
    109                 src->name, src );
    110         assertf( src->last->state != Halted,
    111                 "Attempt by coroutine \"%.256s\" (%p) to suspend back to terminated coroutine \"%.256s\" (%p).\n"
    112                 "Possible cause is terminated coroutine's main routine has already returned.",
    113                 src->name, src, src->last->name, src->last );
     102                assertf( src->last != 0,
     103                        "Attempt to suspend coroutine \"%.256s\" (%p) that has never been resumed.\n"
     104                        "Possible cause is a suspend executed in a member called by a coroutine user rather than by the coroutine main.",
     105                        src->name, src );
     106                assertf( src->last->state != Halted,
     107                        "Attempt by coroutine \"%.256s\" (%p) to suspend back to terminated coroutine \"%.256s\" (%p).\n"
     108                        "Possible cause is terminated coroutine's main routine has already returned.",
     109                        src->name, src, src->last->name, src->last );
    114110
    115         CoroutineCtxSwitch( src, src->last );
     111                $ctx_switch( src, src->last );
     112        }
    116113}
    117114
    … …  
    124121        // will also migrate which means this value will
    125122        // stay in syn with the TLS
    126         coroutine_desc * src = TL_GET( this_thread )->curr_cor;
    127         coroutine_desc * dst = get_coroutine(cor);
     123        $coroutine * src = TL_GET( this_thread )->curr_cor;
     124        $coroutine * dst = get_coroutine(cor);
    128125
    129126        if( unlikely(dst->context.SP == 0p) ) {
    130127                TL_GET( this_thread )->curr_cor = dst;
    131128                __stack_prepare(&dst->stack, 65000);
    132                 CtxStart(main, dst, cor, CtxInvokeCoroutine);
     129                __cfactx_start(main, dst, cor, __cfactx_invoke_coroutine);
    133130                TL_GET( this_thread )->curr_cor = src;
    134131        }
    … …  
    147144
    148145        // always done for performance testing
    149         CoroutineCtxSwitch( src, dst );
     146        $ctx_switch( src, dst );
    150147
    151148        return cor;
    152149}
    153150
    154 static inline void resume(coroutine_desc * dst) {
     151static inline void resume( $coroutine * dst ) __attribute__((nonnull (1))) {
    155152        // optimization : read TLS once and reuse it
    156153        // Safety note: this is preemption safe since if
    … …  
    158155        // will also migrate which means this value will
    159156        // stay in syn with the TLS
    160         coroutine_desc * src = TL_GET( this_thread )->curr_cor;
     157        $coroutine * src = TL_GET( this_thread )->curr_cor;
    161158
    162159        // not resuming self ?
    … …  
    172169
    173170        // always done for performance testing
    174         CoroutineCtxSwitch( src, dst );
     171        $ctx_switch( src, dst );
    175172}
    176173

  • libcfa/src/concurrency/invoke.c

    rb7d6a36 r6a490b2  
    2929// Called from the kernel when starting a coroutine or task so must switch back to user mode.
    3030
    31 extern void __leave_coroutine ( struct coroutine_desc * );
    32 extern struct coroutine_desc * __finish_coroutine(void);
    33 extern void __leave_thread_monitor();
     31extern struct $coroutine * __cfactx_cor_finish(void);
     32extern void __cfactx_cor_leave ( struct $coroutine * );
     33extern void __cfactx_thrd_leave();
     34
    3435extern void disable_interrupts() OPTIONAL_THREAD;
    3536extern void enable_interrupts( __cfaabi_dbg_ctx_param );
    3637
    37 void CtxInvokeCoroutine(
     38void __cfactx_invoke_coroutine(
    3839        void (*main)(void *),
    3940        void *this
    4041) {
    4142        // Finish setting up the coroutine by setting its state
    42         struct coroutine_desc * cor = __finish_coroutine();
     43        struct $coroutine * cor = __cfactx_cor_finish();
    4344
    4445        // Call the main of the coroutine
    … …  
    4647
    4748        //Final suspend, should never return
    48         __leave_coroutine( cor );
     49        __cfactx_cor_leave( cor );
    4950        __cabi_abort( "Resumed dead coroutine" );
    5051}
    5152
    52 static _Unwind_Reason_Code _CtxCoroutine_UnwindStop(
     53static _Unwind_Reason_Code __cfactx_coroutine_unwindstop(
    5354        __attribute((__unused__)) int version,
    5455        _Unwind_Action actions,
    … …  
    6162                // We finished unwinding the coroutine,
    6263                // leave it
    63                 __leave_coroutine( param );
     64                __cfactx_cor_leave( param );
    6465                __cabi_abort( "Resumed dead coroutine" );
    6566        }
    … …  
    6970}
    7071
    71 void _CtxCoroutine_Unwind(struct _Unwind_Exception * storage, struct coroutine_desc * cor) __attribute__ ((__noreturn__));
    72 void _CtxCoroutine_Unwind(struct _Unwind_Exception * storage, struct coroutine_desc * cor) {
    73         _Unwind_Reason_Code ret = _Unwind_ForcedUnwind( storage, _CtxCoroutine_UnwindStop, cor );
     72void __cfactx_coroutine_unwind(struct _Unwind_Exception * storage, struct $coroutine * cor) __attribute__ ((__noreturn__));
     73void __cfactx_coroutine_unwind(struct _Unwind_Exception * storage, struct $coroutine * cor) {
     74        _Unwind_Reason_Code ret = _Unwind_ForcedUnwind( storage, __cfactx_coroutine_unwindstop, cor );
    7475        printf("UNWIND ERROR %d after force unwind\n", ret);
    7576        abort();
    7677}
    7778
    78 void CtxInvokeThread(
     79void __cfactx_invoke_thread(
    7980        void (*main)(void *),
    8081        void *this
    … …  
    9394        // The order of these 4 operations is very important
    9495        //Final suspend, should never return
    95         __leave_thread_monitor();
     96        __cfactx_thrd_leave();
    9697        __cabi_abort( "Resumed dead thread" );
    9798}
    9899
    99 void CtxStart(
     100void __cfactx_start(
    100101        void (*main)(void *),
    101         struct coroutine_desc * cor,
     102        struct $coroutine * cor,
    102103        void *this,
    103104        void (*invoke)(void *)
    … …  
    139140
    140141        fs->dummyReturn = NULL;
    141         fs->rturn = CtxInvokeStub;
     142        fs->rturn = __cfactx_invoke_stub;
    142143        fs->fixedRegisters[0] = main;
    143144        fs->fixedRegisters[1] = this;
    … …  
    157158        struct FakeStack *fs = (struct FakeStack *)cor->context.SP;
    158159
    159         fs->intRegs[8] = CtxInvokeStub;
     160        fs->intRegs[8] = __cfactx_invoke_stub;
    160161        fs->arg[0] = this;
    161162        fs->arg[1] = invoke;

  • libcfa/src/concurrency/invoke.h

    rb7d6a36 r6a490b2  
    4747        extern "Cforall" {
    4848                extern __attribute__((aligned(128))) thread_local struct KernelThreadData {
    49                         struct thread_desc    * volatile this_thread;
     49                        struct $thread    * volatile this_thread;
    5050                        struct processor      * volatile this_processor;
    5151
    … …  
    9292        };
    9393
    94         enum coroutine_state { Halted, Start, Inactive, Active, Primed };
    95 
    96         struct coroutine_desc {
    97                 // context that is switch during a CtxSwitch
     94        enum coroutine_state { Halted, Start, Primed, Blocked, Ready, Active, Rerun };
     95        enum __Preemption_Reason { __NO_PREEMPTION, __ALARM_PREEMPTION, __POLL_PREEMPTION, __MANUAL_PREEMPTION };
     96
     97        struct $coroutine {
     98                // context that is switch during a __cfactx_switch
    9899                struct __stack_context_t context;
    99100
    … …  
    108109
    109110                // first coroutine to resume this one
    110                 struct coroutine_desc * starter;
     111                struct $coroutine * starter;
    111112
    112113                // last coroutine to resume this one
    113                 struct coroutine_desc * last;
     114                struct $coroutine * last;
    114115
    115116                // If non-null stack must be unwound with this exception
    … …  
    117118
    118119        };
     120
     121        static inline struct __stack_t * __get_stack( struct $coroutine * cor ) { return (struct __stack_t*)(((uintptr_t)cor->stack.storage) & ((uintptr_t)-2)); }
    119122
    120123        // struct which calls the monitor is accepting
    … …  
    127130        };
    128131
    129         struct monitor_desc {
     132        struct $monitor {
    130133                // spinlock to protect internal data
    131134                struct __spinlock_t lock;
    132135
    133136                // current owner of the monitor
    134                 struct thread_desc * owner;
     137                struct $thread * owner;
    135138
    136139                // queue of threads that are blocked waiting for the monitor
    137                 __queue_t(struct thread_desc) entry_queue;
     140                __queue_t(struct $thread) entry_queue;
    138141
    139142                // stack of conditions to run next once we exit the monitor
    … …  
    152155        struct __monitor_group_t {
    153156                // currently held monitors
    154                 __cfa_anonymous_object( __small_array_t(monitor_desc*) );
     157                __cfa_anonymous_object( __small_array_t($monitor*) );
    155158
    156159                // last function that acquired monitors
    … …  
    161164        // instrusive link field for threads
    162165        struct __thread_desc_link {
    163                 struct thread_desc * next;
    164                 struct thread_desc * prev;
     166                struct $thread * next;
     167                struct $thread * prev;
    165168                unsigned long long ts;
    166169        };
    167170
    168         struct thread_desc {
     171        struct $thread {
    169172                // Core threading fields
    170                 // context that is switch during a CtxSwitch
     173                // context that is switch during a __cfactx_switch
    171174                struct __stack_context_t context;
    172175
    173176                // current execution status for coroutine
    174                 enum coroutine_state state;
     177                volatile int state;
     178                enum __Preemption_Reason preempted;
    175179
    176180                //SKULLDUGGERY errno is not save in the thread data structure because returnToKernel appears to be the only function to require saving and restoring it
    177181
    178182                // coroutine body used to store context
    179                 struct coroutine_desc  self_cor;
     183                struct $coroutine  self_cor;
    180184
    181185                // current active context
    182                 struct coroutine_desc * curr_cor;
     186                struct $coroutine * curr_cor;
    183187
    184188                // monitor body used for mutual exclusion
    185                 struct monitor_desc    self_mon;
     189                struct $monitor    self_mon;
    186190
    187191                // pointer to monitor with sufficient lifetime for current monitors
    188                 struct monitor_desc *  self_mon_p;
     192                struct $monitor *  self_mon_p;
    189193
    190194                // pointer to the cluster on which the thread is running
    … …  
    199203
    200204                struct {
    201                         struct thread_desc * next;
    202                         struct thread_desc * prev;
     205                        struct $thread * next;
     206                        struct $thread * prev;
    203207                } node;
    204         };
     208
     209                #ifdef __CFA_DEBUG__
     210                        // previous function to park/unpark the thread
     211                        const char * park_caller;
     212                        enum coroutine_state park_result;
     213                        bool park_stale;
     214                        const char * unpark_caller;
     215                        enum coroutine_state unpark_result;
     216                        bool unpark_stale;
     217                #endif
     218        };
     219
     220        #ifdef __CFA_DEBUG__
     221                void __cfaabi_dbg_record_thrd($thread & this, bool park, const char prev_name[]);
     222        #else
     223                #define __cfaabi_dbg_record_thrd(x, y, z)
     224        #endif
    205225
    206226        #ifdef __cforall
    207227        extern "Cforall" {
    208                 static inline thread_desc *& get_next( thread_desc & this ) {
     228
     229                static inline $thread *& get_next( $thread & this ) __attribute__((const)) {
    209230                        return this.link.next;
    210231                }
    211232
    212                 static inline [thread_desc *&, thread_desc *& ] __get( thread_desc & this ) {
     233                static inline [$thread *&, $thread *& ] __get( $thread & this ) __attribute__((const)) {
    213234                        return this.node.[next, prev];
    214235                }
    … …  
    220241                }
    221242
    222                 static inline void ?{}(__monitor_group_t & this, struct monitor_desc ** data, __lock_size_t size, fptr_t func) {
     243                static inline void ?{}(__monitor_group_t & this, struct $monitor ** data, __lock_size_t size, fptr_t func) {
    223244                        (this.data){data};
    224245                        (this.size){size};
    … …  
    226247                }
    227248
    228                 static inline bool ?==?( const __monitor_group_t & lhs, const __monitor_group_t & rhs ) {
     249                static inline bool ?==?( const __monitor_group_t & lhs, const __monitor_group_t & rhs ) __attribute__((const)) {
    229250                        if( (lhs.data != 0) != (rhs.data != 0) ) return false;
    230251                        if( lhs.size != rhs.size ) return false;
    … …  
    260281
    261282        // assembler routines that performs the context switch
    262         extern void CtxInvokeStub( void );
    263         extern void CtxSwitch( struct __stack_context_t * from, struct __stack_context_t * to ) asm ("CtxSwitch");
     283        extern void __cfactx_invoke_stub( void );
     284        extern void __cfactx_switch( struct __stack_context_t * from, struct __stack_context_t * to ) asm ("__cfactx_switch");
    264285        // void CtxStore ( void * this ) asm ("CtxStore");
    265286        // void CtxRet   ( void * dst  ) asm ("CtxRet");

  • libcfa/src/concurrency/kernel.cfa

    rb7d6a36 r6a490b2  
    1515
    1616#define __cforall_thread__
     17// #define __CFA_DEBUG_PRINT_RUNTIME_CORE__
    1718
    1819//C Includes
    … …  
    4041#include "invoke.h"
    4142
     43
    4244//-----------------------------------------------------------------------------
    4345// Some assembly required
    … …  
    110112//-----------------------------------------------------------------------------
    111113//Start and stop routine for the kernel, declared first to make sure they run first
    112 static void kernel_startup(void)  __attribute__(( constructor( STARTUP_PRIORITY_KERNEL ) ));
    113 static void kernel_shutdown(void) __attribute__(( destructor ( STARTUP_PRIORITY_KERNEL ) ));
     114static void __kernel_startup (void) __attribute__(( constructor( STARTUP_PRIORITY_KERNEL ) ));
     115static void __kernel_shutdown(void) __attribute__(( destructor ( STARTUP_PRIORITY_KERNEL ) ));
     116
     117//-----------------------------------------------------------------------------
     118// Kernel Scheduling logic
     119static $thread * __next_thread(cluster * this);
     120static void __run_thread(processor * this, $thread * dst);
     121static $thread * __halt(processor * this);
     122static bool __wake_one(cluster * cltr, bool was_empty);
     123static bool __wake_proc(processor *);
    114124
    115125//-----------------------------------------------------------------------------
    … …  
    117127KERNEL_STORAGE(cluster,         mainCluster);
    118128KERNEL_STORAGE(processor,       mainProcessor);
    119 KERNEL_STORAGE(thread_desc,     mainThread);
     129KERNEL_STORAGE($thread, mainThread);
    120130KERNEL_STORAGE(__stack_t,       mainThreadCtx);
    121131
    122132cluster     * mainCluster;
    123133processor   * mainProcessor;
    124 thread_desc * mainThread;
     134$thread * mainThread;
    125135
    126136extern "C" {
    … …  
    164174// Main thread construction
    165175
    166 void ?{}( coroutine_desc & this, current_stack_info_t * info) with( this ) {
     176void ?{}( $coroutine & this, current_stack_info_t * info) with( this ) {
    167177        stack.storage = info->storage;
    168178        with(*stack.storage) {
    … …  
    179189}
    180190
    181 void ?{}( thread_desc & this, current_stack_info_t * info) with( this ) {
     191void ?{}( $thread & this, current_stack_info_t * info) with( this ) {
    182192        state = Start;
    183193        self_cor{ info };
    … …  
    209219}
    210220
    211 static void start(processor * this);
     221static void * __invoke_processor(void * arg);
     222
    212223void ?{}(processor & this, const char name[], cluster & cltr) with( this ) {
    213224        this.name = name;
    … …  
    215226        id = -1u;
    216227        terminated{ 0 };
     228        destroyer = 0p;
    217229        do_terminate = false;
    218230        preemption_alarm = 0p;
    … …  
    220232        runner.proc = &this;
    221233
    222         idleLock{};
    223 
    224         start( &this );
     234        idle{};
     235
     236        __cfadbg_print_safe(runtime_core, "Kernel : Starting core %p\n", &this);
     237
     238        this.stack = __create_pthread( &this.kernel_thread, __invoke_processor, (void *)&this );
     239
     240        __cfadbg_print_safe(runtime_core, "Kernel : core %p created\n", &this);
    225241}
    226242
    227243void ^?{}(processor & this) with( this ){
    228244        if( ! __atomic_load_n(&do_terminate, __ATOMIC_ACQUIRE) ) {
    229                 __cfaabi_dbg_print_safe("Kernel : core %p signaling termination\n", &this);
     245                __cfadbg_print_safe(runtime_core, "Kernel : core %p signaling termination\n", &this);
    230246
    231247                __atomic_store_n(&do_terminate, true, __ATOMIC_RELAXED);
    232                 wake( &this );
     248                __wake_proc( &this );
    233249
    234250                P( terminated );
    … …  
    236252        }
    237253
    238         pthread_join( kernel_thread, 0p );
     254        int err = pthread_join( kernel_thread, 0p );
     255        if( err != 0 ) abort("KERNEL ERROR: joining processor %p caused error %s\n", &this, strerror(err));
     256
    239257        free( this.stack );
    240258}
    241259
    242 void ?{}(cluster & this, const char name[], Duration preemption_rate) with( this ) {
     260void ?{}(cluster & this, const char name[], Duration preemption_rate, int io_flags) with( this ) {
    243261        this.name = name;
    244262        this.preemption_rate = preemption_rate;
    … …  
    246264        ready_lock{};
    247265
     266        #if !defined(__CFA_NO_STATISTICS__)
     267                print_stats = false;
     268        #endif
     269
     270        procs{ __get };
    248271        idles{ __get };
    249272        threads{ __get };
    250273
     274        __kernel_io_startup( this, io_flags, &this == mainCluster );
     275
    251276        doregister(this);
    252277}
    253278
    254279void ^?{}(cluster & this) {
     280        __kernel_io_shutdown( this, &this == mainCluster );
     281
    255282        unregister(this);
    256283}
    … …  
    259286// Kernel Scheduling logic
    260287//=============================================================================================
    261 static void runThread(processor * this, thread_desc * dst);
    262 static void finishRunning(processor * this);
    263 static void halt(processor * this);
    264 
    265288//Main of the processor contexts
    266289void main(processorCtx_t & runner) {
    … …  
    272295        verify(this);
    273296
    274         __cfaabi_dbg_print_safe("Kernel : core %p starting\n", this);
     297        __cfadbg_print_safe(runtime_core, "Kernel : core %p starting\n", this);
    275298
    276299        // register the processor unless it's the main thread which is handled in the boot sequence
    … …  
    285308                preemption_scope scope = { this };
    286309
    287                 __cfaabi_dbg_print_safe("Kernel : core %p started\n", this);
    288 
    289                 thread_desc * readyThread = 0p;
     310                __cfadbg_print_safe(runtime_core, "Kernel : core %p started\n", this);
     311
     312                $thread * readyThread = 0p;
    290313                for( unsigned int spin_count = 0; ! __atomic_load_n(&this->do_terminate, __ATOMIC_SEQ_CST); spin_count++ ) {
    291                         readyThread = nextThread( this->cltr );
    292 
    293                         if(readyThread) {
    294                                 verify( ! kernelTLS.preemption_state.enabled );
    295 
    296                                 runThread(this, readyThread);
    297 
    298                                 verify( ! kernelTLS.preemption_state.enabled );
    299 
    300                                 //Some actions need to be taken from the kernel
    301                                 finishRunning(this);
    302 
    303                                 spin_count = 0;
    304                         } else {
    305                                 // spin(this, &spin_count);
    306                                 halt(this);
     314                        // Try to get the next thread
     315                        readyThread = __next_thread( this->cltr );
     316
     317                        // If no ready thread
     318                        if( readyThread == 0p ) {
     319                                // Block until a thread is ready
     320                                readyThread = __halt(this);
     321                        }
     322
     323                        // Check if we actually found a thread
     324                        if( readyThread ) {
     325                                /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
     326                                /* paranoid */ verifyf( readyThread->state == Ready || readyThread->preempted != __NO_PREEMPTION, "state : %d, preempted %d\n", readyThread->state, readyThread->preempted);
     327                                /* paranoid */ verifyf( readyThread->next == 0p, "Expected null got %p", readyThread->next );
     328
     329                                // We found a thread run it
     330                                __run_thread(this, readyThread);
     331
     332                                /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
    307333                        }
    308334                }
    309335
    310                 __cfaabi_dbg_print_safe("Kernel : core %p stopping\n", this);
     336                __cfadbg_print_safe(runtime_core, "Kernel : core %p stopping\n", this);
    311337        }
    312338
    313339        V( this->terminated );
    314 
    315340
    316341        // unregister the processor unless it's the main thread which is handled in the boot sequence
    … …  
    319344                unregister(this->cltr, this);
    320345        }
    321 
    322         __cfaabi_dbg_print_safe("Kernel : core %p terminated\n", this);
     346        else {
     347                // HACK : the coroutine context switch expects this_thread to be set
     348                // and it make sense for it to be set in all other cases except here
     349                // fake it
     350                kernelTLS.this_thread = mainThread;
     351        }
     352
     353        __cfadbg_print_safe(runtime_core, "Kernel : core %p terminated\n", this);
    323354
    324355        stats_tls_tally(this->cltr);
    … …  
    331362// runThread runs a thread by context switching
    332363// from the processor coroutine to the target thread
    333 static void runThread(processor * this, thread_desc * thrd_dst) {
    334         coroutine_desc * proc_cor = get_coroutine(this->runner);
    335 
    336         // Reset the terminating actions here
    337         this->finish.action_code = No_Action;
     364static void __run_thread(processor * this, $thread * thrd_dst) {
     365        $coroutine * proc_cor = get_coroutine(this->runner);
    338366
    339367        // Update global state
    340368        kernelTLS.this_thread = thrd_dst;
    341369
    342         // set state of processor coroutine to inactive and the thread to active
    343         proc_cor->state = proc_cor->state == Halted ? Halted : Inactive;
    344         thrd_dst->state = Active;
    345 
    346         // set context switch to the thread that the processor is executing
    347         verify( thrd_dst->context.SP );
    348         CtxSwitch( &proc_cor->context, &thrd_dst->context );
    349         // when CtxSwitch returns we are back in the processor coroutine
    350 
    351         // set state of processor coroutine to active and the thread to inactive
    352         thrd_dst->state = thrd_dst->state == Halted ? Halted : Inactive;
     370        // set state of processor coroutine to inactive
     371        verify(proc_cor->state == Active);
     372        proc_cor->state = Blocked;
     373
     374        // Actually run the thread
     375        RUNNING:  while(true) {
     376                if(unlikely(thrd_dst->preempted)) {
     377                        thrd_dst->preempted = __NO_PREEMPTION;
     378                        verify(thrd_dst->state == Active  || thrd_dst->state == Rerun);
     379                } else {
     380                        verify(thrd_dst->state == Blocked || thrd_dst->state == Ready); // Ready means scheduled normally, blocked means rerun
     381                        thrd_dst->state = Active;
     382                }
     383
     384                __cfaabi_dbg_debug_do(
     385                        thrd_dst->park_stale   = true;
     386                        thrd_dst->unpark_stale = true;
     387                )
     388
     389                /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
     390                /* paranoid */ verify( kernelTLS.this_thread == thrd_dst );
     391                /* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) < ((uintptr_t)__get_stack(thrd_dst->curr_cor)->base ) || thrd_dst->curr_cor == proc_cor, "ERROR : Destination $thread %p has been corrupted.\n StackPointer too small.\n", thrd_dst ); // add escape condition if we are setting up the processor
     392                /* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) > ((uintptr_t)__get_stack(thrd_dst->curr_cor)->limit) || thrd_dst->curr_cor == proc_cor, "ERROR : Destination $thread %p has been corrupted.\n StackPointer too large.\n", thrd_dst ); // add escape condition if we are setting up the processor
     393
     394                // set context switch to the thread that the processor is executing
     395                verify( thrd_dst->context.SP );
     396                __cfactx_switch( &proc_cor->context, &thrd_dst->context );
     397                // when __cfactx_switch returns we are back in the processor coroutine
     398
     399                /* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) > ((uintptr_t)__get_stack(thrd_dst->curr_cor)->limit), "ERROR : Destination $thread %p has been corrupted.\n StackPointer too large.\n", thrd_dst );
     400                /* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) < ((uintptr_t)__get_stack(thrd_dst->curr_cor)->base ), "ERROR : Destination $thread %p has been corrupted.\n StackPointer too small.\n", thrd_dst );
     401                /* paranoid */ verify( kernelTLS.this_thread == thrd_dst );
     402                /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
     403
     404
     405                // We just finished running a thread, there are a few things that could have happened.
     406                // 1 - Regular case : the thread has blocked and now one has scheduled it yet.
     407                // 2 - Racy case    : the thread has blocked but someone has already tried to schedule it.
     408                // 4 - Preempted
     409                // In case 1, we may have won a race so we can't write to the state again.
     410                // In case 2, we lost the race so we now own the thread.
     411
     412                if(unlikely(thrd_dst->preempted != __NO_PREEMPTION)) {
     413                        // The thread was preempted, reschedule it and reset the flag
     414                        __schedule_thread( thrd_dst );
     415                        break RUNNING;
     416                }
     417
     418                // set state of processor coroutine to active and the thread to inactive
     419                static_assert(sizeof(thrd_dst->state) == sizeof(int));
     420                enum coroutine_state old_state = __atomic_exchange_n(&thrd_dst->state, Blocked, __ATOMIC_SEQ_CST);
     421                __cfaabi_dbg_debug_do( thrd_dst->park_result = old_state; )
     422                switch(old_state) {
     423                        case Halted:
     424                                // The thread has halted, it should never be scheduled/run again, leave it back to Halted and move on
     425                                thrd_dst->state = Halted;
     426
     427                                // We may need to wake someone up here since
     428                                unpark( this->destroyer __cfaabi_dbg_ctx2 );
     429                                this->destroyer = 0p;
     430                                break RUNNING;
     431                        case Active:
     432                                // This is case 1, the regular case, nothing more is needed
     433                                break RUNNING;
     434                        case Rerun:
     435                                // This is case 2, the racy case, someone tried to run this thread before it finished blocking
     436                                // In this case, just run it again.
     437                                continue RUNNING;
     438                        default:
     439                                // This makes no sense, something is wrong abort
     440                                abort("Finished running a thread that was Blocked/Start/Primed %d\n", old_state);
     441                }
     442        }
     443
     444        // Just before returning to the processor, set the processor coroutine to active
    353445        proc_cor->state = Active;
     446        kernelTLS.this_thread = 0p;
    354447}
    355448
    356449// KERNEL_ONLY
    357 static void returnToKernel() {
    358         coroutine_desc * proc_cor = get_coroutine(kernelTLS.this_processor->runner);
    359         thread_desc * thrd_src = kernelTLS.this_thread;
    360 
    361         // set state of current coroutine to inactive
    362         thrd_src->state = thrd_src->state == Halted ? Halted : Inactive;
    363         proc_cor->state = Active;
    364         int local_errno = *__volatile_errno();
    365         #if defined( __i386 ) || defined( __x86_64 )
    366                 __x87_store;
    367         #endif
    368 
    369         // set new coroutine that the processor is executing
    370         // and context switch to it
    371         verify( proc_cor->context.SP );
    372         CtxSwitch( &thrd_src->context, &proc_cor->context );
    373 
    374         // set state of new coroutine to active
    375         proc_cor->state = proc_cor->state == Halted ? Halted : Inactive;
    376         thrd_src->state = Active;
    377 
    378         #if defined( __i386 ) || defined( __x86_64 )
    379                 __x87_load;
    380         #endif
    381         *__volatile_errno() = local_errno;
    382 }
    383 
    384 // KERNEL_ONLY
    385 // Once a thread has finished running, some of
    386 // its final actions must be executed from the kernel
    387 static void finishRunning(processor * this) with( this->finish ) {
    388         verify( ! kernelTLS.preemption_state.enabled );
    389         choose( action_code ) {
    390         case No_Action:
    391                 break;
    392         case Release:
    393                 unlock( *lock );
    394         case Schedule:
    395                 ScheduleThread( thrd );
    396         case Release_Schedule:
    397                 unlock( *lock );
    398                 ScheduleThread( thrd );
    399         case Release_Multi:
    400                 for(int i = 0; i < lock_count; i++) {
    401                         unlock( *locks[i] );
    402                 }
    403         case Release_Multi_Schedule:
    404                 for(int i = 0; i < lock_count; i++) {
    405                         unlock( *locks[i] );
    406                 }
    407                 for(int i = 0; i < thrd_count; i++) {
    408                         ScheduleThread( thrds[i] );
    409                 }
    410         case Callback:
    411                 callback();
    412         default:
    413                 abort("KERNEL ERROR: Unexpected action to run after thread");
    414         }
     450void returnToKernel() {
     451        /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
     452        $coroutine * proc_cor = get_coroutine(kernelTLS.this_processor->runner);
     453        $thread * thrd_src = kernelTLS.this_thread;
     454
     455        // Run the thread on this processor
     456        {
     457                int local_errno = *__volatile_errno();
     458                #if defined( __i386 ) || defined( __x86_64 )
     459                        __x87_store;
     460                #endif
     461                verify( proc_cor->context.SP );
     462                __cfactx_switch( &thrd_src->context, &proc_cor->context );
     463                #if defined( __i386 ) || defined( __x86_64 )
     464                        __x87_load;
     465                #endif
     466                *__volatile_errno() = local_errno;
     467        }
     468
     469        /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
     470        /* paranoid */ verifyf( ((uintptr_t)thrd_src->context.SP) < ((uintptr_t)__get_stack(thrd_src->curr_cor)->base ), "ERROR : Returning $thread %p has been corrupted.\n StackPointer too small.\n", thrd_src );
     471        /* paranoid */ verifyf( ((uintptr_t)thrd_src->context.SP) > ((uintptr_t)__get_stack(thrd_src->curr_cor)->limit), "ERROR : Returning $thread %p has been corrupted.\n StackPointer too large.\n", thrd_src );
    415472}
    416473
    … …  
    419476// This is the entry point for processors (kernel threads)
    420477// It effectively constructs a coroutine by stealing the pthread stack
    421 static void * CtxInvokeProcessor(void * arg) {
     478static void * __invoke_processor(void * arg) {
    422479        processor * proc = (processor *) arg;
    423480        kernelTLS.this_processor = proc;
    … …  
    438495
    439496        //We now have a proper context from which to schedule threads
    440         __cfaabi_dbg_print_safe("Kernel : core %p created (%p, %p)\n", proc, &proc->runner, &ctx);
     497        __cfadbg_print_safe(runtime_core, "Kernel : core %p created (%p, %p)\n", proc, &proc->runner, &ctx);
    441498
    442499        // SKULLDUGGERY: Since the coroutine doesn't have its own stack, we can't
    … …  
    449506
    450507        // Main routine of the core returned, the core is now fully terminated
    451         __cfaabi_dbg_print_safe("Kernel : core %p main ended (%p)\n", proc, &proc->runner);
     508        __cfadbg_print_safe(runtime_core, "Kernel : core %p main ended (%p)\n", proc, &proc->runner);
    452509
    453510        return 0p;
    … …  
    460517} // Abort
    461518
    462 void * create_pthread( pthread_t * pthread, void * (*start)(void *), void * arg ) {
     519void * __create_pthread( pthread_t * pthread, void * (*start)(void *), void * arg ) {
    463520        pthread_attr_t attr;
    464521
    … …  
    488545}
    489546
    490 static void start(processor * this) {
    491         __cfaabi_dbg_print_safe("Kernel : Starting core %p\n", this);
    492 
    493         this->stack = create_pthread( &this->kernel_thread, CtxInvokeProcessor, (void *)this );
    494 
    495         __cfaabi_dbg_print_safe("Kernel : core %p started\n", this);
    496 }
    497 
    498547// KERNEL_ONLY
    499 void kernel_first_resume( processor * this ) {
    500         thread_desc * src = mainThread;
    501         coroutine_desc * dst = get_coroutine(this->runner);
     548static void __kernel_first_resume( processor * this ) {
     549        $thread * src = mainThread;
     550        $coroutine * dst = get_coroutine(this->runner);
    502551
    503552        verify( ! kernelTLS.preemption_state.enabled );
    … …  
    505554        kernelTLS.this_thread->curr_cor = dst;
    506555        __stack_prepare( &dst->stack, 65000 );
    507         CtxStart(main, dst, this->runner, CtxInvokeCoroutine);
     556        __cfactx_start(main, dst, this->runner, __cfactx_invoke_coroutine);
    508557
    509558        verify( ! kernelTLS.preemption_state.enabled );
    … …  
    512561        dst->starter = dst->starter ? dst->starter : &src->self_cor;
    513562
    514         // set state of current coroutine to inactive
    515         src->state = src->state == Halted ? Halted : Inactive;
     563        // make sure the current state is still correct
     564        /* paranoid */ verify(src->state == Ready);
    516565
    517566        // context switch to specified coroutine
    518567        verify( dst->context.SP );
    519         CtxSwitch( &src->context, &dst->context );
    520         // when CtxSwitch returns we are back in the src coroutine
     568        __cfactx_switch( &src->context, &dst->context );
     569        // when __cfactx_switch returns we are back in the src coroutine
    521570
    522571        mainThread->curr_cor = &mainThread->self_cor;
    523572
    524         // set state of new coroutine to active
    525         src->state = Active;
     573        // make sure the current state has been update
     574        /* paranoid */ verify(src->state == Active);
    526575
    527576        verify( ! kernelTLS.preemption_state.enabled );
    … …  
    529578
    530579// KERNEL_ONLY
    531 void kernel_last_resume( processor * this ) {
    532         coroutine_desc * src = &mainThread->self_cor;
    533         coroutine_desc * dst = get_coroutine(this->runner);
     580static void __kernel_last_resume( processor * this ) {
     581        $coroutine * src = &mainThread->self_cor;
     582        $coroutine * dst = get_coroutine(this->runner);
    534583
    535584        verify( ! kernelTLS.preemption_state.enabled );
    … …  
    537586        verify( dst->context.SP );
    538587
     588        // SKULLDUGGERY in debug the processors check that the
     589        // stack is still within the limit of the stack limits after running a thread.
     590        // that check doesn't make sense if we context switch to the processor using the
     591        // coroutine semantics. Since this is a special case, use the current context
     592        // info to populate these fields.
     593        __cfaabi_dbg_debug_do(
     594                __stack_context_t ctx;
     595                CtxGet( ctx );
     596                mainThread->context.SP = ctx.SP;
     597                mainThread->context.FP = ctx.FP;
     598        )
     599
    539600        // context switch to the processor
    540         CtxSwitch( &src->context, &dst->context );
     601        __cfactx_switch( &src->context, &dst->context );
    541602}
    542603
    543604//-----------------------------------------------------------------------------
    544605// Scheduler routines
    545 
    546606// KERNEL ONLY
    547 void ScheduleThread( thread_desc * thrd ) {
    548         verify( thrd );
    549         verify( thrd->state != Halted );
    550 
    551         verify( ! kernelTLS.preemption_state.enabled );
    552 
    553         verifyf( thrd->link.next == 0p, "Expected null got %p", thrd->link.next );
    554 
     607void __schedule_thread( $thread * thrd ) {
     608        /* paranoid */ verify( thrd );
     609        /* paranoid */ verify( thrd->state != Halted );
     610        /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
     611        /* paranoid */ #if defined( __CFA_WITH_VERIFY__ )
     612        /* paranoid */ if( thrd->state == Blocked || thrd->state == Start ) assertf( thrd->preempted == __NO_PREEMPTION,
     613                          "Error inactive thread marked as preempted, state %d, preemption %d\n", thrd->state, thrd->preempted );
     614        /* paranoid */ if( thrd->preempted != __NO_PREEMPTION ) assertf(thrd->state == Active || thrd->state == Rerun,
     615                          "Error preempted thread marked as not currently running, state %d, preemption %d\n", thrd->state, thrd->preempted );
     616        /* paranoid */ #endif
     617        /* paranoid */ verifyf( thrd->link.next == 0p, "Expected null got %p", thrd->link.next );
     618
     619        if (thrd->preempted == __NO_PREEMPTION) thrd->state = Ready;
    555620
    556621        ready_schedule_lock(thrd->curr_cluster, kernelTLS.this_processor);
    … …  
    558623        ready_schedule_unlock(thrd->curr_cluster, kernelTLS.this_processor);
    559624
    560         with( *thrd->curr_cluster ) {
    561                 // if(was_empty) {
    562                 //      lock      (proc_list_lock __cfaabi_dbg_ctx2);
    563                 //      if(idles) {
    564                 //              wake_fast(idles.head);
    565                 //      }
    566                 //      unlock    (proc_list_lock);
    567                 // }
    568                 // else if( struct processor * idle = idles.head ) {
    569                 //      wake_fast(idle);
    570                 // }
    571         }
    572 
    573         verify( ! kernelTLS.preemption_state.enabled );
     625        __wake_one(thrd->curr_cluster, was_empty);
     626
     627        /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
    574628}
    575629
    576630// KERNEL ONLY
    577 thread_desc * nextThread(cluster * this) with( *this ) {
    578         verify( ! kernelTLS.preemption_state.enabled );
     631static $thread * __next_thread(cluster * this) with( *this ) {
     632        /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
    579633
    580634        ready_schedule_lock(this, kernelTLS.this_processor);
    581                 thread_desc * head = pop( this );
     635                $thread * head = pop( this );
    582636        ready_schedule_unlock(this, kernelTLS.this_processor);
    583637
    584         verify( ! kernelTLS.preemption_state.enabled );
     638        /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
    585639        return head;
    586640}
    587641
    588 void BlockInternal() {
     642// KERNEL ONLY unpark with out disabling interrupts
     643void __unpark( $thread * thrd __cfaabi_dbg_ctx_param2 ) {
     644        static_assert(sizeof(thrd->state) == sizeof(int));
     645
     646        // record activity
     647        __cfaabi_dbg_record_thrd( *thrd, false, caller );
     648
     649        enum coroutine_state old_state = __atomic_exchange_n(&thrd->state, Rerun, __ATOMIC_SEQ_CST);
     650        __cfaabi_dbg_debug_do( thrd->unpark_result = old_state; )
     651        switch(old_state) {
     652                case Active:
     653                        // Wake won the race, the thread will reschedule/rerun itself
     654                        break;
     655                case Blocked:
     656                        /* paranoid */ verify( ! thrd->preempted != __NO_PREEMPTION );
     657
     658                        // Wake lost the race,
     659                        thrd->state = Blocked;
     660                        __schedule_thread( thrd );
     661                        break;
     662                case Rerun:
     663                        abort("More than one thread attempted to schedule thread %p\n", thrd);
     664                        break;
     665                case Halted:
     666                case Start:
     667                case Primed:
     668                default:
     669                        // This makes no sense, something is wrong abort
     670                        abort();
     671        }
     672}
     673
     674void unpark( $thread * thrd __cfaabi_dbg_ctx_param2 ) {
     675        if( !thrd ) return;
     676
    589677        disable_interrupts();
    590         verify( ! kernelTLS.preemption_state.enabled );
     678        __unpark( thrd __cfaabi_dbg_ctx_fwd2 );
     679        enable_interrupts( __cfaabi_dbg_ctx );
     680}
     681
     682void park( __cfaabi_dbg_ctx_param ) {
     683        /* paranoid */ verify( kernelTLS.preemption_state.enabled );
     684        disable_interrupts();
     685        /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
     686        /* paranoid */ verify( kernelTLS.this_thread->preempted == __NO_PREEMPTION );
     687
     688        // record activity
     689        __cfaabi_dbg_record_thrd( *kernelTLS.this_thread, true, caller );
     690
    591691        returnToKernel();
    592         verify( ! kernelTLS.preemption_state.enabled );
     692
     693        /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
    593694        enable_interrupts( __cfaabi_dbg_ctx );
    594 }
    595 
    596 void BlockInternal( __spinlock_t * lock ) {
     695        /* paranoid */ verify( kernelTLS.preemption_state.enabled );
     696
     697}
     698
     699// KERNEL ONLY
     700void __leave_thread() {
     701        /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
     702        returnToKernel();
     703        abort();
     704}
     705
     706// KERNEL ONLY
     707bool force_yield( __Preemption_Reason reason ) {
     708        /* paranoid */ verify( kernelTLS.preemption_state.enabled );
    597709        disable_interrupts();
    598         with( *kernelTLS.this_processor ) {
    599                 finish.action_code = Release;
    600                 finish.lock        = lock;
    601         }
    602 
    603         verify( ! kernelTLS.preemption_state.enabled );
    604         returnToKernel();
    605         verify( ! kernelTLS.preemption_state.enabled );
    606 
    607         enable_interrupts( __cfaabi_dbg_ctx );
    608 }
    609 
    610 void BlockInternal( thread_desc * thrd ) {
    611         disable_interrupts();
    612         with( * kernelTLS.this_processor ) {
    613                 finish.action_code = Schedule;
    614                 finish.thrd        = thrd;
    615         }
    616 
    617         verify( ! kernelTLS.preemption_state.enabled );
    618         returnToKernel();
    619         verify( ! kernelTLS.preemption_state.enabled );
    620 
    621         enable_interrupts( __cfaabi_dbg_ctx );
    622 }
    623 
    624 void BlockInternal( __spinlock_t * lock, thread_desc * thrd ) {
    625         assert(thrd);
    626         disable_interrupts();
    627         with( * kernelTLS.this_processor ) {
    628                 finish.action_code = Release_Schedule;
    629                 finish.lock        = lock;
    630                 finish.thrd        = thrd;
    631         }
    632 
    633         verify( ! kernelTLS.preemption_state.enabled );
    634         returnToKernel();
    635         verify( ! kernelTLS.preemption_state.enabled );
    636 
    637         enable_interrupts( __cfaabi_dbg_ctx );
    638 }
    639 
    640 void BlockInternal(__spinlock_t * locks [], unsigned short count) {
    641         disable_interrupts();
    642         with( * kernelTLS.this_processor ) {
    643                 finish.action_code = Release_Multi;
    644                 finish.locks       = locks;
    645                 finish.lock_count  = count;
    646         }
    647 
    648         verify( ! kernelTLS.preemption_state.enabled );
    649         returnToKernel();
    650         verify( ! kernelTLS.preemption_state.enabled );
    651 
    652         enable_interrupts( __cfaabi_dbg_ctx );
    653 }
    654 
    655 void BlockInternal(__spinlock_t * locks [], unsigned short lock_count, thread_desc * thrds [], unsigned short thrd_count) {
    656         disable_interrupts();
    657         with( *kernelTLS.this_processor ) {
    658                 finish.action_code = Release_Multi_Schedule;
    659                 finish.locks       = locks;
    660                 finish.lock_count  = lock_count;
    661                 finish.thrds       = thrds;
    662                 finish.thrd_count  = thrd_count;
    663         }
    664 
    665         verify( ! kernelTLS.preemption_state.enabled );
    666         returnToKernel();
    667         verify( ! kernelTLS.preemption_state.enabled );
    668 
    669         enable_interrupts( __cfaabi_dbg_ctx );
    670 }
    671 
    672 void BlockInternal(__finish_callback_fptr_t callback) {
    673         disable_interrupts();
    674         with( *kernelTLS.this_processor ) {
    675                 finish.action_code = Callback;
    676                 finish.callback    = callback;
    677         }
    678 
    679         verify( ! kernelTLS.preemption_state.enabled );
    680         returnToKernel();
    681         verify( ! kernelTLS.preemption_state.enabled );
    682 
    683         enable_interrupts( __cfaabi_dbg_ctx );
    684 }
    685 
    686 // KERNEL ONLY
    687 void LeaveThread(__spinlock_t * lock, thread_desc * thrd) {
    688         verify( ! kernelTLS.preemption_state.enabled );
    689         with( * kernelTLS.this_processor ) {
    690                 finish.action_code = thrd ? Release_Schedule : Release;
    691                 finish.lock        = lock;
    692                 finish.thrd        = thrd;
    693         }
    694 
    695         returnToKernel();
     710        /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
     711
     712        $thread * thrd = kernelTLS.this_thread;
     713        /* paranoid */ verify(thrd->state == Active || thrd->state == Rerun);
     714
     715        // SKULLDUGGERY: It is possible that we are preempting this thread just before
     716        // it was going to park itself. If that is the case and it is already using the
     717        // intrusive fields then we can't use them to preempt the thread
     718        // If that is the case, abandon the preemption.
     719        bool preempted = false;
     720        if(thrd->next == 0p) {
     721                preempted = true;
     722                thrd->preempted = reason;
     723                returnToKernel();
     724        }
     725
     726        /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
     727        enable_interrupts_noPoll();
     728        /* paranoid */ verify( kernelTLS.preemption_state.enabled );
     729
     730        return preempted;
    696731}
    697732
    … …  
    701736//-----------------------------------------------------------------------------
    702737// Kernel boot procedures
    703 static void kernel_startup(void) {
     738static void __kernel_startup(void) {
    704739        verify( ! kernelTLS.preemption_state.enabled );
    705         __cfaabi_dbg_print_safe("Kernel : Starting\n");
     740        __cfadbg_print_safe(runtime_core, "Kernel : Starting\n");
    706741
    707742        __page_size = sysconf( _SC_PAGESIZE );
    … …  
    714749        (*mainCluster){"Main Cluster"};
    715750
    716         __cfaabi_dbg_print_safe("Kernel : Main cluster ready\n");
     751        __cfadbg_print_safe(runtime_core, "Kernel : Main cluster ready\n");
    717752
    718753        // Start by initializing the main thread
    719754        // SKULLDUGGERY: the mainThread steals the process main thread
    720755        // which will then be scheduled by the mainProcessor normally
    721         mainThread = (thread_desc *)&storage_mainThread;
     756        mainThread = ($thread *)&storage_mainThread;
    722757        current_stack_info_t info;
    723758        info.storage = (__stack_t*)&storage_mainThreadCtx;
    724759        (*mainThread){ &info };
    725760
    726         __cfaabi_dbg_print_safe("Kernel : Main thread ready\n");
     761        __cfadbg_print_safe(runtime_core, "Kernel : Main thread ready\n");
    727762
    728763
    … …  
    746781
    747782                runner{ &this };
    748                 __cfaabi_dbg_print_safe("Kernel : constructed main processor context %p\n", &runner);
     783                __cfadbg_print_safe(runtime_core, "Kernel : constructed main processor context %p\n", &runner);
    749784        }
    750785
    … …  
    765800        // Add the main thread to the ready queue
    766801        // once resume is called on mainProcessor->runner the mainThread needs to be scheduled like any normal thread
    767         ScheduleThread(mainThread);
     802        __schedule_thread(mainThread);
    768803
    769804        // SKULLDUGGERY: Force a context switch to the main processor to set the main thread's context to the current UNIX
    770         // context. Hence, the main thread does not begin through CtxInvokeThread, like all other threads. The trick here is that
     805        // context. Hence, the main thread does not begin through __cfactx_invoke_thread, like all other threads. The trick here is that
    771806        // mainThread is on the ready queue when this call is made.
    772         kernel_first_resume( kernelTLS.this_processor );
    773 
     807        __kernel_first_resume( kernelTLS.this_processor );
    774808
    775809
    776810        // THE SYSTEM IS NOW COMPLETELY RUNNING
    777         __cfaabi_dbg_print_safe("Kernel : Started\n--------------------------------------------------\n\n");
     811
     812
     813        // Now that the system is up, finish creating systems that need threading
     814        __kernel_io_finish_start( *mainCluster );
     815
     816
     817        __cfadbg_print_safe(runtime_core, "Kernel : Started\n--------------------------------------------------\n\n");
    778818
    779819        verify( ! kernelTLS.preemption_state.enabled );
    … …  
    782822}
    783823
    784 static void kernel_shutdown(void) {
    785         __cfaabi_dbg_print_safe("\n--------------------------------------------------\nKernel : Shutting down\n");
    786 
    787         verify( TL_GET( preemption_state.enabled ) );
     824static void __kernel_shutdown(void) {
     825        //Before we start shutting things down, wait for systems that need threading to shutdown
     826        __kernel_io_prepare_stop( *mainCluster );
     827
     828        /* paranoid */ verify( TL_GET( preemption_state.enabled ) );
    788829        disable_interrupts();
    789         verify( ! kernelTLS.preemption_state.enabled );
     830        /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
     831
     832        __cfadbg_print_safe(runtime_core, "\n--------------------------------------------------\nKernel : Shutting down\n");
    790833
    791834        // SKULLDUGGERY: Notify the mainProcessor it needs to terminates.
    … …  
    793836        // which is currently here
    794837        __atomic_store_n(&mainProcessor->do_terminate, true, __ATOMIC_RELEASE);
    795         kernel_last_resume( kernelTLS.this_processor );
     838        __kernel_last_resume( kernelTLS.this_processor );
    796839        mainThread->self_cor.state = Halted;
    797840
    … …  
    805848        // Destroy the main processor and its context in reverse order of construction
    806849        // These were manually constructed so we need manually destroy them
    807         void ^?{}(processor & this) with( this ) {
    808                 //don't join the main thread here, that wouldn't make any sense
     850        void ^?{}(processor & this) with( this ){
     851                /* paranoid */ verify( this.do_terminate == true );
    809852                __cfaabi_dbg_print_safe("Kernel : destroyed main processor context %p\n", &runner);
    810853        }
    … …  
    813856
    814857        // Final step, destroy the main thread since it is no longer needed
    815         // Since we provided a stack to this task it will not destroy anything
     858
     859        // Since we provided a stack to this taxk it will not destroy anything
     860        /* paranoid */ verify(mainThread->self_cor.stack.storage == (__stack_t*)(((uintptr_t)&storage_mainThreadCtx)| 0x1));
    816861        ^(*mainThread){};
    817862
    … …  
    821866        ^(__cfa_dbg_global_clusters.lock){};
    822867
    823         __cfaabi_dbg_print_safe("Kernel : Shutdown complete\n");
     868        __cfadbg_print_safe(runtime_core, "Kernel : Shutdown complete\n");
    824869}
    825870
    826871//=============================================================================================
    827 // Kernel Quiescing
     872// Kernel Idle Sleep
    828873//=============================================================================================
    829 static void halt(processor * this) with( *this ) {
    830         // // verify( ! __atomic_load_n(&do_terminate, __ATOMIC_SEQ_CST) );
    831 
    832         // with( *cltr ) {
    833         //      lock      (proc_list_lock __cfaabi_dbg_ctx2);
    834         //      push_front(idles, *this);
    835         //      unlock    (proc_list_lock);
    836         // }
    837 
    838         // __cfaabi_dbg_print_safe("Kernel : Processor %p ready to sleep\n", this);
    839 
    840         // wait( idleLock );
    841 
    842         // __cfaabi_dbg_print_safe("Kernel : Processor %p woke up and ready to run\n", this);
    843 
    844         // with( *cltr ) {
    845         //      lock      (proc_list_lock __cfaabi_dbg_ctx2);
    846         //      remove    (idles, *this);
    847         //      unlock    (proc_list_lock);
    848         // }
     874static $thread * __halt(processor * this) with( *this ) {
     875        if( do_terminate ) return 0p;
     876
     877        // First, lock the cluster idle
     878        lock( cltr->idle_lock __cfaabi_dbg_ctx2 );
     879
     880        // Check if we can find a thread
     881        if( $thread * found = __next_thread( cltr ) ) {
     882                unlock( cltr->idle_lock );
     883                return found;
     884        }
     885
     886        // Move this processor from the active list to the idle list
     887        move_to_front(cltr->procs, cltr->idles, *this);
     888
     889        // Unlock the idle lock so we don't go to sleep with a lock
     890        unlock    (cltr->idle_lock);
     891
     892        // We are ready to sleep
     893        __cfadbg_print_safe(runtime_core, "Kernel : Processor %p ready to sleep\n", this);
     894        wait( idle );
     895
     896        // We have woken up
     897        __cfadbg_print_safe(runtime_core, "Kernel : Processor %p woke up and ready to run\n", this);
     898
     899        // Get ourself off the idle list
     900        with( *cltr ) {
     901                lock  (idle_lock __cfaabi_dbg_ctx2);
     902                move_to_front(idles, procs, *this);
     903                unlock(idle_lock);
     904        }
     905
     906        // Don't check the ready queue again, we may not be in a position to run a thread
     907        return 0p;
     908}
     909
     910// Wake a thread from the front if there are any
     911static bool __wake_one(cluster * this, __attribute__((unused)) bool force) {
     912        // if we don't want to force check if we know it's false
     913        // if( !this->idles.head && !force ) return false;
     914
     915        // First, lock the cluster idle
     916        lock( this->idle_lock __cfaabi_dbg_ctx2 );
     917
     918        // Check if there is someone to wake up
     919        if( !this->idles.head ) {
     920                // Nope unlock and return false
     921                unlock( this->idle_lock );
     922                return false;
     923        }
     924
     925        // Wake them up
     926        __cfadbg_print_safe(runtime_core, "Kernel : waking Processor %p\n", this->idles.head);
     927        /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
     928        post( this->idles.head->idle );
     929
     930        // Unlock and return true
     931        unlock( this->idle_lock );
     932        return true;
     933}
     934
     935// Unconditionnaly wake a thread
     936static bool __wake_proc(processor * this) {
     937        __cfadbg_print_safe(runtime_core, "Kernel : waking Processor %p\n", this);
     938
     939        disable_interrupts();
     940                /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
     941                bool ret = post( this->idle );
     942        enable_interrupts( __cfaabi_dbg_ctx );
     943
     944        return ret;
    849945}
    850946
    … …  
    880976
    881977void kernel_abort_msg( void * kernel_data, char * abort_text, int abort_text_size ) {
    882         thread_desc * thrd = kernel_data;
     978        $thread * thrd = kernel_data;
    883979
    884980        if(thrd) {
    … …  
    9281024void ^?{}(semaphore & this) {}
    9291025
    930 void P(semaphore & this) with( this ){
     1026bool P(semaphore & this) with( this ){
    9311027        lock( lock __cfaabi_dbg_ctx2 );
    9321028        count -= 1;
    … …  
    9361032
    9371033                // atomically release spin lock and block
    938                 BlockInternal( &lock );
     1034                unlock( lock );
     1035                park( __cfaabi_dbg_ctx );
     1036                return true;
    9391037        }
    9401038        else {
    9411039            unlock( lock );
    942         }
    943 }
    944 
    945 void V(semaphore & this) with( this ) {
    946         thread_desc * thrd = 0p;
     1040            return false;
     1041        }
     1042}
     1043
     1044bool V(semaphore & this) with( this ) {
     1045        $thread * thrd = 0p;
    9471046        lock( lock __cfaabi_dbg_ctx2 );
    9481047        count += 1;
    … …  
    9551054
    9561055        // make new owner
    957         WakeThread( thrd );
     1056        unpark( thrd __cfaabi_dbg_ctx2 );
     1057
     1058        return thrd != 0p;
     1059}
     1060
     1061bool V(semaphore & this, unsigned diff) with( this ) {
     1062        $thread * thrd = 0p;
     1063        lock( lock __cfaabi_dbg_ctx2 );
     1064        int release = max(-count, (int)diff);
     1065        count += diff;
     1066        for(release) {
     1067                unpark( pop_head( waiting ) __cfaabi_dbg_ctx2 );
     1068        }
     1069
     1070        unlock( lock );
     1071
     1072        return thrd != 0p;
    9581073}
    9591074
    … …  
    9721087}
    9731088
    974 void doregister( cluster * cltr, thread_desc & thrd ) {
     1089void doregister( cluster * cltr, $thread & thrd ) {
    9751090        lock      (cltr->thread_list_lock __cfaabi_dbg_ctx2);
    9761091        cltr->nthreads += 1;
    … …  
    9791094}
    9801095
    981 void unregister( cluster * cltr, thread_desc & thrd ) {
     1096void unregister( cluster * cltr, $thread & thrd ) {
    9821097        lock  (cltr->thread_list_lock __cfaabi_dbg_ctx2);
    9831098        remove(cltr->threads, thrd );
    … …  
    9901105__cfaabi_dbg_debug_do(
    9911106        extern "C" {
    992                 void __cfaabi_dbg_record(__spinlock_t & this, const char prev_name[]) {
     1107                void __cfaabi_dbg_record_lock(__spinlock_t & this, const char prev_name[]) {
    9931108                        this.prev_name = prev_name;
    9941109                        this.prev_thrd = kernelTLS.this_thread;
    9951110                }
     1111
     1112                void __cfaabi_dbg_record_thrd($thread & this, bool park, const char prev_name[]) {
     1113                        if(park) {
     1114                                this.park_caller   = prev_name;
     1115                                this.park_stale    = false;
     1116                        }
     1117                        else {
     1118                                this.unpark_caller = prev_name;
     1119                                this.unpark_stale  = false;
     1120                        }
     1121                }
    9961122        }
    9971123)
    … …  
    9991125//-----------------------------------------------------------------------------
    10001126// Debug
    1001 bool threading_enabled(void) {
     1127bool threading_enabled(void) __attribute__((const)) {
    10021128        return true;
    10031129}

  • libcfa/src/concurrency/kernel.hfa

    rb7d6a36 r6a490b2  
    1717
    1818#include <stdbool.h>
     19#include <stdint.h>
    1920
    2021#include "invoke.h"
    … …  
    3233        __spinlock_t lock;
    3334        int count;
    34         __queue_t(thread_desc) waiting;
     35        __queue_t($thread) waiting;
    3536};
    3637
    3738void  ?{}(semaphore & this, int count = 1);
    3839void ^?{}(semaphore & this);
    39 void   P (semaphore & this);
    40 void   V (semaphore & this);
     40bool   P (semaphore & this);
     41bool   V (semaphore & this);
     42bool   V (semaphore & this, unsigned count);
    4143
    4244
    … …  
    4446// Processor
    4547extern struct cluster * mainCluster;
    46 
    47 enum FinishOpCode { No_Action, Release, Schedule, Release_Schedule, Release_Multi, Release_Multi_Schedule, Callback };
    48 
    49 typedef void (*__finish_callback_fptr_t)(void);
    50 
    51 //TODO use union, many of these fields are mutually exclusive (i.e. MULTI vs NOMULTI)
    52 struct FinishAction {
    53         FinishOpCode action_code;
    54         /*
    55         // Union of possible actions
    56         union {
    57                 // Option 1 : locks and threads
    58                 struct {
    59                         // 1 thread or N thread
    60                         union {
    61                                 thread_desc * thrd;
    62                                 struct {
    63                                         thread_desc ** thrds;
    64                                         unsigned short thrd_count;
    65                                 };
    66                         };
    67                         // 1 lock or N lock
    68                         union {
    69                                 __spinlock_t * lock;
    70                                 struct {
    71                                         __spinlock_t ** locks;
    72                                         unsigned short lock_count;
    73                                 };
    74                         };
    75                 };
    76                 // Option 2 : action pointer
    77                 __finish_callback_fptr_t callback;
    78         };
    79         /*/
    80         thread_desc * thrd;
    81         thread_desc ** thrds;
    82         unsigned short thrd_count;
    83         __spinlock_t * lock;
    84         __spinlock_t ** locks;
    85         unsigned short lock_count;
    86         __finish_callback_fptr_t callback;
    87         //*/
    88 };
    89 static inline void ?{}(FinishAction & this) {
    90         this.action_code = No_Action;
    91         this.thrd = 0p;
    92         this.lock = 0p;
    93 }
    94 static inline void ^?{}(FinishAction &) {}
    9548
    9649// Processor
    … …  
    11770        // RunThread data
    11871        // Action to do after a thread is ran
    119         struct FinishAction finish;
     72        $thread * destroyer;
    12073
    12174        // Preemption data
    … …  
    12679        bool pending_preemption;
    12780
    128         // Idle lock
    129         __bin_sem_t idleLock;
     81        // Idle lock (kernel semaphore)
     82        __bin_sem_t idle;
    13083
    13184        // Termination
    … …  
    13386        volatile bool do_terminate;
    13487
    135         // Termination synchronisation
     88        // Termination synchronisation (user semaphore)
    13689        semaphore terminated;
    13790
    … …  
    158111static inline void  ?{}(processor & this, const char name[]) { this{name, *mainCluster }; }
    159112
    160 static inline [processor *&, processor *& ] __get( processor & this ) {
    161         return this.node.[next, prev];
    162 }
     113static inline [processor *&, processor *& ] __get( processor & this ) __attribute__((const)) { return this.node.[next, prev]; }
     114
     115//-----------------------------------------------------------------------------
     116// I/O
     117struct __io_data;
     118
     119#define CFA_CLUSTER_IO_POLLER_USER_THREAD 1 << 0
     120// #define CFA_CLUSTER_IO_POLLER_KERNEL_SIDE 1 << 1
    163121
    164122
    … …  
    333291        // List of threads
    334292        __spinlock_t thread_list_lock;
    335         __dllist_t(struct thread_desc) threads;
     293        __dllist_t(struct $thread) threads;
    336294        unsigned int nthreads;
    337295
    … …  
    341299                cluster * prev;
    342300        } node;
     301
     302        struct __io_data * io;
     303
     304        #if !defined(__CFA_NO_STATISTICS__)
     305                bool print_stats;
     306        #endif
    343307};
    344308extern Duration default_preemption();
    345309
    346 void ?{} (cluster & this, const char name[], Duration preemption_rate);
     310void ?{} (cluster & this, const char name[], Duration preemption_rate, int flags);
    347311void ^?{}(cluster & this);
    348312
    349 static inline void ?{} (cluster & this)                           { this{"Anonymous Cluster", default_preemption()}; }
    350 static inline void ?{} (cluster & this, Duration preemption_rate) { this{"Anonymous Cluster", preemption_rate}; }
    351 static inline void ?{} (cluster & this, const char name[])        { this{name, default_preemption()}; }
    352 
    353 static inline [cluster *&, cluster *& ] __get( cluster & this ) {
    354         return this.node.[next, prev];
    355 }
     313static inline void ?{} (cluster & this)                                      { this{"Anonymous Cluster", default_preemption(), 0}; }
     314static inline void ?{} (cluster & this, Duration preemption_rate)            { this{"Anonymous Cluster", preemption_rate, 0}; }
     315static inline void ?{} (cluster & this, const char name[])                   { this{name, default_preemption(), 0}; }
     316static inline void ?{} (cluster & this, int flags)                           { this{"Anonymous Cluster", default_preemption(), flags}; }
     317static inline void ?{} (cluster & this, Duration preemption_rate, int flags) { this{"Anonymous Cluster", preemption_rate, flags}; }
     318static inline void ?{} (cluster & this, const char name[], int flags)        { this{name, default_preemption(), flags}; }
     319
     320static inline [cluster *&, cluster *& ] __get( cluster & this ) __attribute__((const)) { return this.node.[next, prev]; }
    356321
    357322static inline struct processor * active_processor() { return TL_GET( this_processor ); } // UNSAFE
    358323static inline struct cluster   * active_cluster  () { return TL_GET( this_processor )->cltr; }
     324
     325#if !defined(__CFA_NO_STATISTICS__)
     326        static inline void print_stats_at_exit( cluster & this ) {
     327                this.print_stats = true;
     328        }
     329#endif
    359330
    360331// Local Variables: //

  • libcfa/src/concurrency/kernel_private.hfa

    rb7d6a36 r6a490b2  
    3131}
    3232
    33 void ScheduleThread( thread_desc * );
    34 static inline void WakeThread( thread_desc * thrd ) {
    35         if( !thrd ) return;
    36 
    37         verify(thrd->state == Inactive);
    38 
    39         disable_interrupts();
    40         ScheduleThread( thrd );
    41         enable_interrupts( __cfaabi_dbg_ctx );
    42 }
    43 thread_desc * nextThread(cluster * this);
     33void __schedule_thread( $thread * ) __attribute__((nonnull (1)));
    4434
    4535//Block current thread and release/wake-up the following resources
    46 void BlockInternal(void);
    47 void BlockInternal(__spinlock_t * lock);
    48 void BlockInternal(thread_desc * thrd);
    49 void BlockInternal(__spinlock_t * lock, thread_desc * thrd);
    50 void BlockInternal(__spinlock_t * locks [], unsigned short count);
    51 void BlockInternal(__spinlock_t * locks [], unsigned short count, thread_desc * thrds [], unsigned short thrd_count);
    52 void BlockInternal(__finish_callback_fptr_t callback);
    53 void LeaveThread(__spinlock_t * lock, thread_desc * thrd);
     36void __leave_thread() __attribute__((noreturn));
    5437
    5538//-----------------------------------------------------------------------------
    … …  
    5740void main(processorCtx_t *);
    5841
    59 void * create_pthread( pthread_t *, void * (*)(void *), void * );
    60 
    61 static inline void wake_fast(processor * this) {
    62         __cfaabi_dbg_print_safe("Kernel : Waking up processor %p\n", this);
    63         post( this->idleLock );
    64 }
    65 
    66 static inline void wake(processor * this) {
    67         disable_interrupts();
    68         wake_fast(this);
    69         enable_interrupts( __cfaabi_dbg_ctx );
    70 }
     42void * __create_pthread( pthread_t *, void * (*)(void *), void * );
     43
     44
    7145
    7246struct event_kernel_t {
    … …  
    8559extern volatile thread_local __cfa_kernel_preemption_state_t preemption_state __attribute__ ((tls_model ( "initial-exec" )));
    8660
     61extern cluster * mainCluster;
     62
    8763//-----------------------------------------------------------------------------
    8864// Threads
    8965extern "C" {
    90       void CtxInvokeThread(void (*main)(void *), void * this);
    91 }
    92 
    93 extern void ThreadCtxSwitch(coroutine_desc * src, coroutine_desc * dst);
     66      void __cfactx_invoke_thread(void (*main)(void *), void * this);
     67}
    9468
    9569__cfaabi_dbg_debug_do(
    96         extern void __cfaabi_dbg_thread_register  ( thread_desc * thrd );
    97         extern void __cfaabi_dbg_thread_unregister( thread_desc * thrd );
     70        extern void __cfaabi_dbg_thread_register  ( $thread * thrd );
     71        extern void __cfaabi_dbg_thread_unregister( $thread * thrd );
    9872)
     73
     74// KERNEL ONLY unpark with out disabling interrupts
     75void __unpark( $thread * thrd __cfaabi_dbg_ctx_param2 );
     76
     77//-----------------------------------------------------------------------------
     78// I/O
     79void __kernel_io_startup     ( cluster &, int, bool );
     80void __kernel_io_finish_start( cluster & );
     81void __kernel_io_prepare_stop( cluster & );
     82void __kernel_io_shutdown    ( cluster &, bool );
    9983
    10084//-----------------------------------------------------------------------------
    … …  
    10286#define KERNEL_STORAGE(T,X) __attribute((aligned(__alignof__(T)))) static char storage_##X[sizeof(T)]
    10387
    104 static inline uint32_t tls_rand() {
     88static inline uint32_t __tls_rand() {
    10589        kernelTLS.rand_seed ^= kernelTLS.rand_seed << 6;
    10690        kernelTLS.rand_seed ^= kernelTLS.rand_seed >> 21;
    … …  
    11397void unregister( struct cluster & cltr );
    11498
    115 void doregister( struct cluster * cltr, struct thread_desc & thrd );
    116 void unregister( struct cluster * cltr, struct thread_desc & thrd );
     99void doregister( struct cluster * cltr, struct $thread & thrd );
     100void unregister( struct cluster * cltr, struct $thread & thrd );
    117101
    118102//=======================================================================

  • libcfa/src/concurrency/monitor.cfa

    rb7d6a36 r6a490b2  
    55// file "LICENCE" distributed with Cforall.
    66//
    7 // monitor_desc.c --
     7// $monitor.c --
    88//
    99// Author           : Thierry Delisle
    … …  
    2727//-----------------------------------------------------------------------------
    2828// Forward declarations
    29 static inline void set_owner ( monitor_desc * this, thread_desc * owner );
    30 static inline void set_owner ( monitor_desc * storage [], __lock_size_t count, thread_desc * owner );
    31 static inline void set_mask  ( monitor_desc * storage [], __lock_size_t count, const __waitfor_mask_t & mask );
    32 static inline void reset_mask( monitor_desc * this );
    33 
    34 static inline thread_desc * next_thread( monitor_desc * this );
    35 static inline bool is_accepted( monitor_desc * this, const __monitor_group_t & monitors );
     29static inline void __set_owner ( $monitor * this, $thread * owner );
     30static inline void __set_owner ( $monitor * storage [], __lock_size_t count, $thread * owner );
     31static inline void set_mask  ( $monitor * storage [], __lock_size_t count, const __waitfor_mask_t & mask );
     32static inline void reset_mask( $monitor * this );
     33
     34static inline $thread * next_thread( $monitor * this );
     35static inline bool is_accepted( $monitor * this, const __monitor_group_t & monitors );
    3636
    3737static inline void lock_all  ( __spinlock_t * locks [], __lock_size_t count );
    38 static inline void lock_all  ( monitor_desc * source [], __spinlock_t * /*out*/ locks [], __lock_size_t count );
     38static inline void lock_all  ( $monitor * source [], __spinlock_t * /*out*/ locks [], __lock_size_t count );
    3939static inline void unlock_all( __spinlock_t * locks [], __lock_size_t count );
    40 static inline void unlock_all( monitor_desc * locks [], __lock_size_t count );
    41 
    42 static inline void save   ( monitor_desc * ctx [], __lock_size_t count, __spinlock_t * locks [], unsigned int /*out*/ recursions [], __waitfor_mask_t /*out*/ masks [] );
    43 static inline void restore( monitor_desc * ctx [], __lock_size_t count, __spinlock_t * locks [], unsigned int /*in */ recursions [], __waitfor_mask_t /*in */ masks [] );
    44 
    45 static inline void init     ( __lock_size_t count, monitor_desc * monitors [], __condition_node_t & waiter, __condition_criterion_t criteria [] );
    46 static inline void init_push( __lock_size_t count, monitor_desc * monitors [], __condition_node_t & waiter, __condition_criterion_t criteria [] );
    47 
    48 static inline thread_desc *        check_condition   ( __condition_criterion_t * );
     40static inline void unlock_all( $monitor * locks [], __lock_size_t count );
     41
     42static inline void save   ( $monitor * ctx [], __lock_size_t count, __spinlock_t * locks [], unsigned int /*out*/ recursions [], __waitfor_mask_t /*out*/ masks [] );
     43static inline void restore( $monitor * ctx [], __lock_size_t count, __spinlock_t * locks [], unsigned int /*in */ recursions [], __waitfor_mask_t /*in */ masks [] );
     44
     45static inline void init     ( __lock_size_t count, $monitor * monitors [], __condition_node_t & waiter, __condition_criterion_t criteria [] );
     46static inline void init_push( __lock_size_t count, $monitor * monitors [], __condition_node_t & waiter, __condition_criterion_t criteria [] );
     47
     48static inline $thread *        check_condition   ( __condition_criterion_t * );
    4949static inline void                 brand_condition   ( condition & );
    50 static inline [thread_desc *, int] search_entry_queue( const __waitfor_mask_t &, monitor_desc * monitors [], __lock_size_t count );
     50static inline [$thread *, int] search_entry_queue( const __waitfor_mask_t &, $monitor * monitors [], __lock_size_t count );
    5151
    5252forall(dtype T | sized( T ))
    5353static inline __lock_size_t insert_unique( T * array [], __lock_size_t & size, T * val );
    5454static inline __lock_size_t count_max    ( const __waitfor_mask_t & mask );
    55 static inline __lock_size_t aggregate    ( monitor_desc * storage [], const __waitfor_mask_t & mask );
     55static inline __lock_size_t aggregate    ( $monitor * storage [], const __waitfor_mask_t & mask );
    5656
    5757//-----------------------------------------------------------------------------
    … …  
    6868
    6969#define monitor_ctx( mons, cnt )                                /* Define that create the necessary struct for internal/external scheduling operations */ \
    70         monitor_desc ** monitors = mons;                          /* Save the targeted monitors                                                          */ \
     70        $monitor ** monitors = mons;                          /* Save the targeted monitors                                                          */ \
    7171        __lock_size_t count = cnt;                                /* Save the count to a local variable                                                  */ \
    7272        unsigned int recursions[ count ];                         /* Save the current recursion levels to restore them later                             */ \
    … …  
    8080//-----------------------------------------------------------------------------
    8181// Enter/Leave routines
    82 
    83 
    84 extern "C" {
    85         // Enter single monitor
    86         static void __enter_monitor_desc( monitor_desc * this, const __monitor_group_t & group ) {
    87                 // Lock the monitor spinlock
    88                 lock( this->lock __cfaabi_dbg_ctx2 );
    89                 // Interrupts disable inside critical section
    90                 thread_desc * thrd = kernelTLS.this_thread;
    91 
    92                 __cfaabi_dbg_print_safe( "Kernel : %10p Entering mon %p (%p)\n", thrd, this, this->owner);
    93 
    94                 if( !this->owner ) {
    95                         // No one has the monitor, just take it
    96                         set_owner( this, thrd );
    97 
    98                         __cfaabi_dbg_print_safe( "Kernel :  mon is free \n" );
    99                 }
    100                 else if( this->owner == thrd) {
    101                         // We already have the monitor, just note how many times we took it
    102                         this->recursion += 1;
    103 
    104                         __cfaabi_dbg_print_safe( "Kernel :  mon already owned \n" );
    105                 }
    106                 else if( is_accepted( this, group) ) {
    107                         // Some one was waiting for us, enter
    108                         set_owner( this, thrd );
    109 
    110                         // Reset mask
    111                         reset_mask( this );
    112 
    113                         __cfaabi_dbg_print_safe( "Kernel :  mon accepts \n" );
    114                 }
    115                 else {
    116                         __cfaabi_dbg_print_safe( "Kernel :  blocking \n" );
    117 
    118                         // Some one else has the monitor, wait in line for it
    119                         append( this->entry_queue, thrd );
    120 
    121                         BlockInternal( &this->lock );
    122 
    123                         __cfaabi_dbg_print_safe( "Kernel : %10p Entered  mon %p\n", thrd, this);
    124 
    125                         // BlockInternal will unlock spinlock, no need to unlock ourselves
    126                         return;
    127                 }
     82// Enter single monitor
     83static void __enter( $monitor * this, const __monitor_group_t & group ) {
     84        // Lock the monitor spinlock
     85        lock( this->lock __cfaabi_dbg_ctx2 );
     86        // Interrupts disable inside critical section
     87        $thread * thrd = kernelTLS.this_thread;
     88
     89        __cfaabi_dbg_print_safe( "Kernel : %10p Entering mon %p (%p)\n", thrd, this, this->owner);
     90
     91        if( !this->owner ) {
     92                // No one has the monitor, just take it
     93                __set_owner( this, thrd );
     94
     95                __cfaabi_dbg_print_safe( "Kernel :  mon is free \n" );
     96        }
     97        else if( this->owner == thrd) {
     98                // We already have the monitor, just note how many times we took it
     99                this->recursion += 1;
     100
     101                __cfaabi_dbg_print_safe( "Kernel :  mon already owned \n" );
     102        }
     103        else if( is_accepted( this, group) ) {
     104                // Some one was waiting for us, enter
     105                __set_owner( this, thrd );
     106
     107                // Reset mask
     108                reset_mask( this );
     109
     110                __cfaabi_dbg_print_safe( "Kernel :  mon accepts \n" );
     111        }
     112        else {
     113                __cfaabi_dbg_print_safe( "Kernel :  blocking \n" );
     114
     115                // Some one else has the monitor, wait in line for it
     116                /* paranoid */ verify( thrd->next == 0p );
     117                append( this->entry_queue, thrd );
     118                /* paranoid */ verify( thrd->next == 1p );
     119
     120                unlock( this->lock );
     121                park( __cfaabi_dbg_ctx );
    128122
    129123                __cfaabi_dbg_print_safe( "Kernel : %10p Entered  mon %p\n", thrd, this);
    130124
    131                 // Release the lock and leave
     125                /* paranoid */ verifyf( kernelTLS.this_thread == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", kernelTLS.this_thread, this->owner, this->recursion, this );
     126                return;
     127        }
     128
     129        __cfaabi_dbg_print_safe( "Kernel : %10p Entered  mon %p\n", thrd, this);
     130
     131        /* paranoid */ verifyf( kernelTLS.this_thread == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", kernelTLS.this_thread, this->owner, this->recursion, this );
     132        /* paranoid */ verify( this->lock.lock );
     133
     134        // Release the lock and leave
     135        unlock( this->lock );
     136        return;
     137}
     138
     139static void __dtor_enter( $monitor * this, fptr_t func ) {
     140        // Lock the monitor spinlock
     141        lock( this->lock __cfaabi_dbg_ctx2 );
     142        // Interrupts disable inside critical section
     143        $thread * thrd = kernelTLS.this_thread;
     144
     145        __cfaabi_dbg_print_safe( "Kernel : %10p Entering dtor for mon %p (%p)\n", thrd, this, this->owner);
     146
     147
     148        if( !this->owner ) {
     149                __cfaabi_dbg_print_safe( "Kernel : Destroying free mon %p\n", this);
     150
     151                // No one has the monitor, just take it
     152                __set_owner( this, thrd );
     153
     154                verifyf( kernelTLS.this_thread == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", kernelTLS.this_thread, this->owner, this->recursion, this );
     155
    132156                unlock( this->lock );
    133157                return;
    134158        }
    135 
    136         static void __enter_monitor_dtor( monitor_desc * this, fptr_t func ) {
    137                 // Lock the monitor spinlock
    138                 lock( this->lock __cfaabi_dbg_ctx2 );
    139                 // Interrupts disable inside critical section
    140                 thread_desc * thrd = kernelTLS.this_thread;
    141 
    142                 __cfaabi_dbg_print_safe( "Kernel : %10p Entering dtor for mon %p (%p)\n", thrd, this, this->owner);
    143 
    144 
    145                 if( !this->owner ) {
    146                         __cfaabi_dbg_print_safe( "Kernel : Destroying free mon %p\n", this);
    147 
    148                         // No one has the monitor, just take it
    149                         set_owner( this, thrd );
    150 
    151                         unlock( this->lock );
    152                         return;
     159        else if( this->owner == thrd) {
     160                // We already have the monitor... but where about to destroy it so the nesting will fail
     161                // Abort!
     162                abort( "Attempt to destroy monitor %p by thread \"%.256s\" (%p) in nested mutex.", this, thrd->self_cor.name, thrd );
     163        }
     164
     165        __lock_size_t count = 1;
     166        $monitor ** monitors = &this;
     167        __monitor_group_t group = { &this, 1, func };
     168        if( is_accepted( this, group) ) {
     169                __cfaabi_dbg_print_safe( "Kernel :  mon accepts dtor, block and signal it \n" );
     170
     171                // Wake the thread that is waiting for this
     172                __condition_criterion_t * urgent = pop( this->signal_stack );
     173                /* paranoid */ verify( urgent );
     174
     175                // Reset mask
     176                reset_mask( this );
     177
     178                // Create the node specific to this wait operation
     179                wait_ctx_primed( thrd, 0 )
     180
     181                // Some one else has the monitor, wait for him to finish and then run
     182                unlock( this->lock );
     183
     184                // Release the next thread
     185                /* paranoid */ verifyf( urgent->owner->waiting_thread == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", kernelTLS.this_thread, this->owner, this->recursion, this );
     186                unpark( urgent->owner->waiting_thread __cfaabi_dbg_ctx2 );
     187
     188                // Park current thread waiting
     189                park( __cfaabi_dbg_ctx );
     190
     191                // Some one was waiting for us, enter
     192                /* paranoid */ verifyf( kernelTLS.this_thread == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", kernelTLS.this_thread, this->owner, this->recursion, this );
     193        }
     194        else {
     195                __cfaabi_dbg_print_safe( "Kernel :  blocking \n" );
     196
     197                wait_ctx( thrd, 0 )
     198                this->dtor_node = &waiter;
     199
     200                // Some one else has the monitor, wait in line for it
     201                /* paranoid */ verify( thrd->next == 0p );
     202                append( this->entry_queue, thrd );
     203                /* paranoid */ verify( thrd->next == 1p );
     204                unlock( this->lock );
     205
     206                // Park current thread waiting
     207                park( __cfaabi_dbg_ctx );
     208
     209                /* paranoid */ verifyf( kernelTLS.this_thread == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", kernelTLS.this_thread, this->owner, this->recursion, this );
     210                return;
     211        }
     212
     213        __cfaabi_dbg_print_safe( "Kernel : Destroying %p\n", this);
     214
     215}
     216
     217// Leave single monitor
     218void __leave( $monitor * this ) {
     219        // Lock the monitor spinlock
     220        lock( this->lock __cfaabi_dbg_ctx2 );
     221
     222        __cfaabi_dbg_print_safe( "Kernel : %10p Leaving mon %p (%p)\n", kernelTLS.this_thread, this, this->owner);
     223
     224        /* paranoid */ verifyf( kernelTLS.this_thread == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", kernelTLS.this_thread, this->owner, this->recursion, this );
     225
     226        // Leaving a recursion level, decrement the counter
     227        this->recursion -= 1;
     228
     229        // If we haven't left the last level of recursion
     230        // it means we don't need to do anything
     231        if( this->recursion != 0) {
     232                __cfaabi_dbg_print_safe( "Kernel :  recursion still %d\n", this->recursion);
     233                unlock( this->lock );
     234                return;
     235        }
     236
     237        // Get the next thread, will be null on low contention monitor
     238        $thread * new_owner = next_thread( this );
     239
     240        // Check the new owner is consistent with who we wake-up
     241        // new_owner might be null even if someone owns the monitor when the owner is still waiting for another monitor
     242        /* paranoid */ verifyf( !new_owner || new_owner == this->owner, "Expected owner to be %p, got %p (m: %p)", new_owner, this->owner, this );
     243
     244        // We can now let other threads in safely
     245        unlock( this->lock );
     246
     247        //We need to wake-up the thread
     248        /* paranoid */ verifyf( !new_owner || new_owner == this->owner, "Expected owner to be %p, got %p (m: %p)", new_owner, this->owner, this );
     249        unpark( new_owner __cfaabi_dbg_ctx2 );
     250}
     251
     252// Leave single monitor for the last time
     253void __dtor_leave( $monitor * this ) {
     254        __cfaabi_dbg_debug_do(
     255                if( TL_GET( this_thread ) != this->owner ) {
     256                        abort( "Destroyed monitor %p has inconsistent owner, expected %p got %p.\n", this, TL_GET( this_thread ), this->owner);
    153257                }
    154                 else if( this->owner == thrd) {
    155                         // We already have the monitor... but where about to destroy it so the nesting will fail
    156                         // Abort!
    157                         abort( "Attempt to destroy monitor %p by thread \"%.256s\" (%p) in nested mutex.", this, thrd->self_cor.name, thrd );
     258                if( this->recursion != 1 ) {
     259                        abort( "Destroyed monitor %p has %d outstanding nested calls.\n", this, this->recursion - 1);
    158260                }
    159 
    160                 __lock_size_t count = 1;
    161                 monitor_desc ** monitors = &this;
    162                 __monitor_group_t group = { &this, 1, func };
    163                 if( is_accepted( this, group) ) {
    164                         __cfaabi_dbg_print_safe( "Kernel :  mon accepts dtor, block and signal it \n" );
    165 
    166                         // Wake the thread that is waiting for this
    167                         __condition_criterion_t * urgent = pop( this->signal_stack );
    168                         verify( urgent );
    169 
    170                         // Reset mask
    171                         reset_mask( this );
    172 
    173                         // Create the node specific to this wait operation
    174                         wait_ctx_primed( thrd, 0 )
    175 
    176                         // Some one else has the monitor, wait for him to finish and then run
    177                         BlockInternal( &this->lock, urgent->owner->waiting_thread );
    178 
    179                         // Some one was waiting for us, enter
    180                         set_owner( this, thrd );
    181                 }
    182                 else {
    183                         __cfaabi_dbg_print_safe( "Kernel :  blocking \n" );
    184 
    185                         wait_ctx( thrd, 0 )
    186                         this->dtor_node = &waiter;
    187 
    188                         // Some one else has the monitor, wait in line for it
    189                         append( this->entry_queue, thrd );
    190                         BlockInternal( &this->lock );
    191 
    192                         // BlockInternal will unlock spinlock, no need to unlock ourselves
    193                         return;
    194                 }
    195 
    196                 __cfaabi_dbg_print_safe( "Kernel : Destroying %p\n", this);
    197 
    198         }
    199 
    200         // Leave single monitor
    201         void __leave_monitor_desc( monitor_desc * this ) {
    202                 // Lock the monitor spinlock
    203                 lock( this->lock __cfaabi_dbg_ctx2 );
    204 
    205                 __cfaabi_dbg_print_safe( "Kernel : %10p Leaving mon %p (%p)\n", kernelTLS.this_thread, this, this->owner);
    206 
    207                 verifyf( kernelTLS.this_thread == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", kernelTLS.this_thread, this->owner, this->recursion, this );
    208 
    209                 // Leaving a recursion level, decrement the counter
    210                 this->recursion -= 1;
    211 
    212                 // If we haven't left the last level of recursion
    213                 // it means we don't need to do anything
    214                 if( this->recursion != 0) {
    215                         __cfaabi_dbg_print_safe( "Kernel :  recursion still %d\n", this->recursion);
    216                         unlock( this->lock );
    217                         return;
    218                 }
    219 
    220                 // Get the next thread, will be null on low contention monitor
    221                 thread_desc * new_owner = next_thread( this );
    222 
    223                 // We can now let other threads in safely
    224                 unlock( this->lock );
    225 
    226                 //We need to wake-up the thread
    227                 WakeThread( new_owner );
    228         }
    229 
    230         // Leave single monitor for the last time
    231         void __leave_dtor_monitor_desc( monitor_desc * this ) {
    232                 __cfaabi_dbg_debug_do(
    233                         if( TL_GET( this_thread ) != this->owner ) {
    234                                 abort( "Destroyed monitor %p has inconsistent owner, expected %p got %p.\n", this, TL_GET( this_thread ), this->owner);
    235                         }
    236                         if( this->recursion != 1 ) {
    237                                 abort( "Destroyed monitor %p has %d outstanding nested calls.\n", this, this->recursion - 1);
    238                         }
    239                 )
    240         }
    241 
     261        )
     262}
     263
     264extern "C" {
    242265        // Leave the thread monitor
    243266        // last routine called by a thread.
    244267        // Should never return
    245         void __leave_thread_monitor() {
    246                 thread_desc * thrd = TL_GET( this_thread );
    247                 monitor_desc * this = &thrd->self_mon;
     268        void __cfactx_thrd_leave() {
     269                $thread * thrd = TL_GET( this_thread );
     270                $monitor * this = &thrd->self_mon;
    248271
    249272                // Lock the monitor now
    … …  
    252275                disable_interrupts();
    253276
    254                 thrd->self_cor.state = Halted;
    255 
    256                 verifyf( thrd == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", thrd, this->owner, this->recursion, this );
     277                thrd->state = Halted;
     278
     279                /* paranoid */ verifyf( thrd == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", thrd, this->owner, this->recursion, this );
    257280
    258281                // Leaving a recursion level, decrement the counter
    … …  
    264287
    265288                // Fetch the next thread, can be null
    266                 thread_desc * new_owner = next_thread( this );
    267 
    268                 // Leave the thread, this will unlock the spinlock
    269                 // Use leave thread instead of BlockInternal which is
    270                 // specialized for this case and supports null new_owner
    271                 LeaveThread( &this->lock, new_owner );
     289                $thread * new_owner = next_thread( this );
     290
     291                // Release the monitor lock
     292                unlock( this->lock );
     293
     294                // Unpark the next owner if needed
     295                /* paranoid */ verifyf( !new_owner || new_owner == this->owner, "Expected owner to be %p, got %p (m: %p)", new_owner, this->owner, this );
     296                /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
     297                /* paranoid */ verify( ! kernelTLS.this_processor->destroyer );
     298                /* paranoid */ verify( thrd->state == Halted );
     299
     300                kernelTLS.this_processor->destroyer = new_owner;
     301
     302                // Leave the thread
     303                __leave_thread();
    272304
    273305                // Control flow should never reach here!
    … …  
    279311static inline void enter( __monitor_group_t monitors ) {
    280312        for( __lock_size_t i = 0; i < monitors.size; i++) {
    281                 __enter_monitor_desc( monitors[i], monitors );
     313                __enter( monitors[i], monitors );
    282314        }
    283315}
    … …  
    285317// Leave multiple monitor
    286318// relies on the monitor array being sorted
    287 static inline void leave(monitor_desc * monitors [], __lock_size_t count) {
     319static inline void leave($monitor * monitors [], __lock_size_t count) {
    288320        for( __lock_size_t i = count - 1; i >= 0; i--) {
    289                 __leave_monitor_desc( monitors[i] );
     321                __leave( monitors[i] );
    290322        }
    291323}
    … …  
    293325// Ctor for monitor guard
    294326// Sorts monitors before entering
    295 void ?{}( monitor_guard_t & this, monitor_desc * m [], __lock_size_t count, fptr_t func ) {
    296         thread_desc * thrd = TL_GET( this_thread );
     327void ?{}( monitor_guard_t & this, $monitor * m [], __lock_size_t count, fptr_t func ) {
     328        $thread * thrd = TL_GET( this_thread );
    297329
    298330        // Store current array
    … …  
    334366// Ctor for monitor guard
    335367// Sorts monitors before entering
    336 void ?{}( monitor_dtor_guard_t & this, monitor_desc * m [], fptr_t func ) {
     368void ?{}( monitor_dtor_guard_t & this, $monitor * m [], fptr_t func ) {
    337369        // optimization
    338         thread_desc * thrd = TL_GET( this_thread );
     370        $thread * thrd = TL_GET( this_thread );
    339371
    340372        // Store current array
    … …  
    347379        (thrd->monitors){m, 1, func};
    348380
    349         __enter_monitor_dtor( this.m, func );
     381        __dtor_enter( this.m, func );
    350382}
    351383
    … …  
    353385void ^?{}( monitor_dtor_guard_t & this ) {
    354386        // Leave the monitors in order
    355         __leave_dtor_monitor_desc( this.m );
     387        __dtor_leave( this.m );
    356388
    357389        // Restore thread context
    … …  
    361393//-----------------------------------------------------------------------------
    362394// Internal scheduling types
    363 void ?{}(__condition_node_t & this, thread_desc * waiting_thread, __lock_size_t count, uintptr_t user_info ) {
     395void ?{}(__condition_node_t & this, $thread * waiting_thread, __lock_size_t count, uintptr_t user_info ) {
    364396        this.waiting_thread = waiting_thread;
    365397        this.count = count;
    … …  
    375407}
    376408
    377 void ?{}(__condition_criterion_t & this, monitor_desc * target, __condition_node_t & owner ) {
     409void ?{}(__condition_criterion_t & this, $monitor * target, __condition_node_t & owner ) {
    378410        this.ready  = false;
    379411        this.target = target;
    … …  
    400432        // Append the current wait operation to the ones already queued on the condition
    401433        // We don't need locks for that since conditions must always be waited on inside monitor mutual exclusion
     434        /* paranoid */ verify( waiter.next == 0p );
    402435        append( this.blocked, &waiter );
     436        /* paranoid */ verify( waiter.next == 1p );
    403437
    404438        // Lock all monitors (aggregates the locks as well)
    … …  
    407441        // Find the next thread(s) to run
    408442        __lock_size_t thread_count = 0;
    409         thread_desc * threads[ count ];
     443        $thread * threads[ count ];
    410444        __builtin_memset( threads, 0, sizeof( threads ) );
    411445
    … …  
    415449        // Remove any duplicate threads
    416450        for( __lock_size_t i = 0; i < count; i++) {
    417                 thread_desc * new_owner = next_thread( monitors[i] );
     451                $thread * new_owner = next_thread( monitors[i] );
    418452                insert_unique( threads, thread_count, new_owner );
    419453        }
    420454
     455        // Unlock the locks, we don't need them anymore
     456        for(int i = 0; i < count; i++) {
     457                unlock( *locks[i] );
     458        }
     459
     460        // Wake the threads
     461        for(int i = 0; i < thread_count; i++) {
     462                unpark( threads[i] __cfaabi_dbg_ctx2 );
     463        }
     464
    421465        // Everything is ready to go to sleep
    422         BlockInternal( locks, count, threads, thread_count );
     466        park( __cfaabi_dbg_ctx );
    423467
    424468        // We are back, restore the owners and recursions
    … …  
    435479        //Some more checking in debug
    436480        __cfaabi_dbg_debug_do(
    437                 thread_desc * this_thrd = TL_GET( this_thread );
     481                $thread * this_thrd = TL_GET( this_thread );
    438482                if ( this.monitor_count != this_thrd->monitors.size ) {
    439483                        abort( "Signal on condition %p made with different number of monitor(s), expected %zi got %zi", &this, this.monitor_count, this_thrd->monitors.size );
    … …  
    489533
    490534        //Find the thread to run
    491         thread_desc * signallee = pop_head( this.blocked )->waiting_thread;
    492         set_owner( monitors, count, signallee );
     535        $thread * signallee = pop_head( this.blocked )->waiting_thread;
     536        __set_owner( monitors, count, signallee );
    493537
    494538        __cfaabi_dbg_print_buffer_decl( "Kernel : signal_block condition %p (s: %p)\n", &this, signallee );
    495539
     540        // unlock all the monitors
     541        unlock_all( locks, count );
     542
     543        // unpark the thread we signalled
     544        unpark( signallee __cfaabi_dbg_ctx2 );
     545
    496546        //Everything is ready to go to sleep
    497         BlockInternal( locks, count, &signallee, 1 );
     547        park( __cfaabi_dbg_ctx );
    498548
    499549
    … …  
    536586        // Create one!
    537587        __lock_size_t max = count_max( mask );
    538         monitor_desc * mon_storage[max];
     588        $monitor * mon_storage[max];
    539589        __builtin_memset( mon_storage, 0, sizeof( mon_storage ) );
    540590        __lock_size_t actual_count = aggregate( mon_storage, mask );
    … …  
    554604        {
    555605                // Check if the entry queue
    556                 thread_desc * next; int index;
     606                $thread * next; int index;
    557607                [next, index] = search_entry_queue( mask, monitors, count );
    558608
    … …  
    564614                                verifyf( accepted.size == 1,  "ERROR: Accepted dtor has more than 1 mutex parameter." );
    565615
    566                                 monitor_desc * mon2dtor = accepted[0];
     616                                $monitor * mon2dtor = accepted[0];
    567617                                verifyf( mon2dtor->dtor_node, "ERROR: Accepted monitor has no dtor_node." );
    568618
    … …  
    590640
    591641                                // Set the owners to be the next thread
    592                                 set_owner( monitors, count, next );
    593 
    594                                 // Everything is ready to go to sleep
    595                                 BlockInternal( locks, count, &next, 1 );
     642                                __set_owner( monitors, count, next );
     643
     644                                // unlock all the monitors
     645                                unlock_all( locks, count );
     646
     647                                // unpark the thread we signalled
     648                                unpark( next __cfaabi_dbg_ctx2 );
     649
     650                                //Everything is ready to go to sleep
     651                                park( __cfaabi_dbg_ctx );
    596652
    597653                                // We are back, restore the owners and recursions
    … …  
    631687        }
    632688
     689        // unlock all the monitors
     690        unlock_all( locks, count );
     691
    633692        //Everything is ready to go to sleep
    634         BlockInternal( locks, count );
     693        park( __cfaabi_dbg_ctx );
    635694
    636695
    … …  
    649708// Utilities
    650709
    651 static inline void set_owner( monitor_desc * this, thread_desc * owner ) {
    652         // __cfaabi_dbg_print_safe( "Kernal :   Setting owner of %p to %p ( was %p)\n", this, owner, this->owner );
     710static inline void __set_owner( $monitor * this, $thread * owner ) {
     711        /* paranoid */ verify( this->lock.lock );
    653712
    654713        //Pass the monitor appropriately
    … …  
    659718}
    660719
    661 static inline void set_owner( monitor_desc * monitors [], __lock_size_t count, thread_desc * owner ) {
    662         monitors[0]->owner     = owner;
    663         monitors[0]->recursion = 1;
     720static inline void __set_owner( $monitor * monitors [], __lock_size_t count, $thread * owner ) {
     721        /* paranoid */ verify ( monitors[0]->lock.lock );
     722        /* paranoid */ verifyf( monitors[0]->owner == kernelTLS.this_thread, "Expected owner to be %p, got %p (r: %i, m: %p)", kernelTLS.this_thread, monitors[0]->owner, monitors[0]->recursion, monitors[0] );
     723        monitors[0]->owner        = owner;
     724        monitors[0]->recursion    = 1;
    664725        for( __lock_size_t i = 1; i < count; i++ ) {
    665                 monitors[i]->owner     = owner;
    666                 monitors[i]->recursion = 0;
    667         }
    668 }
    669 
    670 static inline void set_mask( monitor_desc * storage [], __lock_size_t count, const __waitfor_mask_t & mask ) {
     726                /* paranoid */ verify ( monitors[i]->lock.lock );
     727                /* paranoid */ verifyf( monitors[i]->owner == kernelTLS.this_thread, "Expected owner to be %p, got %p (r: %i, m: %p)", kernelTLS.this_thread, monitors[i]->owner, monitors[i]->recursion, monitors[i] );
     728                monitors[i]->owner        = owner;
     729                monitors[i]->recursion    = 0;
     730        }
     731}
     732
     733static inline void set_mask( $monitor * storage [], __lock_size_t count, const __waitfor_mask_t & mask ) {
    671734        for( __lock_size_t i = 0; i < count; i++) {
    672735                storage[i]->mask = mask;
    … …  
    674737}
    675738
    676 static inline void reset_mask( monitor_desc * this ) {
     739static inline void reset_mask( $monitor * this ) {
    677740        this->mask.accepted = 0p;
    678741        this->mask.data = 0p;
    … …  
    680743}
    681744
    682 static inline thread_desc * next_thread( monitor_desc * this ) {
     745static inline $thread * next_thread( $monitor * this ) {
    683746        //Check the signaller stack
    684747        __cfaabi_dbg_print_safe( "Kernel :  mon %p AS-stack top %p\n", this, this->signal_stack.top);
    … …  
    688751                //regardless of if we are ready to baton pass,
    689752                //we need to set the monitor as in use
    690                 set_owner( this,  urgent->owner->waiting_thread );
     753                /* paranoid */ verifyf( !this->owner || kernelTLS.this_thread == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", kernelTLS.this_thread, this->owner, this->recursion, this );
     754                __set_owner( this,  urgent->owner->waiting_thread );
    691755
    692756                return check_condition( urgent );
    … …  
    695759        // No signaller thread
    696760        // Get the next thread in the entry_queue
    697         thread_desc * new_owner = pop_head( this->entry_queue );
    698         set_owner( this, new_owner );
     761        $thread * new_owner = pop_head( this->entry_queue );
     762        /* paranoid */ verifyf( !this->owner || kernelTLS.this_thread == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", kernelTLS.this_thread, this->owner, this->recursion, this );
     763        /* paranoid */ verify( !new_owner || new_owner->next == 0p );
     764        __set_owner( this, new_owner );
    699765
    700766        return new_owner;
    701767}
    702768
    703 static inline bool is_accepted( monitor_desc * this, const __monitor_group_t & group ) {
     769static inline bool is_accepted( $monitor * this, const __monitor_group_t & group ) {
    704770        __acceptable_t * it = this->mask.data; // Optim
    705771        __lock_size_t count = this->mask.size;
    … …  
    723789}
    724790
    725 static inline void init( __lock_size_t count, monitor_desc * monitors [], __condition_node_t & waiter, __condition_criterion_t criteria [] ) {
     791static inline void init( __lock_size_t count, $monitor * monitors [], __condition_node_t & waiter, __condition_criterion_t criteria [] ) {
    726792        for( __lock_size_t i = 0; i < count; i++) {
    727793                (criteria[i]){ monitors[i], waiter };
    … …  
    731797}
    732798
    733 static inline void init_push( __lock_size_t count, monitor_desc * monitors [], __condition_node_t & waiter, __condition_criterion_t criteria [] ) {
     799static inline void init_push( __lock_size_t count, $monitor * monitors [], __condition_node_t & waiter, __condition_criterion_t criteria [] ) {
    734800        for( __lock_size_t i = 0; i < count; i++) {
    735801                (criteria[i]){ monitors[i], waiter };
    … …  
    747813}
    748814
    749 static inline void lock_all( monitor_desc * source [], __spinlock_t * /*out*/ locks [], __lock_size_t count ) {
     815static inline void lock_all( $monitor * source [], __spinlock_t * /*out*/ locks [], __lock_size_t count ) {
    750816        for( __lock_size_t i = 0; i < count; i++ ) {
    751817                __spinlock_t * l = &source[i]->lock;
    … …  
    761827}
    762828
    763 static inline void unlock_all( monitor_desc * locks [], __lock_size_t count ) {
     829static inline void unlock_all( $monitor * locks [], __lock_size_t count ) {
    764830        for( __lock_size_t i = 0; i < count; i++ ) {
    765831                unlock( locks[i]->lock );
    … …  
    768834
    769835static inline void save(
    770         monitor_desc * ctx [],
     836        $monitor * ctx [],
    771837        __lock_size_t count,
    772838        __attribute((unused)) __spinlock_t * locks [],
    … …  
    781847
    782848static inline void restore(
    783         monitor_desc * ctx [],
     849        $monitor * ctx [],
    784850        __lock_size_t count,
    785851        __spinlock_t * locks [],
    … …  
    799865// 2 - Checks if all the monitors are ready to run
    800866//     if so return the thread to run
    801 static inline thread_desc * check_condition( __condition_criterion_t * target ) {
     867static inline $thread * check_condition( __condition_criterion_t * target ) {
    802868        __condition_node_t * node = target->owner;
    803869        unsigned short count = node->count;
    … …  
    822888
    823889static inline void brand_condition( condition & this ) {
    824         thread_desc * thrd = TL_GET( this_thread );
     890        $thread * thrd = TL_GET( this_thread );
    825891        if( !this.monitors ) {
    826892                // __cfaabi_dbg_print_safe( "Branding\n" );
    … …  
    828894                this.monitor_count = thrd->monitors.size;
    829895
    830                 this.monitors = (monitor_desc **)malloc( this.monitor_count * sizeof( *this.monitors ) );
     896                this.monitors = ($monitor **)malloc( this.monitor_count * sizeof( *this.monitors ) );
    831897                for( int i = 0; i < this.monitor_count; i++ ) {
    832898                        this.monitors[i] = thrd->monitors[i];
    … …  
    835901}
    836902
    837 static inline [thread_desc *, int] search_entry_queue( const __waitfor_mask_t & mask, monitor_desc * monitors [], __lock_size_t count ) {
    838 
    839         __queue_t(thread_desc) & entry_queue = monitors[0]->entry_queue;
     903static inline [$thread *, int] search_entry_queue( const __waitfor_mask_t & mask, $monitor * monitors [], __lock_size_t count ) {
     904
     905        __queue_t($thread) & entry_queue = monitors[0]->entry_queue;
    840906
    841907        // For each thread in the entry-queue
    842         for(    thread_desc ** thrd_it = &entry_queue.head;
     908        for(    $thread ** thrd_it = &entry_queue.head;
    843909                *thrd_it;
    844910                thrd_it = &(*thrd_it)->link.next
    … …  
    884950}
    885951
    886 static inline __lock_size_t aggregate( monitor_desc * storage [], const __waitfor_mask_t & mask ) {
     952static inline __lock_size_t aggregate( $monitor * storage [], const __waitfor_mask_t & mask ) {
    887953        __lock_size_t size = 0;
    888954        for( __lock_size_t i = 0; i < mask.size; i++ ) {

  • libcfa/src/concurrency/monitor.hfa

    rb7d6a36 r6a490b2  
    2323
    2424trait is_monitor(dtype T) {
    25         monitor_desc * get_monitor( T & );
     25        $monitor * get_monitor( T & );
    2626        void ^?{}( T & mutex );
    2727};
    2828
    29 static inline void ?{}(monitor_desc & this) with( this ) {
     29static inline void ?{}($monitor & this) with( this ) {
    3030        lock{};
    3131        entry_queue{};
    … …  
    3939}
    4040
    41 static inline void ^?{}(monitor_desc & ) {}
     41static inline void ^?{}($monitor & ) {}
    4242
    4343struct monitor_guard_t {
    44         monitor_desc **         m;
     44        $monitor **     m;
    4545        __lock_size_t           count;
    4646        __monitor_group_t prev;
    4747};
    4848
    49 void ?{}( monitor_guard_t & this, monitor_desc ** m, __lock_size_t count, void (*func)() );
     49void ?{}( monitor_guard_t & this, $monitor ** m, __lock_size_t count, void (*func)() );
    5050void ^?{}( monitor_guard_t & this );
    5151
    5252struct monitor_dtor_guard_t {
    53         monitor_desc *    m;
     53        $monitor *    m;
    5454        __monitor_group_t prev;
    5555};
    5656
    57 void ?{}( monitor_dtor_guard_t & this, monitor_desc ** m, void (*func)() );
     57void ?{}( monitor_dtor_guard_t & this, $monitor ** m, void (*func)() );
    5858void ^?{}( monitor_dtor_guard_t & this );
    5959
    … …  
    7272
    7373        // The monitor this criterion concerns
    74         monitor_desc * target;
     74        $monitor * target;
    7575
    7676        // The parent node to which this criterion belongs
    … …  
    8787struct __condition_node_t {
    8888        // Thread that needs to be woken when all criteria are met
    89         thread_desc * waiting_thread;
     89        $thread * waiting_thread;
    9090
    9191        // Array of criteria (Criterions are contiguous in memory)
    … …  
    106106}
    107107
    108 void ?{}(__condition_node_t & this, thread_desc * waiting_thread, __lock_size_t count, uintptr_t user_info );
     108void ?{}(__condition_node_t & this, $thread * waiting_thread, __lock_size_t count, uintptr_t user_info );
    109109void ?{}(__condition_criterion_t & this );
    110 void ?{}(__condition_criterion_t & this, monitor_desc * target, __condition_node_t * owner );
     110void ?{}(__condition_criterion_t & this, $monitor * target, __condition_node_t * owner );
    111111
    112112struct condition {
    … …  
    115115
    116116        // Array of monitor pointers (Monitors are NOT contiguous in memory)
    117         monitor_desc ** monitors;
     117        $monitor ** monitors;
    118118
    119119        // Number of monitors in the array
    … …  
    133133              bool signal      ( condition & this );
    134134              bool signal_block( condition & this );
    135 static inline bool is_empty    ( condition & this ) { return !this.blocked.head; }
     135static inline bool is_empty    ( condition & this ) { return this.blocked.head == 1p; }
    136136         uintptr_t front       ( condition & this );
    137137

  • libcfa/src/concurrency/mutex.cfa

    rb7d6a36 r6a490b2  
    4040        if( is_locked ) {
    4141                append( blocked_threads, kernelTLS.this_thread );
    42                 BlockInternal( &lock );
     42                unlock( lock );
     43                park( __cfaabi_dbg_ctx );
    4344        }
    4445        else {
    … …  
    6263        lock( this.lock __cfaabi_dbg_ctx2 );
    6364        this.is_locked = (this.blocked_threads != 0);
    64         WakeThread(
    65                 pop_head( this.blocked_threads )
     65        unpark(
     66                pop_head( this.blocked_threads ) __cfaabi_dbg_ctx2
    6667        );
    6768        unlock( this.lock );
    … …  
    9495        else {
    9596                append( blocked_threads, kernelTLS.this_thread );
    96                 BlockInternal( &lock );
     97                unlock( lock );
     98                park( __cfaabi_dbg_ctx );
    9799        }
    98100}
    … …  
    118120        recursion_count--;
    119121        if( recursion_count == 0 ) {
    120                 thread_desc * thrd = pop_head( blocked_threads );
     122                $thread * thrd = pop_head( blocked_threads );
    121123                owner = thrd;
    122124                recursion_count = (thrd ? 1 : 0);
    123                 WakeThread( thrd );
     125                unpark( thrd __cfaabi_dbg_ctx2 );
    124126        }
    125127        unlock( lock );
    … …  
    138140void notify_one(condition_variable & this) with(this) {
    139141        lock( lock __cfaabi_dbg_ctx2 );
    140         WakeThread(
    141                 pop_head( this.blocked_threads )
     142        unpark(
     143                pop_head( this.blocked_threads ) __cfaabi_dbg_ctx2
    142144        );
    143145        unlock( lock );
    … …  
    147149        lock( lock __cfaabi_dbg_ctx2 );
    148150        while(this.blocked_threads) {
    149                 WakeThread(
    150                         pop_head( this.blocked_threads )
     151                unpark(
     152                        pop_head( this.blocked_threads ) __cfaabi_dbg_ctx2
    151153                );
    152154        }
    … …  
    157159        lock( this.lock __cfaabi_dbg_ctx2 );
    158160        append( this.blocked_threads, kernelTLS.this_thread );
    159         BlockInternal( &this.lock );
     161        unlock( this.lock );
     162        park( __cfaabi_dbg_ctx );
    160163}
    161164
    … …  
    164167        lock( this.lock __cfaabi_dbg_ctx2 );
    165168        append( this.blocked_threads, kernelTLS.this_thread );
    166         void __unlock(void) {
    167                 unlock(l);
    168                 unlock(this.lock);
    169         }
    170         BlockInternal( __unlock );
     169        unlock(l);
     170        unlock(this.lock);
     171        park( __cfaabi_dbg_ctx );
    171172        lock(l);
    172173}

  • libcfa/src/concurrency/mutex.hfa

    rb7d6a36 r6a490b2  
    3636
    3737        // List of blocked threads
    38         __queue_t(struct thread_desc) blocked_threads;
     38        __queue_t(struct $thread) blocked_threads;
    3939
    4040        // Locked flag
    … …  
    5555
    5656        // List of blocked threads
    57         __queue_t(struct thread_desc) blocked_threads;
     57        __queue_t(struct $thread) blocked_threads;
    5858
    5959        // Current thread owning the lock
    60         struct thread_desc * owner;
     60        struct $thread * owner;
    6161
    6262        // Number of recursion level
    … …  
    8383
    8484        // List of blocked threads
    85         __queue_t(struct thread_desc) blocked_threads;
     85        __queue_t(struct $thread) blocked_threads;
    8686};
    8787

  • libcfa/src/concurrency/preemption.cfa

    rb7d6a36 r6a490b2  
    3939// FwdDeclarations : timeout handlers
    4040static void preempt( processor   * this );
    41 static void timeout( thread_desc * this );
     41static void timeout( $thread * this );
    4242
    4343// FwdDeclarations : Signal handlers
    4444static void sigHandler_ctxSwitch( __CFA_SIGPARMS__ );
     45static void sigHandler_alarm    ( __CFA_SIGPARMS__ );
    4546static void sigHandler_segv     ( __CFA_SIGPARMS__ );
    4647static void sigHandler_ill      ( __CFA_SIGPARMS__ );
    … …  
    8384// Get next expired node
    8485static inline alarm_node_t * get_expired( alarm_list_t * alarms, Time currtime ) {
    85         if( !alarms->head ) return 0p;                                          // If no alarms return null
    86         if( alarms->head->alarm >= currtime ) return 0p;        // If alarms head not expired return null
     86        if( ! & (*alarms)`first ) return 0p;                                            // If no alarms return null
     87        if( (*alarms)`first.alarm >= currtime ) return 0p;      // If alarms head not expired return null
    8788        return pop(alarms);                                                                     // Otherwise just pop head
    8889}
    … …  
    9798        while( node = get_expired( alarms, currtime ) ) {
    9899                // __cfaabi_dbg_print_buffer_decl( " KERNEL: preemption tick.\n" );
     100                Duration period = node->period;
     101                if( period == 0) {
     102                        node->set = false;                  // Node is one-shot, just mark it as not pending
     103                }
    99104
    100105                // Check if this is a kernel
    … …  
    107112
    108113                // Check if this is a periodic alarm
    109                 Duration period = node->period;
    110114                if( period > 0 ) {
    111115                        // __cfaabi_dbg_print_buffer_local( " KERNEL: alarm period is %lu.\n", period.tv );
    … …  
    113117                        insert( alarms, node );             // Reinsert the node for the next time it triggers
    114118                }
    115                 else {
    116                         node->set = false;                  // Node is one-shot, just mark it as not pending
    117                 }
    118119        }
    119120
    120121        // If there are still alarms pending, reset the timer
    121         if( alarms->head ) {
    122                 // __cfaabi_dbg_print_buffer_decl( " KERNEL: @%ju(%ju) resetting alarm to %ju.\n", currtime.tv, __kernel_get_time().tv, (alarms->head->alarm - currtime).tv);
    123                 Duration delta = alarms->head->alarm - currtime;
    124                 Duration caped = max(delta, 50`us);
     122        if( & (*alarms)`first ) {
     123                __cfadbg_print_buffer_decl(preemption, " KERNEL: @%ju(%ju) resetting alarm to %ju.\n", currtime.tv, __kernel_get_time().tv, (alarms->head->alarm - currtime).tv);
     124                Duration delta = (*alarms)`first.alarm - currtime;
     125                Duration capped = max(delta, 50`us);
    125126                // itimerval tim  = { caped };
    126127                // __cfaabi_dbg_print_buffer_local( "    Values are %lu, %lu, %lu %lu.\n", delta.tv, caped.tv, tim.it_value.tv_sec, tim.it_value.tv_usec);
    127128
    128                 __kernel_set_timer( caped );
     129                __kernel_set_timer( capped );
    129130        }
    130131}
    … …  
    184185
    185186        // Enable interrupts by decrementing the counter
    186         // If counter reaches 0, execute any pending CtxSwitch
     187        // If counter reaches 0, execute any pending __cfactx_switch
    187188        void enable_interrupts( __cfaabi_dbg_ctx_param ) {
    188189                processor   * proc = kernelTLS.this_processor; // Cache the processor now since interrupts can start happening after the atomic store
    189                 thread_desc * thrd = kernelTLS.this_thread;       // Cache the thread now since interrupts can start happening after the atomic store
    190190
    191191                with( kernelTLS.preemption_state ){
    … …  
    209209                                if( proc->pending_preemption ) {
    210210                                        proc->pending_preemption = false;
    211                                         BlockInternal( thrd );
     211                                        force_yield( __POLL_PREEMPTION );
    212212                                }
    213213                        }
    … …  
    219219
    220220        // Disable interrupts by incrementint the counter
    221         // Don't execute any pending CtxSwitch even if counter reaches 0
     221        // Don't execute any pending __cfactx_switch even if counter reaches 0
    222222        void enable_interrupts_noPoll() {
    223223                unsigned short prev = kernelTLS.preemption_state.disable_count;
    … …  
    257257
    258258        if ( pthread_sigmask( SIG_BLOCK, &mask, 0p ) == -1 ) {
    259             abort( "internal error, pthread_sigmask" );
     259                abort( "internal error, pthread_sigmask" );
    260260        }
    261261}
    … …  
    268268
    269269// reserved for future use
    270 static void timeout( thread_desc * this ) {
    271         //TODO : implement waking threads
     270static void timeout( $thread * this ) {
     271        __unpark( this __cfaabi_dbg_ctx2 );
    272272}
    273273
    274274// KERNEL ONLY
    275 // Check if a CtxSwitch signal handler shoud defer
     275// Check if a __cfactx_switch signal handler shoud defer
    276276// If true  : preemption is safe
    277277// If false : preemption is unsafe and marked as pending
    … …  
    303303
    304304        // Setup proper signal handlers
    305         __cfaabi_sigaction( SIGUSR1, sigHandler_ctxSwitch, SA_SIGINFO | SA_RESTART ); // CtxSwitch handler
     305        __cfaabi_sigaction( SIGUSR1, sigHandler_ctxSwitch, SA_SIGINFO | SA_RESTART ); // __cfactx_switch handler
     306        __cfaabi_sigaction( SIGALRM, sigHandler_alarm    , SA_SIGINFO | SA_RESTART ); // debug handler
    306307
    307308        signal_block( SIGALRM );
    308309
    309         alarm_stack = create_pthread( &alarm_thread, alarm_loop, 0p );
     310        alarm_stack = __create_pthread( &alarm_thread, alarm_loop, 0p );
    310311}
    311312
    … …  
    394395        // Preemption can occur here
    395396
    396         BlockInternal( kernelTLS.this_thread ); // Do the actual CtxSwitch
     397        force_yield( __ALARM_PREEMPTION ); // Do the actual __cfactx_switch
     398}
     399
     400static void sigHandler_alarm( __CFA_SIGPARMS__ ) {
     401        abort("SIGALRM should never reach the signal handler");
    397402}
    398403

  • libcfa/src/concurrency/thread.cfa

    rb7d6a36 r6a490b2  
    2323#include "invoke.h"
    2424
    25 extern "C" {
    26         #include <fenv.h>
    27         #include <stddef.h>
    28 }
    29 
    30 //extern volatile thread_local processor * this_processor;
    31 
    3225//-----------------------------------------------------------------------------
    3326// Thread ctors and dtors
    34 void ?{}(thread_desc & this, const char * const name, cluster & cl, void * storage, size_t storageSize ) with( this ) {
     27void ?{}($thread & this, const char * const name, cluster & cl, void * storage, size_t storageSize ) with( this ) {
    3528        context{ 0p, 0p };
    3629        self_cor{ name, storage, storageSize };
    3730        state = Start;
     31        preempted = __NO_PREEMPTION;
    3832        curr_cor = &self_cor;
    3933        self_mon.owner = &this;
    … …  
    5145}
    5246
    53 void ^?{}(thread_desc& this) with( this ) {
     47void ^?{}($thread& this) with( this ) {
    5448        unregister(curr_cluster, this);
    5549        ^self_cor{};
    5650}
    5751
     52//-----------------------------------------------------------------------------
     53// Starting and stopping threads
     54forall( dtype T | is_thread(T) )
     55void __thrd_start( T & this, void (*main_p)(T &) ) {
     56        $thread * this_thrd = get_thread(this);
     57
     58        disable_interrupts();
     59        __cfactx_start(main_p, get_coroutine(this), this, __cfactx_invoke_thread);
     60
     61        this_thrd->context.[SP, FP] = this_thrd->self_cor.context.[SP, FP];
     62        verify( this_thrd->context.SP );
     63
     64        __schedule_thread(this_thrd);
     65        enable_interrupts( __cfaabi_dbg_ctx );
     66}
     67
     68//-----------------------------------------------------------------------------
     69// Support for threads that don't ues the thread keyword
    5870forall( dtype T | sized(T) | is_thread(T) | { void ?{}(T&); } )
    5971void ?{}( scoped(T)& this ) with( this ) {
    … …  
    7385}
    7486
    75 //-----------------------------------------------------------------------------
    76 // Starting and stopping threads
    77 forall( dtype T | is_thread(T) )
    78 void __thrd_start( T & this, void (*main_p)(T &) ) {
    79         thread_desc * this_thrd = get_thread(this);
    80 
    81         disable_interrupts();
    82         CtxStart(main_p, get_coroutine(this), this, CtxInvokeThread);
    83 
    84         this_thrd->context.[SP, FP] = this_thrd->self_cor.context.[SP, FP];
    85         verify( this_thrd->context.SP );
    86 
    87         ScheduleThread(this_thrd);
    88         enable_interrupts( __cfaabi_dbg_ctx );
    89 }
    90 
    91 void yield( void ) {
    92         // Safety note : This could cause some false positives due to preemption
    93       verify( TL_GET( preemption_state.enabled ) );
    94         BlockInternal( TL_GET( this_thread ) );
    95         // Safety note : This could cause some false positives due to preemption
    96       verify( TL_GET( preemption_state.enabled ) );
    97 }
    98 
    99 void yield( unsigned times ) {
    100         for( unsigned i = 0; i < times; i++ ) {
    101                 yield();
    102         }
    103 }
    104 
    10587// Local Variables: //
    10688// mode: c //

  • libcfa/src/concurrency/thread.hfa

    rb7d6a36 r6a490b2  
    2828      void ^?{}(T& mutex this);
    2929      void main(T& this);
    30       thread_desc* get_thread(T& this);
     30      $thread* get_thread(T& this);
    3131};
    3232
    33 #define DECL_THREAD(X) thread_desc* get_thread(X& this) { return &this.__thrd; } void main(X& this)
     33// define that satisfies the trait without using the thread keyword
     34#define DECL_THREAD(X) $thread* get_thread(X& this) __attribute__((const)) { return &this.__thrd; } void main(X& this)
     35
     36// Inline getters for threads/coroutines/monitors
     37forall( dtype T | is_thread(T) )
     38static inline $coroutine* get_coroutine(T & this) __attribute__((const)) { return &get_thread(this)->self_cor; }
    3439
    3540forall( dtype T | is_thread(T) )
    36 static inline coroutine_desc* get_coroutine(T & this) {
    37         return &get_thread(this)->self_cor;
    38 }
     41static inline $monitor  * get_monitor  (T & this) __attribute__((const)) { return &get_thread(this)->self_mon; }
    3942
    40 forall( dtype T | is_thread(T) )
    41 static inline monitor_desc* get_monitor(T & this) {
    42         return &get_thread(this)->self_mon;
    43 }
     43static inline $coroutine* get_coroutine($thread * this) __attribute__((const)) { return &this->self_cor; }
     44static inline $monitor  * get_monitor  ($thread * this) __attribute__((const)) { return &this->self_mon; }
    4445
    45 static inline coroutine_desc* get_coroutine(thread_desc * this) {
    46         return &this->self_cor;
    47 }
    48 
    49 static inline monitor_desc* get_monitor(thread_desc * this) {
    50         return &this->self_mon;
    51 }
    52 
     46//-----------------------------------------------------------------------------
     47// forward declarations needed for threads
    5348extern struct cluster * mainCluster;
    5449
    … …  
    5853//-----------------------------------------------------------------------------
    5954// Ctors and dtors
    60 void ?{}(thread_desc & this, const char * const name, struct cluster & cl, void * storage, size_t storageSize );
    61 void ^?{}(thread_desc & this);
     55void ?{}($thread & this, const char * const name, struct cluster & cl, void * storage, size_t storageSize );
     56void ^?{}($thread & this);
    6257
    63 static inline void ?{}(thread_desc & this)                                                                  { this{ "Anonymous Thread", *mainCluster, 0p, 65000 }; }
    64 static inline void ?{}(thread_desc & this, size_t stackSize )                                               { this{ "Anonymous Thread", *mainCluster, 0p, stackSize }; }
    65 static inline void ?{}(thread_desc & this, void * storage, size_t storageSize )                             { this{ "Anonymous Thread", *mainCluster, storage, storageSize }; }
    66 static inline void ?{}(thread_desc & this, struct cluster & cl )                                            { this{ "Anonymous Thread", cl, 0p, 65000 }; }
    67 static inline void ?{}(thread_desc & this, struct cluster & cl, size_t stackSize )                          { this{ "Anonymous Thread", cl, 0p, stackSize }; }
    68 static inline void ?{}(thread_desc & this, struct cluster & cl, void * storage, size_t storageSize )        { this{ "Anonymous Thread", cl, storage, storageSize }; }
    69 static inline void ?{}(thread_desc & this, const char * const name)                                         { this{ name, *mainCluster, 0p, 65000 }; }
    70 static inline void ?{}(thread_desc & this, const char * const name, struct cluster & cl )                   { this{ name, cl, 0p, 65000 }; }
    71 static inline void ?{}(thread_desc & this, const char * const name, struct cluster & cl, size_t stackSize ) { this{ name, cl, 0p, stackSize }; }
     58static inline void ?{}($thread & this)                                                                  { this{ "Anonymous Thread", *mainCluster, 0p, 65000 }; }
     59static inline void ?{}($thread & this, size_t stackSize )                                               { this{ "Anonymous Thread", *mainCluster, 0p, stackSize }; }
     60static inline void ?{}($thread & this, void * storage, size_t storageSize )                             { this{ "Anonymous Thread", *mainCluster, storage, storageSize }; }
     61static inline void ?{}($thread & this, struct cluster & cl )                                            { this{ "Anonymous Thread", cl, 0p, 65000 }; }
     62static inline void ?{}($thread & this, struct cluster & cl, size_t stackSize )                          { this{ "Anonymous Thread", cl, 0p, stackSize }; }
     63static inline void ?{}($thread & this, struct cluster & cl, void * storage, size_t storageSize )        { this{ "Anonymous Thread", cl, storage, storageSize }; }
     64static inline void ?{}($thread & this, const char * const name)                                         { this{ name, *mainCluster, 0p, 65000 }; }
     65static inline void ?{}($thread & this, const char * const name, struct cluster & cl )                   { this{ name, cl, 0p, 65000 }; }
     66static inline void ?{}($thread & this, const char * const name, struct cluster & cl, size_t stackSize ) { this{ name, cl, 0p, stackSize }; }
    7267
    7368//-----------------------------------------------------------------------------
    … …  
    8883void ^?{}( scoped(T)& this );
    8984
    90 void yield();
    91 void yield( unsigned times );
     85//-----------------------------------------------------------------------------
     86// Thread getters
     87static inline struct $thread * active_thread () { return TL_GET( this_thread ); }
    9288
    93 static inline struct thread_desc * active_thread () { return TL_GET( this_thread ); }
     89//-----------------------------------------------------------------------------
     90// Scheduler API
     91
     92//----------
     93// Park thread: block until corresponding call to unpark, won't block if unpark is already called
     94void park( __cfaabi_dbg_ctx_param );
     95
     96//----------
     97// Unpark a thread, if the thread is already blocked, schedule it
     98//                  if the thread is not yet block, signal that it should rerun immediately
     99void unpark( $thread * this __cfaabi_dbg_ctx_param2 );
     100
     101forall( dtype T | is_thread(T) )
     102static inline void unpark( T & this __cfaabi_dbg_ctx_param2 ) { if(!&this) return; unpark( get_thread( this ) __cfaabi_dbg_ctx_fwd2 );}
     103
     104//----------
     105// Yield: force thread to block and be rescheduled
     106bool force_yield( enum __Preemption_Reason );
     107
     108static inline void yield() {
     109        force_yield(__MANUAL_PREEMPTION);
     110}
     111
     112// Yield: yield N times
     113static inline void yield( unsigned times ) {
     114        for( times ) {
     115                yield();
     116        }
     117}
     118
     119//----------
     120// sleep: force thread to block and be rescheduled after Duration duration
     121void sleep( Duration duration );
    94122
    95123// Local Variables: //
Note: See TracChangeset for help on using the changeset viewer.