Ignore:
File:
1 edited

Legend:

Unmodified
Added
Removed

  • libcfa/src/concurrency/locks.hfa

    r1ab773e0 r70a4ed5  
    3030#include "time.hfa"
    3131
     32#include "select.hfa"
     33
    3234#include <fstream.hfa>
    33 
    3435
    3536// futex headers
    … …  
    3839#include <unistd.h>
    3940
    40 // undef to make a number of the locks not reacquire upon waking from a condlock
    41 #define REACQ 1
     41typedef void (*__cfa_pre_park)( void * );
     42
     43static inline void pre_park_noop( void * ) {}
     44
     45//-----------------------------------------------------------------------------
     46// is_blocking_lock
     47forall( L & | sized(L) )
     48trait is_blocking_lock {
     49        // For synchronization locks to use when acquiring
     50        void on_notify( L &, struct thread$ * );
     51
     52        // For synchronization locks to use when releasing
     53        size_t on_wait( L &, __cfa_pre_park pp_fn, void * pp_datum );
     54
     55        // to set recursion count after getting signalled;
     56        void on_wakeup( L &, size_t recursion );
     57};
     58
     59static inline void pre_park_then_park( __cfa_pre_park pp_fn, void * pp_datum ) {
     60    pp_fn( pp_datum );
     61    park();
     62}
     63
     64// macros for default routine impls for is_blocking_lock trait that do not wait-morph
     65
     66#define DEFAULT_ON_NOTIFY( lock_type ) \
     67    static inline void on_notify( lock_type & this, thread$ * t ){ unpark(t); }
     68
     69#define DEFAULT_ON_WAIT( lock_type ) \
     70    static inline size_t on_wait( lock_type & this, __cfa_pre_park pp_fn, void * pp_datum ) { \
     71        unlock( this ); \
     72        pre_park_then_park( pp_fn, pp_datum ); \
     73        return 0; \
     74    }
     75
     76// on_wakeup impl if lock should be reacquired after waking up
     77#define DEFAULT_ON_WAKEUP_REACQ( lock_type ) \
     78    static inline void on_wakeup( lock_type & this, size_t recursion ) { lock( this ); }
     79
     80// on_wakeup impl if lock will not be reacquired after waking up
     81#define DEFAULT_ON_WAKEUP_NO_REACQ( lock_type ) \
     82    static inline void on_wakeup( lock_type & this, size_t recursion ) {}
     83
     84
    4285
    4386//-----------------------------------------------------------------------------
    … …  
    66109static inline bool   try_lock ( single_acquisition_lock & this ) { return try_lock( (blocking_lock &)this ); }
    67110static inline void   unlock   ( single_acquisition_lock & this ) { unlock  ( (blocking_lock &)this ); }
    68 static inline size_t on_wait  ( single_acquisition_lock & this ) { return on_wait ( (blocking_lock &)this ); }
     111static inline size_t on_wait  ( single_acquisition_lock & this, __cfa_pre_park pp_fn, void * pp_datum ) { return on_wait ( (blocking_lock &)this, pp_fn, pp_datum ); }
    69112static inline void   on_wakeup( single_acquisition_lock & this, size_t v ) { on_wakeup ( (blocking_lock &)this, v ); }
    70113static inline void   on_notify( single_acquisition_lock & this, struct thread$ * t ) { on_notify( (blocking_lock &)this, t ); }
     114static inline bool   register_select( single_acquisition_lock & this, select_node & node ) { return register_select( (blocking_lock &)this, node ); }
     115static inline bool   unregister_select( single_acquisition_lock & this, select_node & node ) { return unregister_select( (blocking_lock &)this, node ); }
     116static inline void   on_selected( single_acquisition_lock & this, select_node & node ) { on_selected( (blocking_lock &)this, node ); }
    71117
    72118//----------
    … …  
    80126static inline bool   try_lock ( owner_lock & this ) { return try_lock( (blocking_lock &)this ); }
    81127static inline void   unlock   ( owner_lock & this ) { unlock  ( (blocking_lock &)this ); }
    82 static inline size_t on_wait  ( owner_lock & this ) { return on_wait ( (blocking_lock &)this ); }
     128static inline size_t on_wait  ( owner_lock & this, __cfa_pre_park pp_fn, void * pp_datum ) { return on_wait ( (blocking_lock &)this, pp_fn, pp_datum ); }
    83129static inline void   on_wakeup( owner_lock & this, size_t v ) { on_wakeup ( (blocking_lock &)this, v ); }
    84130static inline void   on_notify( owner_lock & this, struct thread$ * t ) { on_notify( (blocking_lock &)this, t ); }
     131static inline bool   register_select( owner_lock & this, select_node & node ) { return register_select( (blocking_lock &)this, node ); }
     132static inline bool   unregister_select( owner_lock & this, select_node & node ) { return unregister_select( (blocking_lock &)this, node ); }
     133static inline void   on_selected( owner_lock & this, select_node & node ) { on_selected( (blocking_lock &)this, node ); }
    85134
    86135//-----------------------------------------------------------------------------
    … …  
    127176static inline void ?{}(mcs_spin_node & this) { this.next = 0p; this.locked = true; }
    128177
    129 static inline mcs_spin_node * volatile & ?`next ( mcs_spin_node * node ) {
    130         return node->next;
    131 }
    132 
    133178struct mcs_spin_lock {
    134179        mcs_spin_queue queue;
    … …  
    136181
    137182static inline void lock(mcs_spin_lock & l, mcs_spin_node & n) {
     183    n.locked = true;
    138184        mcs_spin_node * prev = __atomic_exchange_n(&l.queue.tail, &n, __ATOMIC_SEQ_CST);
    139         n.locked = true;
    140         if(prev == 0p) return;
     185        if( prev == 0p ) return;
    141186        prev->next = &n;
    142         while(__atomic_load_n(&n.locked, __ATOMIC_RELAXED)) Pause();
     187        while( __atomic_load_n(&n.locked, __ATOMIC_RELAXED) ) Pause();
    143188}
    144189
    … …  
    146191        mcs_spin_node * n_ptr = &n;
    147192        if (__atomic_compare_exchange_n(&l.queue.tail, &n_ptr, 0p, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) return;
    148         while (__atomic_load_n(&n.next, __ATOMIC_RELAXED) == 0p) {}
     193        while (__atomic_load_n(&n.next, __ATOMIC_RELAXED) == 0p) Pause();
    149194        n.next->locked = false;
    150195}
    … …  
    153198// futex_mutex
    154199
    155 // - No cond var support
    156200// - Kernel thd blocking alternative to the spinlock
    157201// - No ownership (will deadlock on reacq)
     202// - no reacq on wakeup
    158203struct futex_mutex {
    159204        // lock state any state other than UNLOCKED is locked
    … …  
    169214}
    170215
    171 static inline void  ?{}( futex_mutex & this ) with(this) { val = 0; }
    172 
    173 static inline bool internal_try_lock(futex_mutex & this, int & compare_val) with(this) {
     216static inline void ?{}( futex_mutex & this ) with(this) { val = 0; }
     217
     218static inline bool internal_try_lock( futex_mutex & this, int & compare_val) with(this) {
    174219        return __atomic_compare_exchange_n((int*)&val, (int*)&compare_val, 1, false, __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE);
    175220}
    176221
    177 static inline int internal_exchange(futex_mutex & this) with(this) {
     222static inline int internal_exchange( futex_mutex & this ) with(this) {
    178223        return __atomic_exchange_n((int*)&val, 2, __ATOMIC_ACQUIRE);
    179224}
    180225
    181226// if this is called recursively IT WILL DEADLOCK!!!!!
    182 static inline void lock(futex_mutex & this) with(this) {
     227static inline void lock( futex_mutex & this ) with(this) {
    183228        int state;
    184229
    185        
    186         // // linear backoff omitted for now
    187         // for( int spin = 4; spin < 1024; spin += spin) {
    188         //      state = 0;
    189         //      // if unlocked, lock and return
    190         //      if (internal_try_lock(this, state)) return;
    191         //      if (2 == state) break;
    192         //      for (int i = 0; i < spin; i++) Pause();
    193         // }
    194 
    195         // no contention try to acquire
    196         if (internal_try_lock(this, state)) return;
     230        for( int spin = 4; spin < 1024; spin += spin) {
     231                state = 0;
     232                // if unlocked, lock and return
     233                if (internal_try_lock(this, state)) return;
     234                if (2 == state) break;
     235                for (int i = 0; i < spin; i++) Pause();
     236        }
    197237       
    198238        // if not in contended state, set to be in contended state
    … …  
    207247
    208248static inline void unlock(futex_mutex & this) with(this) {
    209         // if uncontended do atomice unlock and then return
    210         if (__atomic_fetch_sub(&val, 1, __ATOMIC_RELEASE) == 1) return; // TODO: try acq/rel
     249        // if uncontended do atomic unlock and then return
     250    if (__atomic_exchange_n(&val, 0, __ATOMIC_RELEASE) == 1) return;
    211251       
    212252        // otherwise threads are blocked so we must wake one
    213         __atomic_store_n((int *)&val, 0, __ATOMIC_RELEASE);
    214253        futex((int *)&val, FUTEX_WAKE, 1);
    215254}
    216255
    217 static inline void on_notify( futex_mutex & f, thread$ * t){ unpark(t); }
    218 static inline size_t on_wait( futex_mutex & f ) {unlock(f); return 0;}
    219 
    220 // to set recursion count after getting signalled;
    221 static inline void on_wakeup( futex_mutex & f, size_t recursion ) {}
    222 
    223 //-----------------------------------------------------------------------------
    224 // CLH Spinlock
    225 // - No recursive acquisition
    226 // - Needs to be released by owner
    227 
    228 struct clh_lock {
    229         volatile bool * volatile tail;
    230 };
    231 
    232 static inline void  ?{}( clh_lock & this ) { this.tail = malloc(); *this.tail = true; }
    233 static inline void ^?{}( clh_lock & this ) { free(this.tail); }
    234 
    235 static inline void lock(clh_lock & l) {
    236         thread$ * curr_thd = active_thread();
    237         *(curr_thd->clh_node) = false;
    238         volatile bool * prev = __atomic_exchange_n((bool **)(&l.tail), (bool *)(curr_thd->clh_node), __ATOMIC_SEQ_CST);
    239         while(!__atomic_load_n(prev, __ATOMIC_ACQUIRE)) Pause();
    240         curr_thd->clh_prev = prev;
    241 }
    242 
    243 static inline void unlock(clh_lock & l) {
    244         thread$ * curr_thd = active_thread();
    245         __atomic_store_n(curr_thd->clh_node, true, __ATOMIC_RELEASE);
    246         curr_thd->clh_node = curr_thd->clh_prev;
    247 }
    248 
    249 static inline void on_notify(clh_lock & this, struct thread$ * t ) { unpark(t); }
    250 static inline size_t on_wait(clh_lock & this) { unlock(this); return 0; }
    251 static inline void on_wakeup(clh_lock & this, size_t recursion ) {
    252         #ifdef REACQ
    253         lock(this);
    254         #endif
    255 }
    256 
    257 
    258 //-----------------------------------------------------------------------------
    259 // Linear backoff Spinlock
    260 struct linear_backoff_then_block_lock {
     256DEFAULT_ON_NOTIFY( futex_mutex )
     257DEFAULT_ON_WAIT( futex_mutex )
     258DEFAULT_ON_WAKEUP_NO_REACQ( futex_mutex )
     259
     260//-----------------------------------------------------------------------------
     261// go_mutex
     262
     263// - Kernel thd blocking alternative to the spinlock
     264// - No ownership (will deadlock on reacq)
     265// - Golang's flavour of mutex
     266// - Impl taken from Golang: src/runtime/lock_futex.go
     267struct go_mutex {
     268        // lock state any state other than UNLOCKED is locked
     269        // enum LockState { UNLOCKED = 0, LOCKED = 1, SLEEPING = 2 };
     270       
     271        // stores a lock state
     272        int val;
     273};
     274static inline void  ?{}( go_mutex & this ) with(this) { val = 0; }
     275// static inline void ?{}( go_mutex & this, go_mutex this2 ) = void; // these don't compile correctly at the moment so they should be omitted
     276// static inline void ?=?( go_mutex & this, go_mutex this2 ) = void;
     277
     278static inline bool internal_try_lock(go_mutex & this, int & compare_val, int new_val ) with(this) {
     279        return __atomic_compare_exchange_n((int*)&val, (int*)&compare_val, new_val, false, __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE);
     280}
     281
     282static inline int internal_exchange(go_mutex & this, int swap ) with(this) {
     283        return __atomic_exchange_n((int*)&val, swap, __ATOMIC_ACQUIRE);
     284}
     285
     286// if this is called recursively IT WILL DEADLOCK!!!!!
     287static inline void lock( go_mutex & this ) with( this ) {
     288        int state, init_state;
     289
     290    // speculative grab
     291    state = internal_exchange(this, 1);
     292    if ( !state ) return; // state == 0
     293    init_state = state;
     294    for (;;) {
     295        for( int i = 0; i < 4; i++ ) {
     296            while( !val ) { // lock unlocked
     297                state = 0;
     298                if ( internal_try_lock( this, state, init_state ) ) return;
     299            }
     300            for (int i = 0; i < 30; i++) Pause();
     301        }
     302
     303        while( !val ) { // lock unlocked
     304            state = 0;
     305            if ( internal_try_lock( this, state, init_state ) ) return;
     306        }
     307        sched_yield();
     308       
     309        // if not in contended state, set to be in contended state
     310        state = internal_exchange( this, 2 );
     311        if ( !state ) return; // state == 0
     312        init_state = 2;
     313        futex( (int*)&val, FUTEX_WAIT, 2 ); // if val is not 2 this returns with EWOULDBLOCK
     314    }
     315}
     316
     317static inline void unlock( go_mutex & this ) with(this) {
     318        // if uncontended do atomic unlock and then return
     319    if ( __atomic_exchange_n(&val, 0, __ATOMIC_RELEASE) == 1 ) return;
     320       
     321        // otherwise threads are blocked so we must wake one
     322        futex( (int *)&val, FUTEX_WAKE, 1 );
     323}
     324
     325DEFAULT_ON_NOTIFY( go_mutex )
     326DEFAULT_ON_WAIT( go_mutex )
     327DEFAULT_ON_WAKEUP_NO_REACQ( go_mutex )
     328
     329//-----------------------------------------------------------------------------
     330// Exponential backoff then block lock
     331struct exp_backoff_then_block_lock {
    261332        // Spin lock used for mutual exclusion
    262333        __spinlock_t spinlock;
    … …  
    269340};
    270341
    271 static inline void  ?{}( linear_backoff_then_block_lock & this ) {
     342static inline void  ?{}( exp_backoff_then_block_lock & this ) {
    272343        this.spinlock{};
    273344        this.blocked_threads{};
    274345        this.lock_value = 0;
    275346}
    276 static inline void ^?{}( linear_backoff_then_block_lock & this ) {}
    277 // static inline void ?{}( linear_backoff_then_block_lock & this, linear_backoff_then_block_lock this2 ) = void;
    278 // static inline void ?=?( linear_backoff_then_block_lock & this, linear_backoff_then_block_lock this2 ) = void;
    279 
    280 static inline bool internal_try_lock(linear_backoff_then_block_lock & this, size_t & compare_val) with(this) {
    281         if (__atomic_compare_exchange_n(&lock_value, &compare_val, 1, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
    282                 return true;
    283         }
    284         return false;
    285 }
    286 
    287 static inline bool try_lock(linear_backoff_then_block_lock & this) { size_t compare_val = 0; return internal_try_lock(this, compare_val); }
    288 
    289 static inline bool try_lock_contention(linear_backoff_then_block_lock & this) with(this) {
    290         if (__atomic_exchange_n(&lock_value, 2, __ATOMIC_ACQUIRE) == 0) {
    291                 return true;
    292         }
    293         return false;
    294 }
    295 
    296 static inline bool block(linear_backoff_then_block_lock & this) with(this) {
    297         lock( spinlock __cfaabi_dbg_ctx2 ); // TODO change to lockfree queue (MPSC)
    298         if (lock_value != 2) {
    299                 unlock( spinlock );
    300                 return true;
    301         }
    302         insert_last( blocked_threads, *active_thread() );
    303         unlock( spinlock );
     347static inline void ?{}( exp_backoff_then_block_lock & this, exp_backoff_then_block_lock this2 ) = void;
     348static inline void ?=?( exp_backoff_then_block_lock & this, exp_backoff_then_block_lock this2 ) = void;
     349
     350static inline void  ^?{}( exp_backoff_then_block_lock & this ){}
     351
     352static inline bool internal_try_lock( exp_backoff_then_block_lock & this, size_t & compare_val ) with(this) {
     353        return __atomic_compare_exchange_n(&lock_value, &compare_val, 1, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
     354}
     355
     356static inline bool try_lock( exp_backoff_then_block_lock & this ) { size_t compare_val = 0; return internal_try_lock( this, compare_val ); }
     357
     358static inline bool try_lock_contention( exp_backoff_then_block_lock & this ) with(this) {
     359        return !__atomic_exchange_n( &lock_value, 2, __ATOMIC_ACQUIRE );
     360}
     361
     362static inline bool block( exp_backoff_then_block_lock & this ) with(this) {
     363    lock( spinlock __cfaabi_dbg_ctx2 );
     364    if (__atomic_load_n( &lock_value, __ATOMIC_SEQ_CST) != 2) {
     365        unlock( spinlock );
     366        return true;
     367    }
     368    insert_last( blocked_threads, *active_thread() );
     369    unlock( spinlock );
    304370        park( );
    305371        return true;
    306372}
    307373
    308 static inline void lock(linear_backoff_then_block_lock & this) with(this) {
     374static inline void lock( exp_backoff_then_block_lock & this ) with(this) {
    309375        size_t compare_val = 0;
    310376        int spin = 4;
     377
    311378        // linear backoff
    312379        for( ;; ) {
    … …  
    324391}
    325392
    326 static inline void unlock(linear_backoff_then_block_lock & this) with(this) {
     393static inline void unlock( exp_backoff_then_block_lock & this ) with(this) {
    327394    if (__atomic_exchange_n(&lock_value, 0, __ATOMIC_RELEASE) == 1) return;
    328         lock( spinlock __cfaabi_dbg_ctx2 );
    329         thread$ * t = &try_pop_front( blocked_threads );
    330         unlock( spinlock );
    331         unpark( t );
    332 }
    333 
    334 static inline void on_notify(linear_backoff_then_block_lock & this, struct thread$ * t ) { unpark(t); }
    335 static inline size_t on_wait(linear_backoff_then_block_lock & this) { unlock(this); return 0; }
    336 static inline void on_wakeup(linear_backoff_then_block_lock & this, size_t recursion ) {
    337         #ifdef REACQ
    338         lock(this);
    339         #endif
    340 }
     395    lock( spinlock __cfaabi_dbg_ctx2 );
     396    thread$ * t = &try_pop_front( blocked_threads );
     397    unlock( spinlock );
     398    unpark( t );
     399}
     400
     401DEFAULT_ON_NOTIFY( exp_backoff_then_block_lock )
     402DEFAULT_ON_WAIT( exp_backoff_then_block_lock )
     403DEFAULT_ON_WAKEUP_REACQ( exp_backoff_then_block_lock )
    341404
    342405//-----------------------------------------------------------------------------
    … …  
    368431
    369432// if this is called recursively IT WILL DEADLOCK!!!!!
    370 static inline void lock(fast_block_lock & this) with(this) {
     433static inline void lock( fast_block_lock & this ) with(this) {
    371434        lock( lock __cfaabi_dbg_ctx2 );
    372435        if ( held ) {
    … …  
    380443}
    381444
    382 static inline void unlock(fast_block_lock & this) with(this) {
     445static inline void unlock( fast_block_lock & this ) with(this) {
    383446        lock( lock __cfaabi_dbg_ctx2 );
    384447        /* paranoid */ verifyf( held != false, "Attempt to release lock %p that isn't held", &this );
    … …  
    389452}
    390453
    391 static inline void on_notify(fast_block_lock & this, struct thread$ * t ) with(this) {
    392         #ifdef REACQ
    393                 lock( lock __cfaabi_dbg_ctx2 );
    394                 insert_last( blocked_threads, *t );
    395                 unlock( lock );
    396         #else
    397                 unpark(t);
    398         #endif
    399 }
    400 static inline size_t on_wait(fast_block_lock & this) { unlock(this); return 0; }
    401 static inline void on_wakeup(fast_block_lock & this, size_t recursion ) { }
     454static inline void on_notify( fast_block_lock & this, struct thread$ * t ) with(this) {
     455    lock( lock __cfaabi_dbg_ctx2 );
     456    insert_last( blocked_threads, *t );
     457    unlock( lock );
     458}
     459DEFAULT_ON_WAIT( fast_block_lock )
     460DEFAULT_ON_WAKEUP_NO_REACQ( fast_block_lock )
    402461
    403462//-----------------------------------------------------------------------------
    … …  
    410469struct simple_owner_lock {
    411470        // List of blocked threads
    412         dlist( thread$ ) blocked_threads;
     471        dlist( select_node ) blocked_threads;
    413472
    414473        // Spin lock used for mutual exclusion
    … …  
    431490static inline void ?=?( simple_owner_lock & this, simple_owner_lock this2 ) = void;
    432491
    433 static inline void lock(simple_owner_lock & this) with(this) {
    434         if (owner == active_thread()) {
     492static inline void lock( simple_owner_lock & this ) with(this) {
     493        if ( owner == active_thread() ) {
    435494                recursion_count++;
    436495                return;
    … …  
    438497        lock( lock __cfaabi_dbg_ctx2 );
    439498
    440         if (owner != 0p) {
    441                 insert_last( blocked_threads, *active_thread() );
     499        if ( owner != 0p ) {
     500        select_node node;
     501                insert_last( blocked_threads, node );
    442502                unlock( lock );
    443503                park( );
    … …  
    449509}
    450510
    451 // TODO: fix duplicate def issue and bring this back
    452 // void pop_and_set_new_owner( simple_owner_lock & this ) with( this ) {
    453         // thread$ * t = &try_pop_front( blocked_threads );
    454         // owner = t;
    455         // recursion_count = ( t ? 1 : 0 );
    456         // unpark( t );
    457 // }
    458 
    459 static inline void unlock(simple_owner_lock & this) with(this) {
     511static inline void pop_node( simple_owner_lock & this ) with(this) {
     512    __handle_waituntil_OR( blocked_threads );
     513    select_node * node = &try_pop_front( blocked_threads );
     514    if ( node ) {
     515        owner = node->blocked_thread;
     516        recursion_count = 1;
     517        // if ( !node->clause_status || __make_select_node_available( *node ) ) unpark( node->blocked_thread );
     518        wake_one( blocked_threads, *node );
     519    } else {
     520        owner = 0p;
     521        recursion_count = 0;
     522    }
     523}
     524
     525static inline void unlock( simple_owner_lock & this ) with(this) {
    460526        lock( lock __cfaabi_dbg_ctx2 );
    461527        /* paranoid */ verifyf( owner != 0p, "Attempt to release lock %p that isn't held", &this );
    … …  
    464530        recursion_count--;
    465531        if ( recursion_count == 0 ) {
    466                 // pop_and_set_new_owner( this );
    467                 thread$ * t = &try_pop_front( blocked_threads );
    468                 owner = t;
    469                 recursion_count = ( t ? 1 : 0 );
    470                 unpark( t );
     532                pop_node( this );
    471533        }
    472534        unlock( lock );
    473535}
    474536
    475 static inline void on_notify(simple_owner_lock & this, struct thread$ * t ) with(this) {
     537static inline void on_notify( simple_owner_lock & this, thread$ * t ) with(this) {
    476538        lock( lock __cfaabi_dbg_ctx2 );
    477539        // lock held
    478540        if ( owner != 0p ) {
    479                 insert_last( blocked_threads, *t );
     541                insert_last( blocked_threads, *(select_node *)t->link_node );
    480542        }
    481543        // lock not held
    … …  
    488550}
    489551
    490 static inline size_t on_wait(simple_owner_lock & this) with(this) {
     552static inline size_t on_wait( simple_owner_lock & this, __cfa_pre_park pp_fn, void * pp_datum ) with(this) {
    491553        lock( lock __cfaabi_dbg_ctx2 );
    492554        /* paranoid */ verifyf( owner != 0p, "Attempt to release lock %p that isn't held", &this );
    … …  
    495557        size_t ret = recursion_count;
    496558
    497         // pop_and_set_new_owner( this );
    498 
    499         thread$ * t = &try_pop_front( blocked_threads );
    500         owner = t;
    501         recursion_count = ( t ? 1 : 0 );
    502         unpark( t );
    503 
     559        pop_node( this );
     560
     561    select_node node;
     562    active_thread()->link_node = (void *)&node;
    504563        unlock( lock );
     564
     565    pre_park_then_park( pp_fn, pp_datum );
     566
    505567        return ret;
    506568}
    507569
    508 static inline void on_wakeup(simple_owner_lock & this, size_t recursion ) with(this) { recursion_count = recursion; }
     570static inline void on_wakeup( simple_owner_lock & this, size_t recursion ) with(this) { recursion_count = recursion; }
     571
     572// waituntil() support
     573static inline bool register_select( simple_owner_lock & this, select_node & node ) with(this) {
     574    lock( lock __cfaabi_dbg_ctx2 );
     575
     576    // check if we can complete operation. If so race to establish winner in special OR case
     577    if ( !node.park_counter && ( owner == active_thread() || owner == 0p ) ) {
     578        if ( !__make_select_node_available( node ) ) { // we didn't win the race so give up on registering
     579           unlock( lock );
     580           return false;
     581        }
     582    }
     583
     584    if ( owner == active_thread() ) {
     585                recursion_count++;
     586        if ( node.park_counter ) __make_select_node_available( node );
     587        unlock( lock );
     588                return true;
     589        }
     590
     591    if ( owner != 0p ) {
     592                insert_last( blocked_threads, node );
     593                unlock( lock );
     594                return false;
     595        }
     596   
     597        owner = active_thread();
     598        recursion_count = 1;
     599
     600    if ( node.park_counter ) __make_select_node_available( node );
     601    unlock( lock );
     602    return true;
     603}
     604
     605static inline bool unregister_select( simple_owner_lock & this, select_node & node ) with(this) {
     606    lock( lock __cfaabi_dbg_ctx2 );
     607    if ( node`isListed ) {
     608        remove( node );
     609        unlock( lock );
     610        return false;
     611    }
     612
     613    if ( owner == active_thread() ) {
     614        recursion_count--;
     615        if ( recursion_count == 0 ) {
     616            pop_node( this );
     617        }
     618    }
     619    unlock( lock );
     620    return false;
     621}
     622
     623static inline void on_selected( simple_owner_lock & this, select_node & node ) {}
     624
    509625
    510626//-----------------------------------------------------------------------------
    … …  
    521637        // flag showing if lock is held
    522638        volatile bool held;
    523 
    524         #ifdef __CFA_DEBUG__
    525         // for deadlock detection
    526         struct thread$ * owner;
    527         #endif
    528639};
    529640
    … …  
    536647static inline void ?=?( spin_queue_lock & this, spin_queue_lock this2 ) = void;
    537648
    538 // if this is called recursively IT WILL DEADLOCK!!!!!
    539 static inline void lock(spin_queue_lock & this) with(this) {
     649// if this is called recursively IT WILL DEADLOCK!
     650static inline void lock( spin_queue_lock & this ) with(this) {
    540651        mcs_spin_node node;
    541652        lock( lock, node );
    … …  
    545656}
    546657
    547 static inline void unlock(spin_queue_lock & this) with(this) {
     658static inline void unlock( spin_queue_lock & this ) with(this) {
    548659        __atomic_store_n(&held, false, __ATOMIC_RELEASE);
    549660}
    550661
    551 static inline void on_notify(spin_queue_lock & this, struct thread$ * t ) {
    552         unpark(t);
    553 }
    554 static inline size_t on_wait(spin_queue_lock & this) { unlock(this); return 0; }
    555 static inline void on_wakeup(spin_queue_lock & this, size_t recursion ) {
    556         #ifdef REACQ
    557         lock(this);
    558         #endif
    559 }
    560 
     662DEFAULT_ON_NOTIFY( spin_queue_lock )
     663DEFAULT_ON_WAIT( spin_queue_lock )
     664DEFAULT_ON_WAKEUP_REACQ( spin_queue_lock )
    561665
    562666//-----------------------------------------------------------------------------
    … …  
    584688
    585689// if this is called recursively IT WILL DEADLOCK!!!!!
    586 static inline void lock(mcs_block_spin_lock & this) with(this) {
     690static inline void lock( mcs_block_spin_lock & this ) with(this) {
    587691        mcs_node node;
    588692        lock( lock, node );
    … …  
    596700}
    597701
    598 static inline void on_notify(mcs_block_spin_lock & this, struct thread$ * t ) { unpark(t); }
    599 static inline size_t on_wait(mcs_block_spin_lock & this) { unlock(this); return 0; }
    600 static inline void on_wakeup(mcs_block_spin_lock & this, size_t recursion ) {
    601         #ifdef REACQ
    602         lock(this);
    603         #endif
    604 }
     702DEFAULT_ON_NOTIFY( mcs_block_spin_lock )
     703DEFAULT_ON_WAIT( mcs_block_spin_lock )
     704DEFAULT_ON_WAKEUP_REACQ( mcs_block_spin_lock )
    605705
    606706//-----------------------------------------------------------------------------
    … …  
    628728
    629729// if this is called recursively IT WILL DEADLOCK!!!!!
    630 static inline void lock(block_spin_lock & this) with(this) {
     730static inline void lock( block_spin_lock & this ) with(this) {
    631731        lock( lock );
    632732        while(__atomic_load_n(&held, __ATOMIC_SEQ_CST)) Pause();
    … …  
    635735}
    636736
    637 static inline void unlock(block_spin_lock & this) with(this) {
     737static inline void unlock( block_spin_lock & this ) with(this) {
    638738        __atomic_store_n(&held, false, __ATOMIC_RELEASE);
    639739}
    640740
    641 static inline void on_notify(block_spin_lock & this, struct thread$ * t ) with(this.lock) {
    642   #ifdef REACQ
     741static inline void on_notify( block_spin_lock & this, struct thread$ * t ) with(this.lock) {
    643742        // first we acquire internal fast_block_lock
    644743        lock( lock __cfaabi_dbg_ctx2 );
    … …  
    652751        unlock( lock );
    653752
    654   #endif
    655753        unpark(t);
    656        
    657 }
    658 static inline size_t on_wait(block_spin_lock & this) { unlock(this); return 0; }
    659 static inline void on_wakeup(block_spin_lock & this, size_t recursion ) with(this) {
    660   #ifdef REACQ
     754}
     755DEFAULT_ON_WAIT( block_spin_lock )
     756static inline void on_wakeup( block_spin_lock & this, size_t recursion ) with(this) {
    661757        // now we acquire the entire block_spin_lock upon waking up
    662758        while(__atomic_load_n(&held, __ATOMIC_SEQ_CST)) Pause();
    663759        __atomic_store_n(&held, true, __ATOMIC_RELEASE);
    664760        unlock( lock ); // Now we release the internal fast_spin_lock
    665   #endif
    666 }
    667 
    668 //-----------------------------------------------------------------------------
    669 // is_blocking_lock
    670 trait is_blocking_lock(L & | sized(L)) {
    671         // For synchronization locks to use when acquiring
    672         void on_notify( L &, struct thread$ * );
    673 
    674         // For synchronization locks to use when releasing
    675         size_t on_wait( L & );
    676 
    677         // to set recursion count after getting signalled;
    678         void on_wakeup( L &, size_t recursion );
    679 };
     761}
    680762
    681763//-----------------------------------------------------------------------------
    … …  
    685767forall(L & | is_blocking_lock(L)) {
    686768        struct info_thread;
    687 
    688         // // for use by sequence
    689         // info_thread(L) *& Back( info_thread(L) * this );
    690         // info_thread(L) *& Next( info_thread(L) * this );
    691769}
    692770
Note: See TracChangeset for help on using the changeset viewer.