source: src/libcfa/concurrency/monitor.c @ afd550c

ADTaaron-thesisarm-ehast-experimentalcleanup-dtorsdeferred_resndemanglerenumforall-pointer-decayjacob/cs343-translationjenkins-sandboxnew-astnew-ast-unique-exprnew-envno_listpersistent-indexerpthread-emulationqualifiedEnumwith_gc
Last change on this file since afd550c was afd550c, checked in by Thierry Delisle <tdelisle@…>, 6 years ago

Some more work on TLS macros

  • Property mode set to 100644
File size: 28.9 KB
RevLine 
[f07e037]1//
2// Cforall Version 1.0.0 Copyright (C) 2016 University of Waterloo
3//
4// The contents of this file are covered under the licence agreement in the
5// file "LICENCE" distributed with Cforall.
6//
[84c52a8]7// monitor_desc.c --
[f07e037]8//
9// Author           : Thierry Delisle
10// Created On       : Thd Feb 23 12:27:26 2017
[6b0b624]11// Last Modified By : Peter A. Buhr
[b10affd]12// Last Modified On : Fri Mar 30 14:30:26 2018
13// Update Count     : 9
[f07e037]14//
15
16#include "monitor"
17
[a933dcf4]18#include <stdlib>
[2f6a7e93]19#include <inttypes.h>
[a933dcf4]20
[2ac095d]21#include "kernel_private.h"
[f07e037]22
[de737c8]23#include "bits/algorithms.h"
24
[0c78741]25//-----------------------------------------------------------------------------
26// Forward declarations
[daacf82]27static inline void set_owner ( monitor_desc * this, thread_desc * owner );
[513daec]28static inline void set_owner ( monitor_desc * storage [], __lock_size_t count, thread_desc * owner );
29static inline void set_mask  ( monitor_desc * storage [], __lock_size_t count, const __waitfor_mask_t & mask );
[daacf82]30static inline void reset_mask( monitor_desc * this );
[6ff4507]31
[0c78741]32static inline thread_desc * next_thread( monitor_desc * this );
[6ae8c92]33static inline bool is_accepted( monitor_desc * this, const __monitor_group_t & monitors );
[0c78741]34
[ea7d2b0]35static inline void lock_all  ( __spinlock_t * locks [], __lock_size_t count );
36static inline void lock_all  ( monitor_desc * source [], __spinlock_t * /*out*/ locks [], __lock_size_t count );
37static inline void unlock_all( __spinlock_t * locks [], __lock_size_t count );
[513daec]38static inline void unlock_all( monitor_desc * locks [], __lock_size_t count );
[0c78741]39
[ea7d2b0]40static inline void save   ( monitor_desc * ctx [], __lock_size_t count, __spinlock_t * locks [], unsigned int /*out*/ recursions [], __waitfor_mask_t /*out*/ masks [] );
41static inline void restore( monitor_desc * ctx [], __lock_size_t count, __spinlock_t * locks [], unsigned int /*in */ recursions [], __waitfor_mask_t /*in */ masks [] );
[0c78741]42
[513daec]43static inline void init     ( __lock_size_t count, monitor_desc * monitors [], __condition_node_t & waiter, __condition_criterion_t criteria [] );
44static inline void init_push( __lock_size_t count, monitor_desc * monitors [], __condition_node_t & waiter, __condition_criterion_t criteria [] );
[97e3296]45
[6ff4507]46static inline thread_desc *        check_condition   ( __condition_criterion_t * );
[4cedd9f]47static inline void                 brand_condition   ( condition & );
[59a0bde]48static inline [thread_desc *, int] search_entry_queue( const __waitfor_mask_t &, monitor_desc * monitors [], __lock_size_t count );
[b18830e]49
[6ff4507]50forall(dtype T | sized( T ))
[59a0bde]51static inline __lock_size_t insert_unique( T * array [], __lock_size_t & size, T * val );
52static inline __lock_size_t count_max    ( const __waitfor_mask_t & mask );
53static inline __lock_size_t aggregate    ( monitor_desc * storage [], const __waitfor_mask_t & mask );
[97e3296]54
55//-----------------------------------------------------------------------------
56// Useful defines
[6ff4507]57#define wait_ctx(thrd, user_info)                               /* Create the necessary information to use the signaller stack                         */ \
58        __condition_node_t waiter = { thrd, count, user_info };   /* Create the node specific to this wait operation                                     */ \
59        __condition_criterion_t criteria[count];                  /* Create the creteria this wait operation needs to wake up                            */ \
[8fc45b7]60        init( count, monitors, waiter, criteria );                /* Link everything together                                                            */ \
[6ff4507]61
62#define wait_ctx_primed(thrd, user_info)                        /* Create the necessary information to use the signaller stack                         */ \
63        __condition_node_t waiter = { thrd, count, user_info };   /* Create the node specific to this wait operation                                     */ \
64        __condition_criterion_t criteria[count];                  /* Create the creteria this wait operation needs to wake up                            */ \
[8fc45b7]65        init_push( count, monitors, waiter, criteria );           /* Link everything together and push it to the AS-Stack                                */ \
[6ff4507]66
67#define monitor_ctx( mons, cnt )                                /* Define that create the necessary struct for internal/external scheduling operations */ \
68        monitor_desc ** monitors = mons;                          /* Save the targeted monitors                                                          */ \
[59a0bde]69        __lock_size_t count = cnt;                                /* Save the count to a local variable                                                  */ \
[6ff4507]70        unsigned int recursions[ count ];                         /* Save the current recursion levels to restore them later                             */ \
[4cedd9f]71        __waitfor_mask_t masks [ count ];                         /* Save the current waitfor masks to restore them later                                */ \
[ea7d2b0]72        __spinlock_t *   locks [ count ];                         /* We need to pass-in an array of locks to BlockInternal                               */ \
[6ff4507]73
74#define monitor_save    save   ( monitors, count, locks, recursions, masks )
75#define monitor_restore restore( monitors, count, locks, recursions, masks )
76
[97e3296]77
[0c78741]78//-----------------------------------------------------------------------------
79// Enter/Leave routines
[690f13c]80
81
[cb0e6de]82extern "C" {
[97e3296]83        // Enter single monitor
[a843067]84        static void __enter_monitor_desc( monitor_desc * this, const __monitor_group_t & group ) {
[34c6c767]85                // Lock the monitor spinlock
[2e9aed4]86                lock( this->lock __cfaabi_dbg_ctx2 );
[14a61b5]87                // Interrupts disable inside critical section
88                thread_desc * thrd = kernelTLS.this_thread;
[f07e037]89
[169d944]90                __cfaabi_dbg_print_safe( "Kernel : %10p Entering mon %p (%p)\n", thrd, this, this->owner);
[90c4df0]91
[cb0e6de]92                if( !this->owner ) {
[97e3296]93                        // No one has the monitor, just take it
[cd348e7]94                        set_owner( this, thrd );
[90c4df0]95
[169d944]96                        __cfaabi_dbg_print_safe( "Kernel :  mon is free \n" );
[cb0e6de]97                }
98                else if( this->owner == thrd) {
[549c006]99                        // We already have the monitor, just note how many times we took it
[cb0e6de]100                        this->recursion += 1;
[90c4df0]101
[169d944]102                        __cfaabi_dbg_print_safe( "Kernel :  mon already owned \n" );
[cb0e6de]103                }
[6ae8c92]104                else if( is_accepted( this, group) ) {
[97e3296]105                        // Some one was waiting for us, enter
106                        set_owner( this, thrd );
[90c4df0]107
[daacf82]108                        // Reset mask
109                        reset_mask( this );
110
[169d944]111                        __cfaabi_dbg_print_safe( "Kernel :  mon accepts \n" );
[97e3296]112                }
[cb0e6de]113                else {
[169d944]114                        __cfaabi_dbg_print_safe( "Kernel :  blocking \n" );
[90c4df0]115
[97e3296]116                        // Some one else has the monitor, wait in line for it
[8fc45b7]117                        append( this->entry_queue, thrd );
[2e9aed4]118
[82ff5845]119                        BlockInternal( &this->lock );
[cc7f4b1]120
[169d944]121                        __cfaabi_dbg_print_safe( "Kernel : %10p Entered  mon %p\n", thrd, this);
[90c4df0]122
[97e3296]123                        // BlockInternal will unlock spinlock, no need to unlock ourselves
[2ac095d]124                        return;
[cb0e6de]125                }
[f07e037]126
[169d944]127                __cfaabi_dbg_print_safe( "Kernel : %10p Entered  mon %p\n", thrd, this);
[90c4df0]128
[97e3296]129                // Release the lock and leave
[ea7d2b0]130                unlock( this->lock );
[5ea06d6]131                return;
[cb0e6de]132        }
[f07e037]133
[549c006]134        static void __enter_monitor_dtor( monitor_desc * this, fptr_t func ) {
[34c6c767]135                // Lock the monitor spinlock
[2e9aed4]136                lock( this->lock __cfaabi_dbg_ctx2 );
[14a61b5]137                // Interrupts disable inside critical section
138                thread_desc * thrd = kernelTLS.this_thread;
[549c006]139
[169d944]140                __cfaabi_dbg_print_safe( "Kernel : %10p Entering dtor for mon %p (%p)\n", thrd, this, this->owner);
[549c006]141
142
143                if( !this->owner ) {
[169d944]144                        __cfaabi_dbg_print_safe( "Kernel : Destroying free mon %p\n", this);
[549c006]145
146                        // No one has the monitor, just take it
147                        set_owner( this, thrd );
148
[ea7d2b0]149                        unlock( this->lock );
[549c006]150                        return;
151                }
152                else if( this->owner == thrd) {
153                        // We already have the monitor... but where about to destroy it so the nesting will fail
154                        // Abort!
[2fdbb3b]155                        abort( "Attempt to destroy monitor %p by thread \"%.256s\" (%p) in nested mutex.", this, thrd->self_cor.name, thrd );
[549c006]156                }
157
[59a0bde]158                __lock_size_t count = 1;
[549c006]159                monitor_desc ** monitors = &this;
160                __monitor_group_t group = { &this, 1, func };
161                if( is_accepted( this, group) ) {
[169d944]162                        __cfaabi_dbg_print_safe( "Kernel :  mon accepts dtor, block and signal it \n" );
[549c006]163
[b8116cd]164                        // Wake the thread that is waiting for this
[8fc45b7]165                        __condition_criterion_t * urgent = pop( this->signal_stack );
[b8116cd]166                        verify( urgent );
167
[549c006]168                        // Reset mask
169                        reset_mask( this );
170
171                        // Create the node specific to this wait operation
[14a61b5]172                        wait_ctx_primed( thrd, 0 )
[549c006]173
174                        // Some one else has the monitor, wait for him to finish and then run
[b8116cd]175                        BlockInternal( &this->lock, urgent->owner->waiting_thread );
[549c006]176
177                        // Some one was waiting for us, enter
178                        set_owner( this, thrd );
179                }
180                else {
[169d944]181                        __cfaabi_dbg_print_safe( "Kernel :  blocking \n" );
[549c006]182
[14a61b5]183                        wait_ctx( thrd, 0 )
[549c006]184                        this->dtor_node = &waiter;
185
186                        // Some one else has the monitor, wait in line for it
[8fc45b7]187                        append( this->entry_queue, thrd );
[549c006]188                        BlockInternal( &this->lock );
189
190                        // BlockInternal will unlock spinlock, no need to unlock ourselves
191                        return;
192                }
193
[169d944]194                __cfaabi_dbg_print_safe( "Kernel : Destroying %p\n", this);
[549c006]195
196        }
197
[97e3296]198        // Leave single monitor
[1c273d0]199        void __leave_monitor_desc( monitor_desc * this ) {
[2e9aed4]200                // Lock the monitor spinlock
201                lock( this->lock __cfaabi_dbg_ctx2 );
[f07e037]202
[14a61b5]203                __cfaabi_dbg_print_safe( "Kernel : %10p Leaving mon %p (%p)\n", kernelTLS.this_thread, this, this->owner);
[a843067]204
[14a61b5]205                verifyf( kernelTLS.this_thread == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", kernelTLS.this_thread, this->owner, this->recursion, this );
[cc7f4b1]206
[97e3296]207                // Leaving a recursion level, decrement the counter
[cb0e6de]208                this->recursion -= 1;
[f07e037]209
[97e3296]210                // If we haven't left the last level of recursion
211                // it means we don't need to do anything
[690f13c]212                if( this->recursion != 0) {
[169d944]213                        __cfaabi_dbg_print_safe( "Kernel :  recursion still %d\n", this->recursion);
[ea7d2b0]214                        unlock( this->lock );
[690f13c]215                        return;
216                }
[f07e037]217
[97e3296]218                // Get the next thread, will be null on low contention monitor
[0c78741]219                thread_desc * new_owner = next_thread( this );
[5ea06d6]220
[97e3296]221                // We can now let other threads in safely
[ea7d2b0]222                unlock( this->lock );
[51f3798]223
[690f13c]224                //We need to wake-up the thread
[1c273d0]225                WakeThread( new_owner );
226        }
227
[549c006]228        // Leave single monitor for the last time
229        void __leave_dtor_monitor_desc( monitor_desc * this ) {
[36982fc]230                __cfaabi_dbg_debug_do(
[b10affd]231                        if( TL_GET( this_thread ) != this->owner ) {
232                                abort( "Destroyed monitor %p has inconsistent owner, expected %p got %p.\n", this, TL_GET( this_thread ), this->owner);
[549c006]233                        }
234                        if( this->recursion != 1 ) {
[169d944]235                                abort( "Destroyed monitor %p has %d outstanding nested calls.\n", this, this->recursion - 1);
[549c006]236                        }
237                )
238        }
239
[97e3296]240        // Leave the thread monitor
241        // last routine called by a thread.
242        // Should never return
[1c273d0]243        void __leave_thread_monitor( thread_desc * thrd ) {
[b18830e]244                monitor_desc * this = &thrd->self_mon;
[97e3296]245
246                // Lock the monitor now
[2e9aed4]247                lock( this->lock __cfaabi_dbg_ctx2 );
[1c273d0]248
249                disable_interrupts();
250
[b18830e]251                thrd->self_cor.state = Halted;
[1c273d0]252
[a843067]253                verifyf( thrd == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", thrd, this->owner, this->recursion, this );
[1c273d0]254
[97e3296]255                // Leaving a recursion level, decrement the counter
[1c273d0]256                this->recursion -= 1;
257
[97e3296]258                // If we haven't left the last level of recursion
259                // it must mean there is an error
[169d944]260                if( this->recursion != 0) { abort( "Thread internal monitor has unbalanced recursion" ); }
[1c273d0]261
[97e3296]262                // Fetch the next thread, can be null
[1c273d0]263                thread_desc * new_owner = next_thread( this );
264
[97e3296]265                // Leave the thread, this will unlock the spinlock
266                // Use leave thread instead of BlockInternal which is
267                // specialized for this case and supports null new_owner
[f2b12406]268                LeaveThread( &this->lock, new_owner );
[97e3296]269
270                // Control flow should never reach here!
[cc7f4b1]271        }
[2781e65]272}
273
[97e3296]274// Enter multiple monitor
275// relies on the monitor array being sorted
[6ae8c92]276static inline void enter( __monitor_group_t monitors ) {
[59a0bde]277        for( __lock_size_t i = 0; i < monitors.size; i++) {
[0cf5b79]278                __enter_monitor_desc( monitors[i], monitors );
[97e3296]279        }
[2781e65]280}
281
[97e3296]282// Leave multiple monitor
283// relies on the monitor array being sorted
[59a0bde]284static inline void leave(monitor_desc * monitors [], __lock_size_t count) {
285        for( __lock_size_t i = count - 1; i >= 0; i--) {
[0c78741]286                __leave_monitor_desc( monitors[i] );
[2781e65]287        }
[5ea06d6]288}
289
[97e3296]290// Ctor for monitor guard
291// Sorts monitors before entering
[59a0bde]292void ?{}( monitor_guard_t & this, monitor_desc * m [], __lock_size_t count, fptr_t func ) {
[14a61b5]293        thread_desc * thrd = TL_GET( this_thread );
294
[97e3296]295        // Store current array
[242a902]296        this.m = m;
297        this.count = count;
[97e3296]298
299        // Sort monitors based on address -> TODO use a sort specialized for small numbers
[de737c8]300        __libcfa_small_sort(this.m, count);
[5ea06d6]301
[97e3296]302        // Save previous thread context
[14a61b5]303        this.prev = thrd->monitors;
[5ea06d6]304
[97e3296]305        // Update thread context (needed for conditions)
[14a61b5]306        (thrd->monitors){m, count, func};
[90c4df0]307
[169d944]308        // __cfaabi_dbg_print_safe( "MGUARD : enter %d\n", count);
[a843067]309
[90c4df0]310        // Enter the monitors in order
[6ae8c92]311        __monitor_group_t group = {this.m, this.count, func};
[b18830e]312        enter( group );
[a843067]313
[169d944]314        // __cfaabi_dbg_print_safe( "MGUARD : entered\n" );
[5ea06d6]315}
316
[6b224a52]317
[97e3296]318// Dtor for monitor guard
[242a902]319void ^?{}( monitor_guard_t & this ) {
[169d944]320        // __cfaabi_dbg_print_safe( "MGUARD : leaving %d\n", this.count);
[a843067]321
[97e3296]322        // Leave the monitors in order
[242a902]323        leave( this.m, this.count );
[5ea06d6]324
[169d944]325        // __cfaabi_dbg_print_safe( "MGUARD : left\n" );
[a843067]326
[97e3296]327        // Restore thread context
[b10affd]328        TL_GET( this_thread )->monitors = this.prev;
[5ea06d6]329}
330
[549c006]331// Ctor for monitor guard
332// Sorts monitors before entering
[8fc45b7]333void ?{}( monitor_dtor_guard_t & this, monitor_desc * m [], fptr_t func ) {
[14a61b5]334        // optimization
335        thread_desc * thrd = TL_GET( this_thread );
336
[549c006]337        // Store current array
338        this.m = *m;
339
340        // Save previous thread context
[14a61b5]341        this.prev = thrd->monitors;
[549c006]342
343        // Update thread context (needed for conditions)
[14a61b5]344        (thrd->monitors){m, 1, func};
[549c006]345
346        __enter_monitor_dtor( this.m, func );
347}
348
349// Dtor for monitor guard
350void ^?{}( monitor_dtor_guard_t & this ) {
351        // Leave the monitors in order
352        __leave_dtor_monitor_desc( this.m );
353
354        // Restore thread context
[b10affd]355        TL_GET( this_thread )->monitors = this.prev;
[549c006]356}
357
[97e3296]358//-----------------------------------------------------------------------------
359// Internal scheduling types
[59a0bde]360void ?{}(__condition_node_t & this, thread_desc * waiting_thread, __lock_size_t count, uintptr_t user_info ) {
[242a902]361        this.waiting_thread = waiting_thread;
362        this.count = count;
363        this.next = NULL;
364        this.user_info = user_info;
[be3d020]365}
366
[c40e7c5]367void ?{}(__condition_criterion_t & this ) with( this ) {
368        ready  = false;
369        target = NULL;
370        owner  = NULL;
371        next   = NULL;
[be3d020]372}
373
[8fc45b7]374void ?{}(__condition_criterion_t & this, monitor_desc * target, __condition_node_t & owner ) {
[242a902]375        this.ready  = false;
376        this.target = target;
[8fc45b7]377        this.owner  = &owner;
[242a902]378        this.next   = NULL;
[ad1a8dd]379}
380
[5ea06d6]381//-----------------------------------------------------------------------------
382// Internal scheduling
[4cedd9f]383void wait( condition & this, uintptr_t user_info = 0 ) {
[0c78741]384        brand_condition( this );
[5ea06d6]385
[97e3296]386        // Check that everything is as expected
[4cedd9f]387        assertf( this.monitors != NULL, "Waiting with no monitors (%p)", this.monitors );
[2f6a7e93]388        verifyf( this.monitor_count != 0, "Waiting with 0 monitors (%"PRIiFAST16")", this.monitor_count );
389        verifyf( this.monitor_count < 32u, "Excessive monitor count (%"PRIiFAST16")", this.monitor_count );
[5ea06d6]390
[97e3296]391        // Create storage for monitor context
[4cedd9f]392        monitor_ctx( this.monitors, this.monitor_count );
[0c78741]393
[97e3296]394        // Create the node specific to this wait operation
[b10affd]395        wait_ctx( TL_GET( this_thread ), user_info );
[0c78741]396
[97e3296]397        // Append the current wait operation to the ones already queued on the condition
398        // We don't need locks for that since conditions must always be waited on inside monitor mutual exclusion
[8fc45b7]399        append( this.blocked, &waiter );
[0c78741]400
[6ff4507]401        // Lock all monitors (aggregates the locks as well)
[97e3296]402        lock_all( monitors, locks, count );
[5ea06d6]403
[97e3296]404        // Find the next thread(s) to run
[59a0bde]405        __lock_size_t thread_count = 0;
[0c78741]406        thread_desc * threads[ count ];
[66298de]407        __builtin_memset( threads, 0, sizeof( threads ) );
[ad1a8dd]408
[a843067]409        // Save monitor states
410        monitor_save;
411
[97e3296]412        // Remove any duplicate threads
[59a0bde]413        for( __lock_size_t i = 0; i < count; i++) {
[97e3296]414                thread_desc * new_owner = next_thread( monitors[i] );
[6ff4507]415                insert_unique( threads, thread_count, new_owner );
[5ea06d6]416        }
417
[a843067]418        // Everything is ready to go to sleep
419        BlockInternal( locks, count, threads, thread_count );
420
421        // We are back, restore the owners and recursions
422        monitor_restore;
[5ea06d6]423}
424
[4cedd9f]425bool signal( condition & this ) {
[97e3296]426        if( is_empty( this ) ) { return false; }
[5ea06d6]427
428        //Check that everything is as expected
[4cedd9f]429        verify( this.monitors );
430        verify( this.monitor_count != 0 );
[0c78741]431
[44264c5]432        //Some more checking in debug
[36982fc]433        __cfaabi_dbg_debug_do(
[b10affd]434                thread_desc * this_thrd = TL_GET( this_thread );
[4cedd9f]435                if ( this.monitor_count != this_thrd->monitors.size ) {
[c2ca04d]436                        abort( "Signal on condition %p made with different number of monitor(s), expected %zi got %zi", &this, this.monitor_count, this_thrd->monitors.size );
[97e3296]437                }
[0c78741]438
[4cedd9f]439                for(int i = 0; i < this.monitor_count; i++) {
[0cf5b79]440                        if ( this.monitors[i] != this_thrd->monitors[i] ) {
[2fdbb3b]441                                abort( "Signal on condition %p made with different monitor, expected %p got %p", &this, this.monitors[i], this_thrd->monitors[i] );
[97e3296]442                        }
[0c78741]443                }
[5ea06d6]444        );
445
[59a0bde]446        __lock_size_t count = this.monitor_count;
[97e3296]447
448        // Lock all monitors
[4cedd9f]449        lock_all( this.monitors, NULL, count );
[0c78741]450
[44264c5]451        //Pop the head of the waiting queue
[8fc45b7]452        __condition_node_t * node = pop_head( this.blocked );
[44264c5]453
454        //Add the thread to the proper AS stack
[0c78741]455        for(int i = 0; i < count; i++) {
456                __condition_criterion_t * crit = &node->criteria[i];
457                assert( !crit->ready );
[8fc45b7]458                push( crit->target->signal_stack, crit );
[5ea06d6]459        }
[0c78741]460
[44264c5]461        //Release
[4cedd9f]462        unlock_all( this.monitors, count );
[be3d020]463
464        return true;
[5ea06d6]465}
466
[4cedd9f]467bool signal_block( condition & this ) {
468        if( !this.blocked.head ) { return false; }
[44264c5]469
470        //Check that everything is as expected
[4cedd9f]471        verifyf( this.monitors != NULL, "Waiting with no monitors (%p)", this.monitors );
[2f6a7e93]472        verifyf( this.monitor_count != 0, "Waiting with 0 monitors (%"PRIiFAST16")", this.monitor_count );
[44264c5]473
[97e3296]474        // Create storage for monitor context
[4cedd9f]475        monitor_ctx( this.monitors, this.monitor_count );
[44264c5]476
[97e3296]477        // Lock all monitors (aggregates the locks them as well)
478        lock_all( monitors, locks, count );
[44264c5]479
[2e9aed4]480
[97e3296]481        // Create the node specific to this wait operation
[afd550c]482        wait_ctx_primed( kernelTLS.this_thread, 0 )
[44264c5]483
484        //save contexts
[6ff4507]485        monitor_save;
[44264c5]486
487        //Find the thread to run
[8fc45b7]488        thread_desc * signallee = pop_head( this.blocked )->waiting_thread;
[6ff4507]489        set_owner( monitors, count, signallee );
[44264c5]490
[36982fc]491        __cfaabi_dbg_print_buffer_decl( "Kernel : signal_block condition %p (s: %p)\n", &this, signallee );
[de737c8]492
[44264c5]493        //Everything is ready to go to sleep
[82ff5845]494        BlockInternal( locks, count, &signallee, 1 );
[44264c5]495
[c81ebf9]496
[97e3296]497        // WE WOKE UP
[c81ebf9]498
499
[36982fc]500        __cfaabi_dbg_print_buffer_local( "Kernel :   signal_block returned\n" );
[de737c8]501
[6ff4507]502        //We are back, restore the masks and recursions
503        monitor_restore;
[be3d020]504
505        return true;
506}
507
[97e3296]508// Access the user_info of the thread waiting at the front of the queue
[4cedd9f]509uintptr_t front( condition & this ) {
[2ac095d]510        verifyf( !is_empty(this),
[4aa2fb2]511                "Attempt to access user data on an empty condition.\n"
512                "Possible cause is not checking if the condition is empty before reading stored data."
[be3d020]513        );
[0cf5b79]514        return ((typeof(this.blocked.head))this.blocked.head)->user_info;
[44264c5]515}
516
[c81ebf9]517//-----------------------------------------------------------------------------
[b18830e]518// External scheduling
519// cases to handle :
520//      - target already there :
521//              block and wake
522//      - dtor already there
523//              put thread on signaller stack
524//      - non-blocking
525//              return else
526//      - timeout
527//              return timeout
528//      - block
529//              setup mask
530//              block
[6ae8c92]531void __waitfor_internal( const __waitfor_mask_t & mask, int duration ) {
[b18830e]532        // This statment doesn't have a contiguous list of monitors...
533        // Create one!
[59a0bde]534        __lock_size_t max = count_max( mask );
[b18830e]535        monitor_desc * mon_storage[max];
[66298de]536        __builtin_memset( mon_storage, 0, sizeof( mon_storage ) );
[59a0bde]537        __lock_size_t actual_count = aggregate( mon_storage, mask );
[97e3296]538
[523232d]539        __cfaabi_dbg_print_buffer_decl( "Kernel : waitfor %"PRIdFAST16" (s: %"PRIdFAST16", m: %"PRIdFAST16")\n", actual_count, mask.size, (__lock_size_t)max);
[66298de]540
[daacf82]541        if(actual_count == 0) return;
[19c43b7]542
[169d944]543        __cfaabi_dbg_print_buffer_local( "Kernel : waitfor internal proceeding\n" );
[4cc9b13]544
[97e3296]545        // Create storage for monitor context
[b18830e]546        monitor_ctx( mon_storage, actual_count );
[c81ebf9]547
[6ff4507]548        // Lock all monitors (aggregates the locks as well)
[97e3296]549        lock_all( monitors, locks, count );
[c81ebf9]550
[b18830e]551        {
552                // Check if the entry queue
[6ae8c92]553                thread_desc * next; int index;
554                [next, index] = search_entry_queue( mask, monitors, count );
[b18830e]555
556                if( next ) {
[19c43b7]557                        *mask.accepted = index;
[0cf5b79]558                        __acceptable_t& accepted = mask[index];
559                        if( accepted.is_dtor ) {
[169d944]560                                __cfaabi_dbg_print_buffer_local( "Kernel : dtor already there\n" );
[0cf5b79]561                                verifyf( accepted.size == 1,  "ERROR: Accepted dtor has more than 1 mutex parameter." );
[549c006]562
[0cf5b79]563                                monitor_desc * mon2dtor = accepted[0];
[549c006]564                                verifyf( mon2dtor->dtor_node, "ERROR: Accepted monitor has no dtor_node." );
565
566                                __condition_criterion_t * dtor_crit = mon2dtor->dtor_node->criteria;
[8fc45b7]567                                push( mon2dtor->signal_stack, dtor_crit );
[549c006]568
569                                unlock_all( locks, count );
[b18830e]570                        }
571                        else {
[169d944]572                                __cfaabi_dbg_print_buffer_local( "Kernel : thread present, baton-passing\n" );
[19c43b7]573
574                                // Create the node specific to this wait operation
[14a61b5]575                                wait_ctx_primed( kernelTLS.this_thread, 0 );
[19c43b7]576
577                                // Save monitor states
578                                monitor_save;
579
[523232d]580                                __cfaabi_dbg_print_buffer_local( "Kernel :  baton of %"PRIdFAST16" monitors : ", count );
[6a5be52]581                                #ifdef __CFA_DEBUG_PRINT__
582                                        for( int i = 0; i < count; i++) {
[36982fc]583                                                __cfaabi_dbg_print_buffer_local( "%p %p ", monitors[i], monitors[i]->signal_stack.top );
[6a5be52]584                                        }
585                                #endif
[169d944]586                                __cfaabi_dbg_print_buffer_local( "\n" );
[6a5be52]587
[19c43b7]588                                // Set the owners to be the next thread
589                                set_owner( monitors, count, next );
590
591                                // Everything is ready to go to sleep
592                                BlockInternal( locks, count, &next, 1 );
593
594                                // We are back, restore the owners and recursions
595                                monitor_restore;
596
[169d944]597                                __cfaabi_dbg_print_buffer_local( "Kernel : thread present, returned\n" );
[b18830e]598                        }
599
[36982fc]600                        __cfaabi_dbg_print_buffer_local( "Kernel : accepted %d\n", *mask.accepted);
[19c43b7]601                        return;
[90c4df0]602                }
603        }
604
[c81ebf9]605
[4cc9b13]606        if( duration == 0 ) {
[169d944]607                __cfaabi_dbg_print_buffer_local( "Kernel : non-blocking, exiting\n" );
[19c43b7]608
[4cc9b13]609                unlock_all( locks, count );
[19c43b7]610
[36982fc]611                __cfaabi_dbg_print_buffer_local( "Kernel : accepted %d\n", *mask.accepted);
[4cc9b13]612                return;
613        }
[b18830e]614
615
[169d944]616        verifyf( duration < 0, "Timeout on waitfor statments not supported yet." );
[b18830e]617
[169d944]618        __cfaabi_dbg_print_buffer_local( "Kernel : blocking waitfor\n" );
[19c43b7]619
620        // Create the node specific to this wait operation
[14a61b5]621        wait_ctx_primed( kernelTLS.this_thread, 0 );
[b18830e]622
[6ff4507]623        monitor_save;
[6ae8c92]624        set_mask( monitors, count, mask );
[c81ebf9]625
[59a0bde]626        for( __lock_size_t i = 0; i < count; i++) {
[14a61b5]627                verify( monitors[i]->owner == kernelTLS.this_thread );
[daacf82]628        }
629
[19c43b7]630        //Everything is ready to go to sleep
631        BlockInternal( locks, count );
632
633
634        // WE WOKE UP
635
636
637        //We are back, restore the masks and recursions
638        monitor_restore;
639
[169d944]640        __cfaabi_dbg_print_buffer_local( "Kernel : exiting\n" );
[19c43b7]641
[36982fc]642        __cfaabi_dbg_print_buffer_local( "Kernel : accepted %d\n", *mask.accepted);
[c81ebf9]643}
644
[0c78741]645//-----------------------------------------------------------------------------
646// Utilities
647
648static inline void set_owner( monitor_desc * this, thread_desc * owner ) {
[169d944]649        // __cfaabi_dbg_print_safe( "Kernal :   Setting owner of %p to %p ( was %p)\n", this, owner, this->owner );
[a843067]650
[0c78741]651        //Pass the monitor appropriately
652        this->owner = owner;
653
654        //We are passing the monitor to someone else, which means recursion level is not 0
655        this->recursion = owner ? 1 : 0;
656}
657
[513daec]658static inline void set_owner( monitor_desc * monitors [], __lock_size_t count, thread_desc * owner ) {
[6a5be52]659        monitors[0]->owner     = owner;
660        monitors[0]->recursion = 1;
[513daec]661        for( __lock_size_t i = 1; i < count; i++ ) {
[6a5be52]662                monitors[i]->owner     = owner;
663                monitors[i]->recursion = 0;
[6ff4507]664        }
665}
666
[513daec]667static inline void set_mask( monitor_desc * storage [], __lock_size_t count, const __waitfor_mask_t & mask ) {
668        for( __lock_size_t i = 0; i < count; i++) {
[6ff4507]669                storage[i]->mask = mask;
670        }
671}
672
[daacf82]673static inline void reset_mask( monitor_desc * this ) {
674        this->mask.accepted = NULL;
[0cf5b79]675        this->mask.data = NULL;
[daacf82]676        this->mask.size = 0;
677}
678
[0c78741]679static inline thread_desc * next_thread( monitor_desc * this ) {
680        //Check the signaller stack
[169d944]681        __cfaabi_dbg_print_safe( "Kernel :  mon %p AS-stack top %p\n", this, this->signal_stack.top);
[8fc45b7]682        __condition_criterion_t * urgent = pop( this->signal_stack );
[0c78741]683        if( urgent ) {
684                //The signaller stack is not empty,
685                //regardless of if we are ready to baton pass,
686                //we need to set the monitor as in use
687                set_owner( this,  urgent->owner->waiting_thread );
688
689                return check_condition( urgent );
690        }
691
692        // No signaller thread
693        // Get the next thread in the entry_queue
[8fc45b7]694        thread_desc * new_owner = pop_head( this->entry_queue );
[0c78741]695        set_owner( this, new_owner );
696
697        return new_owner;
698}
699
[6ff4507]700static inline bool is_accepted( monitor_desc * this, const __monitor_group_t & group ) {
[0cf5b79]701        __acceptable_t * it = this->mask.data; // Optim
[59a0bde]702        __lock_size_t count = this->mask.size;
[6ff4507]703
704        // Check if there are any acceptable functions
[a843067]705        if( !it ) return false;
[6ff4507]706
707        // If this isn't the first monitor to test this, there is no reason to repeat the test.
708        if( this != group[0] ) return group[0]->mask.accepted >= 0;
709
710        // For all acceptable functions check if this is the current function.
[59a0bde]711        for( __lock_size_t i = 0; i < count; i++, it++ ) {
[6ff4507]712                if( *it == group ) {
713                        *this->mask.accepted = i;
714                        return true;
715                }
716        }
717
718        // No function matched
719        return false;
720}
721
[513daec]722static inline void init( __lock_size_t count, monitor_desc * monitors [], __condition_node_t & waiter, __condition_criterion_t criteria [] ) {
723        for( __lock_size_t i = 0; i < count; i++) {
[6b224a52]724                (criteria[i]){ monitors[i], waiter };
[97e3296]725        }
726
[8fc45b7]727        waiter.criteria = criteria;
[97e3296]728}
729
[513daec]730static inline void init_push( __lock_size_t count, monitor_desc * monitors [], __condition_node_t & waiter, __condition_criterion_t criteria [] ) {
731        for( __lock_size_t i = 0; i < count; i++) {
[6b224a52]732                (criteria[i]){ monitors[i], waiter };
[36982fc]733                __cfaabi_dbg_print_safe( "Kernel :  target %p = %p\n", criteria[i].target, &criteria[i] );
[8fc45b7]734                push( criteria[i].target->signal_stack, &criteria[i] );
[97e3296]735        }
736
[8fc45b7]737        waiter.criteria = criteria;
[97e3296]738}
739
[ea7d2b0]740static inline void lock_all( __spinlock_t * locks [], __lock_size_t count ) {
[513daec]741        for( __lock_size_t i = 0; i < count; i++ ) {
[2e9aed4]742                lock( *locks[i] __cfaabi_dbg_ctx2 );
[0c78741]743        }
744}
745
[ea7d2b0]746static inline void lock_all( monitor_desc * source [], __spinlock_t * /*out*/ locks [], __lock_size_t count ) {
[513daec]747        for( __lock_size_t i = 0; i < count; i++ ) {
[ea7d2b0]748                __spinlock_t * l = &source[i]->lock;
[2e9aed4]749                lock( *l __cfaabi_dbg_ctx2 );
[0c78741]750                if(locks) locks[i] = l;
751        }
752}
753
[ea7d2b0]754static inline void unlock_all( __spinlock_t * locks [], __lock_size_t count ) {
[513daec]755        for( __lock_size_t i = 0; i < count; i++ ) {
[ea7d2b0]756                unlock( *locks[i] );
[0c78741]757        }
758}
759
[513daec]760static inline void unlock_all( monitor_desc * locks [], __lock_size_t count ) {
761        for( __lock_size_t i = 0; i < count; i++ ) {
[ea7d2b0]762                unlock( locks[i]->lock );
[0c78741]763        }
764}
765
[8fc45b7]766static inline void save(
767        monitor_desc * ctx [],
[513daec]768        __lock_size_t count,
[ea7d2b0]769        __attribute((unused)) __spinlock_t * locks [],
[8fc45b7]770        unsigned int /*out*/ recursions [],
771        __waitfor_mask_t /*out*/ masks []
772) {
[513daec]773        for( __lock_size_t i = 0; i < count; i++ ) {
[0c78741]774                recursions[i] = ctx[i]->recursion;
[6ff4507]775                masks[i]      = ctx[i]->mask;
[0c78741]776        }
777}
778
[8fc45b7]779static inline void restore(
780        monitor_desc * ctx [],
[513daec]781        __lock_size_t count,
[ea7d2b0]782        __spinlock_t * locks [],
[8fc45b7]783        unsigned int /*out*/ recursions [],
784        __waitfor_mask_t /*out*/ masks []
785) {
[6ff4507]786        lock_all( locks, count );
[513daec]787        for( __lock_size_t i = 0; i < count; i++ ) {
[0c78741]788                ctx[i]->recursion = recursions[i];
[6ff4507]789                ctx[i]->mask      = masks[i];
[0c78741]790        }
[6ff4507]791        unlock_all( locks, count );
[0c78741]792}
793
794// Function has 2 different behavior
795// 1 - Marks a monitors as being ready to run
796// 2 - Checks if all the monitors are ready to run
797//     if so return the thread to run
798static inline thread_desc * check_condition( __condition_criterion_t * target ) {
799        __condition_node_t * node = target->owner;
800        unsigned short count = node->count;
801        __condition_criterion_t * criteria = node->criteria;
802
803        bool ready2run = true;
804
805        for(    int i = 0; i < count; i++ ) {
[44264c5]806
[36982fc]807                // __cfaabi_dbg_print_safe( "Checking %p for %p\n", &criteria[i], target );
[0c78741]808                if( &criteria[i] == target ) {
809                        criteria[i].ready = true;
[36982fc]810                        // __cfaabi_dbg_print_safe( "True\n" );
[0c78741]811                }
812
813                ready2run = criteria[i].ready && ready2run;
814        }
815
[36982fc]816        __cfaabi_dbg_print_safe( "Kernel :  Runing %i (%p)\n", ready2run, ready2run ? node->waiting_thread : NULL );
[0c78741]817        return ready2run ? node->waiting_thread : NULL;
818}
819
[4cedd9f]820static inline void brand_condition( condition & this ) {
[b10affd]821        thread_desc * thrd = TL_GET( this_thread );
[4cedd9f]822        if( !this.monitors ) {
[169d944]823                // __cfaabi_dbg_print_safe( "Branding\n" );
[0cf5b79]824                assertf( thrd->monitors.data != NULL, "No current monitor to brand condition %p", thrd->monitors.data );
[4cedd9f]825                this.monitor_count = thrd->monitors.size;
[a933dcf4]826
[cdbfab0]827                this.monitors = (monitor_desc **)malloc( this.monitor_count * sizeof( *this.monitors ) );
[4cedd9f]828                for( int i = 0; i < this.monitor_count; i++ ) {
[0cf5b79]829                        this.monitors[i] = thrd->monitors[i];
[a933dcf4]830                }
[0c78741]831        }
832}
833
[59a0bde]834static inline [thread_desc *, int] search_entry_queue( const __waitfor_mask_t & mask, monitor_desc * monitors [], __lock_size_t count ) {
[90c4df0]835
[0cf5b79]836        __queue_t(thread_desc) & entry_queue = monitors[0]->entry_queue;
[90c4df0]837
838        // For each thread in the entry-queue
[8fc45b7]839        for(    thread_desc ** thrd_it = &entry_queue.head;
[90c4df0]840                *thrd_it;
[4cc9b13]841                thrd_it = &(*thrd_it)->next
842        ) {
[90c4df0]843                // For each acceptable check if it matches
[4cc9b13]844                int i = 0;
[0cf5b79]845                __acceptable_t * end   = end  (mask);
846                __acceptable_t * begin = begin(mask);
847                for( __acceptable_t * it = begin; it != end; it++, i++ ) {
[90c4df0]848                        // Check if we have a match
[aaa4f93]849                        if( *it == (*thrd_it)->monitors ) {
[90c4df0]850
851                                // If we have a match return it
852                                // after removeing it from the entry queue
[b18830e]853                                return [remove( entry_queue, thrd_it ), i];
[90c4df0]854                        }
855                }
856        }
857
[b18830e]858        return [0, -1];
859}
860
[6ff4507]861forall(dtype T | sized( T ))
[59a0bde]862static inline __lock_size_t insert_unique( T * array [], __lock_size_t & size, T * val ) {
[6ff4507]863        if( !val ) return size;
864
[59a0bde]865        for( __lock_size_t i = 0; i <= size; i++) {
[6ff4507]866                if( array[i] == val ) return size;
867        }
868
869        array[size] = val;
870        size = size + 1;
871        return size;
872}
873
[59a0bde]874static inline __lock_size_t count_max( const __waitfor_mask_t & mask ) {
875        __lock_size_t max = 0;
876        for( __lock_size_t i = 0; i < mask.size; i++ ) {
[0cf5b79]877                __acceptable_t & accepted = mask[i];
878                max += accepted.size;
[b18830e]879        }
880        return max;
[97e3296]881}
[b18830e]882
[59a0bde]883static inline __lock_size_t aggregate( monitor_desc * storage [], const __waitfor_mask_t & mask ) {
884        __lock_size_t size = 0;
885        for( __lock_size_t i = 0; i < mask.size; i++ ) {
[0cf5b79]886                __acceptable_t & accepted = mask[i];
887                __libcfa_small_sort( accepted.data, accepted.size );
888                for( __lock_size_t j = 0; j < accepted.size; j++) {
889                        insert_unique( storage, size, accepted[j] );
[6ff4507]890                }
[b18830e]891        }
[6a5be52]892        // TODO insertion sort instead of this
[de737c8]893        __libcfa_small_sort( storage, size );
[6ff4507]894        return size;
[b18830e]895}
896
[6b0b624]897// Local Variables: //
898// mode: c //
899// tab-width: 4 //
900// End: //
Note: See TracBrowser for help on using the repository browser.