[f07e037] | 1 | // |
---|
| 2 | // Cforall Version 1.0.0 Copyright (C) 2016 University of Waterloo |
---|
| 3 | // |
---|
| 4 | // The contents of this file are covered under the licence agreement in the |
---|
| 5 | // file "LICENCE" distributed with Cforall. |
---|
| 6 | // |
---|
[ac2b598] | 7 | // $monitor.c -- |
---|
[f07e037] | 8 | // |
---|
| 9 | // Author : Thierry Delisle |
---|
| 10 | // Created On : Thd Feb 23 12:27:26 2017 |
---|
[6b0b624] | 11 | // Last Modified By : Peter A. Buhr |
---|
[121be3e] | 12 | // Last Modified On : Wed Dec 4 07:55:14 2019 |
---|
| 13 | // Update Count : 10 |
---|
[f07e037] | 14 | // |
---|
| 15 | |
---|
[2026bb6] | 16 | #define __cforall_thread__ |
---|
| 17 | |
---|
[58b6d1b] | 18 | #include "monitor.hfa" |
---|
[f07e037] | 19 | |
---|
[73abe95] | 20 | #include <stdlib.hfa> |
---|
[2f6a7e93] | 21 | #include <inttypes.h> |
---|
[a933dcf4] | 22 | |
---|
[73abe95] | 23 | #include "kernel_private.hfa" |
---|
[f07e037] | 24 | |
---|
[58b6d1b] | 25 | #include "bits/algorithm.hfa" |
---|
[de737c8] | 26 | |
---|
[0c78741] | 27 | //----------------------------------------------------------------------------- |
---|
| 28 | // Forward declarations |
---|
[ac2b598] | 29 | static inline void __set_owner ( $monitor * this, $thread * owner ); |
---|
| 30 | static inline void __set_owner ( $monitor * storage [], __lock_size_t count, $thread * owner ); |
---|
| 31 | static inline void set_mask ( $monitor * storage [], __lock_size_t count, const __waitfor_mask_t & mask ); |
---|
| 32 | static inline void reset_mask( $monitor * this ); |
---|
[6ff4507] | 33 | |
---|
[ac2b598] | 34 | static inline $thread * next_thread( $monitor * this ); |
---|
| 35 | static inline bool is_accepted( $monitor * this, const __monitor_group_t & monitors ); |
---|
[0c78741] | 36 | |
---|
[ea7d2b0] | 37 | static inline void lock_all ( __spinlock_t * locks [], __lock_size_t count ); |
---|
[ac2b598] | 38 | static inline void lock_all ( $monitor * source [], __spinlock_t * /*out*/ locks [], __lock_size_t count ); |
---|
[ea7d2b0] | 39 | static inline void unlock_all( __spinlock_t * locks [], __lock_size_t count ); |
---|
[ac2b598] | 40 | static inline void unlock_all( $monitor * locks [], __lock_size_t count ); |
---|
[0c78741] | 41 | |
---|
[ac2b598] | 42 | static inline void save ( $monitor * ctx [], __lock_size_t count, __spinlock_t * locks [], unsigned int /*out*/ recursions [], __waitfor_mask_t /*out*/ masks [] ); |
---|
| 43 | static inline void restore( $monitor * ctx [], __lock_size_t count, __spinlock_t * locks [], unsigned int /*in */ recursions [], __waitfor_mask_t /*in */ masks [] ); |
---|
[0c78741] | 44 | |
---|
[ac2b598] | 45 | static inline void init ( __lock_size_t count, $monitor * monitors [], __condition_node_t & waiter, __condition_criterion_t criteria [] ); |
---|
| 46 | static inline void init_push( __lock_size_t count, $monitor * monitors [], __condition_node_t & waiter, __condition_criterion_t criteria [] ); |
---|
[97e3296] | 47 | |
---|
[ac2b598] | 48 | static inline $thread * check_condition ( __condition_criterion_t * ); |
---|
[4cedd9f] | 49 | static inline void brand_condition ( condition & ); |
---|
[ac2b598] | 50 | static inline [$thread *, int] search_entry_queue( const __waitfor_mask_t &, $monitor * monitors [], __lock_size_t count ); |
---|
[b18830e] | 51 | |
---|
[6ff4507] | 52 | forall(dtype T | sized( T )) |
---|
[59a0bde] | 53 | static inline __lock_size_t insert_unique( T * array [], __lock_size_t & size, T * val ); |
---|
| 54 | static inline __lock_size_t count_max ( const __waitfor_mask_t & mask ); |
---|
[ac2b598] | 55 | static inline __lock_size_t aggregate ( $monitor * storage [], const __waitfor_mask_t & mask ); |
---|
[97e3296] | 56 | |
---|
| 57 | //----------------------------------------------------------------------------- |
---|
| 58 | // Useful defines |
---|
[6ff4507] | 59 | #define wait_ctx(thrd, user_info) /* Create the necessary information to use the signaller stack */ \ |
---|
| 60 | __condition_node_t waiter = { thrd, count, user_info }; /* Create the node specific to this wait operation */ \ |
---|
| 61 | __condition_criterion_t criteria[count]; /* Create the creteria this wait operation needs to wake up */ \ |
---|
[8fc45b7] | 62 | init( count, monitors, waiter, criteria ); /* Link everything together */ \ |
---|
[6ff4507] | 63 | |
---|
| 64 | #define wait_ctx_primed(thrd, user_info) /* Create the necessary information to use the signaller stack */ \ |
---|
| 65 | __condition_node_t waiter = { thrd, count, user_info }; /* Create the node specific to this wait operation */ \ |
---|
| 66 | __condition_criterion_t criteria[count]; /* Create the creteria this wait operation needs to wake up */ \ |
---|
[8fc45b7] | 67 | init_push( count, monitors, waiter, criteria ); /* Link everything together and push it to the AS-Stack */ \ |
---|
[6ff4507] | 68 | |
---|
| 69 | #define monitor_ctx( mons, cnt ) /* Define that create the necessary struct for internal/external scheduling operations */ \ |
---|
[ac2b598] | 70 | $monitor ** monitors = mons; /* Save the targeted monitors */ \ |
---|
[59a0bde] | 71 | __lock_size_t count = cnt; /* Save the count to a local variable */ \ |
---|
[6ff4507] | 72 | unsigned int recursions[ count ]; /* Save the current recursion levels to restore them later */ \ |
---|
[4cedd9f] | 73 | __waitfor_mask_t masks [ count ]; /* Save the current waitfor masks to restore them later */ \ |
---|
[ea7d2b0] | 74 | __spinlock_t * locks [ count ]; /* We need to pass-in an array of locks to BlockInternal */ \ |
---|
[6ff4507] | 75 | |
---|
| 76 | #define monitor_save save ( monitors, count, locks, recursions, masks ) |
---|
| 77 | #define monitor_restore restore( monitors, count, locks, recursions, masks ) |
---|
| 78 | |
---|
[97e3296] | 79 | |
---|
[0c78741] | 80 | //----------------------------------------------------------------------------- |
---|
| 81 | // Enter/Leave routines |
---|
[c7a900a] | 82 | // Enter single monitor |
---|
[ac2b598] | 83 | static void __enter( $monitor * this, const __monitor_group_t & group ) { |
---|
[c7a900a] | 84 | // Lock the monitor spinlock |
---|
| 85 | lock( this->lock __cfaabi_dbg_ctx2 ); |
---|
| 86 | // Interrupts disable inside critical section |
---|
[ac2b598] | 87 | $thread * thrd = kernelTLS.this_thread; |
---|
[690f13c] | 88 | |
---|
[c7a900a] | 89 | __cfaabi_dbg_print_safe( "Kernel : %10p Entering mon %p (%p)\n", thrd, this, this->owner); |
---|
[690f13c] | 90 | |
---|
[77a2994] | 91 | if( unlikely(0 != (0x1 & (uintptr_t)this->owner)) ) { |
---|
| 92 | abort( "Attempt by thread \"%.256s\" (%p) to access joined monitor %p.", thrd->self_cor.name, thrd, this ); |
---|
| 93 | } |
---|
| 94 | else if( !this->owner ) { |
---|
[c7a900a] | 95 | // No one has the monitor, just take it |
---|
| 96 | __set_owner( this, thrd ); |
---|
[f07e037] | 97 | |
---|
[c7a900a] | 98 | __cfaabi_dbg_print_safe( "Kernel : mon is free \n" ); |
---|
| 99 | } |
---|
| 100 | else if( this->owner == thrd) { |
---|
| 101 | // We already have the monitor, just note how many times we took it |
---|
| 102 | this->recursion += 1; |
---|
[90c4df0] | 103 | |
---|
[c7a900a] | 104 | __cfaabi_dbg_print_safe( "Kernel : mon already owned \n" ); |
---|
| 105 | } |
---|
| 106 | else if( is_accepted( this, group) ) { |
---|
| 107 | // Some one was waiting for us, enter |
---|
| 108 | __set_owner( this, thrd ); |
---|
[90c4df0] | 109 | |
---|
[c7a900a] | 110 | // Reset mask |
---|
| 111 | reset_mask( this ); |
---|
[90c4df0] | 112 | |
---|
[c7a900a] | 113 | __cfaabi_dbg_print_safe( "Kernel : mon accepts \n" ); |
---|
| 114 | } |
---|
| 115 | else { |
---|
| 116 | __cfaabi_dbg_print_safe( "Kernel : blocking \n" ); |
---|
[90c4df0] | 117 | |
---|
[c7a900a] | 118 | // Some one else has the monitor, wait in line for it |
---|
[504a7dc] | 119 | /* paranoid */ verify( thrd->link.next == 0p ); |
---|
[c7a900a] | 120 | append( this->entry_queue, thrd ); |
---|
[504a7dc] | 121 | /* paranoid */ verify( thrd->link.next == 1p ); |
---|
[daacf82] | 122 | |
---|
[c7a900a] | 123 | unlock( this->lock ); |
---|
[ae66348] | 124 | park( __cfaabi_dbg_ctx ); |
---|
[90c4df0] | 125 | |
---|
[169d944] | 126 | __cfaabi_dbg_print_safe( "Kernel : %10p Entered mon %p\n", thrd, this); |
---|
[2e9aed4] | 127 | |
---|
[3381ed7] | 128 | /* paranoid */ verifyf( kernelTLS.this_thread == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", kernelTLS.this_thread, this->owner, this->recursion, this ); |
---|
[5ea06d6] | 129 | return; |
---|
[cb0e6de] | 130 | } |
---|
[cc7f4b1] | 131 | |
---|
[c7a900a] | 132 | __cfaabi_dbg_print_safe( "Kernel : %10p Entered mon %p\n", thrd, this); |
---|
[90c4df0] | 133 | |
---|
[c7a900a] | 134 | /* paranoid */ verifyf( kernelTLS.this_thread == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", kernelTLS.this_thread, this->owner, this->recursion, this ); |
---|
| 135 | /* paranoid */ verify( this->lock.lock ); |
---|
[f07e037] | 136 | |
---|
[c7a900a] | 137 | // Release the lock and leave |
---|
| 138 | unlock( this->lock ); |
---|
| 139 | return; |
---|
| 140 | } |
---|
[90c4df0] | 141 | |
---|
[77a2994] | 142 | static void __dtor_enter( $monitor * this, fptr_t func, bool join ) { |
---|
[c7a900a] | 143 | // Lock the monitor spinlock |
---|
| 144 | lock( this->lock __cfaabi_dbg_ctx2 ); |
---|
| 145 | // Interrupts disable inside critical section |
---|
[ac2b598] | 146 | $thread * thrd = kernelTLS.this_thread; |
---|
[f07e037] | 147 | |
---|
[c7a900a] | 148 | __cfaabi_dbg_print_safe( "Kernel : %10p Entering dtor for mon %p (%p)\n", thrd, this, this->owner); |
---|
[549c006] | 149 | |
---|
| 150 | |
---|
[c7a900a] | 151 | if( !this->owner ) { |
---|
| 152 | __cfaabi_dbg_print_safe( "Kernel : Destroying free mon %p\n", this); |
---|
[549c006] | 153 | |
---|
[c7a900a] | 154 | // No one has the monitor, just take it |
---|
| 155 | __set_owner( this, thrd ); |
---|
[549c006] | 156 | |
---|
[c7a900a] | 157 | verifyf( kernelTLS.this_thread == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", kernelTLS.this_thread, this->owner, this->recursion, this ); |
---|
[549c006] | 158 | |
---|
[c7a900a] | 159 | unlock( this->lock ); |
---|
| 160 | return; |
---|
| 161 | } |
---|
[77a2994] | 162 | else if( this->owner == thrd && !join) { |
---|
[c7a900a] | 163 | // We already have the monitor... but where about to destroy it so the nesting will fail |
---|
| 164 | // Abort! |
---|
| 165 | abort( "Attempt to destroy monitor %p by thread \"%.256s\" (%p) in nested mutex.", this, thrd->self_cor.name, thrd ); |
---|
| 166 | } |
---|
[77a2994] | 167 | // SKULLDUGGERY: join will act as a dtor so it would normally trigger to above check |
---|
| 168 | // to avoid that it sets the owner to the special value thrd | 1p before exiting |
---|
| 169 | else if( this->owner == ($thread*)(1 | (uintptr_t)thrd) ) { |
---|
| 170 | // restore the owner and just return |
---|
| 171 | __cfaabi_dbg_print_safe( "Kernel : Destroying free mon %p\n", this); |
---|
| 172 | |
---|
| 173 | // No one has the monitor, just take it |
---|
| 174 | this->owner = thrd; |
---|
| 175 | |
---|
| 176 | verifyf( kernelTLS.this_thread == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", kernelTLS.this_thread, this->owner, this->recursion, this ); |
---|
| 177 | |
---|
| 178 | unlock( this->lock ); |
---|
| 179 | return; |
---|
| 180 | } |
---|
[549c006] | 181 | |
---|
[c7a900a] | 182 | __lock_size_t count = 1; |
---|
[ac2b598] | 183 | $monitor ** monitors = &this; |
---|
[c7a900a] | 184 | __monitor_group_t group = { &this, 1, func }; |
---|
| 185 | if( is_accepted( this, group) ) { |
---|
| 186 | __cfaabi_dbg_print_safe( "Kernel : mon accepts dtor, block and signal it \n" ); |
---|
[549c006] | 187 | |
---|
[c7a900a] | 188 | // Wake the thread that is waiting for this |
---|
| 189 | __condition_criterion_t * urgent = pop( this->signal_stack ); |
---|
| 190 | /* paranoid */ verify( urgent ); |
---|
[b8116cd] | 191 | |
---|
[c7a900a] | 192 | // Reset mask |
---|
| 193 | reset_mask( this ); |
---|
[549c006] | 194 | |
---|
[c7a900a] | 195 | // Create the node specific to this wait operation |
---|
| 196 | wait_ctx_primed( thrd, 0 ) |
---|
[549c006] | 197 | |
---|
[c7a900a] | 198 | // Some one else has the monitor, wait for him to finish and then run |
---|
| 199 | unlock( this->lock ); |
---|
[549c006] | 200 | |
---|
[c7a900a] | 201 | // Release the next thread |
---|
| 202 | /* paranoid */ verifyf( urgent->owner->waiting_thread == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", kernelTLS.this_thread, this->owner, this->recursion, this ); |
---|
[ae66348] | 203 | unpark( urgent->owner->waiting_thread __cfaabi_dbg_ctx2 ); |
---|
[549c006] | 204 | |
---|
[c7a900a] | 205 | // Park current thread waiting |
---|
[ae66348] | 206 | park( __cfaabi_dbg_ctx ); |
---|
[549c006] | 207 | |
---|
[c7a900a] | 208 | // Some one was waiting for us, enter |
---|
| 209 | /* paranoid */ verifyf( kernelTLS.this_thread == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", kernelTLS.this_thread, this->owner, this->recursion, this ); |
---|
| 210 | } |
---|
| 211 | else { |
---|
| 212 | __cfaabi_dbg_print_safe( "Kernel : blocking \n" ); |
---|
[549c006] | 213 | |
---|
[c7a900a] | 214 | wait_ctx( thrd, 0 ) |
---|
| 215 | this->dtor_node = &waiter; |
---|
[549c006] | 216 | |
---|
[c7a900a] | 217 | // Some one else has the monitor, wait in line for it |
---|
[504a7dc] | 218 | /* paranoid */ verify( thrd->link.next == 0p ); |
---|
[c7a900a] | 219 | append( this->entry_queue, thrd ); |
---|
[504a7dc] | 220 | /* paranoid */ verify( thrd->link.next == 1p ); |
---|
[c7a900a] | 221 | unlock( this->lock ); |
---|
[549c006] | 222 | |
---|
[c7a900a] | 223 | // Park current thread waiting |
---|
[ae66348] | 224 | park( __cfaabi_dbg_ctx ); |
---|
[549c006] | 225 | |
---|
[c7a900a] | 226 | /* paranoid */ verifyf( kernelTLS.this_thread == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", kernelTLS.this_thread, this->owner, this->recursion, this ); |
---|
| 227 | return; |
---|
[549c006] | 228 | } |
---|
| 229 | |
---|
[c7a900a] | 230 | __cfaabi_dbg_print_safe( "Kernel : Destroying %p\n", this); |
---|
[f07e037] | 231 | |
---|
[c7a900a] | 232 | } |
---|
[a843067] | 233 | |
---|
[c7a900a] | 234 | // Leave single monitor |
---|
[ac2b598] | 235 | void __leave( $monitor * this ) { |
---|
[c7a900a] | 236 | // Lock the monitor spinlock |
---|
| 237 | lock( this->lock __cfaabi_dbg_ctx2 ); |
---|
[cc7f4b1] | 238 | |
---|
[c7a900a] | 239 | __cfaabi_dbg_print_safe( "Kernel : %10p Leaving mon %p (%p)\n", kernelTLS.this_thread, this, this->owner); |
---|
[f07e037] | 240 | |
---|
[c7a900a] | 241 | /* paranoid */ verifyf( kernelTLS.this_thread == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", kernelTLS.this_thread, this->owner, this->recursion, this ); |
---|
[f07e037] | 242 | |
---|
[c7a900a] | 243 | // Leaving a recursion level, decrement the counter |
---|
| 244 | this->recursion -= 1; |
---|
[5ea06d6] | 245 | |
---|
[c7a900a] | 246 | // If we haven't left the last level of recursion |
---|
| 247 | // it means we don't need to do anything |
---|
| 248 | if( this->recursion != 0) { |
---|
| 249 | __cfaabi_dbg_print_safe( "Kernel : recursion still %d\n", this->recursion); |
---|
[ea7d2b0] | 250 | unlock( this->lock ); |
---|
[c7a900a] | 251 | return; |
---|
[1c273d0] | 252 | } |
---|
| 253 | |
---|
[c7a900a] | 254 | // Get the next thread, will be null on low contention monitor |
---|
[ac2b598] | 255 | $thread * new_owner = next_thread( this ); |
---|
[c7a900a] | 256 | |
---|
| 257 | // Check the new owner is consistent with who we wake-up |
---|
| 258 | // new_owner might be null even if someone owns the monitor when the owner is still waiting for another monitor |
---|
| 259 | /* paranoid */ verifyf( !new_owner || new_owner == this->owner, "Expected owner to be %p, got %p (m: %p)", new_owner, this->owner, this ); |
---|
| 260 | |
---|
| 261 | // We can now let other threads in safely |
---|
| 262 | unlock( this->lock ); |
---|
[549c006] | 263 | |
---|
[c7a900a] | 264 | //We need to wake-up the thread |
---|
| 265 | /* paranoid */ verifyf( !new_owner || new_owner == this->owner, "Expected owner to be %p, got %p (m: %p)", new_owner, this->owner, this ); |
---|
[ae66348] | 266 | unpark( new_owner __cfaabi_dbg_ctx2 ); |
---|
[c7a900a] | 267 | } |
---|
| 268 | |
---|
| 269 | // Leave single monitor for the last time |
---|
[77a2994] | 270 | void __dtor_leave( $monitor * this, bool join ) { |
---|
[c7a900a] | 271 | __cfaabi_dbg_debug_do( |
---|
| 272 | if( TL_GET( this_thread ) != this->owner ) { |
---|
| 273 | abort( "Destroyed monitor %p has inconsistent owner, expected %p got %p.\n", this, TL_GET( this_thread ), this->owner); |
---|
| 274 | } |
---|
[77a2994] | 275 | if( this->recursion != 1 && !join ) { |
---|
[c7a900a] | 276 | abort( "Destroyed monitor %p has %d outstanding nested calls.\n", this, this->recursion - 1); |
---|
| 277 | } |
---|
| 278 | ) |
---|
[77a2994] | 279 | |
---|
| 280 | this->owner = ($thread*)(1 | (uintptr_t)this->owner); |
---|
[c7a900a] | 281 | } |
---|
[549c006] | 282 | |
---|
[c7a900a] | 283 | extern "C" { |
---|
[97e3296] | 284 | // Leave the thread monitor |
---|
| 285 | // last routine called by a thread. |
---|
| 286 | // Should never return |
---|
[c7a900a] | 287 | void __cfactx_thrd_leave() { |
---|
[ac2b598] | 288 | $thread * thrd = TL_GET( this_thread ); |
---|
| 289 | $monitor * this = &thrd->self_mon; |
---|
[97e3296] | 290 | |
---|
| 291 | // Lock the monitor now |
---|
[2e9aed4] | 292 | lock( this->lock __cfaabi_dbg_ctx2 ); |
---|
[1c273d0] | 293 | |
---|
| 294 | disable_interrupts(); |
---|
| 295 | |
---|
[b0c7419] | 296 | thrd->state = Halted; |
---|
[1c273d0] | 297 | |
---|
[3381ed7] | 298 | /* paranoid */ verifyf( thrd == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", thrd, this->owner, this->recursion, this ); |
---|
[1c273d0] | 299 | |
---|
[97e3296] | 300 | // Leaving a recursion level, decrement the counter |
---|
[1c273d0] | 301 | this->recursion -= 1; |
---|
| 302 | |
---|
[97e3296] | 303 | // If we haven't left the last level of recursion |
---|
| 304 | // it must mean there is an error |
---|
[169d944] | 305 | if( this->recursion != 0) { abort( "Thread internal monitor has unbalanced recursion" ); } |
---|
[1c273d0] | 306 | |
---|
[97e3296] | 307 | // Fetch the next thread, can be null |
---|
[ac2b598] | 308 | $thread * new_owner = next_thread( this ); |
---|
[3381ed7] | 309 | |
---|
| 310 | // Release the monitor lock |
---|
| 311 | unlock( this->lock ); |
---|
| 312 | |
---|
| 313 | // Unpark the next owner if needed |
---|
| 314 | /* paranoid */ verifyf( !new_owner || new_owner == this->owner, "Expected owner to be %p, got %p (m: %p)", new_owner, this->owner, this ); |
---|
[b0c7419] | 315 | /* paranoid */ verify( ! kernelTLS.preemption_state.enabled ); |
---|
| 316 | /* paranoid */ verify( ! kernelTLS.this_processor->destroyer ); |
---|
| 317 | /* paranoid */ verify( thrd->state == Halted ); |
---|
| 318 | |
---|
| 319 | kernelTLS.this_processor->destroyer = new_owner; |
---|
[1c273d0] | 320 | |
---|
[b0c7419] | 321 | // Leave the thread |
---|
| 322 | __leave_thread(); |
---|
[97e3296] | 323 | |
---|
| 324 | // Control flow should never reach here! |
---|
[cc7f4b1] | 325 | } |
---|
[2781e65] | 326 | } |
---|
| 327 | |
---|
[77a2994] | 328 | // Join a thread |
---|
| 329 | forall( dtype T | is_thread(T) ) |
---|
| 330 | T & join( T & this ) { |
---|
| 331 | $monitor * m = get_monitor(this); |
---|
| 332 | void (*dtor)(T& mutex this) = ^?{}; |
---|
| 333 | monitor_dtor_guard_t __guard = { &m, (fptr_t)dtor, true }; |
---|
| 334 | { |
---|
| 335 | return this; |
---|
| 336 | } |
---|
| 337 | } |
---|
| 338 | |
---|
[97e3296] | 339 | // Enter multiple monitor |
---|
| 340 | // relies on the monitor array being sorted |
---|
[6ae8c92] | 341 | static inline void enter( __monitor_group_t monitors ) { |
---|
[59a0bde] | 342 | for( __lock_size_t i = 0; i < monitors.size; i++) { |
---|
[c7a900a] | 343 | __enter( monitors[i], monitors ); |
---|
[97e3296] | 344 | } |
---|
[2781e65] | 345 | } |
---|
| 346 | |
---|
[97e3296] | 347 | // Leave multiple monitor |
---|
| 348 | // relies on the monitor array being sorted |
---|
[ac2b598] | 349 | static inline void leave($monitor * monitors [], __lock_size_t count) { |
---|
[59a0bde] | 350 | for( __lock_size_t i = count - 1; i >= 0; i--) { |
---|
[c7a900a] | 351 | __leave( monitors[i] ); |
---|
[2781e65] | 352 | } |
---|
[5ea06d6] | 353 | } |
---|
| 354 | |
---|
[97e3296] | 355 | // Ctor for monitor guard |
---|
| 356 | // Sorts monitors before entering |
---|
[ac2b598] | 357 | void ?{}( monitor_guard_t & this, $monitor * m [], __lock_size_t count, fptr_t func ) { |
---|
| 358 | $thread * thrd = TL_GET( this_thread ); |
---|
[14a61b5] | 359 | |
---|
[97e3296] | 360 | // Store current array |
---|
[242a902] | 361 | this.m = m; |
---|
| 362 | this.count = count; |
---|
[97e3296] | 363 | |
---|
[09800e9] | 364 | // Sort monitors based on address |
---|
[de737c8] | 365 | __libcfa_small_sort(this.m, count); |
---|
[5ea06d6] | 366 | |
---|
[97e3296] | 367 | // Save previous thread context |
---|
[14a61b5] | 368 | this.prev = thrd->monitors; |
---|
[5ea06d6] | 369 | |
---|
[97e3296] | 370 | // Update thread context (needed for conditions) |
---|
[14a61b5] | 371 | (thrd->monitors){m, count, func}; |
---|
[90c4df0] | 372 | |
---|
[169d944] | 373 | // __cfaabi_dbg_print_safe( "MGUARD : enter %d\n", count); |
---|
[a843067] | 374 | |
---|
[90c4df0] | 375 | // Enter the monitors in order |
---|
[6ae8c92] | 376 | __monitor_group_t group = {this.m, this.count, func}; |
---|
[b18830e] | 377 | enter( group ); |
---|
[a843067] | 378 | |
---|
[169d944] | 379 | // __cfaabi_dbg_print_safe( "MGUARD : entered\n" ); |
---|
[5ea06d6] | 380 | } |
---|
| 381 | |
---|
[6b224a52] | 382 | |
---|
[97e3296] | 383 | // Dtor for monitor guard |
---|
[242a902] | 384 | void ^?{}( monitor_guard_t & this ) { |
---|
[169d944] | 385 | // __cfaabi_dbg_print_safe( "MGUARD : leaving %d\n", this.count); |
---|
[a843067] | 386 | |
---|
[97e3296] | 387 | // Leave the monitors in order |
---|
[242a902] | 388 | leave( this.m, this.count ); |
---|
[5ea06d6] | 389 | |
---|
[169d944] | 390 | // __cfaabi_dbg_print_safe( "MGUARD : left\n" ); |
---|
[a843067] | 391 | |
---|
[97e3296] | 392 | // Restore thread context |
---|
[b10affd] | 393 | TL_GET( this_thread )->monitors = this.prev; |
---|
[5ea06d6] | 394 | } |
---|
| 395 | |
---|
[549c006] | 396 | // Ctor for monitor guard |
---|
| 397 | // Sorts monitors before entering |
---|
[77a2994] | 398 | void ?{}( monitor_dtor_guard_t & this, $monitor * m [], fptr_t func, bool join ) { |
---|
[14a61b5] | 399 | // optimization |
---|
[ac2b598] | 400 | $thread * thrd = TL_GET( this_thread ); |
---|
[14a61b5] | 401 | |
---|
[549c006] | 402 | // Store current array |
---|
| 403 | this.m = *m; |
---|
| 404 | |
---|
| 405 | // Save previous thread context |
---|
[14a61b5] | 406 | this.prev = thrd->monitors; |
---|
[549c006] | 407 | |
---|
[77a2994] | 408 | // Save whether we are in a join or not |
---|
| 409 | this.join = join; |
---|
| 410 | |
---|
[549c006] | 411 | // Update thread context (needed for conditions) |
---|
[14a61b5] | 412 | (thrd->monitors){m, 1, func}; |
---|
[549c006] | 413 | |
---|
[77a2994] | 414 | __dtor_enter( this.m, func, join ); |
---|
[549c006] | 415 | } |
---|
| 416 | |
---|
| 417 | // Dtor for monitor guard |
---|
| 418 | void ^?{}( monitor_dtor_guard_t & this ) { |
---|
| 419 | // Leave the monitors in order |
---|
[77a2994] | 420 | __dtor_leave( this.m, this.join ); |
---|
[549c006] | 421 | |
---|
| 422 | // Restore thread context |
---|
[b10affd] | 423 | TL_GET( this_thread )->monitors = this.prev; |
---|
[549c006] | 424 | } |
---|
| 425 | |
---|
[97e3296] | 426 | //----------------------------------------------------------------------------- |
---|
| 427 | // Internal scheduling types |
---|
[ac2b598] | 428 | void ?{}(__condition_node_t & this, $thread * waiting_thread, __lock_size_t count, uintptr_t user_info ) { |
---|
[242a902] | 429 | this.waiting_thread = waiting_thread; |
---|
| 430 | this.count = count; |
---|
[121be3e] | 431 | this.next = 0p; |
---|
[242a902] | 432 | this.user_info = user_info; |
---|
[be3d020] | 433 | } |
---|
| 434 | |
---|
[c40e7c5] | 435 | void ?{}(__condition_criterion_t & this ) with( this ) { |
---|
| 436 | ready = false; |
---|
[121be3e] | 437 | target = 0p; |
---|
| 438 | owner = 0p; |
---|
| 439 | next = 0p; |
---|
[be3d020] | 440 | } |
---|
| 441 | |
---|
[ac2b598] | 442 | void ?{}(__condition_criterion_t & this, $monitor * target, __condition_node_t & owner ) { |
---|
[242a902] | 443 | this.ready = false; |
---|
| 444 | this.target = target; |
---|
[8fc45b7] | 445 | this.owner = &owner; |
---|
[121be3e] | 446 | this.next = 0p; |
---|
[ad1a8dd] | 447 | } |
---|
| 448 | |
---|
[5ea06d6] | 449 | //----------------------------------------------------------------------------- |
---|
| 450 | // Internal scheduling |
---|
[4cedd9f] | 451 | void wait( condition & this, uintptr_t user_info = 0 ) { |
---|
[0c78741] | 452 | brand_condition( this ); |
---|
[5ea06d6] | 453 | |
---|
[97e3296] | 454 | // Check that everything is as expected |
---|
[121be3e] | 455 | assertf( this.monitors != 0p, "Waiting with no monitors (%p)", this.monitors ); |
---|
[2f6a7e93] | 456 | verifyf( this.monitor_count != 0, "Waiting with 0 monitors (%"PRIiFAST16")", this.monitor_count ); |
---|
| 457 | verifyf( this.monitor_count < 32u, "Excessive monitor count (%"PRIiFAST16")", this.monitor_count ); |
---|
[5ea06d6] | 458 | |
---|
[97e3296] | 459 | // Create storage for monitor context |
---|
[4cedd9f] | 460 | monitor_ctx( this.monitors, this.monitor_count ); |
---|
[0c78741] | 461 | |
---|
[97e3296] | 462 | // Create the node specific to this wait operation |
---|
[b10affd] | 463 | wait_ctx( TL_GET( this_thread ), user_info ); |
---|
[0c78741] | 464 | |
---|
[97e3296] | 465 | // Append the current wait operation to the ones already queued on the condition |
---|
| 466 | // We don't need locks for that since conditions must always be waited on inside monitor mutual exclusion |
---|
[3381ed7] | 467 | /* paranoid */ verify( waiter.next == 0p ); |
---|
[8fc45b7] | 468 | append( this.blocked, &waiter ); |
---|
[3381ed7] | 469 | /* paranoid */ verify( waiter.next == 1p ); |
---|
[0c78741] | 470 | |
---|
[6ff4507] | 471 | // Lock all monitors (aggregates the locks as well) |
---|
[97e3296] | 472 | lock_all( monitors, locks, count ); |
---|
[5ea06d6] | 473 | |
---|
[97e3296] | 474 | // Find the next thread(s) to run |
---|
[59a0bde] | 475 | __lock_size_t thread_count = 0; |
---|
[ac2b598] | 476 | $thread * threads[ count ]; |
---|
[66298de] | 477 | __builtin_memset( threads, 0, sizeof( threads ) ); |
---|
[ad1a8dd] | 478 | |
---|
[a843067] | 479 | // Save monitor states |
---|
| 480 | monitor_save; |
---|
| 481 | |
---|
[97e3296] | 482 | // Remove any duplicate threads |
---|
[59a0bde] | 483 | for( __lock_size_t i = 0; i < count; i++) { |
---|
[ac2b598] | 484 | $thread * new_owner = next_thread( monitors[i] ); |
---|
[6ff4507] | 485 | insert_unique( threads, thread_count, new_owner ); |
---|
[5ea06d6] | 486 | } |
---|
| 487 | |
---|
[3381ed7] | 488 | // Unlock the locks, we don't need them anymore |
---|
| 489 | for(int i = 0; i < count; i++) { |
---|
| 490 | unlock( *locks[i] ); |
---|
| 491 | } |
---|
| 492 | |
---|
| 493 | // Wake the threads |
---|
| 494 | for(int i = 0; i < thread_count; i++) { |
---|
[ae66348] | 495 | unpark( threads[i] __cfaabi_dbg_ctx2 ); |
---|
[3381ed7] | 496 | } |
---|
| 497 | |
---|
[a843067] | 498 | // Everything is ready to go to sleep |
---|
[ae66348] | 499 | park( __cfaabi_dbg_ctx ); |
---|
[a843067] | 500 | |
---|
| 501 | // We are back, restore the owners and recursions |
---|
| 502 | monitor_restore; |
---|
[5ea06d6] | 503 | } |
---|
| 504 | |
---|
[4cedd9f] | 505 | bool signal( condition & this ) { |
---|
[97e3296] | 506 | if( is_empty( this ) ) { return false; } |
---|
[5ea06d6] | 507 | |
---|
| 508 | //Check that everything is as expected |
---|
[4cedd9f] | 509 | verify( this.monitors ); |
---|
| 510 | verify( this.monitor_count != 0 ); |
---|
[0c78741] | 511 | |
---|
[44264c5] | 512 | //Some more checking in debug |
---|
[36982fc] | 513 | __cfaabi_dbg_debug_do( |
---|
[ac2b598] | 514 | $thread * this_thrd = TL_GET( this_thread ); |
---|
[4cedd9f] | 515 | if ( this.monitor_count != this_thrd->monitors.size ) { |
---|
[c2ca04d] | 516 | abort( "Signal on condition %p made with different number of monitor(s), expected %zi got %zi", &this, this.monitor_count, this_thrd->monitors.size ); |
---|
[97e3296] | 517 | } |
---|
[0c78741] | 518 | |
---|
[4cedd9f] | 519 | for(int i = 0; i < this.monitor_count; i++) { |
---|
[0cf5b79] | 520 | if ( this.monitors[i] != this_thrd->monitors[i] ) { |
---|
[2fdbb3b] | 521 | abort( "Signal on condition %p made with different monitor, expected %p got %p", &this, this.monitors[i], this_thrd->monitors[i] ); |
---|
[97e3296] | 522 | } |
---|
[0c78741] | 523 | } |
---|
[5ea06d6] | 524 | ); |
---|
| 525 | |
---|
[59a0bde] | 526 | __lock_size_t count = this.monitor_count; |
---|
[97e3296] | 527 | |
---|
| 528 | // Lock all monitors |
---|
[121be3e] | 529 | lock_all( this.monitors, 0p, count ); |
---|
[0c78741] | 530 | |
---|
[44264c5] | 531 | //Pop the head of the waiting queue |
---|
[8fc45b7] | 532 | __condition_node_t * node = pop_head( this.blocked ); |
---|
[44264c5] | 533 | |
---|
| 534 | //Add the thread to the proper AS stack |
---|
[0c78741] | 535 | for(int i = 0; i < count; i++) { |
---|
| 536 | __condition_criterion_t * crit = &node->criteria[i]; |
---|
| 537 | assert( !crit->ready ); |
---|
[8fc45b7] | 538 | push( crit->target->signal_stack, crit ); |
---|
[5ea06d6] | 539 | } |
---|
[0c78741] | 540 | |
---|
[44264c5] | 541 | //Release |
---|
[4cedd9f] | 542 | unlock_all( this.monitors, count ); |
---|
[be3d020] | 543 | |
---|
| 544 | return true; |
---|
[5ea06d6] | 545 | } |
---|
| 546 | |
---|
[4cedd9f] | 547 | bool signal_block( condition & this ) { |
---|
| 548 | if( !this.blocked.head ) { return false; } |
---|
[44264c5] | 549 | |
---|
| 550 | //Check that everything is as expected |
---|
[121be3e] | 551 | verifyf( this.monitors != 0p, "Waiting with no monitors (%p)", this.monitors ); |
---|
[2f6a7e93] | 552 | verifyf( this.monitor_count != 0, "Waiting with 0 monitors (%"PRIiFAST16")", this.monitor_count ); |
---|
[44264c5] | 553 | |
---|
[97e3296] | 554 | // Create storage for monitor context |
---|
[4cedd9f] | 555 | monitor_ctx( this.monitors, this.monitor_count ); |
---|
[44264c5] | 556 | |
---|
[97e3296] | 557 | // Lock all monitors (aggregates the locks them as well) |
---|
| 558 | lock_all( monitors, locks, count ); |
---|
[44264c5] | 559 | |
---|
[2e9aed4] | 560 | |
---|
[97e3296] | 561 | // Create the node specific to this wait operation |
---|
[afd550c] | 562 | wait_ctx_primed( kernelTLS.this_thread, 0 ) |
---|
[44264c5] | 563 | |
---|
| 564 | //save contexts |
---|
[6ff4507] | 565 | monitor_save; |
---|
[44264c5] | 566 | |
---|
| 567 | //Find the thread to run |
---|
[ac2b598] | 568 | $thread * signallee = pop_head( this.blocked )->waiting_thread; |
---|
[c7a900a] | 569 | __set_owner( monitors, count, signallee ); |
---|
[44264c5] | 570 | |
---|
[36982fc] | 571 | __cfaabi_dbg_print_buffer_decl( "Kernel : signal_block condition %p (s: %p)\n", &this, signallee ); |
---|
[de737c8] | 572 | |
---|
[3381ed7] | 573 | // unlock all the monitors |
---|
| 574 | unlock_all( locks, count ); |
---|
| 575 | |
---|
| 576 | // unpark the thread we signalled |
---|
[ae66348] | 577 | unpark( signallee __cfaabi_dbg_ctx2 ); |
---|
[3381ed7] | 578 | |
---|
[44264c5] | 579 | //Everything is ready to go to sleep |
---|
[ae66348] | 580 | park( __cfaabi_dbg_ctx ); |
---|
[44264c5] | 581 | |
---|
[c81ebf9] | 582 | |
---|
[97e3296] | 583 | // WE WOKE UP |
---|
[c81ebf9] | 584 | |
---|
| 585 | |
---|
[36982fc] | 586 | __cfaabi_dbg_print_buffer_local( "Kernel : signal_block returned\n" ); |
---|
[de737c8] | 587 | |
---|
[6ff4507] | 588 | //We are back, restore the masks and recursions |
---|
| 589 | monitor_restore; |
---|
[be3d020] | 590 | |
---|
| 591 | return true; |
---|
| 592 | } |
---|
| 593 | |
---|
[97e3296] | 594 | // Access the user_info of the thread waiting at the front of the queue |
---|
[4cedd9f] | 595 | uintptr_t front( condition & this ) { |
---|
[2ac095d] | 596 | verifyf( !is_empty(this), |
---|
[4aa2fb2] | 597 | "Attempt to access user data on an empty condition.\n" |
---|
| 598 | "Possible cause is not checking if the condition is empty before reading stored data." |
---|
[be3d020] | 599 | ); |
---|
[0cf5b79] | 600 | return ((typeof(this.blocked.head))this.blocked.head)->user_info; |
---|
[44264c5] | 601 | } |
---|
| 602 | |
---|
[c81ebf9] | 603 | //----------------------------------------------------------------------------- |
---|
[b18830e] | 604 | // External scheduling |
---|
| 605 | // cases to handle : |
---|
| 606 | // - target already there : |
---|
| 607 | // block and wake |
---|
| 608 | // - dtor already there |
---|
| 609 | // put thread on signaller stack |
---|
| 610 | // - non-blocking |
---|
| 611 | // return else |
---|
| 612 | // - timeout |
---|
| 613 | // return timeout |
---|
| 614 | // - block |
---|
| 615 | // setup mask |
---|
| 616 | // block |
---|
[6ae8c92] | 617 | void __waitfor_internal( const __waitfor_mask_t & mask, int duration ) { |
---|
[b18830e] | 618 | // This statment doesn't have a contiguous list of monitors... |
---|
| 619 | // Create one! |
---|
[59a0bde] | 620 | __lock_size_t max = count_max( mask ); |
---|
[ac2b598] | 621 | $monitor * mon_storage[max]; |
---|
[66298de] | 622 | __builtin_memset( mon_storage, 0, sizeof( mon_storage ) ); |
---|
[59a0bde] | 623 | __lock_size_t actual_count = aggregate( mon_storage, mask ); |
---|
[97e3296] | 624 | |
---|
[523232d] | 625 | __cfaabi_dbg_print_buffer_decl( "Kernel : waitfor %"PRIdFAST16" (s: %"PRIdFAST16", m: %"PRIdFAST16")\n", actual_count, mask.size, (__lock_size_t)max); |
---|
[66298de] | 626 | |
---|
[daacf82] | 627 | if(actual_count == 0) return; |
---|
[19c43b7] | 628 | |
---|
[169d944] | 629 | __cfaabi_dbg_print_buffer_local( "Kernel : waitfor internal proceeding\n" ); |
---|
[4cc9b13] | 630 | |
---|
[97e3296] | 631 | // Create storage for monitor context |
---|
[b18830e] | 632 | monitor_ctx( mon_storage, actual_count ); |
---|
[c81ebf9] | 633 | |
---|
[6ff4507] | 634 | // Lock all monitors (aggregates the locks as well) |
---|
[97e3296] | 635 | lock_all( monitors, locks, count ); |
---|
[c81ebf9] | 636 | |
---|
[b18830e] | 637 | { |
---|
| 638 | // Check if the entry queue |
---|
[ac2b598] | 639 | $thread * next; int index; |
---|
[6ae8c92] | 640 | [next, index] = search_entry_queue( mask, monitors, count ); |
---|
[b18830e] | 641 | |
---|
| 642 | if( next ) { |
---|
[19c43b7] | 643 | *mask.accepted = index; |
---|
[0cf5b79] | 644 | __acceptable_t& accepted = mask[index]; |
---|
| 645 | if( accepted.is_dtor ) { |
---|
[169d944] | 646 | __cfaabi_dbg_print_buffer_local( "Kernel : dtor already there\n" ); |
---|
[0cf5b79] | 647 | verifyf( accepted.size == 1, "ERROR: Accepted dtor has more than 1 mutex parameter." ); |
---|
[549c006] | 648 | |
---|
[ac2b598] | 649 | $monitor * mon2dtor = accepted[0]; |
---|
[549c006] | 650 | verifyf( mon2dtor->dtor_node, "ERROR: Accepted monitor has no dtor_node." ); |
---|
| 651 | |
---|
| 652 | __condition_criterion_t * dtor_crit = mon2dtor->dtor_node->criteria; |
---|
[8fc45b7] | 653 | push( mon2dtor->signal_stack, dtor_crit ); |
---|
[549c006] | 654 | |
---|
| 655 | unlock_all( locks, count ); |
---|
[b18830e] | 656 | } |
---|
| 657 | else { |
---|
[169d944] | 658 | __cfaabi_dbg_print_buffer_local( "Kernel : thread present, baton-passing\n" ); |
---|
[19c43b7] | 659 | |
---|
| 660 | // Create the node specific to this wait operation |
---|
[14a61b5] | 661 | wait_ctx_primed( kernelTLS.this_thread, 0 ); |
---|
[19c43b7] | 662 | |
---|
| 663 | // Save monitor states |
---|
| 664 | monitor_save; |
---|
| 665 | |
---|
[523232d] | 666 | __cfaabi_dbg_print_buffer_local( "Kernel : baton of %"PRIdFAST16" monitors : ", count ); |
---|
[6a5be52] | 667 | #ifdef __CFA_DEBUG_PRINT__ |
---|
| 668 | for( int i = 0; i < count; i++) { |
---|
[36982fc] | 669 | __cfaabi_dbg_print_buffer_local( "%p %p ", monitors[i], monitors[i]->signal_stack.top ); |
---|
[6a5be52] | 670 | } |
---|
| 671 | #endif |
---|
[169d944] | 672 | __cfaabi_dbg_print_buffer_local( "\n" ); |
---|
[6a5be52] | 673 | |
---|
[19c43b7] | 674 | // Set the owners to be the next thread |
---|
[c7a900a] | 675 | __set_owner( monitors, count, next ); |
---|
[19c43b7] | 676 | |
---|
[3381ed7] | 677 | // unlock all the monitors |
---|
| 678 | unlock_all( locks, count ); |
---|
[19c43b7] | 679 | |
---|
[3381ed7] | 680 | // unpark the thread we signalled |
---|
[ae66348] | 681 | unpark( next __cfaabi_dbg_ctx2 ); |
---|
[3381ed7] | 682 | |
---|
| 683 | //Everything is ready to go to sleep |
---|
[ae66348] | 684 | park( __cfaabi_dbg_ctx ); |
---|
[19c43b7] | 685 | |
---|
| 686 | // We are back, restore the owners and recursions |
---|
| 687 | monitor_restore; |
---|
| 688 | |
---|
[169d944] | 689 | __cfaabi_dbg_print_buffer_local( "Kernel : thread present, returned\n" ); |
---|
[b18830e] | 690 | } |
---|
| 691 | |
---|
[36982fc] | 692 | __cfaabi_dbg_print_buffer_local( "Kernel : accepted %d\n", *mask.accepted); |
---|
[19c43b7] | 693 | return; |
---|
[90c4df0] | 694 | } |
---|
| 695 | } |
---|
| 696 | |
---|
[c81ebf9] | 697 | |
---|
[4cc9b13] | 698 | if( duration == 0 ) { |
---|
[169d944] | 699 | __cfaabi_dbg_print_buffer_local( "Kernel : non-blocking, exiting\n" ); |
---|
[19c43b7] | 700 | |
---|
[4cc9b13] | 701 | unlock_all( locks, count ); |
---|
[19c43b7] | 702 | |
---|
[36982fc] | 703 | __cfaabi_dbg_print_buffer_local( "Kernel : accepted %d\n", *mask.accepted); |
---|
[4cc9b13] | 704 | return; |
---|
| 705 | } |
---|
[b18830e] | 706 | |
---|
| 707 | |
---|
[169d944] | 708 | verifyf( duration < 0, "Timeout on waitfor statments not supported yet." ); |
---|
[b18830e] | 709 | |
---|
[169d944] | 710 | __cfaabi_dbg_print_buffer_local( "Kernel : blocking waitfor\n" ); |
---|
[19c43b7] | 711 | |
---|
| 712 | // Create the node specific to this wait operation |
---|
[14a61b5] | 713 | wait_ctx_primed( kernelTLS.this_thread, 0 ); |
---|
[b18830e] | 714 | |
---|
[6ff4507] | 715 | monitor_save; |
---|
[6ae8c92] | 716 | set_mask( monitors, count, mask ); |
---|
[c81ebf9] | 717 | |
---|
[59a0bde] | 718 | for( __lock_size_t i = 0; i < count; i++) { |
---|
[14a61b5] | 719 | verify( monitors[i]->owner == kernelTLS.this_thread ); |
---|
[daacf82] | 720 | } |
---|
| 721 | |
---|
[3381ed7] | 722 | // unlock all the monitors |
---|
| 723 | unlock_all( locks, count ); |
---|
| 724 | |
---|
[19c43b7] | 725 | //Everything is ready to go to sleep |
---|
[ae66348] | 726 | park( __cfaabi_dbg_ctx ); |
---|
[19c43b7] | 727 | |
---|
| 728 | |
---|
| 729 | // WE WOKE UP |
---|
| 730 | |
---|
| 731 | |
---|
| 732 | //We are back, restore the masks and recursions |
---|
| 733 | monitor_restore; |
---|
| 734 | |
---|
[169d944] | 735 | __cfaabi_dbg_print_buffer_local( "Kernel : exiting\n" ); |
---|
[19c43b7] | 736 | |
---|
[36982fc] | 737 | __cfaabi_dbg_print_buffer_local( "Kernel : accepted %d\n", *mask.accepted); |
---|
[c81ebf9] | 738 | } |
---|
| 739 | |
---|
[0c78741] | 740 | //----------------------------------------------------------------------------- |
---|
| 741 | // Utilities |
---|
| 742 | |
---|
[ac2b598] | 743 | static inline void __set_owner( $monitor * this, $thread * owner ) { |
---|
[3381ed7] | 744 | /* paranoid */ verify( this->lock.lock ); |
---|
[a843067] | 745 | |
---|
[0c78741] | 746 | //Pass the monitor appropriately |
---|
| 747 | this->owner = owner; |
---|
| 748 | |
---|
| 749 | //We are passing the monitor to someone else, which means recursion level is not 0 |
---|
| 750 | this->recursion = owner ? 1 : 0; |
---|
| 751 | } |
---|
| 752 | |
---|
[ac2b598] | 753 | static inline void __set_owner( $monitor * monitors [], __lock_size_t count, $thread * owner ) { |
---|
[3381ed7] | 754 | /* paranoid */ verify ( monitors[0]->lock.lock ); |
---|
| 755 | /* paranoid */ verifyf( monitors[0]->owner == kernelTLS.this_thread, "Expected owner to be %p, got %p (r: %i, m: %p)", kernelTLS.this_thread, monitors[0]->owner, monitors[0]->recursion, monitors[0] ); |
---|
| 756 | monitors[0]->owner = owner; |
---|
| 757 | monitors[0]->recursion = 1; |
---|
[513daec] | 758 | for( __lock_size_t i = 1; i < count; i++ ) { |
---|
[3381ed7] | 759 | /* paranoid */ verify ( monitors[i]->lock.lock ); |
---|
| 760 | /* paranoid */ verifyf( monitors[i]->owner == kernelTLS.this_thread, "Expected owner to be %p, got %p (r: %i, m: %p)", kernelTLS.this_thread, monitors[i]->owner, monitors[i]->recursion, monitors[i] ); |
---|
| 761 | monitors[i]->owner = owner; |
---|
| 762 | monitors[i]->recursion = 0; |
---|
[6ff4507] | 763 | } |
---|
| 764 | } |
---|
| 765 | |
---|
[ac2b598] | 766 | static inline void set_mask( $monitor * storage [], __lock_size_t count, const __waitfor_mask_t & mask ) { |
---|
[513daec] | 767 | for( __lock_size_t i = 0; i < count; i++) { |
---|
[6ff4507] | 768 | storage[i]->mask = mask; |
---|
| 769 | } |
---|
| 770 | } |
---|
| 771 | |
---|
[ac2b598] | 772 | static inline void reset_mask( $monitor * this ) { |
---|
[121be3e] | 773 | this->mask.accepted = 0p; |
---|
| 774 | this->mask.data = 0p; |
---|
[daacf82] | 775 | this->mask.size = 0; |
---|
| 776 | } |
---|
| 777 | |
---|
[ac2b598] | 778 | static inline $thread * next_thread( $monitor * this ) { |
---|
[0c78741] | 779 | //Check the signaller stack |
---|
[169d944] | 780 | __cfaabi_dbg_print_safe( "Kernel : mon %p AS-stack top %p\n", this, this->signal_stack.top); |
---|
[8fc45b7] | 781 | __condition_criterion_t * urgent = pop( this->signal_stack ); |
---|
[0c78741] | 782 | if( urgent ) { |
---|
| 783 | //The signaller stack is not empty, |
---|
| 784 | //regardless of if we are ready to baton pass, |
---|
| 785 | //we need to set the monitor as in use |
---|
[3381ed7] | 786 | /* paranoid */ verifyf( !this->owner || kernelTLS.this_thread == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", kernelTLS.this_thread, this->owner, this->recursion, this ); |
---|
[c7a900a] | 787 | __set_owner( this, urgent->owner->waiting_thread ); |
---|
[0c78741] | 788 | |
---|
| 789 | return check_condition( urgent ); |
---|
| 790 | } |
---|
| 791 | |
---|
| 792 | // No signaller thread |
---|
| 793 | // Get the next thread in the entry_queue |
---|
[ac2b598] | 794 | $thread * new_owner = pop_head( this->entry_queue ); |
---|
[3381ed7] | 795 | /* paranoid */ verifyf( !this->owner || kernelTLS.this_thread == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", kernelTLS.this_thread, this->owner, this->recursion, this ); |
---|
[504a7dc] | 796 | /* paranoid */ verify( !new_owner || new_owner->link.next == 0p ); |
---|
[c7a900a] | 797 | __set_owner( this, new_owner ); |
---|
[0c78741] | 798 | |
---|
| 799 | return new_owner; |
---|
| 800 | } |
---|
| 801 | |
---|
[ac2b598] | 802 | static inline bool is_accepted( $monitor * this, const __monitor_group_t & group ) { |
---|
[0cf5b79] | 803 | __acceptable_t * it = this->mask.data; // Optim |
---|
[59a0bde] | 804 | __lock_size_t count = this->mask.size; |
---|
[6ff4507] | 805 | |
---|
| 806 | // Check if there are any acceptable functions |
---|
[a843067] | 807 | if( !it ) return false; |
---|
[6ff4507] | 808 | |
---|
| 809 | // If this isn't the first monitor to test this, there is no reason to repeat the test. |
---|
| 810 | if( this != group[0] ) return group[0]->mask.accepted >= 0; |
---|
| 811 | |
---|
| 812 | // For all acceptable functions check if this is the current function. |
---|
[59a0bde] | 813 | for( __lock_size_t i = 0; i < count; i++, it++ ) { |
---|
[6ff4507] | 814 | if( *it == group ) { |
---|
| 815 | *this->mask.accepted = i; |
---|
| 816 | return true; |
---|
| 817 | } |
---|
| 818 | } |
---|
| 819 | |
---|
| 820 | // No function matched |
---|
| 821 | return false; |
---|
| 822 | } |
---|
| 823 | |
---|
[ac2b598] | 824 | static inline void init( __lock_size_t count, $monitor * monitors [], __condition_node_t & waiter, __condition_criterion_t criteria [] ) { |
---|
[513daec] | 825 | for( __lock_size_t i = 0; i < count; i++) { |
---|
[6b224a52] | 826 | (criteria[i]){ monitors[i], waiter }; |
---|
[97e3296] | 827 | } |
---|
| 828 | |
---|
[8fc45b7] | 829 | waiter.criteria = criteria; |
---|
[97e3296] | 830 | } |
---|
| 831 | |
---|
[ac2b598] | 832 | static inline void init_push( __lock_size_t count, $monitor * monitors [], __condition_node_t & waiter, __condition_criterion_t criteria [] ) { |
---|
[513daec] | 833 | for( __lock_size_t i = 0; i < count; i++) { |
---|
[6b224a52] | 834 | (criteria[i]){ monitors[i], waiter }; |
---|
[36982fc] | 835 | __cfaabi_dbg_print_safe( "Kernel : target %p = %p\n", criteria[i].target, &criteria[i] ); |
---|
[8fc45b7] | 836 | push( criteria[i].target->signal_stack, &criteria[i] ); |
---|
[97e3296] | 837 | } |
---|
| 838 | |
---|
[8fc45b7] | 839 | waiter.criteria = criteria; |
---|
[97e3296] | 840 | } |
---|
| 841 | |
---|
[ea7d2b0] | 842 | static inline void lock_all( __spinlock_t * locks [], __lock_size_t count ) { |
---|
[513daec] | 843 | for( __lock_size_t i = 0; i < count; i++ ) { |
---|
[2e9aed4] | 844 | lock( *locks[i] __cfaabi_dbg_ctx2 ); |
---|
[0c78741] | 845 | } |
---|
| 846 | } |
---|
| 847 | |
---|
[ac2b598] | 848 | static inline void lock_all( $monitor * source [], __spinlock_t * /*out*/ locks [], __lock_size_t count ) { |
---|
[513daec] | 849 | for( __lock_size_t i = 0; i < count; i++ ) { |
---|
[ea7d2b0] | 850 | __spinlock_t * l = &source[i]->lock; |
---|
[2e9aed4] | 851 | lock( *l __cfaabi_dbg_ctx2 ); |
---|
[0c78741] | 852 | if(locks) locks[i] = l; |
---|
| 853 | } |
---|
| 854 | } |
---|
| 855 | |
---|
[ea7d2b0] | 856 | static inline void unlock_all( __spinlock_t * locks [], __lock_size_t count ) { |
---|
[513daec] | 857 | for( __lock_size_t i = 0; i < count; i++ ) { |
---|
[ea7d2b0] | 858 | unlock( *locks[i] ); |
---|
[0c78741] | 859 | } |
---|
| 860 | } |
---|
| 861 | |
---|
[ac2b598] | 862 | static inline void unlock_all( $monitor * locks [], __lock_size_t count ) { |
---|
[513daec] | 863 | for( __lock_size_t i = 0; i < count; i++ ) { |
---|
[ea7d2b0] | 864 | unlock( locks[i]->lock ); |
---|
[0c78741] | 865 | } |
---|
| 866 | } |
---|
| 867 | |
---|
[8fc45b7] | 868 | static inline void save( |
---|
[ac2b598] | 869 | $monitor * ctx [], |
---|
[513daec] | 870 | __lock_size_t count, |
---|
[ea7d2b0] | 871 | __attribute((unused)) __spinlock_t * locks [], |
---|
[8fc45b7] | 872 | unsigned int /*out*/ recursions [], |
---|
| 873 | __waitfor_mask_t /*out*/ masks [] |
---|
| 874 | ) { |
---|
[513daec] | 875 | for( __lock_size_t i = 0; i < count; i++ ) { |
---|
[0c78741] | 876 | recursions[i] = ctx[i]->recursion; |
---|
[6ff4507] | 877 | masks[i] = ctx[i]->mask; |
---|
[0c78741] | 878 | } |
---|
| 879 | } |
---|
| 880 | |
---|
[8fc45b7] | 881 | static inline void restore( |
---|
[ac2b598] | 882 | $monitor * ctx [], |
---|
[513daec] | 883 | __lock_size_t count, |
---|
[ea7d2b0] | 884 | __spinlock_t * locks [], |
---|
[8fc45b7] | 885 | unsigned int /*out*/ recursions [], |
---|
| 886 | __waitfor_mask_t /*out*/ masks [] |
---|
| 887 | ) { |
---|
[6ff4507] | 888 | lock_all( locks, count ); |
---|
[513daec] | 889 | for( __lock_size_t i = 0; i < count; i++ ) { |
---|
[0c78741] | 890 | ctx[i]->recursion = recursions[i]; |
---|
[6ff4507] | 891 | ctx[i]->mask = masks[i]; |
---|
[0c78741] | 892 | } |
---|
[6ff4507] | 893 | unlock_all( locks, count ); |
---|
[0c78741] | 894 | } |
---|
| 895 | |
---|
| 896 | // Function has 2 different behavior |
---|
| 897 | // 1 - Marks a monitors as being ready to run |
---|
| 898 | // 2 - Checks if all the monitors are ready to run |
---|
| 899 | // if so return the thread to run |
---|
[ac2b598] | 900 | static inline $thread * check_condition( __condition_criterion_t * target ) { |
---|
[0c78741] | 901 | __condition_node_t * node = target->owner; |
---|
| 902 | unsigned short count = node->count; |
---|
| 903 | __condition_criterion_t * criteria = node->criteria; |
---|
| 904 | |
---|
| 905 | bool ready2run = true; |
---|
| 906 | |
---|
| 907 | for( int i = 0; i < count; i++ ) { |
---|
[44264c5] | 908 | |
---|
[36982fc] | 909 | // __cfaabi_dbg_print_safe( "Checking %p for %p\n", &criteria[i], target ); |
---|
[0c78741] | 910 | if( &criteria[i] == target ) { |
---|
| 911 | criteria[i].ready = true; |
---|
[36982fc] | 912 | // __cfaabi_dbg_print_safe( "True\n" ); |
---|
[0c78741] | 913 | } |
---|
| 914 | |
---|
| 915 | ready2run = criteria[i].ready && ready2run; |
---|
| 916 | } |
---|
| 917 | |
---|
[504a7dc] | 918 | __cfaabi_dbg_print_safe( "Kernel : Runing %i (%p)\n", ready2run, ready2run ? (thread*)node->waiting_thread : (thread*)0p ); |
---|
[121be3e] | 919 | return ready2run ? node->waiting_thread : 0p; |
---|
[0c78741] | 920 | } |
---|
| 921 | |
---|
[4cedd9f] | 922 | static inline void brand_condition( condition & this ) { |
---|
[ac2b598] | 923 | $thread * thrd = TL_GET( this_thread ); |
---|
[4cedd9f] | 924 | if( !this.monitors ) { |
---|
[169d944] | 925 | // __cfaabi_dbg_print_safe( "Branding\n" ); |
---|
[121be3e] | 926 | assertf( thrd->monitors.data != 0p, "No current monitor to brand condition %p", thrd->monitors.data ); |
---|
[4cedd9f] | 927 | this.monitor_count = thrd->monitors.size; |
---|
[a933dcf4] | 928 | |
---|
[ac2b598] | 929 | this.monitors = ($monitor **)malloc( this.monitor_count * sizeof( *this.monitors ) ); |
---|
[4cedd9f] | 930 | for( int i = 0; i < this.monitor_count; i++ ) { |
---|
[0cf5b79] | 931 | this.monitors[i] = thrd->monitors[i]; |
---|
[a933dcf4] | 932 | } |
---|
[0c78741] | 933 | } |
---|
| 934 | } |
---|
| 935 | |
---|
[ac2b598] | 936 | static inline [$thread *, int] search_entry_queue( const __waitfor_mask_t & mask, $monitor * monitors [], __lock_size_t count ) { |
---|
[90c4df0] | 937 | |
---|
[ac2b598] | 938 | __queue_t($thread) & entry_queue = monitors[0]->entry_queue; |
---|
[90c4df0] | 939 | |
---|
| 940 | // For each thread in the entry-queue |
---|
[ac2b598] | 941 | for( $thread ** thrd_it = &entry_queue.head; |
---|
[1b143de] | 942 | (*thrd_it) != 1p; |
---|
[b798713] | 943 | thrd_it = &(*thrd_it)->link.next |
---|
[4cc9b13] | 944 | ) { |
---|
[90c4df0] | 945 | // For each acceptable check if it matches |
---|
[4cc9b13] | 946 | int i = 0; |
---|
[0cf5b79] | 947 | __acceptable_t * end = end (mask); |
---|
| 948 | __acceptable_t * begin = begin(mask); |
---|
| 949 | for( __acceptable_t * it = begin; it != end; it++, i++ ) { |
---|
[90c4df0] | 950 | // Check if we have a match |
---|
[aaa4f93] | 951 | if( *it == (*thrd_it)->monitors ) { |
---|
[90c4df0] | 952 | |
---|
| 953 | // If we have a match return it |
---|
| 954 | // after removeing it from the entry queue |
---|
[b18830e] | 955 | return [remove( entry_queue, thrd_it ), i]; |
---|
[90c4df0] | 956 | } |
---|
| 957 | } |
---|
| 958 | } |
---|
| 959 | |
---|
[b18830e] | 960 | return [0, -1]; |
---|
| 961 | } |
---|
| 962 | |
---|
[6ff4507] | 963 | forall(dtype T | sized( T )) |
---|
[59a0bde] | 964 | static inline __lock_size_t insert_unique( T * array [], __lock_size_t & size, T * val ) { |
---|
[6ff4507] | 965 | if( !val ) return size; |
---|
| 966 | |
---|
[59a0bde] | 967 | for( __lock_size_t i = 0; i <= size; i++) { |
---|
[6ff4507] | 968 | if( array[i] == val ) return size; |
---|
| 969 | } |
---|
| 970 | |
---|
| 971 | array[size] = val; |
---|
| 972 | size = size + 1; |
---|
| 973 | return size; |
---|
| 974 | } |
---|
| 975 | |
---|
[59a0bde] | 976 | static inline __lock_size_t count_max( const __waitfor_mask_t & mask ) { |
---|
| 977 | __lock_size_t max = 0; |
---|
| 978 | for( __lock_size_t i = 0; i < mask.size; i++ ) { |
---|
[0cf5b79] | 979 | __acceptable_t & accepted = mask[i]; |
---|
| 980 | max += accepted.size; |
---|
[b18830e] | 981 | } |
---|
| 982 | return max; |
---|
[97e3296] | 983 | } |
---|
[b18830e] | 984 | |
---|
[ac2b598] | 985 | static inline __lock_size_t aggregate( $monitor * storage [], const __waitfor_mask_t & mask ) { |
---|
[59a0bde] | 986 | __lock_size_t size = 0; |
---|
| 987 | for( __lock_size_t i = 0; i < mask.size; i++ ) { |
---|
[0cf5b79] | 988 | __acceptable_t & accepted = mask[i]; |
---|
| 989 | __libcfa_small_sort( accepted.data, accepted.size ); |
---|
| 990 | for( __lock_size_t j = 0; j < accepted.size; j++) { |
---|
| 991 | insert_unique( storage, size, accepted[j] ); |
---|
[6ff4507] | 992 | } |
---|
[b18830e] | 993 | } |
---|
[6a5be52] | 994 | // TODO insertion sort instead of this |
---|
[de737c8] | 995 | __libcfa_small_sort( storage, size ); |
---|
[6ff4507] | 996 | return size; |
---|
[b18830e] | 997 | } |
---|
| 998 | |
---|
[6b0b624] | 999 | // Local Variables: // |
---|
| 1000 | // mode: c // |
---|
| 1001 | // tab-width: 4 // |
---|
| 1002 | // End: // |
---|