[f07e037] | 1 | // |
---|
| 2 | // Cforall Version 1.0.0 Copyright (C) 2016 University of Waterloo |
---|
| 3 | // |
---|
| 4 | // The contents of this file are covered under the licence agreement in the |
---|
| 5 | // file "LICENCE" distributed with Cforall. |
---|
| 6 | // |
---|
[ac2b598] | 7 | // $monitor.c -- |
---|
[f07e037] | 8 | // |
---|
| 9 | // Author : Thierry Delisle |
---|
| 10 | // Created On : Thd Feb 23 12:27:26 2017 |
---|
[6b0b624] | 11 | // Last Modified By : Peter A. Buhr |
---|
[121be3e] | 12 | // Last Modified On : Wed Dec 4 07:55:14 2019 |
---|
| 13 | // Update Count : 10 |
---|
[f07e037] | 14 | // |
---|
| 15 | |
---|
[2026bb6] | 16 | #define __cforall_thread__ |
---|
| 17 | |
---|
[58b6d1b] | 18 | #include "monitor.hfa" |
---|
[f07e037] | 19 | |
---|
[73abe95] | 20 | #include <stdlib.hfa> |
---|
[2f6a7e93] | 21 | #include <inttypes.h> |
---|
[a933dcf4] | 22 | |
---|
[73abe95] | 23 | #include "kernel_private.hfa" |
---|
[f07e037] | 24 | |
---|
[58b6d1b] | 25 | #include "bits/algorithm.hfa" |
---|
[de737c8] | 26 | |
---|
[0c78741] | 27 | //----------------------------------------------------------------------------- |
---|
| 28 | // Forward declarations |
---|
[ac2b598] | 29 | static inline void __set_owner ( $monitor * this, $thread * owner ); |
---|
| 30 | static inline void __set_owner ( $monitor * storage [], __lock_size_t count, $thread * owner ); |
---|
| 31 | static inline void set_mask ( $monitor * storage [], __lock_size_t count, const __waitfor_mask_t & mask ); |
---|
| 32 | static inline void reset_mask( $monitor * this ); |
---|
[6ff4507] | 33 | |
---|
[ac2b598] | 34 | static inline $thread * next_thread( $monitor * this ); |
---|
| 35 | static inline bool is_accepted( $monitor * this, const __monitor_group_t & monitors ); |
---|
[0c78741] | 36 | |
---|
[ea7d2b0] | 37 | static inline void lock_all ( __spinlock_t * locks [], __lock_size_t count ); |
---|
[ac2b598] | 38 | static inline void lock_all ( $monitor * source [], __spinlock_t * /*out*/ locks [], __lock_size_t count ); |
---|
[ea7d2b0] | 39 | static inline void unlock_all( __spinlock_t * locks [], __lock_size_t count ); |
---|
[ac2b598] | 40 | static inline void unlock_all( $monitor * locks [], __lock_size_t count ); |
---|
[0c78741] | 41 | |
---|
[ac2b598] | 42 | static inline void save ( $monitor * ctx [], __lock_size_t count, __spinlock_t * locks [], unsigned int /*out*/ recursions [], __waitfor_mask_t /*out*/ masks [] ); |
---|
| 43 | static inline void restore( $monitor * ctx [], __lock_size_t count, __spinlock_t * locks [], unsigned int /*in */ recursions [], __waitfor_mask_t /*in */ masks [] ); |
---|
[0c78741] | 44 | |
---|
[ac2b598] | 45 | static inline void init ( __lock_size_t count, $monitor * monitors [], __condition_node_t & waiter, __condition_criterion_t criteria [] ); |
---|
| 46 | static inline void init_push( __lock_size_t count, $monitor * monitors [], __condition_node_t & waiter, __condition_criterion_t criteria [] ); |
---|
[97e3296] | 47 | |
---|
[ac2b598] | 48 | static inline $thread * check_condition ( __condition_criterion_t * ); |
---|
[4cedd9f] | 49 | static inline void brand_condition ( condition & ); |
---|
[ac2b598] | 50 | static inline [$thread *, int] search_entry_queue( const __waitfor_mask_t &, $monitor * monitors [], __lock_size_t count ); |
---|
[b18830e] | 51 | |
---|
[6ff4507] | 52 | forall(dtype T | sized( T )) |
---|
[59a0bde] | 53 | static inline __lock_size_t insert_unique( T * array [], __lock_size_t & size, T * val ); |
---|
| 54 | static inline __lock_size_t count_max ( const __waitfor_mask_t & mask ); |
---|
[ac2b598] | 55 | static inline __lock_size_t aggregate ( $monitor * storage [], const __waitfor_mask_t & mask ); |
---|
[97e3296] | 56 | |
---|
| 57 | //----------------------------------------------------------------------------- |
---|
| 58 | // Useful defines |
---|
[6ff4507] | 59 | #define wait_ctx(thrd, user_info) /* Create the necessary information to use the signaller stack */ \ |
---|
| 60 | __condition_node_t waiter = { thrd, count, user_info }; /* Create the node specific to this wait operation */ \ |
---|
| 61 | __condition_criterion_t criteria[count]; /* Create the creteria this wait operation needs to wake up */ \ |
---|
[8fc45b7] | 62 | init( count, monitors, waiter, criteria ); /* Link everything together */ \ |
---|
[6ff4507] | 63 | |
---|
| 64 | #define wait_ctx_primed(thrd, user_info) /* Create the necessary information to use the signaller stack */ \ |
---|
| 65 | __condition_node_t waiter = { thrd, count, user_info }; /* Create the node specific to this wait operation */ \ |
---|
| 66 | __condition_criterion_t criteria[count]; /* Create the creteria this wait operation needs to wake up */ \ |
---|
[8fc45b7] | 67 | init_push( count, monitors, waiter, criteria ); /* Link everything together and push it to the AS-Stack */ \ |
---|
[6ff4507] | 68 | |
---|
| 69 | #define monitor_ctx( mons, cnt ) /* Define that create the necessary struct for internal/external scheduling operations */ \ |
---|
[ac2b598] | 70 | $monitor ** monitors = mons; /* Save the targeted monitors */ \ |
---|
[59a0bde] | 71 | __lock_size_t count = cnt; /* Save the count to a local variable */ \ |
---|
[6ff4507] | 72 | unsigned int recursions[ count ]; /* Save the current recursion levels to restore them later */ \ |
---|
[4cedd9f] | 73 | __waitfor_mask_t masks [ count ]; /* Save the current waitfor masks to restore them later */ \ |
---|
[ea7d2b0] | 74 | __spinlock_t * locks [ count ]; /* We need to pass-in an array of locks to BlockInternal */ \ |
---|
[6ff4507] | 75 | |
---|
| 76 | #define monitor_save save ( monitors, count, locks, recursions, masks ) |
---|
| 77 | #define monitor_restore restore( monitors, count, locks, recursions, masks ) |
---|
| 78 | |
---|
[97e3296] | 79 | |
---|
[0c78741] | 80 | //----------------------------------------------------------------------------- |
---|
| 81 | // Enter/Leave routines |
---|
[c7a900a] | 82 | // Enter single monitor |
---|
[ac2b598] | 83 | static void __enter( $monitor * this, const __monitor_group_t & group ) { |
---|
[8fc652e0] | 84 | $thread * thrd = active_thread(); |
---|
| 85 | |
---|
[c7a900a] | 86 | // Lock the monitor spinlock |
---|
| 87 | lock( this->lock __cfaabi_dbg_ctx2 ); |
---|
[690f13c] | 88 | |
---|
[c7a900a] | 89 | __cfaabi_dbg_print_safe( "Kernel : %10p Entering mon %p (%p)\n", thrd, this, this->owner); |
---|
[690f13c] | 90 | |
---|
[77a2994] | 91 | if( unlikely(0 != (0x1 & (uintptr_t)this->owner)) ) { |
---|
| 92 | abort( "Attempt by thread \"%.256s\" (%p) to access joined monitor %p.", thrd->self_cor.name, thrd, this ); |
---|
| 93 | } |
---|
| 94 | else if( !this->owner ) { |
---|
[c7a900a] | 95 | // No one has the monitor, just take it |
---|
| 96 | __set_owner( this, thrd ); |
---|
[f07e037] | 97 | |
---|
[c7a900a] | 98 | __cfaabi_dbg_print_safe( "Kernel : mon is free \n" ); |
---|
| 99 | } |
---|
| 100 | else if( this->owner == thrd) { |
---|
| 101 | // We already have the monitor, just note how many times we took it |
---|
| 102 | this->recursion += 1; |
---|
[90c4df0] | 103 | |
---|
[c7a900a] | 104 | __cfaabi_dbg_print_safe( "Kernel : mon already owned \n" ); |
---|
| 105 | } |
---|
| 106 | else if( is_accepted( this, group) ) { |
---|
| 107 | // Some one was waiting for us, enter |
---|
| 108 | __set_owner( this, thrd ); |
---|
[90c4df0] | 109 | |
---|
[c7a900a] | 110 | // Reset mask |
---|
| 111 | reset_mask( this ); |
---|
[90c4df0] | 112 | |
---|
[c7a900a] | 113 | __cfaabi_dbg_print_safe( "Kernel : mon accepts \n" ); |
---|
| 114 | } |
---|
| 115 | else { |
---|
| 116 | __cfaabi_dbg_print_safe( "Kernel : blocking \n" ); |
---|
[90c4df0] | 117 | |
---|
[c7a900a] | 118 | // Some one else has the monitor, wait in line for it |
---|
[504a7dc] | 119 | /* paranoid */ verify( thrd->link.next == 0p ); |
---|
[c7a900a] | 120 | append( this->entry_queue, thrd ); |
---|
[504a7dc] | 121 | /* paranoid */ verify( thrd->link.next == 1p ); |
---|
[daacf82] | 122 | |
---|
[c7a900a] | 123 | unlock( this->lock ); |
---|
[e235429] | 124 | park(); |
---|
[90c4df0] | 125 | |
---|
[169d944] | 126 | __cfaabi_dbg_print_safe( "Kernel : %10p Entered mon %p\n", thrd, this); |
---|
[2e9aed4] | 127 | |
---|
[8fc652e0] | 128 | /* paranoid */ verifyf( active_thread() == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", active_thread(), this->owner, this->recursion, this ); |
---|
[5ea06d6] | 129 | return; |
---|
[cb0e6de] | 130 | } |
---|
[cc7f4b1] | 131 | |
---|
[c7a900a] | 132 | __cfaabi_dbg_print_safe( "Kernel : %10p Entered mon %p\n", thrd, this); |
---|
[90c4df0] | 133 | |
---|
[8fc652e0] | 134 | /* paranoid */ verifyf( active_thread() == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", active_thread(), this->owner, this->recursion, this ); |
---|
[c7a900a] | 135 | /* paranoid */ verify( this->lock.lock ); |
---|
[f07e037] | 136 | |
---|
[c7a900a] | 137 | // Release the lock and leave |
---|
| 138 | unlock( this->lock ); |
---|
| 139 | return; |
---|
| 140 | } |
---|
[90c4df0] | 141 | |
---|
[77a2994] | 142 | static void __dtor_enter( $monitor * this, fptr_t func, bool join ) { |
---|
[8fc652e0] | 143 | $thread * thrd = active_thread(); |
---|
| 144 | |
---|
[c7a900a] | 145 | // Lock the monitor spinlock |
---|
| 146 | lock( this->lock __cfaabi_dbg_ctx2 ); |
---|
[f07e037] | 147 | |
---|
[c7a900a] | 148 | __cfaabi_dbg_print_safe( "Kernel : %10p Entering dtor for mon %p (%p)\n", thrd, this, this->owner); |
---|
[549c006] | 149 | |
---|
| 150 | |
---|
[c7a900a] | 151 | if( !this->owner ) { |
---|
| 152 | __cfaabi_dbg_print_safe( "Kernel : Destroying free mon %p\n", this); |
---|
[549c006] | 153 | |
---|
[c7a900a] | 154 | // No one has the monitor, just take it |
---|
| 155 | __set_owner( this, thrd ); |
---|
[549c006] | 156 | |
---|
[8fc652e0] | 157 | verifyf( active_thread() == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", active_thread(), this->owner, this->recursion, this ); |
---|
[549c006] | 158 | |
---|
[c7a900a] | 159 | unlock( this->lock ); |
---|
| 160 | return; |
---|
| 161 | } |
---|
[77a2994] | 162 | else if( this->owner == thrd && !join) { |
---|
[c7a900a] | 163 | // We already have the monitor... but where about to destroy it so the nesting will fail |
---|
| 164 | // Abort! |
---|
| 165 | abort( "Attempt to destroy monitor %p by thread \"%.256s\" (%p) in nested mutex.", this, thrd->self_cor.name, thrd ); |
---|
| 166 | } |
---|
[77a2994] | 167 | // SKULLDUGGERY: join will act as a dtor so it would normally trigger to above check |
---|
| 168 | // to avoid that it sets the owner to the special value thrd | 1p before exiting |
---|
| 169 | else if( this->owner == ($thread*)(1 | (uintptr_t)thrd) ) { |
---|
| 170 | // restore the owner and just return |
---|
| 171 | __cfaabi_dbg_print_safe( "Kernel : Destroying free mon %p\n", this); |
---|
| 172 | |
---|
| 173 | // No one has the monitor, just take it |
---|
| 174 | this->owner = thrd; |
---|
| 175 | |
---|
[8fc652e0] | 176 | verifyf( active_thread() == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", active_thread(), this->owner, this->recursion, this ); |
---|
[77a2994] | 177 | |
---|
| 178 | unlock( this->lock ); |
---|
| 179 | return; |
---|
| 180 | } |
---|
[549c006] | 181 | |
---|
[c7a900a] | 182 | __lock_size_t count = 1; |
---|
[ac2b598] | 183 | $monitor ** monitors = &this; |
---|
[c7a900a] | 184 | __monitor_group_t group = { &this, 1, func }; |
---|
| 185 | if( is_accepted( this, group) ) { |
---|
| 186 | __cfaabi_dbg_print_safe( "Kernel : mon accepts dtor, block and signal it \n" ); |
---|
[549c006] | 187 | |
---|
[c7a900a] | 188 | // Wake the thread that is waiting for this |
---|
| 189 | __condition_criterion_t * urgent = pop( this->signal_stack ); |
---|
| 190 | /* paranoid */ verify( urgent ); |
---|
[b8116cd] | 191 | |
---|
[c7a900a] | 192 | // Reset mask |
---|
| 193 | reset_mask( this ); |
---|
[549c006] | 194 | |
---|
[c7a900a] | 195 | // Create the node specific to this wait operation |
---|
| 196 | wait_ctx_primed( thrd, 0 ) |
---|
[549c006] | 197 | |
---|
[c7a900a] | 198 | // Some one else has the monitor, wait for him to finish and then run |
---|
| 199 | unlock( this->lock ); |
---|
[549c006] | 200 | |
---|
[c7a900a] | 201 | // Release the next thread |
---|
[8fc652e0] | 202 | /* paranoid */ verifyf( urgent->owner->waiting_thread == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", active_thread(), this->owner, this->recursion, this ); |
---|
[e235429] | 203 | unpark( urgent->owner->waiting_thread ); |
---|
[549c006] | 204 | |
---|
[c7a900a] | 205 | // Park current thread waiting |
---|
[e235429] | 206 | park(); |
---|
[549c006] | 207 | |
---|
[c7a900a] | 208 | // Some one was waiting for us, enter |
---|
[8fc652e0] | 209 | /* paranoid */ verifyf( active_thread() == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", active_thread(), this->owner, this->recursion, this ); |
---|
[c7a900a] | 210 | } |
---|
| 211 | else { |
---|
| 212 | __cfaabi_dbg_print_safe( "Kernel : blocking \n" ); |
---|
[549c006] | 213 | |
---|
[c7a900a] | 214 | wait_ctx( thrd, 0 ) |
---|
| 215 | this->dtor_node = &waiter; |
---|
[549c006] | 216 | |
---|
[c7a900a] | 217 | // Some one else has the monitor, wait in line for it |
---|
[504a7dc] | 218 | /* paranoid */ verify( thrd->link.next == 0p ); |
---|
[c7a900a] | 219 | append( this->entry_queue, thrd ); |
---|
[504a7dc] | 220 | /* paranoid */ verify( thrd->link.next == 1p ); |
---|
[c7a900a] | 221 | unlock( this->lock ); |
---|
[549c006] | 222 | |
---|
[c7a900a] | 223 | // Park current thread waiting |
---|
[e235429] | 224 | park(); |
---|
[549c006] | 225 | |
---|
[8fc652e0] | 226 | /* paranoid */ verifyf( active_thread() == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", active_thread(), this->owner, this->recursion, this ); |
---|
[c7a900a] | 227 | return; |
---|
[549c006] | 228 | } |
---|
| 229 | |
---|
[c7a900a] | 230 | __cfaabi_dbg_print_safe( "Kernel : Destroying %p\n", this); |
---|
[f07e037] | 231 | |
---|
[c7a900a] | 232 | } |
---|
[a843067] | 233 | |
---|
[c7a900a] | 234 | // Leave single monitor |
---|
[ac2b598] | 235 | void __leave( $monitor * this ) { |
---|
[c7a900a] | 236 | // Lock the monitor spinlock |
---|
| 237 | lock( this->lock __cfaabi_dbg_ctx2 ); |
---|
[cc7f4b1] | 238 | |
---|
[8fc652e0] | 239 | __cfaabi_dbg_print_safe( "Kernel : %10p Leaving mon %p (%p)\n", active_thread(), this, this->owner); |
---|
[f07e037] | 240 | |
---|
[8fc652e0] | 241 | /* paranoid */ verifyf( active_thread() == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", active_thread(), this->owner, this->recursion, this ); |
---|
[f07e037] | 242 | |
---|
[c7a900a] | 243 | // Leaving a recursion level, decrement the counter |
---|
| 244 | this->recursion -= 1; |
---|
[5ea06d6] | 245 | |
---|
[c7a900a] | 246 | // If we haven't left the last level of recursion |
---|
| 247 | // it means we don't need to do anything |
---|
| 248 | if( this->recursion != 0) { |
---|
| 249 | __cfaabi_dbg_print_safe( "Kernel : recursion still %d\n", this->recursion); |
---|
[ea7d2b0] | 250 | unlock( this->lock ); |
---|
[c7a900a] | 251 | return; |
---|
[1c273d0] | 252 | } |
---|
| 253 | |
---|
[c7a900a] | 254 | // Get the next thread, will be null on low contention monitor |
---|
[ac2b598] | 255 | $thread * new_owner = next_thread( this ); |
---|
[c7a900a] | 256 | |
---|
| 257 | // Check the new owner is consistent with who we wake-up |
---|
| 258 | // new_owner might be null even if someone owns the monitor when the owner is still waiting for another monitor |
---|
| 259 | /* paranoid */ verifyf( !new_owner || new_owner == this->owner, "Expected owner to be %p, got %p (m: %p)", new_owner, this->owner, this ); |
---|
| 260 | |
---|
| 261 | // We can now let other threads in safely |
---|
| 262 | unlock( this->lock ); |
---|
[549c006] | 263 | |
---|
[c7a900a] | 264 | //We need to wake-up the thread |
---|
| 265 | /* paranoid */ verifyf( !new_owner || new_owner == this->owner, "Expected owner to be %p, got %p (m: %p)", new_owner, this->owner, this ); |
---|
[e235429] | 266 | unpark( new_owner ); |
---|
[c7a900a] | 267 | } |
---|
| 268 | |
---|
| 269 | // Leave single monitor for the last time |
---|
[77a2994] | 270 | void __dtor_leave( $monitor * this, bool join ) { |
---|
[c7a900a] | 271 | __cfaabi_dbg_debug_do( |
---|
[8fc652e0] | 272 | if( active_thread() != this->owner ) { |
---|
| 273 | abort( "Destroyed monitor %p has inconsistent owner, expected %p got %p.\n", this, active_thread(), this->owner); |
---|
[c7a900a] | 274 | } |
---|
[77a2994] | 275 | if( this->recursion != 1 && !join ) { |
---|
[c7a900a] | 276 | abort( "Destroyed monitor %p has %d outstanding nested calls.\n", this, this->recursion - 1); |
---|
| 277 | } |
---|
| 278 | ) |
---|
[77a2994] | 279 | |
---|
| 280 | this->owner = ($thread*)(1 | (uintptr_t)this->owner); |
---|
[c7a900a] | 281 | } |
---|
[549c006] | 282 | |
---|
[5afb49a] | 283 | void __thread_finish( $thread * thrd ) { |
---|
| 284 | $monitor * this = &thrd->self_mon; |
---|
[97e3296] | 285 | |
---|
[5afb49a] | 286 | // Lock the monitor now |
---|
| 287 | /* paranoid */ verify( this->lock.lock ); |
---|
| 288 | /* paranoid */ verifyf( thrd == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", thrd, this->owner, this->recursion, this ); |
---|
[8fc652e0] | 289 | /* paranoid */ verify( ! __preemption_enabled() ); |
---|
[5afb49a] | 290 | /* paranoid */ verify( thrd->state == Halted ); |
---|
| 291 | /* paranoid */ verify( this->recursion == 1 ); |
---|
[3381ed7] | 292 | |
---|
[5afb49a] | 293 | // Leaving a recursion level, decrement the counter |
---|
| 294 | this->recursion -= 1; |
---|
| 295 | this->owner = 0p; |
---|
[b0c7419] | 296 | |
---|
[5afb49a] | 297 | // Fetch the next thread, can be null |
---|
| 298 | $thread * new_owner = next_thread( this ); |
---|
[1c273d0] | 299 | |
---|
[5afb49a] | 300 | // Release the monitor lock |
---|
| 301 | unlock( this->lock ); |
---|
[97e3296] | 302 | |
---|
[5afb49a] | 303 | // Unpark the next owner if needed |
---|
| 304 | /* paranoid */ verifyf( !new_owner || new_owner == this->owner, "Expected owner to be %p, got %p (m: %p)", new_owner, this->owner, this ); |
---|
[8fc652e0] | 305 | /* paranoid */ verify( ! __preemption_enabled() ); |
---|
[5afb49a] | 306 | /* paranoid */ verify( thrd->state == Halted ); |
---|
| 307 | unpark( new_owner ); |
---|
[2781e65] | 308 | } |
---|
| 309 | |
---|
[97e3296] | 310 | // Enter multiple monitor |
---|
| 311 | // relies on the monitor array being sorted |
---|
[6ae8c92] | 312 | static inline void enter( __monitor_group_t monitors ) { |
---|
[59a0bde] | 313 | for( __lock_size_t i = 0; i < monitors.size; i++) { |
---|
[c7a900a] | 314 | __enter( monitors[i], monitors ); |
---|
[97e3296] | 315 | } |
---|
[2781e65] | 316 | } |
---|
| 317 | |
---|
[97e3296] | 318 | // Leave multiple monitor |
---|
| 319 | // relies on the monitor array being sorted |
---|
[ac2b598] | 320 | static inline void leave($monitor * monitors [], __lock_size_t count) { |
---|
[59a0bde] | 321 | for( __lock_size_t i = count - 1; i >= 0; i--) { |
---|
[c7a900a] | 322 | __leave( monitors[i] ); |
---|
[2781e65] | 323 | } |
---|
[5ea06d6] | 324 | } |
---|
| 325 | |
---|
[97e3296] | 326 | // Ctor for monitor guard |
---|
| 327 | // Sorts monitors before entering |
---|
[ac2b598] | 328 | void ?{}( monitor_guard_t & this, $monitor * m [], __lock_size_t count, fptr_t func ) { |
---|
[8fc652e0] | 329 | $thread * thrd = active_thread(); |
---|
[14a61b5] | 330 | |
---|
[97e3296] | 331 | // Store current array |
---|
[242a902] | 332 | this.m = m; |
---|
| 333 | this.count = count; |
---|
[97e3296] | 334 | |
---|
[09800e9] | 335 | // Sort monitors based on address |
---|
[de737c8] | 336 | __libcfa_small_sort(this.m, count); |
---|
[5ea06d6] | 337 | |
---|
[97e3296] | 338 | // Save previous thread context |
---|
[14a61b5] | 339 | this.prev = thrd->monitors; |
---|
[5ea06d6] | 340 | |
---|
[97e3296] | 341 | // Update thread context (needed for conditions) |
---|
[14a61b5] | 342 | (thrd->monitors){m, count, func}; |
---|
[90c4df0] | 343 | |
---|
[169d944] | 344 | // __cfaabi_dbg_print_safe( "MGUARD : enter %d\n", count); |
---|
[a843067] | 345 | |
---|
[90c4df0] | 346 | // Enter the monitors in order |
---|
[6ae8c92] | 347 | __monitor_group_t group = {this.m, this.count, func}; |
---|
[b18830e] | 348 | enter( group ); |
---|
[a843067] | 349 | |
---|
[169d944] | 350 | // __cfaabi_dbg_print_safe( "MGUARD : entered\n" ); |
---|
[5ea06d6] | 351 | } |
---|
| 352 | |
---|
[6b224a52] | 353 | |
---|
[97e3296] | 354 | // Dtor for monitor guard |
---|
[242a902] | 355 | void ^?{}( monitor_guard_t & this ) { |
---|
[169d944] | 356 | // __cfaabi_dbg_print_safe( "MGUARD : leaving %d\n", this.count); |
---|
[a843067] | 357 | |
---|
[97e3296] | 358 | // Leave the monitors in order |
---|
[242a902] | 359 | leave( this.m, this.count ); |
---|
[5ea06d6] | 360 | |
---|
[169d944] | 361 | // __cfaabi_dbg_print_safe( "MGUARD : left\n" ); |
---|
[a843067] | 362 | |
---|
[97e3296] | 363 | // Restore thread context |
---|
[8fc652e0] | 364 | active_thread()->monitors = this.prev; |
---|
[5ea06d6] | 365 | } |
---|
| 366 | |
---|
[549c006] | 367 | // Ctor for monitor guard |
---|
| 368 | // Sorts monitors before entering |
---|
[77a2994] | 369 | void ?{}( monitor_dtor_guard_t & this, $monitor * m [], fptr_t func, bool join ) { |
---|
[14a61b5] | 370 | // optimization |
---|
[8fc652e0] | 371 | $thread * thrd = active_thread(); |
---|
[14a61b5] | 372 | |
---|
[549c006] | 373 | // Store current array |
---|
| 374 | this.m = *m; |
---|
| 375 | |
---|
| 376 | // Save previous thread context |
---|
[14a61b5] | 377 | this.prev = thrd->monitors; |
---|
[549c006] | 378 | |
---|
[77a2994] | 379 | // Save whether we are in a join or not |
---|
| 380 | this.join = join; |
---|
| 381 | |
---|
[549c006] | 382 | // Update thread context (needed for conditions) |
---|
[14a61b5] | 383 | (thrd->monitors){m, 1, func}; |
---|
[549c006] | 384 | |
---|
[77a2994] | 385 | __dtor_enter( this.m, func, join ); |
---|
[549c006] | 386 | } |
---|
| 387 | |
---|
| 388 | // Dtor for monitor guard |
---|
| 389 | void ^?{}( monitor_dtor_guard_t & this ) { |
---|
| 390 | // Leave the monitors in order |
---|
[77a2994] | 391 | __dtor_leave( this.m, this.join ); |
---|
[549c006] | 392 | |
---|
| 393 | // Restore thread context |
---|
[8fc652e0] | 394 | active_thread()->monitors = this.prev; |
---|
[549c006] | 395 | } |
---|
| 396 | |
---|
[97e3296] | 397 | //----------------------------------------------------------------------------- |
---|
| 398 | // Internal scheduling types |
---|
[ac2b598] | 399 | void ?{}(__condition_node_t & this, $thread * waiting_thread, __lock_size_t count, uintptr_t user_info ) { |
---|
[242a902] | 400 | this.waiting_thread = waiting_thread; |
---|
| 401 | this.count = count; |
---|
[121be3e] | 402 | this.next = 0p; |
---|
[242a902] | 403 | this.user_info = user_info; |
---|
[be3d020] | 404 | } |
---|
| 405 | |
---|
[c40e7c5] | 406 | void ?{}(__condition_criterion_t & this ) with( this ) { |
---|
| 407 | ready = false; |
---|
[121be3e] | 408 | target = 0p; |
---|
| 409 | owner = 0p; |
---|
| 410 | next = 0p; |
---|
[be3d020] | 411 | } |
---|
| 412 | |
---|
[ac2b598] | 413 | void ?{}(__condition_criterion_t & this, $monitor * target, __condition_node_t & owner ) { |
---|
[242a902] | 414 | this.ready = false; |
---|
| 415 | this.target = target; |
---|
[8fc45b7] | 416 | this.owner = &owner; |
---|
[121be3e] | 417 | this.next = 0p; |
---|
[ad1a8dd] | 418 | } |
---|
| 419 | |
---|
[5ea06d6] | 420 | //----------------------------------------------------------------------------- |
---|
| 421 | // Internal scheduling |
---|
[4cedd9f] | 422 | void wait( condition & this, uintptr_t user_info = 0 ) { |
---|
[0c78741] | 423 | brand_condition( this ); |
---|
[5ea06d6] | 424 | |
---|
[97e3296] | 425 | // Check that everything is as expected |
---|
[121be3e] | 426 | assertf( this.monitors != 0p, "Waiting with no monitors (%p)", this.monitors ); |
---|
[2f6a7e93] | 427 | verifyf( this.monitor_count != 0, "Waiting with 0 monitors (%"PRIiFAST16")", this.monitor_count ); |
---|
| 428 | verifyf( this.monitor_count < 32u, "Excessive monitor count (%"PRIiFAST16")", this.monitor_count ); |
---|
[5ea06d6] | 429 | |
---|
[97e3296] | 430 | // Create storage for monitor context |
---|
[4cedd9f] | 431 | monitor_ctx( this.monitors, this.monitor_count ); |
---|
[0c78741] | 432 | |
---|
[97e3296] | 433 | // Create the node specific to this wait operation |
---|
[8fc652e0] | 434 | wait_ctx( active_thread(), user_info ); |
---|
[0c78741] | 435 | |
---|
[97e3296] | 436 | // Append the current wait operation to the ones already queued on the condition |
---|
| 437 | // We don't need locks for that since conditions must always be waited on inside monitor mutual exclusion |
---|
[3381ed7] | 438 | /* paranoid */ verify( waiter.next == 0p ); |
---|
[8fc45b7] | 439 | append( this.blocked, &waiter ); |
---|
[3381ed7] | 440 | /* paranoid */ verify( waiter.next == 1p ); |
---|
[0c78741] | 441 | |
---|
[6ff4507] | 442 | // Lock all monitors (aggregates the locks as well) |
---|
[97e3296] | 443 | lock_all( monitors, locks, count ); |
---|
[5ea06d6] | 444 | |
---|
[97e3296] | 445 | // Find the next thread(s) to run |
---|
[59a0bde] | 446 | __lock_size_t thread_count = 0; |
---|
[ac2b598] | 447 | $thread * threads[ count ]; |
---|
[66298de] | 448 | __builtin_memset( threads, 0, sizeof( threads ) ); |
---|
[ad1a8dd] | 449 | |
---|
[a843067] | 450 | // Save monitor states |
---|
| 451 | monitor_save; |
---|
| 452 | |
---|
[97e3296] | 453 | // Remove any duplicate threads |
---|
[59a0bde] | 454 | for( __lock_size_t i = 0; i < count; i++) { |
---|
[ac2b598] | 455 | $thread * new_owner = next_thread( monitors[i] ); |
---|
[6ff4507] | 456 | insert_unique( threads, thread_count, new_owner ); |
---|
[5ea06d6] | 457 | } |
---|
| 458 | |
---|
[3381ed7] | 459 | // Unlock the locks, we don't need them anymore |
---|
| 460 | for(int i = 0; i < count; i++) { |
---|
| 461 | unlock( *locks[i] ); |
---|
| 462 | } |
---|
| 463 | |
---|
| 464 | // Wake the threads |
---|
| 465 | for(int i = 0; i < thread_count; i++) { |
---|
[e235429] | 466 | unpark( threads[i] ); |
---|
[3381ed7] | 467 | } |
---|
| 468 | |
---|
[a843067] | 469 | // Everything is ready to go to sleep |
---|
[e235429] | 470 | park(); |
---|
[a843067] | 471 | |
---|
| 472 | // We are back, restore the owners and recursions |
---|
| 473 | monitor_restore; |
---|
[5ea06d6] | 474 | } |
---|
| 475 | |
---|
[4cedd9f] | 476 | bool signal( condition & this ) { |
---|
[97e3296] | 477 | if( is_empty( this ) ) { return false; } |
---|
[5ea06d6] | 478 | |
---|
| 479 | //Check that everything is as expected |
---|
[4cedd9f] | 480 | verify( this.monitors ); |
---|
| 481 | verify( this.monitor_count != 0 ); |
---|
[0c78741] | 482 | |
---|
[44264c5] | 483 | //Some more checking in debug |
---|
[36982fc] | 484 | __cfaabi_dbg_debug_do( |
---|
[8fc652e0] | 485 | $thread * this_thrd = active_thread(); |
---|
[4cedd9f] | 486 | if ( this.monitor_count != this_thrd->monitors.size ) { |
---|
[c2ca04d] | 487 | abort( "Signal on condition %p made with different number of monitor(s), expected %zi got %zi", &this, this.monitor_count, this_thrd->monitors.size ); |
---|
[97e3296] | 488 | } |
---|
[0c78741] | 489 | |
---|
[4cedd9f] | 490 | for(int i = 0; i < this.monitor_count; i++) { |
---|
[0cf5b79] | 491 | if ( this.monitors[i] != this_thrd->monitors[i] ) { |
---|
[2fdbb3b] | 492 | abort( "Signal on condition %p made with different monitor, expected %p got %p", &this, this.monitors[i], this_thrd->monitors[i] ); |
---|
[97e3296] | 493 | } |
---|
[0c78741] | 494 | } |
---|
[5ea06d6] | 495 | ); |
---|
| 496 | |
---|
[59a0bde] | 497 | __lock_size_t count = this.monitor_count; |
---|
[97e3296] | 498 | |
---|
| 499 | // Lock all monitors |
---|
[121be3e] | 500 | lock_all( this.monitors, 0p, count ); |
---|
[0c78741] | 501 | |
---|
[44264c5] | 502 | //Pop the head of the waiting queue |
---|
[8fc45b7] | 503 | __condition_node_t * node = pop_head( this.blocked ); |
---|
[44264c5] | 504 | |
---|
| 505 | //Add the thread to the proper AS stack |
---|
[0c78741] | 506 | for(int i = 0; i < count; i++) { |
---|
| 507 | __condition_criterion_t * crit = &node->criteria[i]; |
---|
| 508 | assert( !crit->ready ); |
---|
[8fc45b7] | 509 | push( crit->target->signal_stack, crit ); |
---|
[5ea06d6] | 510 | } |
---|
[0c78741] | 511 | |
---|
[44264c5] | 512 | //Release |
---|
[4cedd9f] | 513 | unlock_all( this.monitors, count ); |
---|
[be3d020] | 514 | |
---|
| 515 | return true; |
---|
[5ea06d6] | 516 | } |
---|
| 517 | |
---|
[4cedd9f] | 518 | bool signal_block( condition & this ) { |
---|
| 519 | if( !this.blocked.head ) { return false; } |
---|
[44264c5] | 520 | |
---|
| 521 | //Check that everything is as expected |
---|
[121be3e] | 522 | verifyf( this.monitors != 0p, "Waiting with no monitors (%p)", this.monitors ); |
---|
[2f6a7e93] | 523 | verifyf( this.monitor_count != 0, "Waiting with 0 monitors (%"PRIiFAST16")", this.monitor_count ); |
---|
[44264c5] | 524 | |
---|
[97e3296] | 525 | // Create storage for monitor context |
---|
[4cedd9f] | 526 | monitor_ctx( this.monitors, this.monitor_count ); |
---|
[44264c5] | 527 | |
---|
[97e3296] | 528 | // Lock all monitors (aggregates the locks them as well) |
---|
| 529 | lock_all( monitors, locks, count ); |
---|
[44264c5] | 530 | |
---|
[2e9aed4] | 531 | |
---|
[97e3296] | 532 | // Create the node specific to this wait operation |
---|
[8fc652e0] | 533 | wait_ctx_primed( active_thread(), 0 ) |
---|
[44264c5] | 534 | |
---|
| 535 | //save contexts |
---|
[6ff4507] | 536 | monitor_save; |
---|
[44264c5] | 537 | |
---|
| 538 | //Find the thread to run |
---|
[ac2b598] | 539 | $thread * signallee = pop_head( this.blocked )->waiting_thread; |
---|
[c7a900a] | 540 | __set_owner( monitors, count, signallee ); |
---|
[44264c5] | 541 | |
---|
[36982fc] | 542 | __cfaabi_dbg_print_buffer_decl( "Kernel : signal_block condition %p (s: %p)\n", &this, signallee ); |
---|
[de737c8] | 543 | |
---|
[3381ed7] | 544 | // unlock all the monitors |
---|
| 545 | unlock_all( locks, count ); |
---|
| 546 | |
---|
| 547 | // unpark the thread we signalled |
---|
[e235429] | 548 | unpark( signallee ); |
---|
[3381ed7] | 549 | |
---|
[44264c5] | 550 | //Everything is ready to go to sleep |
---|
[e235429] | 551 | park(); |
---|
[44264c5] | 552 | |
---|
[c81ebf9] | 553 | |
---|
[97e3296] | 554 | // WE WOKE UP |
---|
[c81ebf9] | 555 | |
---|
| 556 | |
---|
[36982fc] | 557 | __cfaabi_dbg_print_buffer_local( "Kernel : signal_block returned\n" ); |
---|
[de737c8] | 558 | |
---|
[6ff4507] | 559 | //We are back, restore the masks and recursions |
---|
| 560 | monitor_restore; |
---|
[be3d020] | 561 | |
---|
| 562 | return true; |
---|
| 563 | } |
---|
| 564 | |
---|
[97e3296] | 565 | // Access the user_info of the thread waiting at the front of the queue |
---|
[4cedd9f] | 566 | uintptr_t front( condition & this ) { |
---|
[2ac095d] | 567 | verifyf( !is_empty(this), |
---|
[4aa2fb2] | 568 | "Attempt to access user data on an empty condition.\n" |
---|
| 569 | "Possible cause is not checking if the condition is empty before reading stored data." |
---|
[be3d020] | 570 | ); |
---|
[0cf5b79] | 571 | return ((typeof(this.blocked.head))this.blocked.head)->user_info; |
---|
[44264c5] | 572 | } |
---|
| 573 | |
---|
[c81ebf9] | 574 | //----------------------------------------------------------------------------- |
---|
[b18830e] | 575 | // External scheduling |
---|
| 576 | // cases to handle : |
---|
| 577 | // - target already there : |
---|
| 578 | // block and wake |
---|
| 579 | // - dtor already there |
---|
| 580 | // put thread on signaller stack |
---|
| 581 | // - non-blocking |
---|
| 582 | // return else |
---|
| 583 | // - timeout |
---|
| 584 | // return timeout |
---|
| 585 | // - block |
---|
| 586 | // setup mask |
---|
| 587 | // block |
---|
[6ae8c92] | 588 | void __waitfor_internal( const __waitfor_mask_t & mask, int duration ) { |
---|
[b18830e] | 589 | // This statment doesn't have a contiguous list of monitors... |
---|
| 590 | // Create one! |
---|
[59a0bde] | 591 | __lock_size_t max = count_max( mask ); |
---|
[ac2b598] | 592 | $monitor * mon_storage[max]; |
---|
[66298de] | 593 | __builtin_memset( mon_storage, 0, sizeof( mon_storage ) ); |
---|
[59a0bde] | 594 | __lock_size_t actual_count = aggregate( mon_storage, mask ); |
---|
[97e3296] | 595 | |
---|
[523232d] | 596 | __cfaabi_dbg_print_buffer_decl( "Kernel : waitfor %"PRIdFAST16" (s: %"PRIdFAST16", m: %"PRIdFAST16")\n", actual_count, mask.size, (__lock_size_t)max); |
---|
[66298de] | 597 | |
---|
[daacf82] | 598 | if(actual_count == 0) return; |
---|
[19c43b7] | 599 | |
---|
[169d944] | 600 | __cfaabi_dbg_print_buffer_local( "Kernel : waitfor internal proceeding\n" ); |
---|
[4cc9b13] | 601 | |
---|
[97e3296] | 602 | // Create storage for monitor context |
---|
[b18830e] | 603 | monitor_ctx( mon_storage, actual_count ); |
---|
[c81ebf9] | 604 | |
---|
[6ff4507] | 605 | // Lock all monitors (aggregates the locks as well) |
---|
[97e3296] | 606 | lock_all( monitors, locks, count ); |
---|
[c81ebf9] | 607 | |
---|
[b18830e] | 608 | { |
---|
| 609 | // Check if the entry queue |
---|
[ac2b598] | 610 | $thread * next; int index; |
---|
[6ae8c92] | 611 | [next, index] = search_entry_queue( mask, monitors, count ); |
---|
[b18830e] | 612 | |
---|
| 613 | if( next ) { |
---|
[19c43b7] | 614 | *mask.accepted = index; |
---|
[0cf5b79] | 615 | __acceptable_t& accepted = mask[index]; |
---|
| 616 | if( accepted.is_dtor ) { |
---|
[169d944] | 617 | __cfaabi_dbg_print_buffer_local( "Kernel : dtor already there\n" ); |
---|
[0cf5b79] | 618 | verifyf( accepted.size == 1, "ERROR: Accepted dtor has more than 1 mutex parameter." ); |
---|
[549c006] | 619 | |
---|
[ac2b598] | 620 | $monitor * mon2dtor = accepted[0]; |
---|
[549c006] | 621 | verifyf( mon2dtor->dtor_node, "ERROR: Accepted monitor has no dtor_node." ); |
---|
| 622 | |
---|
| 623 | __condition_criterion_t * dtor_crit = mon2dtor->dtor_node->criteria; |
---|
[8fc45b7] | 624 | push( mon2dtor->signal_stack, dtor_crit ); |
---|
[549c006] | 625 | |
---|
| 626 | unlock_all( locks, count ); |
---|
[b18830e] | 627 | } |
---|
| 628 | else { |
---|
[169d944] | 629 | __cfaabi_dbg_print_buffer_local( "Kernel : thread present, baton-passing\n" ); |
---|
[19c43b7] | 630 | |
---|
| 631 | // Create the node specific to this wait operation |
---|
[8fc652e0] | 632 | wait_ctx_primed( active_thread(), 0 ); |
---|
[19c43b7] | 633 | |
---|
| 634 | // Save monitor states |
---|
| 635 | monitor_save; |
---|
| 636 | |
---|
[523232d] | 637 | __cfaabi_dbg_print_buffer_local( "Kernel : baton of %"PRIdFAST16" monitors : ", count ); |
---|
[6a5be52] | 638 | #ifdef __CFA_DEBUG_PRINT__ |
---|
| 639 | for( int i = 0; i < count; i++) { |
---|
[36982fc] | 640 | __cfaabi_dbg_print_buffer_local( "%p %p ", monitors[i], monitors[i]->signal_stack.top ); |
---|
[6a5be52] | 641 | } |
---|
| 642 | #endif |
---|
[169d944] | 643 | __cfaabi_dbg_print_buffer_local( "\n" ); |
---|
[6a5be52] | 644 | |
---|
[19c43b7] | 645 | // Set the owners to be the next thread |
---|
[c7a900a] | 646 | __set_owner( monitors, count, next ); |
---|
[19c43b7] | 647 | |
---|
[3381ed7] | 648 | // unlock all the monitors |
---|
| 649 | unlock_all( locks, count ); |
---|
[19c43b7] | 650 | |
---|
[3381ed7] | 651 | // unpark the thread we signalled |
---|
[e235429] | 652 | unpark( next ); |
---|
[3381ed7] | 653 | |
---|
| 654 | //Everything is ready to go to sleep |
---|
[e235429] | 655 | park(); |
---|
[19c43b7] | 656 | |
---|
| 657 | // We are back, restore the owners and recursions |
---|
| 658 | monitor_restore; |
---|
| 659 | |
---|
[169d944] | 660 | __cfaabi_dbg_print_buffer_local( "Kernel : thread present, returned\n" ); |
---|
[b18830e] | 661 | } |
---|
| 662 | |
---|
[36982fc] | 663 | __cfaabi_dbg_print_buffer_local( "Kernel : accepted %d\n", *mask.accepted); |
---|
[19c43b7] | 664 | return; |
---|
[90c4df0] | 665 | } |
---|
| 666 | } |
---|
| 667 | |
---|
[c81ebf9] | 668 | |
---|
[4cc9b13] | 669 | if( duration == 0 ) { |
---|
[169d944] | 670 | __cfaabi_dbg_print_buffer_local( "Kernel : non-blocking, exiting\n" ); |
---|
[19c43b7] | 671 | |
---|
[4cc9b13] | 672 | unlock_all( locks, count ); |
---|
[19c43b7] | 673 | |
---|
[36982fc] | 674 | __cfaabi_dbg_print_buffer_local( "Kernel : accepted %d\n", *mask.accepted); |
---|
[4cc9b13] | 675 | return; |
---|
| 676 | } |
---|
[b18830e] | 677 | |
---|
| 678 | |
---|
[169d944] | 679 | verifyf( duration < 0, "Timeout on waitfor statments not supported yet." ); |
---|
[b18830e] | 680 | |
---|
[169d944] | 681 | __cfaabi_dbg_print_buffer_local( "Kernel : blocking waitfor\n" ); |
---|
[19c43b7] | 682 | |
---|
| 683 | // Create the node specific to this wait operation |
---|
[8fc652e0] | 684 | wait_ctx_primed( active_thread(), 0 ); |
---|
[b18830e] | 685 | |
---|
[6ff4507] | 686 | monitor_save; |
---|
[6ae8c92] | 687 | set_mask( monitors, count, mask ); |
---|
[c81ebf9] | 688 | |
---|
[59a0bde] | 689 | for( __lock_size_t i = 0; i < count; i++) { |
---|
[8fc652e0] | 690 | verify( monitors[i]->owner == active_thread() ); |
---|
[daacf82] | 691 | } |
---|
| 692 | |
---|
[3381ed7] | 693 | // unlock all the monitors |
---|
| 694 | unlock_all( locks, count ); |
---|
| 695 | |
---|
[19c43b7] | 696 | //Everything is ready to go to sleep |
---|
[e235429] | 697 | park(); |
---|
[19c43b7] | 698 | |
---|
| 699 | |
---|
| 700 | // WE WOKE UP |
---|
| 701 | |
---|
| 702 | |
---|
| 703 | //We are back, restore the masks and recursions |
---|
| 704 | monitor_restore; |
---|
| 705 | |
---|
[169d944] | 706 | __cfaabi_dbg_print_buffer_local( "Kernel : exiting\n" ); |
---|
[19c43b7] | 707 | |
---|
[36982fc] | 708 | __cfaabi_dbg_print_buffer_local( "Kernel : accepted %d\n", *mask.accepted); |
---|
[c81ebf9] | 709 | } |
---|
| 710 | |
---|
[0c78741] | 711 | //----------------------------------------------------------------------------- |
---|
| 712 | // Utilities |
---|
| 713 | |
---|
[ac2b598] | 714 | static inline void __set_owner( $monitor * this, $thread * owner ) { |
---|
[3381ed7] | 715 | /* paranoid */ verify( this->lock.lock ); |
---|
[a843067] | 716 | |
---|
[0c78741] | 717 | //Pass the monitor appropriately |
---|
| 718 | this->owner = owner; |
---|
| 719 | |
---|
| 720 | //We are passing the monitor to someone else, which means recursion level is not 0 |
---|
| 721 | this->recursion = owner ? 1 : 0; |
---|
| 722 | } |
---|
| 723 | |
---|
[ac2b598] | 724 | static inline void __set_owner( $monitor * monitors [], __lock_size_t count, $thread * owner ) { |
---|
[3381ed7] | 725 | /* paranoid */ verify ( monitors[0]->lock.lock ); |
---|
[8fc652e0] | 726 | /* paranoid */ verifyf( monitors[0]->owner == active_thread(), "Expected owner to be %p, got %p (r: %i, m: %p)", active_thread(), monitors[0]->owner, monitors[0]->recursion, monitors[0] ); |
---|
[3381ed7] | 727 | monitors[0]->owner = owner; |
---|
| 728 | monitors[0]->recursion = 1; |
---|
[513daec] | 729 | for( __lock_size_t i = 1; i < count; i++ ) { |
---|
[3381ed7] | 730 | /* paranoid */ verify ( monitors[i]->lock.lock ); |
---|
[8fc652e0] | 731 | /* paranoid */ verifyf( monitors[i]->owner == active_thread(), "Expected owner to be %p, got %p (r: %i, m: %p)", active_thread(), monitors[i]->owner, monitors[i]->recursion, monitors[i] ); |
---|
[3381ed7] | 732 | monitors[i]->owner = owner; |
---|
| 733 | monitors[i]->recursion = 0; |
---|
[6ff4507] | 734 | } |
---|
| 735 | } |
---|
| 736 | |
---|
[ac2b598] | 737 | static inline void set_mask( $monitor * storage [], __lock_size_t count, const __waitfor_mask_t & mask ) { |
---|
[513daec] | 738 | for( __lock_size_t i = 0; i < count; i++) { |
---|
[6ff4507] | 739 | storage[i]->mask = mask; |
---|
| 740 | } |
---|
| 741 | } |
---|
| 742 | |
---|
[ac2b598] | 743 | static inline void reset_mask( $monitor * this ) { |
---|
[121be3e] | 744 | this->mask.accepted = 0p; |
---|
| 745 | this->mask.data = 0p; |
---|
[daacf82] | 746 | this->mask.size = 0; |
---|
| 747 | } |
---|
| 748 | |
---|
[ac2b598] | 749 | static inline $thread * next_thread( $monitor * this ) { |
---|
[0c78741] | 750 | //Check the signaller stack |
---|
[169d944] | 751 | __cfaabi_dbg_print_safe( "Kernel : mon %p AS-stack top %p\n", this, this->signal_stack.top); |
---|
[8fc45b7] | 752 | __condition_criterion_t * urgent = pop( this->signal_stack ); |
---|
[0c78741] | 753 | if( urgent ) { |
---|
| 754 | //The signaller stack is not empty, |
---|
| 755 | //regardless of if we are ready to baton pass, |
---|
| 756 | //we need to set the monitor as in use |
---|
[8fc652e0] | 757 | /* paranoid */ verifyf( !this->owner || active_thread() == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", active_thread(), this->owner, this->recursion, this ); |
---|
[c7a900a] | 758 | __set_owner( this, urgent->owner->waiting_thread ); |
---|
[0c78741] | 759 | |
---|
| 760 | return check_condition( urgent ); |
---|
| 761 | } |
---|
| 762 | |
---|
| 763 | // No signaller thread |
---|
| 764 | // Get the next thread in the entry_queue |
---|
[ac2b598] | 765 | $thread * new_owner = pop_head( this->entry_queue ); |
---|
[8fc652e0] | 766 | /* paranoid */ verifyf( !this->owner || active_thread() == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", active_thread(), this->owner, this->recursion, this ); |
---|
[504a7dc] | 767 | /* paranoid */ verify( !new_owner || new_owner->link.next == 0p ); |
---|
[c7a900a] | 768 | __set_owner( this, new_owner ); |
---|
[0c78741] | 769 | |
---|
| 770 | return new_owner; |
---|
| 771 | } |
---|
| 772 | |
---|
[ac2b598] | 773 | static inline bool is_accepted( $monitor * this, const __monitor_group_t & group ) { |
---|
[0cf5b79] | 774 | __acceptable_t * it = this->mask.data; // Optim |
---|
[59a0bde] | 775 | __lock_size_t count = this->mask.size; |
---|
[6ff4507] | 776 | |
---|
| 777 | // Check if there are any acceptable functions |
---|
[a843067] | 778 | if( !it ) return false; |
---|
[6ff4507] | 779 | |
---|
| 780 | // If this isn't the first monitor to test this, there is no reason to repeat the test. |
---|
| 781 | if( this != group[0] ) return group[0]->mask.accepted >= 0; |
---|
| 782 | |
---|
| 783 | // For all acceptable functions check if this is the current function. |
---|
[59a0bde] | 784 | for( __lock_size_t i = 0; i < count; i++, it++ ) { |
---|
[6ff4507] | 785 | if( *it == group ) { |
---|
| 786 | *this->mask.accepted = i; |
---|
| 787 | return true; |
---|
| 788 | } |
---|
| 789 | } |
---|
| 790 | |
---|
| 791 | // No function matched |
---|
| 792 | return false; |
---|
| 793 | } |
---|
| 794 | |
---|
[ac2b598] | 795 | static inline void init( __lock_size_t count, $monitor * monitors [], __condition_node_t & waiter, __condition_criterion_t criteria [] ) { |
---|
[513daec] | 796 | for( __lock_size_t i = 0; i < count; i++) { |
---|
[6b224a52] | 797 | (criteria[i]){ monitors[i], waiter }; |
---|
[97e3296] | 798 | } |
---|
| 799 | |
---|
[8fc45b7] | 800 | waiter.criteria = criteria; |
---|
[97e3296] | 801 | } |
---|
| 802 | |
---|
[ac2b598] | 803 | static inline void init_push( __lock_size_t count, $monitor * monitors [], __condition_node_t & waiter, __condition_criterion_t criteria [] ) { |
---|
[513daec] | 804 | for( __lock_size_t i = 0; i < count; i++) { |
---|
[6b224a52] | 805 | (criteria[i]){ monitors[i], waiter }; |
---|
[36982fc] | 806 | __cfaabi_dbg_print_safe( "Kernel : target %p = %p\n", criteria[i].target, &criteria[i] ); |
---|
[8fc45b7] | 807 | push( criteria[i].target->signal_stack, &criteria[i] ); |
---|
[97e3296] | 808 | } |
---|
| 809 | |
---|
[8fc45b7] | 810 | waiter.criteria = criteria; |
---|
[97e3296] | 811 | } |
---|
| 812 | |
---|
[ea7d2b0] | 813 | static inline void lock_all( __spinlock_t * locks [], __lock_size_t count ) { |
---|
[513daec] | 814 | for( __lock_size_t i = 0; i < count; i++ ) { |
---|
[2e9aed4] | 815 | lock( *locks[i] __cfaabi_dbg_ctx2 ); |
---|
[0c78741] | 816 | } |
---|
| 817 | } |
---|
| 818 | |
---|
[ac2b598] | 819 | static inline void lock_all( $monitor * source [], __spinlock_t * /*out*/ locks [], __lock_size_t count ) { |
---|
[513daec] | 820 | for( __lock_size_t i = 0; i < count; i++ ) { |
---|
[ea7d2b0] | 821 | __spinlock_t * l = &source[i]->lock; |
---|
[2e9aed4] | 822 | lock( *l __cfaabi_dbg_ctx2 ); |
---|
[0c78741] | 823 | if(locks) locks[i] = l; |
---|
| 824 | } |
---|
| 825 | } |
---|
| 826 | |
---|
[ea7d2b0] | 827 | static inline void unlock_all( __spinlock_t * locks [], __lock_size_t count ) { |
---|
[513daec] | 828 | for( __lock_size_t i = 0; i < count; i++ ) { |
---|
[ea7d2b0] | 829 | unlock( *locks[i] ); |
---|
[0c78741] | 830 | } |
---|
| 831 | } |
---|
| 832 | |
---|
[ac2b598] | 833 | static inline void unlock_all( $monitor * locks [], __lock_size_t count ) { |
---|
[513daec] | 834 | for( __lock_size_t i = 0; i < count; i++ ) { |
---|
[ea7d2b0] | 835 | unlock( locks[i]->lock ); |
---|
[0c78741] | 836 | } |
---|
| 837 | } |
---|
| 838 | |
---|
[8fc45b7] | 839 | static inline void save( |
---|
[ac2b598] | 840 | $monitor * ctx [], |
---|
[513daec] | 841 | __lock_size_t count, |
---|
[ea7d2b0] | 842 | __attribute((unused)) __spinlock_t * locks [], |
---|
[8fc45b7] | 843 | unsigned int /*out*/ recursions [], |
---|
| 844 | __waitfor_mask_t /*out*/ masks [] |
---|
| 845 | ) { |
---|
[513daec] | 846 | for( __lock_size_t i = 0; i < count; i++ ) { |
---|
[0c78741] | 847 | recursions[i] = ctx[i]->recursion; |
---|
[6ff4507] | 848 | masks[i] = ctx[i]->mask; |
---|
[0c78741] | 849 | } |
---|
| 850 | } |
---|
| 851 | |
---|
[8fc45b7] | 852 | static inline void restore( |
---|
[ac2b598] | 853 | $monitor * ctx [], |
---|
[513daec] | 854 | __lock_size_t count, |
---|
[ea7d2b0] | 855 | __spinlock_t * locks [], |
---|
[8fc45b7] | 856 | unsigned int /*out*/ recursions [], |
---|
| 857 | __waitfor_mask_t /*out*/ masks [] |
---|
| 858 | ) { |
---|
[6ff4507] | 859 | lock_all( locks, count ); |
---|
[513daec] | 860 | for( __lock_size_t i = 0; i < count; i++ ) { |
---|
[0c78741] | 861 | ctx[i]->recursion = recursions[i]; |
---|
[6ff4507] | 862 | ctx[i]->mask = masks[i]; |
---|
[0c78741] | 863 | } |
---|
[6ff4507] | 864 | unlock_all( locks, count ); |
---|
[0c78741] | 865 | } |
---|
| 866 | |
---|
| 867 | // Function has 2 different behavior |
---|
| 868 | // 1 - Marks a monitors as being ready to run |
---|
| 869 | // 2 - Checks if all the monitors are ready to run |
---|
| 870 | // if so return the thread to run |
---|
[ac2b598] | 871 | static inline $thread * check_condition( __condition_criterion_t * target ) { |
---|
[0c78741] | 872 | __condition_node_t * node = target->owner; |
---|
| 873 | unsigned short count = node->count; |
---|
| 874 | __condition_criterion_t * criteria = node->criteria; |
---|
| 875 | |
---|
| 876 | bool ready2run = true; |
---|
| 877 | |
---|
| 878 | for( int i = 0; i < count; i++ ) { |
---|
[44264c5] | 879 | |
---|
[36982fc] | 880 | // __cfaabi_dbg_print_safe( "Checking %p for %p\n", &criteria[i], target ); |
---|
[0c78741] | 881 | if( &criteria[i] == target ) { |
---|
| 882 | criteria[i].ready = true; |
---|
[36982fc] | 883 | // __cfaabi_dbg_print_safe( "True\n" ); |
---|
[0c78741] | 884 | } |
---|
| 885 | |
---|
| 886 | ready2run = criteria[i].ready && ready2run; |
---|
| 887 | } |
---|
| 888 | |
---|
[504a7dc] | 889 | __cfaabi_dbg_print_safe( "Kernel : Runing %i (%p)\n", ready2run, ready2run ? (thread*)node->waiting_thread : (thread*)0p ); |
---|
[121be3e] | 890 | return ready2run ? node->waiting_thread : 0p; |
---|
[0c78741] | 891 | } |
---|
| 892 | |
---|
[4cedd9f] | 893 | static inline void brand_condition( condition & this ) { |
---|
[8fc652e0] | 894 | $thread * thrd = active_thread(); |
---|
[4cedd9f] | 895 | if( !this.monitors ) { |
---|
[169d944] | 896 | // __cfaabi_dbg_print_safe( "Branding\n" ); |
---|
[121be3e] | 897 | assertf( thrd->monitors.data != 0p, "No current monitor to brand condition %p", thrd->monitors.data ); |
---|
[4cedd9f] | 898 | this.monitor_count = thrd->monitors.size; |
---|
[a933dcf4] | 899 | |
---|
[ac2b598] | 900 | this.monitors = ($monitor **)malloc( this.monitor_count * sizeof( *this.monitors ) ); |
---|
[4cedd9f] | 901 | for( int i = 0; i < this.monitor_count; i++ ) { |
---|
[0cf5b79] | 902 | this.monitors[i] = thrd->monitors[i]; |
---|
[a933dcf4] | 903 | } |
---|
[0c78741] | 904 | } |
---|
| 905 | } |
---|
| 906 | |
---|
[ac2b598] | 907 | static inline [$thread *, int] search_entry_queue( const __waitfor_mask_t & mask, $monitor * monitors [], __lock_size_t count ) { |
---|
[90c4df0] | 908 | |
---|
[ac2b598] | 909 | __queue_t($thread) & entry_queue = monitors[0]->entry_queue; |
---|
[90c4df0] | 910 | |
---|
| 911 | // For each thread in the entry-queue |
---|
[ac2b598] | 912 | for( $thread ** thrd_it = &entry_queue.head; |
---|
[1b143de] | 913 | (*thrd_it) != 1p; |
---|
[b798713] | 914 | thrd_it = &(*thrd_it)->link.next |
---|
[4cc9b13] | 915 | ) { |
---|
[90c4df0] | 916 | // For each acceptable check if it matches |
---|
[4cc9b13] | 917 | int i = 0; |
---|
[0cf5b79] | 918 | __acceptable_t * end = end (mask); |
---|
| 919 | __acceptable_t * begin = begin(mask); |
---|
| 920 | for( __acceptable_t * it = begin; it != end; it++, i++ ) { |
---|
[90c4df0] | 921 | // Check if we have a match |
---|
[aaa4f93] | 922 | if( *it == (*thrd_it)->monitors ) { |
---|
[90c4df0] | 923 | |
---|
| 924 | // If we have a match return it |
---|
| 925 | // after removeing it from the entry queue |
---|
[b18830e] | 926 | return [remove( entry_queue, thrd_it ), i]; |
---|
[90c4df0] | 927 | } |
---|
| 928 | } |
---|
| 929 | } |
---|
| 930 | |
---|
[b18830e] | 931 | return [0, -1]; |
---|
| 932 | } |
---|
| 933 | |
---|
[6ff4507] | 934 | forall(dtype T | sized( T )) |
---|
[59a0bde] | 935 | static inline __lock_size_t insert_unique( T * array [], __lock_size_t & size, T * val ) { |
---|
[6ff4507] | 936 | if( !val ) return size; |
---|
| 937 | |
---|
[59a0bde] | 938 | for( __lock_size_t i = 0; i <= size; i++) { |
---|
[6ff4507] | 939 | if( array[i] == val ) return size; |
---|
| 940 | } |
---|
| 941 | |
---|
| 942 | array[size] = val; |
---|
| 943 | size = size + 1; |
---|
| 944 | return size; |
---|
| 945 | } |
---|
| 946 | |
---|
[59a0bde] | 947 | static inline __lock_size_t count_max( const __waitfor_mask_t & mask ) { |
---|
| 948 | __lock_size_t max = 0; |
---|
| 949 | for( __lock_size_t i = 0; i < mask.size; i++ ) { |
---|
[0cf5b79] | 950 | __acceptable_t & accepted = mask[i]; |
---|
| 951 | max += accepted.size; |
---|
[b18830e] | 952 | } |
---|
| 953 | return max; |
---|
[97e3296] | 954 | } |
---|
[b18830e] | 955 | |
---|
[ac2b598] | 956 | static inline __lock_size_t aggregate( $monitor * storage [], const __waitfor_mask_t & mask ) { |
---|
[59a0bde] | 957 | __lock_size_t size = 0; |
---|
| 958 | for( __lock_size_t i = 0; i < mask.size; i++ ) { |
---|
[0cf5b79] | 959 | __acceptable_t & accepted = mask[i]; |
---|
| 960 | __libcfa_small_sort( accepted.data, accepted.size ); |
---|
| 961 | for( __lock_size_t j = 0; j < accepted.size; j++) { |
---|
| 962 | insert_unique( storage, size, accepted[j] ); |
---|
[6ff4507] | 963 | } |
---|
[b18830e] | 964 | } |
---|
[6a5be52] | 965 | // TODO insertion sort instead of this |
---|
[de737c8] | 966 | __libcfa_small_sort( storage, size ); |
---|
[6ff4507] | 967 | return size; |
---|
[b18830e] | 968 | } |
---|
| 969 | |
---|
[6b0b624] | 970 | // Local Variables: // |
---|
| 971 | // mode: c // |
---|
| 972 | // tab-width: 4 // |
---|
| 973 | // End: // |
---|