Changeset 58fe85a for libcfa/src/concurrency/kernel.cfa
- Timestamp:
- Jan 7, 2021, 3:27:00 PM (5 years ago)
- Branches:
- ADT, arm-eh, ast-experimental, enum, forall-pointer-decay, jacob/cs343-translation, master, new-ast-unique-expr, pthread-emulation, qualifiedEnum
- Children:
- 2b4daf2, 64aeca0
- Parents:
- 3c64c668 (diff), eef8dfb (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the(diff)links above to see all the changes relative to each parent. - File:
-
- 1 edited
-
libcfa/src/concurrency/kernel.cfa (modified) (23 diffs)
Legend:
- Unmodified
- Added
- Removed
-
libcfa/src/concurrency/kernel.cfa
r3c64c668 r58fe85a 10 10 // Created On : Tue Jan 17 12:27:26 2017 11 11 // Last Modified By : Peter A. Buhr 12 // Last Modified On : Tue Feb 4 13:03:15202013 // Update Count : 5812 // Last Modified On : Mon Aug 31 07:08:20 2020 13 // Update Count : 71 14 14 // 15 15 16 16 #define __cforall_thread__ 17 // #define __CFA_DEBUG_PRINT_RUNTIME_CORE__ 17 18 18 19 //C Includes 19 #include <stddef.h>20 20 #include <errno.h> 21 #include <string.h>22 extern "C" {23 21 #include <stdio.h> 24 #include <fenv.h>25 #include <sys/resource.h>26 22 #include <signal.h> 27 23 #include <unistd.h> 28 #include <limits.h> // PTHREAD_STACK_MIN29 #include <sys/mman.h> // mprotect30 }31 24 32 25 //CFA Includes 33 #include "time.hfa"34 26 #include "kernel_private.hfa" 35 27 #include "preemption.hfa" 36 #include "startup.hfa"37 28 38 29 //Private includes … … 40 31 #include "invoke.h" 41 32 33 42 34 //----------------------------------------------------------------------------- 43 35 // Some assembly required 44 36 #if defined( __i386 ) 45 #define CtxGet( ctx ) \46 __asm__ volatile ( \47 "movl %%esp,%0\n"\48 "movl %%ebp,%1\n"\49 : "=rm" (ctx.SP),\50 "=rm" (ctx.FP) \51 )52 53 37 // mxcr : SSE Status and Control bits (control bits are preserved across function calls) 54 38 // fcw : X87 FPU control word (preserved across function calls) … … 72 56 73 57 #elif defined( __x86_64 ) 74 #define CtxGet( ctx ) \75 __asm__ volatile ( \76 "movq %%rsp,%0\n"\77 "movq %%rbp,%1\n"\78 : "=rm" (ctx.SP),\79 "=rm" (ctx.FP) \80 )81 82 58 #define __x87_store \ 83 59 uint32_t __mxcr; \ … … 98 74 ) 99 75 100 101 #elif defined( __ARM_ARCH ) 102 #define CtxGet( ctx ) __asm__ ( \ 103 "mov %0,%%sp\n" \ 104 "mov %1,%%r11\n" \ 105 : "=rm" (ctx.SP), "=rm" (ctx.FP) ) 76 #elif defined( __arm__ ) 77 #define __x87_store 78 #define __x87_load 79 80 #elif defined( __aarch64__ ) 81 #define __x87_store \ 82 uint32_t __fpcntl[2]; \ 83 __asm__ volatile ( \ 84 "mrs x9, FPCR\n" \ 85 "mrs x10, FPSR\n" \ 86 "stp x9, x10, %0\n" \ 87 : "=m" (__fpcntl) : : "x9", "x10" \ 88 ) 89 90 #define __x87_load \ 91 __asm__ volatile ( \ 92 "ldp x9, x10, %0\n" \ 93 "msr FPSR, x10\n" \ 94 "msr FPCR, x9\n" \ 95 : "=m" (__fpcntl) : : "x9", "x10" \ 96 ) 97 106 98 #else 107 #error un knownhardware architecture99 #error unsupported hardware architecture 108 100 #endif 109 101 102 extern $thread * mainThread; 103 extern processor * mainProcessor; 104 110 105 //----------------------------------------------------------------------------- 111 //Start and stop routine for the kernel, declared first to make sure they run first 112 static void __kernel_startup (void) __attribute__(( constructor( STARTUP_PRIORITY_KERNEL ) )); 113 static void __kernel_shutdown(void) __attribute__(( destructor ( STARTUP_PRIORITY_KERNEL ) )); 114 115 //----------------------------------------------------------------------------- 116 // Kernel storage 117 KERNEL_STORAGE(cluster, mainCluster); 118 KERNEL_STORAGE(processor, mainProcessor); 119 KERNEL_STORAGE($thread, mainThread); 120 KERNEL_STORAGE(__stack_t, mainThreadCtx); 121 122 cluster * mainCluster; 123 processor * mainProcessor; 124 $thread * mainThread; 125 126 extern "C" { 127 struct { __dllist_t(cluster) list; __spinlock_t lock; } __cfa_dbg_global_clusters; 128 } 129 130 size_t __page_size = 0; 131 132 //----------------------------------------------------------------------------- 133 // Global state 134 thread_local struct KernelThreadData kernelTLS __attribute__ ((tls_model ( "initial-exec" ))) = { 135 NULL, // cannot use 0p 136 NULL, 137 { 1, false, false }, 138 6u //this should be seeded better but due to a bug calling rdtsc doesn't work 139 }; 140 141 //----------------------------------------------------------------------------- 142 // Struct to steal stack 143 struct current_stack_info_t { 144 __stack_t * storage; // pointer to stack object 145 void * base; // base of stack 146 void * limit; // stack grows towards stack limit 147 void * context; // address of cfa_context_t 148 }; 149 150 void ?{}( current_stack_info_t & this ) { 151 __stack_context_t ctx; 152 CtxGet( ctx ); 153 this.base = ctx.FP; 154 155 rlimit r; 156 getrlimit( RLIMIT_STACK, &r); 157 size_t size = r.rlim_cur; 158 159 this.limit = (void *)(((intptr_t)this.base) - size); 160 this.context = &storage_mainThreadCtx; 161 } 162 163 //----------------------------------------------------------------------------- 164 // Main thread construction 165 166 void ?{}( $coroutine & this, current_stack_info_t * info) with( this ) { 167 stack.storage = info->storage; 168 with(*stack.storage) { 169 limit = info->limit; 170 base = info->base; 171 } 172 __attribute__((may_alias)) intptr_t * istorage = (intptr_t*) &stack.storage; 173 *istorage |= 0x1; 174 name = "Main Thread"; 175 state = Start; 176 starter = 0p; 177 last = 0p; 178 cancellation = 0p; 179 } 180 181 void ?{}( $thread & this, current_stack_info_t * info) with( this ) { 182 state = Start; 183 self_cor{ info }; 184 curr_cor = &self_cor; 185 curr_cluster = mainCluster; 186 self_mon.owner = &this; 187 self_mon.recursion = 1; 188 self_mon_p = &self_mon; 189 next = 0p; 190 191 node.next = 0p; 192 node.prev = 0p; 193 doregister(curr_cluster, this); 194 195 monitors{ &self_mon_p, 1, (fptr_t)0 }; 196 } 197 198 //----------------------------------------------------------------------------- 199 // Processor coroutine 200 void ?{}(processorCtx_t & this) { 201 202 } 203 204 // Construct the processor context of non-main processors 205 static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info) { 206 (this.__cor){ info }; 207 this.proc = proc; 208 } 209 210 static void * __invoke_processor(void * arg); 211 212 void ?{}(processor & this, const char name[], cluster & cltr) with( this ) { 213 this.name = name; 214 this.cltr = &cltr; 215 terminated{ 0 }; 216 destroyer = 0p; 217 do_terminate = false; 218 preemption_alarm = 0p; 219 pending_preemption = false; 220 runner.proc = &this; 221 222 idleLock{}; 223 224 __cfaabi_dbg_print_safe("Kernel : Starting core %p\n", &this); 225 226 this.stack = __create_pthread( &this.kernel_thread, __invoke_processor, (void *)&this ); 227 228 __cfaabi_dbg_print_safe("Kernel : core %p started\n", &this); 229 } 230 231 void ^?{}(processor & this) with( this ){ 232 if( ! __atomic_load_n(&do_terminate, __ATOMIC_ACQUIRE) ) { 233 __cfaabi_dbg_print_safe("Kernel : core %p signaling termination\n", &this); 234 235 __atomic_store_n(&do_terminate, true, __ATOMIC_RELAXED); 236 wake( &this ); 237 238 P( terminated ); 239 verify( kernelTLS.this_processor != &this); 240 } 241 242 pthread_join( kernel_thread, 0p ); 243 free( this.stack ); 244 } 245 246 void ?{}(cluster & this, const char name[], Duration preemption_rate) with( this ) { 247 this.name = name; 248 this.preemption_rate = preemption_rate; 249 ready_queue{}; 250 ready_queue_lock{}; 251 252 procs{ __get }; 253 idles{ __get }; 254 threads{ __get }; 255 256 doregister(this); 257 } 258 259 void ^?{}(cluster & this) { 260 unregister(this); 261 } 106 // Kernel Scheduling logic 107 static $thread * __next_thread(cluster * this); 108 static $thread * __next_thread_slow(cluster * this); 109 static void __run_thread(processor * this, $thread * dst); 110 static void __wake_one(cluster * cltr); 111 112 static void push (__cluster_idles & idles, processor & proc); 113 static void remove(__cluster_idles & idles, processor & proc); 114 static [unsigned idle, unsigned total, * processor] query( & __cluster_idles idles ); 115 262 116 263 117 //============================================================================================= 264 118 // Kernel Scheduling logic 265 119 //============================================================================================= 266 static $thread * __next_thread(cluster * this);267 static void __run_thread(processor * this, $thread * dst);268 static void __halt(processor * this);269 270 120 //Main of the processor contexts 271 121 void main(processorCtx_t & runner) { 272 122 // Because of a bug, we couldn't initialized the seed on construction 273 123 // Do it here 274 kernelTLS.rand_seed ^= rdtscl(); 124 __cfaabi_tls.rand_seed ^= rdtscl(); 125 __cfaabi_tls.ready_rng.fwd_seed = 25214903917_l64u * (rdtscl() ^ (uintptr_t)&runner); 126 __tls_rand_advance_bck(); 275 127 276 128 processor * this = runner.proc; 277 129 verify(this); 278 130 279 __cfaabi_dbg_print_safe("Kernel : core %p starting\n", this); 280 281 doregister(this->cltr, this); 131 __cfadbg_print_safe(runtime_core, "Kernel : core %p starting\n", this); 132 #if !defined(__CFA_NO_STATISTICS__) 133 if( this->print_halts ) { 134 __cfaabi_bits_print_safe( STDOUT_FILENO, "Processor : %d - %s (%p)\n", this->id, this->name, (void*)this); 135 } 136 #endif 282 137 283 138 { … … 285 140 preemption_scope scope = { this }; 286 141 287 __cfa abi_dbg_print_safe("Kernel : core %p started\n", this);142 __cfadbg_print_safe(runtime_core, "Kernel : core %p started\n", this); 288 143 289 144 $thread * readyThread = 0p; 290 for( unsigned int spin_count = 0; ! __atomic_load_n(&this->do_terminate, __ATOMIC_SEQ_CST); spin_count++ ) { 145 MAIN_LOOP: 146 for() { 147 // Try to get the next thread 291 148 readyThread = __next_thread( this->cltr ); 292 149 293 if(readyThread) { 294 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled ); 295 /* paranoid */ verifyf( readyThread->state == Inactive || readyThread->state == Start || readyThread->preempted != __NO_PREEMPTION, "state : %d, preempted %d\n", readyThread->state, readyThread->preempted); 296 /* paranoid */ verifyf( readyThread->next == 0p, "Expected null got %p", readyThread->next ); 297 298 __run_thread(this, readyThread); 299 300 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled ); 301 302 spin_count = 0; 303 } else { 304 // spin(this, &spin_count); 305 __halt(this); 150 if( !readyThread ) { 151 readyThread = __next_thread_slow( this->cltr ); 306 152 } 307 } 308 309 __cfaabi_dbg_print_safe("Kernel : core %p stopping\n", this); 310 } 311 312 unregister(this->cltr, this); 153 154 HALT: 155 if( !readyThread ) { 156 // Don't block if we are done 157 if( __atomic_load_n(&this->do_terminate, __ATOMIC_SEQ_CST) ) break MAIN_LOOP; 158 159 #if !defined(__CFA_NO_STATISTICS__) 160 __tls_stats()->ready.sleep.halts++; 161 #endif 162 163 // Push self to idle stack 164 push(this->cltr->idles, * this); 165 166 // Confirm the ready-queue is empty 167 readyThread = __next_thread_slow( this->cltr ); 168 if( readyThread ) { 169 // A thread was found, cancel the halt 170 remove(this->cltr->idles, * this); 171 172 #if !defined(__CFA_NO_STATISTICS__) 173 __tls_stats()->ready.sleep.cancels++; 174 #endif 175 176 // continue the mai loop 177 break HALT; 178 } 179 180 #if !defined(__CFA_NO_STATISTICS__) 181 if(this->print_halts) { 182 __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 0\n", this->id, rdtscl()); 183 } 184 #endif 185 186 wait( this->idle ); 187 188 #if !defined(__CFA_NO_STATISTICS__) 189 if(this->print_halts) { 190 __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 1\n", this->id, rdtscl()); 191 } 192 #endif 193 194 // We were woken up, remove self from idle 195 remove(this->cltr->idles, * this); 196 197 // DON'T just proceed, start looking again 198 continue MAIN_LOOP; 199 } 200 201 /* paranoid */ verify( readyThread ); 202 203 // We found a thread run it 204 __run_thread(this, readyThread); 205 206 // Are we done? 207 if( __atomic_load_n(&this->do_terminate, __ATOMIC_SEQ_CST) ) break MAIN_LOOP; 208 } 209 210 __cfadbg_print_safe(runtime_core, "Kernel : core %p stopping\n", this); 211 } 313 212 314 213 V( this->terminated ); 315 214 316 __cfaabi_dbg_print_safe("Kernel : core %p terminated\n", this); 215 if(this == mainProcessor) { 216 // HACK : the coroutine context switch expects this_thread to be set 217 // and it make sense for it to be set in all other cases except here 218 // fake it 219 __cfaabi_tls.this_thread = mainThread; 220 } 221 222 __cfadbg_print_safe(runtime_core, "Kernel : core %p terminated\n", this); 317 223 } 318 224 … … 324 230 // from the processor coroutine to the target thread 325 231 static void __run_thread(processor * this, $thread * thrd_dst) { 232 /* paranoid */ verify( ! __preemption_enabled() ); 233 /* paranoid */ verifyf( thrd_dst->state == Ready || thrd_dst->preempted != __NO_PREEMPTION, "state : %d, preempted %d\n", thrd_dst->state, thrd_dst->preempted); 234 /* paranoid */ verifyf( thrd_dst->link.next == 0p, "Expected null got %p", thrd_dst->link.next ); 235 __builtin_prefetch( thrd_dst->context.SP ); 236 326 237 $coroutine * proc_cor = get_coroutine(this->runner); 327 328 // Update global state329 kernelTLS.this_thread = thrd_dst;330 238 331 239 // set state of processor coroutine to inactive 332 240 verify(proc_cor->state == Active); 333 proc_cor->state = Inactive;241 proc_cor->state = Blocked; 334 242 335 243 // Actually run the thread 336 244 RUNNING: while(true) { 337 if(unlikely(thrd_dst->preempted)) { 338 thrd_dst->preempted = __NO_PREEMPTION; 339 verify(thrd_dst->state == Active || thrd_dst->state == Rerun); 340 } else { 341 verify(thrd_dst->state == Start || thrd_dst->state == Primed || thrd_dst->state == Inactive); 342 thrd_dst->state = Active; 343 } 344 345 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled ); 245 thrd_dst->preempted = __NO_PREEMPTION; 246 thrd_dst->state = Active; 247 248 // Update global state 249 kernelTLS().this_thread = thrd_dst; 250 251 /* paranoid */ verify( ! __preemption_enabled() ); 252 /* paranoid */ verify( kernelTLS().this_thread == thrd_dst ); 253 /* paranoid */ verify( thrd_dst->curr_cluster == this->cltr ); 254 /* paranoid */ verify( thrd_dst->context.SP ); 255 /* paranoid */ verify( thrd_dst->state != Halted ); 256 /* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) < ((uintptr_t)__get_stack(thrd_dst->curr_cor)->base ) || thrd_dst->curr_cor == proc_cor, "ERROR : Destination $thread %p has been corrupted.\n StackPointer too small.\n", thrd_dst ); // add escape condition if we are setting up the processor 257 /* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) > ((uintptr_t)__get_stack(thrd_dst->curr_cor)->limit) || thrd_dst->curr_cor == proc_cor, "ERROR : Destination $thread %p has been corrupted.\n StackPointer too large.\n", thrd_dst ); // add escape condition if we are setting up the processor 258 /* paranoid */ verify( 0x0D15EA5E0D15EA5Ep == thrd_dst->canary ); 259 260 346 261 347 262 // set context switch to the thread that the processor is executing 348 verify( thrd_dst->context.SP );349 263 __cfactx_switch( &proc_cor->context, &thrd_dst->context ); 350 264 // when __cfactx_switch returns we are back in the processor coroutine 351 265 352 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled ); 353 266 /* paranoid */ verify( 0x0D15EA5E0D15EA5Ep == thrd_dst->canary ); 267 /* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) > ((uintptr_t)__get_stack(thrd_dst->curr_cor)->limit), "ERROR : Destination $thread %p has been corrupted.\n StackPointer too large.\n", thrd_dst ); 268 /* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) < ((uintptr_t)__get_stack(thrd_dst->curr_cor)->base ), "ERROR : Destination $thread %p has been corrupted.\n StackPointer too small.\n", thrd_dst ); 269 /* paranoid */ verify( thrd_dst->context.SP ); 270 /* paranoid */ verify( thrd_dst->curr_cluster == this->cltr ); 271 /* paranoid */ verify( kernelTLS().this_thread == thrd_dst ); 272 /* paranoid */ verify( ! __preemption_enabled() ); 273 274 // Reset global state 275 kernelTLS().this_thread = 0p; 354 276 355 277 // We just finished running a thread, there are a few things that could have happened. 356 278 // 1 - Regular case : the thread has blocked and now one has scheduled it yet. 357 279 // 2 - Racy case : the thread has blocked but someone has already tried to schedule it. 358 // 3 - Polite Racy case : the thread has blocked, someone has already tried to schedule it, but the thread is nice and wants to go through the ready-queue any way359 280 // 4 - Preempted 360 281 // In case 1, we may have won a race so we can't write to the state again. 361 282 // In case 2, we lost the race so we now own the thread. 362 // In case 3, we lost the race but can just reschedule the thread.363 283 364 284 if(unlikely(thrd_dst->preempted != __NO_PREEMPTION)) { … … 368 288 } 369 289 290 if(unlikely(thrd_dst->state == Halting)) { 291 // The thread has halted, it should never be scheduled/run again 292 // finish the thread 293 __thread_finish( thrd_dst ); 294 break RUNNING; 295 } 296 297 /* paranoid */ verify( thrd_dst->state == Active ); 298 thrd_dst->state = Blocked; 299 370 300 // set state of processor coroutine to active and the thread to inactive 371 static_assert(sizeof(thrd_dst->state) == sizeof(int)); 372 enum coroutine_state old_state = __atomic_exchange_n(&thrd_dst->state, Inactive, __ATOMIC_SEQ_CST); 373 switch(old_state) { 374 case Halted: 375 // The thread has halted, it should never be scheduled/run again, leave it back to Halted and move on 376 thrd_dst->state = Halted; 377 378 // We may need to wake someone up here since 379 unpark( this->destroyer ); 380 this->destroyer = 0p; 381 break RUNNING; 382 case Active: 301 int old_ticket = __atomic_fetch_sub(&thrd_dst->ticket, 1, __ATOMIC_SEQ_CST); 302 switch(old_ticket) { 303 case TICKET_RUNNING: 383 304 // This is case 1, the regular case, nothing more is needed 384 305 break RUNNING; 385 case Rerun:306 case TICKET_UNBLOCK: 386 307 // This is case 2, the racy case, someone tried to run this thread before it finished blocking 387 308 // In this case, just run it again. … … 389 310 default: 390 311 // This makes no sense, something is wrong abort 391 abort( "Finished running a thread that was Inactive/Start/Primed %d\n", old_state);312 abort(); 392 313 } 393 314 } … … 395 316 // Just before returning to the processor, set the processor coroutine to active 396 317 proc_cor->state = Active; 318 319 /* paranoid */ verify( ! __preemption_enabled() ); 397 320 } 398 321 399 322 // KERNEL_ONLY 400 323 void returnToKernel() { 401 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled ); 402 $coroutine * proc_cor = get_coroutine(kernelTLS.this_processor->runner); 403 $thread * thrd_src = kernelTLS.this_thread; 324 /* paranoid */ verify( ! __preemption_enabled() ); 325 $coroutine * proc_cor = get_coroutine(kernelTLS().this_processor->runner); 326 $thread * thrd_src = kernelTLS().this_thread; 327 328 #if !defined(__CFA_NO_STATISTICS__) 329 struct processor * last_proc = kernelTLS().this_processor; 330 #endif 404 331 405 332 // Run the thread on this processor … … 409 336 __x87_store; 410 337 #endif 411 verify( proc_cor->context.SP ); 338 /* paranoid */ verify( proc_cor->context.SP ); 339 /* paranoid */ verify( 0x0D15EA5E0D15EA5Ep == thrd_src->canary ); 412 340 __cfactx_switch( &thrd_src->context, &proc_cor->context ); 341 /* paranoid */ verify( 0x0D15EA5E0D15EA5Ep == thrd_src->canary ); 413 342 #if defined( __i386 ) || defined( __x86_64 ) 414 343 __x87_load; … … 417 346 } 418 347 419 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled ); 420 } 421 422 // KERNEL_ONLY 423 // Context invoker for processors 424 // This is the entry point for processors (kernel threads) 425 // It effectively constructs a coroutine by stealing the pthread stack 426 static void * __invoke_processor(void * arg) { 427 processor * proc = (processor *) arg; 428 kernelTLS.this_processor = proc; 429 kernelTLS.this_thread = 0p; 430 kernelTLS.preemption_state.[enabled, disable_count] = [false, 1]; 431 // SKULLDUGGERY: We want to create a context for the processor coroutine 432 // which is needed for the 2-step context switch. However, there is no reason 433 // to waste the perfectly valid stack create by pthread. 434 current_stack_info_t info; 435 __stack_t ctx; 436 info.storage = &ctx; 437 (proc->runner){ proc, &info }; 438 439 __cfaabi_dbg_print_safe("Coroutine : created stack %p\n", get_coroutine(proc->runner)->stack.storage); 440 441 //Set global state 442 kernelTLS.this_thread = 0p; 443 444 //We now have a proper context from which to schedule threads 445 __cfaabi_dbg_print_safe("Kernel : core %p created (%p, %p)\n", proc, &proc->runner, &ctx); 446 447 // SKULLDUGGERY: Since the coroutine doesn't have its own stack, we can't 448 // resume it to start it like it normally would, it will just context switch 449 // back to here. Instead directly call the main since we already are on the 450 // appropriate stack. 451 get_coroutine(proc->runner)->state = Active; 452 main( proc->runner ); 453 get_coroutine(proc->runner)->state = Halted; 454 455 // Main routine of the core returned, the core is now fully terminated 456 __cfaabi_dbg_print_safe("Kernel : core %p main ended (%p)\n", proc, &proc->runner); 457 458 return 0p; 459 } 460 461 static void Abort( int ret, const char func[] ) { 462 if ( ret ) { // pthread routines return errno values 463 abort( "%s : internal error, error(%d) %s.", func, ret, strerror( ret ) ); 464 } // if 465 } // Abort 466 467 void * __create_pthread( pthread_t * pthread, void * (*start)(void *), void * arg ) { 468 pthread_attr_t attr; 469 470 Abort( pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute 471 472 size_t stacksize; 473 // default stack size, normally defined by shell limit 474 Abort( pthread_attr_getstacksize( &attr, &stacksize ), "pthread_attr_getstacksize" ); 475 assert( stacksize >= PTHREAD_STACK_MIN ); 476 477 void * stack; 478 __cfaabi_dbg_debug_do( 479 stack = memalign( __page_size, stacksize + __page_size ); 480 // pthread has no mechanism to create the guard page in user supplied stack. 481 if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) { 482 abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) ); 483 } // if 484 ); 485 __cfaabi_dbg_no_debug_do( 486 stack = malloc( stacksize ); 487 ); 488 489 Abort( pthread_attr_setstack( &attr, stack, stacksize ), "pthread_attr_setstack" ); 490 491 Abort( pthread_create( pthread, &attr, start, arg ), "pthread_create" ); 492 return stack; 493 } 494 495 // KERNEL_ONLY 496 static void __kernel_first_resume( processor * this ) { 497 $thread * src = mainThread; 498 $coroutine * dst = get_coroutine(this->runner); 499 500 verify( ! kernelTLS.preemption_state.enabled ); 501 502 kernelTLS.this_thread->curr_cor = dst; 503 __stack_prepare( &dst->stack, 65000 ); 504 __cfactx_start(main, dst, this->runner, __cfactx_invoke_coroutine); 505 506 verify( ! kernelTLS.preemption_state.enabled ); 507 508 dst->last = &src->self_cor; 509 dst->starter = dst->starter ? dst->starter : &src->self_cor; 510 511 // set state of current coroutine to inactive 512 src->state = src->state == Halted ? Halted : Inactive; 513 514 // context switch to specified coroutine 515 verify( dst->context.SP ); 516 __cfactx_switch( &src->context, &dst->context ); 517 // when __cfactx_switch returns we are back in the src coroutine 518 519 mainThread->curr_cor = &mainThread->self_cor; 520 521 // set state of new coroutine to active 522 src->state = Active; 523 524 verify( ! kernelTLS.preemption_state.enabled ); 525 } 526 527 // KERNEL_ONLY 528 static void __kernel_last_resume( processor * this ) { 529 $coroutine * src = &mainThread->self_cor; 530 $coroutine * dst = get_coroutine(this->runner); 531 532 verify( ! kernelTLS.preemption_state.enabled ); 533 verify( dst->starter == src ); 534 verify( dst->context.SP ); 535 536 // context switch to the processor 537 __cfactx_switch( &src->context, &dst->context ); 348 #if !defined(__CFA_NO_STATISTICS__) 349 if(last_proc != kernelTLS().this_processor) { 350 __tls_stats()->ready.threads.migration++; 351 } 352 #endif 353 354 /* paranoid */ verify( ! __preemption_enabled() ); 355 /* paranoid */ verifyf( ((uintptr_t)thrd_src->context.SP) < ((uintptr_t)__get_stack(thrd_src->curr_cor)->base ), "ERROR : Returning $thread %p has been corrupted.\n StackPointer too small.\n", thrd_src ); 356 /* paranoid */ verifyf( ((uintptr_t)thrd_src->context.SP) > ((uintptr_t)__get_stack(thrd_src->curr_cor)->limit), "ERROR : Returning $thread %p has been corrupted.\n StackPointer too large.\n", thrd_src ); 538 357 } 539 358 … … 541 360 // Scheduler routines 542 361 // KERNEL ONLY 543 void __schedule_thread( $thread * thrd ) with( *thrd->curr_cluster ) { 544 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled ); 362 void __schedule_thread( $thread * thrd ) { 363 /* paranoid */ verify( ! __preemption_enabled() ); 364 /* paranoid */ verify( kernelTLS().this_proc_id ); 365 /* paranoid */ verify( thrd ); 366 /* paranoid */ verify( thrd->state != Halted ); 367 /* paranoid */ verify( thrd->curr_cluster ); 545 368 /* paranoid */ #if defined( __CFA_WITH_VERIFY__ ) 546 /* paranoid */ if( thrd->state == Inactive|| thrd->state == Start ) assertf( thrd->preempted == __NO_PREEMPTION,547 548 /* paranoid */ if( thrd->preempted != __NO_PREEMPTION ) assertf(thrd->state == Active || thrd->state == Rerun,549 369 /* paranoid */ if( thrd->state == Blocked || thrd->state == Start ) assertf( thrd->preempted == __NO_PREEMPTION, 370 "Error inactive thread marked as preempted, state %d, preemption %d\n", thrd->state, thrd->preempted ); 371 /* paranoid */ if( thrd->preempted != __NO_PREEMPTION ) assertf(thrd->state == Active, 372 "Error preempted thread marked as not currently running, state %d, preemption %d\n", thrd->state, thrd->preempted ); 550 373 /* paranoid */ #endif 551 /* paranoid */ verifyf( thrd->next == 0p, "Expected null got %p", thrd->next ); 552 553 lock ( ready_queue_lock __cfaabi_dbg_ctx2 ); 554 bool was_empty = !(ready_queue != 0); 555 append( ready_queue, thrd ); 556 unlock( ready_queue_lock ); 557 558 if(was_empty) { 559 lock (proc_list_lock __cfaabi_dbg_ctx2); 560 if(idles) { 561 wake_fast(idles.head); 562 } 563 unlock (proc_list_lock); 564 } 565 else if( struct processor * idle = idles.head ) { 566 wake_fast(idle); 567 } 568 569 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled ); 374 /* paranoid */ verifyf( thrd->link.next == 0p, "Expected null got %p", thrd->link.next ); 375 /* paranoid */ verify( 0x0D15EA5E0D15EA5Ep == thrd->canary ); 376 377 378 if (thrd->preempted == __NO_PREEMPTION) thrd->state = Ready; 379 380 ready_schedule_lock(); 381 // Dereference the thread now because once we push it, there is not guaranteed it's still valid. 382 struct cluster * cl = thrd->curr_cluster; 383 384 // push the thread to the cluster ready-queue 385 push( cl, thrd ); 386 387 // variable thrd is no longer safe to use 388 389 // wake the cluster using the save variable. 390 __wake_one( cl ); 391 ready_schedule_unlock(); 392 393 /* paranoid */ verify( ! __preemption_enabled() ); 570 394 } 571 395 572 396 // KERNEL ONLY 573 static $thread * __next_thread(cluster * this) with( *this ) { 574 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled ); 575 576 lock( ready_queue_lock __cfaabi_dbg_ctx2 ); 577 $thread * head = pop_head( ready_queue ); 578 unlock( ready_queue_lock ); 579 580 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled ); 581 return head; 397 static inline $thread * __next_thread(cluster * this) with( *this ) { 398 /* paranoid */ verify( ! __preemption_enabled() ); 399 /* paranoid */ verify( kernelTLS().this_proc_id ); 400 401 ready_schedule_lock(); 402 $thread * thrd = pop( this ); 403 ready_schedule_unlock(); 404 405 /* paranoid */ verify( kernelTLS().this_proc_id ); 406 /* paranoid */ verify( ! __preemption_enabled() ); 407 return thrd; 408 } 409 410 // KERNEL ONLY 411 static inline $thread * __next_thread_slow(cluster * this) with( *this ) { 412 /* paranoid */ verify( ! __preemption_enabled() ); 413 /* paranoid */ verify( kernelTLS().this_proc_id ); 414 415 ready_schedule_lock(); 416 $thread * thrd = pop_slow( this ); 417 ready_schedule_unlock(); 418 419 /* paranoid */ verify( kernelTLS().this_proc_id ); 420 /* paranoid */ verify( ! __preemption_enabled() ); 421 return thrd; 582 422 } 583 423 … … 585 425 if( !thrd ) return; 586 426 587 disable_interrupts(); 588 static_assert(sizeof(thrd->state) == sizeof(int)); 589 enum coroutine_state old_state = __atomic_exchange_n(&thrd->state, Rerun, __ATOMIC_SEQ_CST); 590 switch(old_state) { 591 case Active: 427 int old_ticket = __atomic_fetch_add(&thrd->ticket, 1, __ATOMIC_SEQ_CST); 428 switch(old_ticket) { 429 case TICKET_RUNNING: 592 430 // Wake won the race, the thread will reschedule/rerun itself 593 431 break; 594 case Inactive:432 case TICKET_BLOCKED: 595 433 /* paranoid */ verify( ! thrd->preempted != __NO_PREEMPTION ); 596 597 // Wake lost the race, 598 thrd->state = Inactive; 599 __schedule_thread( thrd ); 434 /* paranoid */ verify( thrd->state == Blocked ); 435 436 { 437 /* paranoid */ verify( publicTLS_get(this_proc_id) ); 438 bool full = publicTLS_get(this_proc_id)->full_proc; 439 if(full) disable_interrupts(); 440 441 /* paranoid */ verify( ! __preemption_enabled() ); 442 443 // Wake lost the race, 444 __schedule_thread( thrd ); 445 446 /* paranoid */ verify( ! __preemption_enabled() ); 447 448 if(full) enable_interrupts( __cfaabi_dbg_ctx ); 449 /* paranoid */ verify( publicTLS_get(this_proc_id) ); 450 } 451 600 452 break; 601 case Rerun:602 abort("More than one thread attempted to schedule thread %p\n", thrd);603 break;604 case Halted:605 case Start:606 case Primed:607 453 default: 608 454 // This makes no sense, something is wrong abort 609 abort(); 610 } 455 abort("Thread %p (%s) has mismatch park/unpark\n", thrd, thrd->self_cor.name); 456 } 457 } 458 459 void park( void ) { 460 /* paranoid */ verify( __preemption_enabled() ); 461 disable_interrupts(); 462 /* paranoid */ verify( ! __preemption_enabled() ); 463 /* paranoid */ verify( kernelTLS().this_thread->preempted == __NO_PREEMPTION ); 464 465 returnToKernel(); 466 467 /* paranoid */ verify( ! __preemption_enabled() ); 611 468 enable_interrupts( __cfaabi_dbg_ctx ); 612 } 613 614 void park( void ) { 615 /* paranoid */ verify( kernelTLS.preemption_state.enabled ); 616 disable_interrupts(); 617 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled ); 618 /* paranoid */ verify( kernelTLS.this_thread->preempted == __NO_PREEMPTION ); 619 620 returnToKernel(); 621 622 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled ); 623 enable_interrupts( __cfaabi_dbg_ctx ); 624 /* paranoid */ verify( kernelTLS.preemption_state.enabled ); 625 626 } 627 628 // KERNEL ONLY 629 void __leave_thread() { 630 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled ); 631 returnToKernel(); 632 abort(); 469 /* paranoid */ verify( __preemption_enabled() ); 470 471 } 472 473 extern "C" { 474 // Leave the thread monitor 475 // last routine called by a thread. 476 // Should never return 477 void __cfactx_thrd_leave() { 478 $thread * thrd = active_thread(); 479 $monitor * this = &thrd->self_mon; 480 481 // Lock the monitor now 482 lock( this->lock __cfaabi_dbg_ctx2 ); 483 484 disable_interrupts(); 485 486 /* paranoid */ verify( ! __preemption_enabled() ); 487 /* paranoid */ verify( thrd->state == Active ); 488 /* paranoid */ verify( 0x0D15EA5E0D15EA5Ep == thrd->canary ); 489 /* paranoid */ verify( kernelTLS().this_thread == thrd ); 490 /* paranoid */ verify( thrd->context.SP ); 491 /* paranoid */ verifyf( ((uintptr_t)thrd->context.SP) > ((uintptr_t)__get_stack(thrd->curr_cor)->limit), "ERROR : $thread %p has been corrupted.\n StackPointer too large.\n", thrd ); 492 /* paranoid */ verifyf( ((uintptr_t)thrd->context.SP) < ((uintptr_t)__get_stack(thrd->curr_cor)->base ), "ERROR : $thread %p has been corrupted.\n StackPointer too small.\n", thrd ); 493 494 thrd->state = Halting; 495 if( TICKET_RUNNING != thrd->ticket ) { abort( "Thread terminated with pending unpark" ); } 496 if( thrd != this->owner ) { abort( "Thread internal monitor has incorrect owner" ); } 497 if( this->recursion != 1) { abort( "Thread internal monitor has unbalanced recursion" ); } 498 499 // Leave the thread 500 returnToKernel(); 501 502 // Control flow should never reach here! 503 abort(); 504 } 633 505 } 634 506 635 507 // KERNEL ONLY 636 508 bool force_yield( __Preemption_Reason reason ) { 637 /* paranoid */ verify( kernelTLS.preemption_state.enabled);509 /* paranoid */ verify( __preemption_enabled() ); 638 510 disable_interrupts(); 639 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled);640 641 $thread * thrd = kernelTLS .this_thread;642 /* paranoid */ verify(thrd->state == Active || thrd->state == Rerun);511 /* paranoid */ verify( ! __preemption_enabled() ); 512 513 $thread * thrd = kernelTLS().this_thread; 514 /* paranoid */ verify(thrd->state == Active); 643 515 644 516 // SKULLDUGGERY: It is possible that we are preempting this thread just before … … 647 519 // If that is the case, abandon the preemption. 648 520 bool preempted = false; 649 if(thrd-> next == 0p) {521 if(thrd->link.next == 0p) { 650 522 preempted = true; 651 523 thrd->preempted = reason; … … 653 525 } 654 526 655 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled);527 /* paranoid */ verify( ! __preemption_enabled() ); 656 528 enable_interrupts_noPoll(); 657 /* paranoid */ verify( kernelTLS.preemption_state.enabled);529 /* paranoid */ verify( __preemption_enabled() ); 658 530 659 531 return preempted; … … 661 533 662 534 //============================================================================================= 663 // Kernel Setup logic535 // Kernel Idle Sleep 664 536 //============================================================================================= 665 //----------------------------------------------------------------------------- 666 // Kernel boot procedures 667 static void __kernel_startup(void) { 668 verify( ! kernelTLS.preemption_state.enabled ); 669 __cfaabi_dbg_print_safe("Kernel : Starting\n"); 670 671 __page_size = sysconf( _SC_PAGESIZE ); 672 673 __cfa_dbg_global_clusters.list{ __get }; 674 __cfa_dbg_global_clusters.lock{}; 675 676 // Initialize the main cluster 677 mainCluster = (cluster *)&storage_mainCluster; 678 (*mainCluster){"Main Cluster"}; 679 680 __cfaabi_dbg_print_safe("Kernel : Main cluster ready\n"); 681 682 // Start by initializing the main thread 683 // SKULLDUGGERY: the mainThread steals the process main thread 684 // which will then be scheduled by the mainProcessor normally 685 mainThread = ($thread *)&storage_mainThread; 686 current_stack_info_t info; 687 info.storage = (__stack_t*)&storage_mainThreadCtx; 688 (*mainThread){ &info }; 689 690 __cfaabi_dbg_print_safe("Kernel : Main thread ready\n"); 691 692 693 694 // Construct the processor context of the main processor 695 void ?{}(processorCtx_t & this, processor * proc) { 696 (this.__cor){ "Processor" }; 697 this.__cor.starter = 0p; 698 this.proc = proc; 699 } 700 701 void ?{}(processor & this) with( this ) { 702 name = "Main Processor"; 703 cltr = mainCluster; 704 terminated{ 0 }; 705 do_terminate = false; 706 preemption_alarm = 0p; 707 pending_preemption = false; 708 kernel_thread = pthread_self(); 709 710 runner{ &this }; 711 __cfaabi_dbg_print_safe("Kernel : constructed main processor context %p\n", &runner); 712 } 713 714 // Initialize the main processor and the main processor ctx 715 // (the coroutine that contains the processing control flow) 716 mainProcessor = (processor *)&storage_mainProcessor; 717 (*mainProcessor){}; 718 719 //initialize the global state variables 720 kernelTLS.this_processor = mainProcessor; 721 kernelTLS.this_thread = mainThread; 722 723 // Enable preemption 724 kernel_start_preemption(); 725 726 // Add the main thread to the ready queue 727 // once resume is called on mainProcessor->runner the mainThread needs to be scheduled like any normal thread 728 __schedule_thread(mainThread); 729 730 // SKULLDUGGERY: Force a context switch to the main processor to set the main thread's context to the current UNIX 731 // context. Hence, the main thread does not begin through __cfactx_invoke_thread, like all other threads. The trick here is that 732 // mainThread is on the ready queue when this call is made. 733 __kernel_first_resume( kernelTLS.this_processor ); 734 735 736 737 // THE SYSTEM IS NOW COMPLETELY RUNNING 738 __cfaabi_dbg_print_safe("Kernel : Started\n--------------------------------------------------\n\n"); 739 740 verify( ! kernelTLS.preemption_state.enabled ); 537 // Wake a thread from the front if there are any 538 static void __wake_one(cluster * this) { 539 /* paranoid */ verify( ! __preemption_enabled() ); 540 /* paranoid */ verify( ready_schedule_islocked() ); 541 542 // Check if there is a sleeping processor 543 processor * p; 544 unsigned idle; 545 unsigned total; 546 [idle, total, p] = query(this->idles); 547 548 // If no one is sleeping, we are done 549 if( idle == 0 ) return; 550 551 // We found a processor, wake it up 552 post( p->idle ); 553 554 #if !defined(__CFA_NO_STATISTICS__) 555 __tls_stats()->ready.sleep.wakes++; 556 #endif 557 558 /* paranoid */ verify( ready_schedule_islocked() ); 559 /* paranoid */ verify( ! __preemption_enabled() ); 560 561 return; 562 } 563 564 // Unconditionnaly wake a thread 565 void __wake_proc(processor * this) { 566 __cfadbg_print_safe(runtime_core, "Kernel : waking Processor %p\n", this); 567 568 disable_interrupts(); 569 /* paranoid */ verify( ! __preemption_enabled() ); 570 post( this->idle ); 741 571 enable_interrupts( __cfaabi_dbg_ctx ); 742 verify( TL_GET( preemption_state.enabled ) ); 743 } 744 745 static void __kernel_shutdown(void) { 746 __cfaabi_dbg_print_safe("\n--------------------------------------------------\nKernel : Shutting down\n"); 747 748 verify( TL_GET( preemption_state.enabled ) ); 749 disable_interrupts(); 750 verify( ! kernelTLS.preemption_state.enabled ); 751 752 // SKULLDUGGERY: Notify the mainProcessor it needs to terminates. 753 // When its coroutine terminates, it return control to the mainThread 754 // which is currently here 755 __atomic_store_n(&mainProcessor->do_terminate, true, __ATOMIC_RELEASE); 756 __kernel_last_resume( kernelTLS.this_processor ); 757 mainThread->self_cor.state = Halted; 758 759 // THE SYSTEM IS NOW COMPLETELY STOPPED 760 761 // Disable preemption 762 kernel_stop_preemption(); 763 764 // Destroy the main processor and its context in reverse order of construction 765 // These were manually constructed so we need manually destroy them 766 ^(mainProcessor->runner){}; 767 ^(mainProcessor){}; 768 769 // Final step, destroy the main thread since it is no longer needed 770 // Since we provided a stack to this taxk it will not destroy anything 771 ^(mainThread){}; 772 773 ^(__cfa_dbg_global_clusters.list){}; 774 ^(__cfa_dbg_global_clusters.lock){}; 775 776 __cfaabi_dbg_print_safe("Kernel : Shutdown complete\n"); 777 } 778 779 //============================================================================================= 780 // Kernel Quiescing 781 //============================================================================================= 782 static void __halt(processor * this) with( *this ) { 783 // verify( ! __atomic_load_n(&do_terminate, __ATOMIC_SEQ_CST) ); 784 785 with( *cltr ) { 786 lock (proc_list_lock __cfaabi_dbg_ctx2); 787 remove (procs, *this); 788 push_front(idles, *this); 789 unlock (proc_list_lock); 790 } 791 792 __cfaabi_dbg_print_safe("Kernel : Processor %p ready to sleep\n", this); 793 794 wait( idleLock ); 795 796 __cfaabi_dbg_print_safe("Kernel : Processor %p woke up and ready to run\n", this); 797 798 with( *cltr ) { 799 lock (proc_list_lock __cfaabi_dbg_ctx2); 800 remove (idles, *this); 801 push_front(procs, *this); 802 unlock (proc_list_lock); 572 } 573 574 static void push (__cluster_idles & this, processor & proc) { 575 /* paranoid */ verify( ! __preemption_enabled() ); 576 lock( this ); 577 this.idle++; 578 /* paranoid */ verify( this.idle <= this.total ); 579 580 insert_first(this.list, proc); 581 unlock( this ); 582 /* paranoid */ verify( ! __preemption_enabled() ); 583 } 584 585 static void remove(__cluster_idles & this, processor & proc) { 586 /* paranoid */ verify( ! __preemption_enabled() ); 587 lock( this ); 588 this.idle--; 589 /* paranoid */ verify( this.idle >= 0 ); 590 591 remove(proc); 592 unlock( this ); 593 /* paranoid */ verify( ! __preemption_enabled() ); 594 } 595 596 static [unsigned idle, unsigned total, * processor] query( & __cluster_idles this ) { 597 for() { 598 uint64_t l = __atomic_load_n(&this.lock, __ATOMIC_SEQ_CST); 599 if( 1 == (l % 2) ) { Pause(); continue; } 600 unsigned idle = this.idle; 601 unsigned total = this.total; 602 processor * proc = &this.list`first; 603 // Compiler fence is unnecessary, but gcc-8 and older incorrectly reorder code without it 604 asm volatile("": : :"memory"); 605 if(l != __atomic_load_n(&this.lock, __ATOMIC_SEQ_CST)) { Pause(); continue; } 606 return [idle, total, proc]; 803 607 } 804 608 } … … 814 618 // the globalAbort flag is true. 815 619 lock( kernel_abort_lock __cfaabi_dbg_ctx2 ); 620 621 // disable interrupts, it no longer makes sense to try to interrupt this processor 622 disable_interrupts(); 816 623 817 624 // first task to abort ? … … 831 638 } 832 639 833 return kernelTLS.this_thread;640 return __cfaabi_tls.this_thread; 834 641 } 835 642 836 643 void kernel_abort_msg( void * kernel_data, char * abort_text, int abort_text_size ) { 837 $thread * thrd = kernel_data;644 $thread * thrd = ( $thread * ) kernel_data; 838 645 839 646 if(thrd) { … … 856 663 857 664 int kernel_abort_lastframe( void ) __attribute__ ((__nothrow__)) { 858 return get_coroutine(kernelTLS .this_thread) == get_coroutine(mainThread) ? 4 : 2;665 return get_coroutine(kernelTLS().this_thread) == get_coroutine(mainThread) ? 4 : 2; 859 666 } 860 667 … … 883 690 void ^?{}(semaphore & this) {} 884 691 885 voidP(semaphore & this) with( this ){692 bool P(semaphore & this) with( this ){ 886 693 lock( lock __cfaabi_dbg_ctx2 ); 887 694 count -= 1; 888 695 if ( count < 0 ) { 889 696 // queue current task 890 append( waiting, kernelTLS.this_thread);697 append( waiting, active_thread() ); 891 698 892 699 // atomically release spin lock and block 893 700 unlock( lock ); 894 701 park(); 702 return true; 895 703 } 896 704 else { 897 705 unlock( lock ); 898 } 899 } 900 901 void V(semaphore & this) with( this ) { 706 return false; 707 } 708 } 709 710 bool V(semaphore & this) with( this ) { 902 711 $thread * thrd = 0p; 903 712 lock( lock __cfaabi_dbg_ctx2 ); … … 912 721 // make new owner 913 722 unpark( thrd ); 914 } 915 916 //----------------------------------------------------------------------------- 917 // Global Queues 918 void doregister( cluster & cltr ) { 919 lock ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2); 920 push_front( __cfa_dbg_global_clusters.list, cltr ); 921 unlock ( __cfa_dbg_global_clusters.lock ); 922 } 923 924 void unregister( cluster & cltr ) { 925 lock ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2); 926 remove( __cfa_dbg_global_clusters.list, cltr ); 927 unlock( __cfa_dbg_global_clusters.lock ); 928 } 929 930 void doregister( cluster * cltr, $thread & thrd ) { 931 lock (cltr->thread_list_lock __cfaabi_dbg_ctx2); 932 cltr->nthreads += 1; 933 push_front(cltr->threads, thrd); 934 unlock (cltr->thread_list_lock); 935 } 936 937 void unregister( cluster * cltr, $thread & thrd ) { 938 lock (cltr->thread_list_lock __cfaabi_dbg_ctx2); 939 remove(cltr->threads, thrd ); 940 cltr->nthreads -= 1; 941 unlock(cltr->thread_list_lock); 942 } 943 944 void doregister( cluster * cltr, processor * proc ) { 945 lock (cltr->proc_list_lock __cfaabi_dbg_ctx2); 946 cltr->nprocessors += 1; 947 push_front(cltr->procs, *proc); 948 unlock (cltr->proc_list_lock); 949 } 950 951 void unregister( cluster * cltr, processor * proc ) { 952 lock (cltr->proc_list_lock __cfaabi_dbg_ctx2); 953 remove(cltr->procs, *proc ); 954 cltr->nprocessors -= 1; 955 unlock(cltr->proc_list_lock); 723 724 return thrd != 0p; 725 } 726 727 bool V(semaphore & this, unsigned diff) with( this ) { 728 $thread * thrd = 0p; 729 lock( lock __cfaabi_dbg_ctx2 ); 730 int release = max(-count, (int)diff); 731 count += diff; 732 for(release) { 733 unpark( pop_head( waiting ) ); 734 } 735 736 unlock( lock ); 737 738 return thrd != 0p; 956 739 } 957 740 … … 960 743 __cfaabi_dbg_debug_do( 961 744 extern "C" { 962 void __cfaabi_dbg_record (__spinlock_t & this, const char prev_name[]) {745 void __cfaabi_dbg_record_lock(__spinlock_t & this, const char prev_name[]) { 963 746 this.prev_name = prev_name; 964 this.prev_thrd = kernelTLS .this_thread;747 this.prev_thrd = kernelTLS().this_thread; 965 748 } 966 749 } … … 972 755 return true; 973 756 } 757 758 //----------------------------------------------------------------------------- 759 // Statistics 760 #if !defined(__CFA_NO_STATISTICS__) 761 void print_halts( processor & this ) { 762 this.print_halts = true; 763 } 764 765 void print_stats_now( cluster & this, int flags ) { 766 __print_stats( this.stats, this.print_stats, "Cluster", this.name, (void*)&this ); 767 } 768 769 extern int __print_alarm_stats; 770 void print_alarm_stats() { 771 __print_alarm_stats = -1; 772 } 773 #endif 974 774 // Local Variables: // 975 775 // mode: c //
Note:
See TracChangeset
for help on using the changeset viewer.
