- File:
-
- 1 edited
-
libcfa/src/concurrency/kernel.cfa (modified) (40 diffs)
Legend:
- Unmodified
- Added
- Removed
-
libcfa/src/concurrency/kernel.cfa
rac2b598 r2026bb6 10 10 // Created On : Tue Jan 17 12:27:26 2017 11 11 // Last Modified By : Peter A. Buhr 12 // Last Modified On : T ue Feb 4 13:03:15 202013 // Update Count : 5812 // Last Modified On : Thu Jun 20 17:21:23 2019 13 // Update Count : 25 14 14 // 15 15 … … 26 26 #include <signal.h> 27 27 #include <unistd.h> 28 #include <limits.h> // PTHREAD_STACK_MIN29 #include <sys/mman.h> // mprotect30 28 } 31 29 … … 42 40 //----------------------------------------------------------------------------- 43 41 // Some assembly required 44 #if defined( __i386 )42 #if defined( __i386 ) 45 43 #define CtxGet( ctx ) \ 46 44 __asm__ volatile ( \ … … 110 108 //----------------------------------------------------------------------------- 111 109 //Start and stop routine for the kernel, declared first to make sure they run first 112 static void __kernel_startup (void)__attribute__(( constructor( STARTUP_PRIORITY_KERNEL ) ));113 static void __kernel_shutdown(void) __attribute__(( destructor ( STARTUP_PRIORITY_KERNEL ) ));110 static void kernel_startup(void) __attribute__(( constructor( STARTUP_PRIORITY_KERNEL ) )); 111 static void kernel_shutdown(void) __attribute__(( destructor ( STARTUP_PRIORITY_KERNEL ) )); 114 112 115 113 //----------------------------------------------------------------------------- … … 117 115 KERNEL_STORAGE(cluster, mainCluster); 118 116 KERNEL_STORAGE(processor, mainProcessor); 119 KERNEL_STORAGE( $thread, mainThread);117 KERNEL_STORAGE(thread_desc, mainThread); 120 118 KERNEL_STORAGE(__stack_t, mainThreadCtx); 121 119 122 120 cluster * mainCluster; 123 121 processor * mainProcessor; 124 $thread* mainThread;122 thread_desc * mainThread; 125 123 126 124 extern "C" { 127 125 struct { __dllist_t(cluster) list; __spinlock_t lock; } __cfa_dbg_global_clusters; 128 126 } 129 127 … … 133 131 // Global state 134 132 thread_local struct KernelThreadData kernelTLS __attribute__ ((tls_model ( "initial-exec" ))) = { 135 NULL, // cannot use 0p136 133 NULL, 137 { 1, false, false },138 6u //this should be seeded better but due to a bug calling rdtsc doesn't work134 NULL, 135 { 1, false, false } 139 136 }; 140 137 … … 142 139 // Struct to steal stack 143 140 struct current_stack_info_t { 144 __stack_t * storage; 145 void * base;// base of stack146 void * limit;// stack grows towards stack limit147 void * context;// address of cfa_context_t141 __stack_t * storage; // pointer to stack object 142 void *base; // base of stack 143 void *limit; // stack grows towards stack limit 144 void *context; // address of cfa_context_t 148 145 }; 149 146 … … 164 161 // Main thread construction 165 162 166 void ?{}( $coroutine& this, current_stack_info_t * info) with( this ) {163 void ?{}( coroutine_desc & this, current_stack_info_t * info) with( this ) { 167 164 stack.storage = info->storage; 168 165 with(*stack.storage) { … … 174 171 name = "Main Thread"; 175 172 state = Start; 176 starter = 0p;177 last = 0p;178 cancellation = 0p;179 } 180 181 void ?{}( $thread& this, current_stack_info_t * info) with( this ) {173 starter = NULL; 174 last = NULL; 175 cancellation = NULL; 176 } 177 178 void ?{}( thread_desc & this, current_stack_info_t * info) with( this ) { 182 179 state = Start; 183 180 self_cor{ info }; … … 187 184 self_mon.recursion = 1; 188 185 self_mon_p = &self_mon; 189 next = 0p;190 191 node.next = 0p;192 node.prev = 0p;186 next = NULL; 187 188 node.next = NULL; 189 node.prev = NULL; 193 190 doregister(curr_cluster, this); 194 191 … … 208 205 } 209 206 210 static void * __invoke_processor(void * arg); 211 212 void ?{}(processor & this, const char name[], cluster & cltr) with( this ) { 207 static void start(processor * this); 208 void ?{}(processor & this, const char * name, cluster & cltr) with( this ) { 213 209 this.name = name; 214 210 this.cltr = &cltr; 215 211 terminated{ 0 }; 216 destroyer = 0p;217 212 do_terminate = false; 218 preemption_alarm = 0p;213 preemption_alarm = NULL; 219 214 pending_preemption = false; 220 215 runner.proc = &this; … … 222 217 idleLock{}; 223 218 224 __cfaabi_dbg_print_safe("Kernel : Starting core %p\n", &this); 225 226 this.stack = __create_pthread( &this.kernel_thread, __invoke_processor, (void *)&this ); 227 228 __cfaabi_dbg_print_safe("Kernel : core %p started\n", &this); 219 start( &this ); 229 220 } 230 221 … … 240 231 } 241 232 242 pthread_join( kernel_thread, 0p ); 243 free( this.stack ); 244 } 245 246 void ?{}(cluster & this, const char name[], Duration preemption_rate) with( this ) { 233 pthread_join( kernel_thread, NULL ); 234 } 235 236 void ?{}(cluster & this, const char * name, Duration preemption_rate) with( this ) { 247 237 this.name = name; 248 238 this.preemption_rate = preemption_rate; … … 264 254 // Kernel Scheduling logic 265 255 //============================================================================================= 266 static $thread * __next_thread(cluster * this);267 static void __run_thread(processor * this, $thread * dst);268 static void __halt(processor * this);256 static void runThread(processor * this, thread_desc * dst); 257 static void finishRunning(processor * this); 258 static void halt(processor * this); 269 259 270 260 //Main of the processor contexts 271 261 void main(processorCtx_t & runner) { 272 // Because of a bug, we couldn't initialized the seed on construction273 // Do it here274 kernelTLS.rand_seed ^= rdtscl();275 276 262 processor * this = runner.proc; 277 263 verify(this); … … 287 273 __cfaabi_dbg_print_safe("Kernel : core %p started\n", this); 288 274 289 $thread * readyThread = 0p; 290 for( unsigned int spin_count = 0; ! __atomic_load_n(&this->do_terminate, __ATOMIC_SEQ_CST); spin_count++ ) { 291 readyThread = __next_thread( this->cltr ); 292 293 if(readyThread) { 294 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled ); 295 /* paranoid */ verifyf( readyThread->state == Inactive || readyThread->state == Start || readyThread->preempted != __NO_PREEMPTION, "state : %d, preempted %d\n", readyThread->state, readyThread->preempted); 296 /* paranoid */ verifyf( readyThread->next == 0p, "Expected null got %p", readyThread->next ); 297 298 __run_thread(this, readyThread); 299 300 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled ); 275 thread_desc * readyThread = NULL; 276 for( unsigned int spin_count = 0; ! __atomic_load_n(&this->do_terminate, __ATOMIC_SEQ_CST); spin_count++ ) 277 { 278 readyThread = nextThread( this->cltr ); 279 280 if(readyThread) 281 { 282 verify( ! kernelTLS.preemption_state.enabled ); 283 284 runThread(this, readyThread); 285 286 verify( ! kernelTLS.preemption_state.enabled ); 287 288 //Some actions need to be taken from the kernel 289 finishRunning(this); 301 290 302 291 spin_count = 0; 303 } else { 292 } 293 else 294 { 304 295 // spin(this, &spin_count); 305 __halt(this);296 halt(this); 306 297 } 307 298 } … … 323 314 // runThread runs a thread by context switching 324 315 // from the processor coroutine to the target thread 325 static void __run_thread(processor * this, $thread * thrd_dst) { 326 $coroutine * proc_cor = get_coroutine(this->runner); 316 static void runThread(processor * this, thread_desc * thrd_dst) { 317 coroutine_desc * proc_cor = get_coroutine(this->runner); 318 319 // Reset the terminating actions here 320 this->finish.action_code = No_Action; 327 321 328 322 // Update global state 329 323 kernelTLS.this_thread = thrd_dst; 330 324 331 // set state of processor coroutine to inactive 332 verify(proc_cor->state == Active); 333 proc_cor->state = Inactive; 334 335 // Actually run the thread 336 RUNNING: while(true) { 337 if(unlikely(thrd_dst->preempted)) { 338 thrd_dst->preempted = __NO_PREEMPTION; 339 verify(thrd_dst->state == Active || thrd_dst->state == Rerun); 340 } else { 341 verify(thrd_dst->state == Start || thrd_dst->state == Primed || thrd_dst->state == Inactive); 342 thrd_dst->state = Active; 325 // set state of processor coroutine to inactive and the thread to active 326 proc_cor->state = proc_cor->state == Halted ? Halted : Inactive; 327 thrd_dst->state = Active; 328 329 // set context switch to the thread that the processor is executing 330 verify( thrd_dst->context.SP ); 331 CtxSwitch( &proc_cor->context, &thrd_dst->context ); 332 // when CtxSwitch returns we are back in the processor coroutine 333 334 // set state of processor coroutine to active and the thread to inactive 335 thrd_dst->state = thrd_dst->state == Halted ? Halted : Inactive; 336 proc_cor->state = Active; 337 } 338 339 // KERNEL_ONLY 340 static void returnToKernel() { 341 coroutine_desc * proc_cor = get_coroutine(kernelTLS.this_processor->runner); 342 thread_desc * thrd_src = kernelTLS.this_thread; 343 344 // set state of current coroutine to inactive 345 thrd_src->state = thrd_src->state == Halted ? Halted : Inactive; 346 proc_cor->state = Active; 347 int local_errno = *__volatile_errno(); 348 #if defined( __i386 ) || defined( __x86_64 ) 349 __x87_store; 350 #endif 351 352 // set new coroutine that the processor is executing 353 // and context switch to it 354 verify( proc_cor->context.SP ); 355 CtxSwitch( &thrd_src->context, &proc_cor->context ); 356 357 // set state of new coroutine to active 358 proc_cor->state = proc_cor->state == Halted ? Halted : Inactive; 359 thrd_src->state = Active; 360 361 #if defined( __i386 ) || defined( __x86_64 ) 362 __x87_load; 363 #endif 364 *__volatile_errno() = local_errno; 365 } 366 367 // KERNEL_ONLY 368 // Once a thread has finished running, some of 369 // its final actions must be executed from the kernel 370 static void finishRunning(processor * this) with( this->finish ) { 371 verify( ! kernelTLS.preemption_state.enabled ); 372 choose( action_code ) { 373 case No_Action: 374 break; 375 case Release: 376 unlock( *lock ); 377 case Schedule: 378 ScheduleThread( thrd ); 379 case Release_Schedule: 380 unlock( *lock ); 381 ScheduleThread( thrd ); 382 case Release_Multi: 383 for(int i = 0; i < lock_count; i++) { 384 unlock( *locks[i] ); 343 385 } 344 345 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled ); 346 347 // set context switch to the thread that the processor is executing 348 verify( thrd_dst->context.SP ); 349 __cfactx_switch( &proc_cor->context, &thrd_dst->context ); 350 // when __cfactx_switch returns we are back in the processor coroutine 351 352 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled ); 353 354 355 // We just finished running a thread, there are a few things that could have happened. 356 // 1 - Regular case : the thread has blocked and now one has scheduled it yet. 357 // 2 - Racy case : the thread has blocked but someone has already tried to schedule it. 358 // 3 - Polite Racy case : the thread has blocked, someone has already tried to schedule it, but the thread is nice and wants to go through the ready-queue any way 359 // 4 - Preempted 360 // In case 1, we may have won a race so we can't write to the state again. 361 // In case 2, we lost the race so we now own the thread. 362 // In case 3, we lost the race but can just reschedule the thread. 363 364 if(unlikely(thrd_dst->preempted != __NO_PREEMPTION)) { 365 // The thread was preempted, reschedule it and reset the flag 366 __schedule_thread( thrd_dst ); 367 break RUNNING; 386 case Release_Multi_Schedule: 387 for(int i = 0; i < lock_count; i++) { 388 unlock( *locks[i] ); 368 389 } 369 370 // set state of processor coroutine to active and the thread to inactive 371 static_assert(sizeof(thrd_dst->state) == sizeof(int)); 372 enum coroutine_state old_state = __atomic_exchange_n(&thrd_dst->state, Inactive, __ATOMIC_SEQ_CST); 373 switch(old_state) { 374 case Halted: 375 // The thread has halted, it should never be scheduled/run again, leave it back to Halted and move on 376 thrd_dst->state = Halted; 377 378 // We may need to wake someone up here since 379 unpark( this->destroyer ); 380 this->destroyer = 0p; 381 break RUNNING; 382 case Active: 383 // This is case 1, the regular case, nothing more is needed 384 break RUNNING; 385 case Rerun: 386 // This is case 2, the racy case, someone tried to run this thread before it finished blocking 387 // In this case, just run it again. 388 continue RUNNING; 389 default: 390 // This makes no sense, something is wrong abort 391 abort("Finished running a thread that was Inactive/Start/Primed %d\n", old_state); 390 for(int i = 0; i < thrd_count; i++) { 391 ScheduleThread( thrds[i] ); 392 392 } 393 } 394 395 // Just before returning to the processor, set the processor coroutine to active 396 proc_cor->state = Active; 397 } 398 399 // KERNEL_ONLY 400 void returnToKernel() { 401 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled ); 402 $coroutine * proc_cor = get_coroutine(kernelTLS.this_processor->runner); 403 $thread * thrd_src = kernelTLS.this_thread; 404 405 // Run the thread on this processor 406 { 407 int local_errno = *__volatile_errno(); 408 #if defined( __i386 ) || defined( __x86_64 ) 409 __x87_store; 410 #endif 411 verify( proc_cor->context.SP ); 412 __cfactx_switch( &thrd_src->context, &proc_cor->context ); 413 #if defined( __i386 ) || defined( __x86_64 ) 414 __x87_load; 415 #endif 416 *__volatile_errno() = local_errno; 417 } 418 419 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled ); 393 case Callback: 394 callback(); 395 default: 396 abort("KERNEL ERROR: Unexpected action to run after thread"); 397 } 420 398 } 421 399 … … 424 402 // This is the entry point for processors (kernel threads) 425 403 // It effectively constructs a coroutine by stealing the pthread stack 426 static void * __invoke_processor(void * arg) {404 static void * CtxInvokeProcessor(void * arg) { 427 405 processor * proc = (processor *) arg; 428 406 kernelTLS.this_processor = proc; 429 kernelTLS.this_thread = 0p;407 kernelTLS.this_thread = NULL; 430 408 kernelTLS.preemption_state.[enabled, disable_count] = [false, 1]; 431 409 // SKULLDUGGERY: We want to create a context for the processor coroutine … … 440 418 441 419 //Set global state 442 kernelTLS.this_thread = 0p;420 kernelTLS.this_thread = NULL; 443 421 444 422 //We now have a proper context from which to schedule threads … … 456 434 __cfaabi_dbg_print_safe("Kernel : core %p main ended (%p)\n", proc, &proc->runner); 457 435 458 return 0p; 459 } 460 461 static void Abort( int ret, const char func[] ) { 462 if ( ret ) { // pthread routines return errno values 463 abort( "%s : internal error, error(%d) %s.", func, ret, strerror( ret ) ); 464 } // if 465 } // Abort 466 467 void * __create_pthread( pthread_t * pthread, void * (*start)(void *), void * arg ) { 468 pthread_attr_t attr; 469 470 Abort( pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute 471 472 size_t stacksize; 473 // default stack size, normally defined by shell limit 474 Abort( pthread_attr_getstacksize( &attr, &stacksize ), "pthread_attr_getstacksize" ); 475 assert( stacksize >= PTHREAD_STACK_MIN ); 476 477 void * stack; 478 __cfaabi_dbg_debug_do( 479 stack = memalign( __page_size, stacksize + __page_size ); 480 // pthread has no mechanism to create the guard page in user supplied stack. 481 if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) { 482 abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) ); 483 } // if 484 ); 485 __cfaabi_dbg_no_debug_do( 486 stack = malloc( stacksize ); 487 ); 488 489 Abort( pthread_attr_setstack( &attr, stack, stacksize ), "pthread_attr_setstack" ); 490 491 Abort( pthread_create( pthread, &attr, start, arg ), "pthread_create" ); 492 return stack; 436 return NULL; 437 } 438 439 static void start(processor * this) { 440 __cfaabi_dbg_print_safe("Kernel : Starting core %p\n", this); 441 442 pthread_create( &this->kernel_thread, NULL, CtxInvokeProcessor, (void*)this ); 443 444 __cfaabi_dbg_print_safe("Kernel : core %p started\n", this); 493 445 } 494 446 495 447 // KERNEL_ONLY 496 static void __kernel_first_resume( processor * this ) { 497 $thread * src = mainThread; 498 $coroutine * dst = get_coroutine(this->runner); 499 500 verify( ! kernelTLS.preemption_state.enabled ); 501 502 kernelTLS.this_thread->curr_cor = dst; 448 void kernel_first_resume( processor * this ) { 449 thread_desc * src = mainThread; 450 coroutine_desc * dst = get_coroutine(this->runner); 451 452 verify( ! kernelTLS.preemption_state.enabled ); 453 503 454 __stack_prepare( &dst->stack, 65000 ); 504 __cfactx_start(main, dst, this->runner, __cfactx_invoke_coroutine);455 CtxStart(&this->runner, CtxInvokeCoroutine); 505 456 506 457 verify( ! kernelTLS.preemption_state.enabled ); … … 514 465 // context switch to specified coroutine 515 466 verify( dst->context.SP ); 516 __cfactx_switch( &src->context, &dst->context ); 517 // when __cfactx_switch returns we are back in the src coroutine 518 519 mainThread->curr_cor = &mainThread->self_cor; 467 CtxSwitch( &src->context, &dst->context ); 468 // when CtxSwitch returns we are back in the src coroutine 520 469 521 470 // set state of new coroutine to active … … 526 475 527 476 // KERNEL_ONLY 528 static void __kernel_last_resume( processor * this ) {529 $coroutine* src = &mainThread->self_cor;530 $coroutine* dst = get_coroutine(this->runner);477 void kernel_last_resume( processor * this ) { 478 coroutine_desc * src = &mainThread->self_cor; 479 coroutine_desc * dst = get_coroutine(this->runner); 531 480 532 481 verify( ! kernelTLS.preemption_state.enabled ); … … 535 484 536 485 // context switch to the processor 537 __cfactx_switch( &src->context, &dst->context );486 CtxSwitch( &src->context, &dst->context ); 538 487 } 539 488 540 489 //----------------------------------------------------------------------------- 541 490 // Scheduler routines 491 542 492 // KERNEL ONLY 543 void __schedule_thread( $thread * thrd ) with( *thrd->curr_cluster ) { 544 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled ); 545 /* paranoid */ #if defined( __CFA_WITH_VERIFY__ ) 546 /* paranoid */ if( thrd->state == Inactive || thrd->state == Start ) assertf( thrd->preempted == __NO_PREEMPTION, 547 "Error inactive thread marked as preempted, state %d, preemption %d\n", thrd->state, thrd->preempted ); 548 /* paranoid */ if( thrd->preempted != __NO_PREEMPTION ) assertf(thrd->state == Active || thrd->state == Rerun, 549 "Error preempted thread marked as not currently running, state %d, preemption %d\n", thrd->state, thrd->preempted ); 550 /* paranoid */ #endif 551 /* paranoid */ verifyf( thrd->next == 0p, "Expected null got %p", thrd->next ); 552 553 lock ( ready_queue_lock __cfaabi_dbg_ctx2 ); 554 bool was_empty = !(ready_queue != 0); 555 append( ready_queue, thrd ); 493 void ScheduleThread( thread_desc * thrd ) { 494 verify( thrd ); 495 verify( thrd->state != Halted ); 496 497 verify( ! kernelTLS.preemption_state.enabled ); 498 499 verifyf( thrd->next == NULL, "Expected null got %p", thrd->next ); 500 501 with( *thrd->curr_cluster ) { 502 lock ( ready_queue_lock __cfaabi_dbg_ctx2 ); 503 bool was_empty = !(ready_queue != 0); 504 append( ready_queue, thrd ); 505 unlock( ready_queue_lock ); 506 507 if(was_empty) { 508 lock (proc_list_lock __cfaabi_dbg_ctx2); 509 if(idles) { 510 wake_fast(idles.head); 511 } 512 unlock (proc_list_lock); 513 } 514 else if( struct processor * idle = idles.head ) { 515 wake_fast(idle); 516 } 517 518 } 519 520 verify( ! kernelTLS.preemption_state.enabled ); 521 } 522 523 // KERNEL ONLY 524 thread_desc * nextThread(cluster * this) with( *this ) { 525 verify( ! kernelTLS.preemption_state.enabled ); 526 lock( ready_queue_lock __cfaabi_dbg_ctx2 ); 527 thread_desc * head = pop_head( ready_queue ); 556 528 unlock( ready_queue_lock ); 557 558 if(was_empty) { 559 lock (proc_list_lock __cfaabi_dbg_ctx2); 560 if(idles) { 561 wake_fast(idles.head); 562 } 563 unlock (proc_list_lock); 564 } 565 else if( struct processor * idle = idles.head ) { 566 wake_fast(idle); 567 } 568 569 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled ); 529 verify( ! kernelTLS.preemption_state.enabled ); 530 return head; 531 } 532 533 void BlockInternal() { 534 disable_interrupts(); 535 verify( ! kernelTLS.preemption_state.enabled ); 536 returnToKernel(); 537 verify( ! kernelTLS.preemption_state.enabled ); 538 enable_interrupts( __cfaabi_dbg_ctx ); 539 } 540 541 void BlockInternal( __spinlock_t * lock ) { 542 disable_interrupts(); 543 with( *kernelTLS.this_processor ) { 544 finish.action_code = Release; 545 finish.lock = lock; 546 } 547 548 verify( ! kernelTLS.preemption_state.enabled ); 549 returnToKernel(); 550 verify( ! kernelTLS.preemption_state.enabled ); 551 552 enable_interrupts( __cfaabi_dbg_ctx ); 553 } 554 555 void BlockInternal( thread_desc * thrd ) { 556 disable_interrupts(); 557 with( * kernelTLS.this_processor ) { 558 finish.action_code = Schedule; 559 finish.thrd = thrd; 560 } 561 562 verify( ! kernelTLS.preemption_state.enabled ); 563 returnToKernel(); 564 verify( ! kernelTLS.preemption_state.enabled ); 565 566 enable_interrupts( __cfaabi_dbg_ctx ); 567 } 568 569 void BlockInternal( __spinlock_t * lock, thread_desc * thrd ) { 570 assert(thrd); 571 disable_interrupts(); 572 with( * kernelTLS.this_processor ) { 573 finish.action_code = Release_Schedule; 574 finish.lock = lock; 575 finish.thrd = thrd; 576 } 577 578 verify( ! kernelTLS.preemption_state.enabled ); 579 returnToKernel(); 580 verify( ! kernelTLS.preemption_state.enabled ); 581 582 enable_interrupts( __cfaabi_dbg_ctx ); 583 } 584 585 void BlockInternal(__spinlock_t * locks [], unsigned short count) { 586 disable_interrupts(); 587 with( * kernelTLS.this_processor ) { 588 finish.action_code = Release_Multi; 589 finish.locks = locks; 590 finish.lock_count = count; 591 } 592 593 verify( ! kernelTLS.preemption_state.enabled ); 594 returnToKernel(); 595 verify( ! kernelTLS.preemption_state.enabled ); 596 597 enable_interrupts( __cfaabi_dbg_ctx ); 598 } 599 600 void BlockInternal(__spinlock_t * locks [], unsigned short lock_count, thread_desc * thrds [], unsigned short thrd_count) { 601 disable_interrupts(); 602 with( *kernelTLS.this_processor ) { 603 finish.action_code = Release_Multi_Schedule; 604 finish.locks = locks; 605 finish.lock_count = lock_count; 606 finish.thrds = thrds; 607 finish.thrd_count = thrd_count; 608 } 609 610 verify( ! kernelTLS.preemption_state.enabled ); 611 returnToKernel(); 612 verify( ! kernelTLS.preemption_state.enabled ); 613 614 enable_interrupts( __cfaabi_dbg_ctx ); 615 } 616 617 void BlockInternal(__finish_callback_fptr_t callback) { 618 disable_interrupts(); 619 with( *kernelTLS.this_processor ) { 620 finish.action_code = Callback; 621 finish.callback = callback; 622 } 623 624 verify( ! kernelTLS.preemption_state.enabled ); 625 returnToKernel(); 626 verify( ! kernelTLS.preemption_state.enabled ); 627 628 enable_interrupts( __cfaabi_dbg_ctx ); 570 629 } 571 630 572 631 // KERNEL ONLY 573 static $thread * __next_thread(cluster * this) with( *this ) { 574 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled ); 575 576 lock( ready_queue_lock __cfaabi_dbg_ctx2 ); 577 $thread * head = pop_head( ready_queue ); 578 unlock( ready_queue_lock ); 579 580 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled ); 581 return head; 582 } 583 584 void unpark( $thread * thrd ) { 585 if( !thrd ) return; 586 587 disable_interrupts(); 588 static_assert(sizeof(thrd->state) == sizeof(int)); 589 enum coroutine_state old_state = __atomic_exchange_n(&thrd->state, Rerun, __ATOMIC_SEQ_CST); 590 switch(old_state) { 591 case Active: 592 // Wake won the race, the thread will reschedule/rerun itself 593 break; 594 case Inactive: 595 /* paranoid */ verify( ! thrd->preempted != __NO_PREEMPTION ); 596 597 // Wake lost the race, 598 thrd->state = Inactive; 599 __schedule_thread( thrd ); 600 break; 601 case Rerun: 602 abort("More than one thread attempted to schedule thread %p\n", thrd); 603 break; 604 case Halted: 605 case Start: 606 case Primed: 607 default: 608 // This makes no sense, something is wrong abort 609 abort(); 610 } 611 enable_interrupts( __cfaabi_dbg_ctx ); 612 } 613 614 void park( void ) { 615 /* paranoid */ verify( kernelTLS.preemption_state.enabled ); 616 disable_interrupts(); 617 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled ); 618 /* paranoid */ verify( kernelTLS.this_thread->preempted == __NO_PREEMPTION ); 632 void LeaveThread(__spinlock_t * lock, thread_desc * thrd) { 633 verify( ! kernelTLS.preemption_state.enabled ); 634 with( * kernelTLS.this_processor ) { 635 finish.action_code = thrd ? Release_Schedule : Release; 636 finish.lock = lock; 637 finish.thrd = thrd; 638 } 619 639 620 640 returnToKernel(); 621 622 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );623 enable_interrupts( __cfaabi_dbg_ctx );624 /* paranoid */ verify( kernelTLS.preemption_state.enabled );625 626 }627 628 // KERNEL ONLY629 void __leave_thread() {630 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );631 returnToKernel();632 abort();633 }634 635 // KERNEL ONLY636 bool force_yield( __Preemption_Reason reason ) {637 /* paranoid */ verify( kernelTLS.preemption_state.enabled );638 disable_interrupts();639 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );640 641 $thread * thrd = kernelTLS.this_thread;642 /* paranoid */ verify(thrd->state == Active || thrd->state == Rerun);643 644 // SKULLDUGGERY: It is possible that we are preempting this thread just before645 // it was going to park itself. If that is the case and it is already using the646 // intrusive fields then we can't use them to preempt the thread647 // If that is the case, abandon the preemption.648 bool preempted = false;649 if(thrd->next == 0p) {650 preempted = true;651 thrd->preempted = reason;652 returnToKernel();653 }654 655 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );656 enable_interrupts_noPoll();657 /* paranoid */ verify( kernelTLS.preemption_state.enabled );658 659 return preempted;660 641 } 661 642 … … 665 646 //----------------------------------------------------------------------------- 666 647 // Kernel boot procedures 667 static void __kernel_startup(void) {648 static void kernel_startup(void) { 668 649 verify( ! kernelTLS.preemption_state.enabled ); 669 650 __cfaabi_dbg_print_safe("Kernel : Starting\n"); … … 683 664 // SKULLDUGGERY: the mainThread steals the process main thread 684 665 // which will then be scheduled by the mainProcessor normally 685 mainThread = ( $thread*)&storage_mainThread;666 mainThread = (thread_desc *)&storage_mainThread; 686 667 current_stack_info_t info; 687 668 info.storage = (__stack_t*)&storage_mainThreadCtx; … … 695 676 void ?{}(processorCtx_t & this, processor * proc) { 696 677 (this.__cor){ "Processor" }; 697 this.__cor.starter = 0p;678 this.__cor.starter = NULL; 698 679 this.proc = proc; 699 680 } … … 704 685 terminated{ 0 }; 705 686 do_terminate = false; 706 preemption_alarm = 0p;687 preemption_alarm = NULL; 707 688 pending_preemption = false; 708 689 kernel_thread = pthread_self(); … … 726 707 // Add the main thread to the ready queue 727 708 // once resume is called on mainProcessor->runner the mainThread needs to be scheduled like any normal thread 728 __schedule_thread(mainThread);709 ScheduleThread(mainThread); 729 710 730 711 // SKULLDUGGERY: Force a context switch to the main processor to set the main thread's context to the current UNIX 731 // context. Hence, the main thread does not begin through __cfactx_invoke_thread, like all other threads. The trick here is that712 // context. Hence, the main thread does not begin through CtxInvokeThread, like all other threads. The trick here is that 732 713 // mainThread is on the ready queue when this call is made. 733 __kernel_first_resume( kernelTLS.this_processor );714 kernel_first_resume( kernelTLS.this_processor ); 734 715 735 716 … … 743 724 } 744 725 745 static void __kernel_shutdown(void) {726 static void kernel_shutdown(void) { 746 727 __cfaabi_dbg_print_safe("\n--------------------------------------------------\nKernel : Shutting down\n"); 747 728 … … 754 735 // which is currently here 755 736 __atomic_store_n(&mainProcessor->do_terminate, true, __ATOMIC_RELEASE); 756 __kernel_last_resume( kernelTLS.this_processor );737 kernel_last_resume( kernelTLS.this_processor ); 757 738 mainThread->self_cor.state = Halted; 758 739 … … 780 761 // Kernel Quiescing 781 762 //============================================================================================= 782 static void __halt(processor * this) with( *this ) {763 static void halt(processor * this) with( *this ) { 783 764 // verify( ! __atomic_load_n(&do_terminate, __ATOMIC_SEQ_CST) ); 784 765 … … 822 803 sigemptyset( &mask ); 823 804 sigaddset( &mask, SIGALRM ); // block SIGALRM signals 824 sigaddset( &mask, SIGUSR1 ); // block SIGALRM signals 825 sigsuspend( &mask ); // block the processor to prevent further damage during abort 826 _exit( EXIT_FAILURE ); // if processor unblocks before it is killed, terminate it 805 sigsuspend( &mask ); // block the processor to prevent further damage during abort 806 _exit( EXIT_FAILURE ); // if processor unblocks before it is killed, terminate it 827 807 } 828 808 else { … … 835 815 836 816 void kernel_abort_msg( void * kernel_data, char * abort_text, int abort_text_size ) { 837 $thread* thrd = kernel_data;817 thread_desc * thrd = kernel_data; 838 818 839 819 if(thrd) { 840 820 int len = snprintf( abort_text, abort_text_size, "Error occurred while executing thread %.256s (%p)", thrd->self_cor.name, thrd ); 841 __cfaabi_ bits_write( STDERR_FILENO,abort_text, len );821 __cfaabi_dbg_bits_write( abort_text, len ); 842 822 843 823 if ( &thrd->self_cor != thrd->curr_cor ) { 844 824 len = snprintf( abort_text, abort_text_size, " in coroutine %.256s (%p).\n", thrd->curr_cor->name, thrd->curr_cor ); 845 __cfaabi_ bits_write( STDERR_FILENO,abort_text, len );825 __cfaabi_dbg_bits_write( abort_text, len ); 846 826 } 847 827 else { 848 __cfaabi_ bits_write( STDERR_FILENO,".\n", 2 );828 __cfaabi_dbg_bits_write( ".\n", 2 ); 849 829 } 850 830 } 851 831 else { 852 832 int len = snprintf( abort_text, abort_text_size, "Error occurred outside of any thread.\n" ); 853 __cfaabi_ bits_write( STDERR_FILENO,abort_text, len );833 __cfaabi_dbg_bits_write( abort_text, len ); 854 834 } 855 835 } … … 862 842 863 843 extern "C" { 864 void __cfaabi_ bits_acquire() {844 void __cfaabi_dbg_bits_acquire() { 865 845 lock( kernel_debug_lock __cfaabi_dbg_ctx2 ); 866 846 } 867 847 868 void __cfaabi_ bits_release() {848 void __cfaabi_dbg_bits_release() { 869 849 unlock( kernel_debug_lock ); 870 850 } … … 891 871 892 872 // atomically release spin lock and block 893 unlock( lock ); 894 park(); 873 BlockInternal( &lock ); 895 874 } 896 875 else { … … 900 879 901 880 void V(semaphore & this) with( this ) { 902 $thread * thrd = 0p;881 thread_desc * thrd = NULL; 903 882 lock( lock __cfaabi_dbg_ctx2 ); 904 883 count += 1; … … 911 890 912 891 // make new owner 913 unpark( thrd );892 WakeThread( thrd ); 914 893 } 915 894 … … 928 907 } 929 908 930 void doregister( cluster * cltr, $thread& thrd ) {909 void doregister( cluster * cltr, thread_desc & thrd ) { 931 910 lock (cltr->thread_list_lock __cfaabi_dbg_ctx2); 932 911 cltr->nthreads += 1; … … 935 914 } 936 915 937 void unregister( cluster * cltr, $thread& thrd ) {916 void unregister( cluster * cltr, thread_desc & thrd ) { 938 917 lock (cltr->thread_list_lock __cfaabi_dbg_ctx2); 939 918 remove(cltr->threads, thrd ); … … 960 939 __cfaabi_dbg_debug_do( 961 940 extern "C" { 962 void __cfaabi_dbg_record(__spinlock_t & this, const char prev_name[]) {941 void __cfaabi_dbg_record(__spinlock_t & this, const char * prev_name) { 963 942 this.prev_name = prev_name; 964 943 this.prev_thrd = kernelTLS.this_thread; … … 969 948 //----------------------------------------------------------------------------- 970 949 // Debug 971 bool threading_enabled(void) __attribute__((const)){950 bool threading_enabled(void) { 972 951 return true; 973 952 }
Note:
See TracChangeset
for help on using the changeset viewer.
