- File:
-
- 1 edited
-
libcfa/src/concurrency/kernel.cfa (modified) (14 diffs)
Legend:
- Unmodified
- Added
- Removed
-
libcfa/src/concurrency/kernel.cfa
r6011658 r09f357ec 10 10 // Created On : Tue Jan 17 12:27:26 2017 11 11 // Last Modified By : Peter A. Buhr 12 // Last Modified On : Mon Aug 31 07:08:20 202013 // Update Count : 7112 // Last Modified On : Thu Dec 5 16:25:52 2019 13 // Update Count : 52 14 14 // 15 15 16 16 #define __cforall_thread__ 17 // #define __CFA_DEBUG_PRINT_RUNTIME_CORE__18 17 19 18 //C Includes 19 #include <stddef.h> 20 20 #include <errno.h> 21 #include <string.h> 22 extern "C" { 21 23 #include <stdio.h> 24 #include <fenv.h> 25 #include <sys/resource.h> 22 26 #include <signal.h> 23 27 #include <unistd.h> 28 #include <limits.h> // PTHREAD_STACK_MIN 29 #include <sys/mman.h> // mprotect 30 } 24 31 25 32 //CFA Includes 33 #include "time.hfa" 26 34 #include "kernel_private.hfa" 27 35 #include "preemption.hfa" 36 #include "startup.hfa" 28 37 29 38 //Private includes … … 31 40 #include "invoke.h" 32 41 33 34 42 //----------------------------------------------------------------------------- 35 43 // Some assembly required 36 44 #if defined( __i386 ) 45 #define CtxGet( ctx ) \ 46 __asm__ volatile ( \ 47 "movl %%esp,%0\n"\ 48 "movl %%ebp,%1\n"\ 49 : "=rm" (ctx.SP),\ 50 "=rm" (ctx.FP) \ 51 ) 52 37 53 // mxcr : SSE Status and Control bits (control bits are preserved across function calls) 38 54 // fcw : X87 FPU control word (preserved across function calls) … … 56 72 57 73 #elif defined( __x86_64 ) 74 #define CtxGet( ctx ) \ 75 __asm__ volatile ( \ 76 "movq %%rsp,%0\n"\ 77 "movq %%rbp,%1\n"\ 78 : "=rm" (ctx.SP),\ 79 "=rm" (ctx.FP) \ 80 ) 81 58 82 #define __x87_store \ 59 83 uint32_t __mxcr; \ … … 74 98 ) 75 99 76 #elif defined( __arm__ ) 77 #define __x87_store 78 #define __x87_load 79 80 #elif defined( __aarch64__ ) 81 #define __x87_store \ 82 uint32_t __fpcntl[2]; \ 83 __asm__ volatile ( \ 84 "mrs x9, FPCR\n" \ 85 "mrs x10, FPSR\n" \ 86 "stp x9, x10, %0\n" \ 87 : "=m" (__fpcntl) : : "x9", "x10" \ 88 ) 89 90 #define __x87_load \ 91 __asm__ volatile ( \ 92 "ldp x9, x10, %0\n" \ 93 "msr FPSR, x10\n" \ 94 "msr FPCR, x9\n" \ 95 : "=m" (__fpcntl) : : "x9", "x10" \ 96 ) 97 100 101 #elif defined( __ARM_ARCH ) 102 #define CtxGet( ctx ) __asm__ ( \ 103 "mov %0,%%sp\n" \ 104 "mov %1,%%r11\n" \ 105 : "=rm" (ctx.SP), "=rm" (ctx.FP) ) 98 106 #else 99 #error un supportedhardware architecture107 #error unknown hardware architecture 100 108 #endif 101 109 102 extern $thread * mainThread; 103 extern processor * mainProcessor; 104 105 //----------------------------------------------------------------------------- 106 // Kernel Scheduling logic 107 static $thread * __next_thread(cluster * this); 108 static $thread * __next_thread_slow(cluster * this); 109 static void __run_thread(processor * this, $thread * dst); 110 static void __wake_one(cluster * cltr); 111 112 static void push (__cluster_idles & idles, processor & proc); 113 static void remove(__cluster_idles & idles, processor & proc); 114 static [unsigned idle, unsigned total, * processor] query( & __cluster_idles idles ); 115 110 //----------------------------------------------------------------------------- 111 //Start and stop routine for the kernel, declared first to make sure they run first 112 static void kernel_startup(void) __attribute__(( constructor( STARTUP_PRIORITY_KERNEL ) )); 113 static void kernel_shutdown(void) __attribute__(( destructor ( STARTUP_PRIORITY_KERNEL ) )); 114 115 //----------------------------------------------------------------------------- 116 // Kernel storage 117 KERNEL_STORAGE(cluster, mainCluster); 118 KERNEL_STORAGE(processor, mainProcessor); 119 KERNEL_STORAGE(thread_desc, mainThread); 120 KERNEL_STORAGE(__stack_t, mainThreadCtx); 121 122 cluster * mainCluster; 123 processor * mainProcessor; 124 thread_desc * mainThread; 125 126 extern "C" { 127 struct { __dllist_t(cluster) list; __spinlock_t lock; } __cfa_dbg_global_clusters; 128 } 129 130 size_t __page_size = 0; 131 132 //----------------------------------------------------------------------------- 133 // Global state 134 thread_local struct KernelThreadData kernelTLS __attribute__ ((tls_model ( "initial-exec" ))) = { 135 NULL, // cannot use 0p 136 NULL, 137 { 1, false, false }, 138 6u //this should be seeded better but due to a bug calling rdtsc doesn't work 139 }; 140 141 //----------------------------------------------------------------------------- 142 // Struct to steal stack 143 struct current_stack_info_t { 144 __stack_t * storage; // pointer to stack object 145 void * base; // base of stack 146 void * limit; // stack grows towards stack limit 147 void * context; // address of cfa_context_t 148 }; 149 150 void ?{}( current_stack_info_t & this ) { 151 __stack_context_t ctx; 152 CtxGet( ctx ); 153 this.base = ctx.FP; 154 155 rlimit r; 156 getrlimit( RLIMIT_STACK, &r); 157 size_t size = r.rlim_cur; 158 159 this.limit = (void *)(((intptr_t)this.base) - size); 160 this.context = &storage_mainThreadCtx; 161 } 162 163 //----------------------------------------------------------------------------- 164 // Main thread construction 165 166 void ?{}( coroutine_desc & this, current_stack_info_t * info) with( this ) { 167 stack.storage = info->storage; 168 with(*stack.storage) { 169 limit = info->limit; 170 base = info->base; 171 } 172 __attribute__((may_alias)) intptr_t * istorage = (intptr_t*) &stack.storage; 173 *istorage |= 0x1; 174 name = "Main Thread"; 175 state = Start; 176 starter = 0p; 177 last = 0p; 178 cancellation = 0p; 179 } 180 181 void ?{}( thread_desc & this, current_stack_info_t * info) with( this ) { 182 state = Start; 183 self_cor{ info }; 184 curr_cor = &self_cor; 185 curr_cluster = mainCluster; 186 self_mon.owner = &this; 187 self_mon.recursion = 1; 188 self_mon_p = &self_mon; 189 next = 0p; 190 191 node.next = 0p; 192 node.prev = 0p; 193 doregister(curr_cluster, this); 194 195 monitors{ &self_mon_p, 1, (fptr_t)0 }; 196 } 197 198 //----------------------------------------------------------------------------- 199 // Processor coroutine 200 void ?{}(processorCtx_t & this) { 201 202 } 203 204 // Construct the processor context of non-main processors 205 static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info) { 206 (this.__cor){ info }; 207 this.proc = proc; 208 } 209 210 static void start(processor * this); 211 void ?{}(processor & this, const char * name, cluster & cltr) with( this ) { 212 this.name = name; 213 this.cltr = &cltr; 214 terminated{ 0 }; 215 do_terminate = false; 216 preemption_alarm = 0p; 217 pending_preemption = false; 218 runner.proc = &this; 219 220 idleLock{}; 221 222 start( &this ); 223 } 224 225 void ^?{}(processor & this) with( this ){ 226 if( ! __atomic_load_n(&do_terminate, __ATOMIC_ACQUIRE) ) { 227 __cfaabi_dbg_print_safe("Kernel : core %p signaling termination\n", &this); 228 229 __atomic_store_n(&do_terminate, true, __ATOMIC_RELAXED); 230 wake( &this ); 231 232 P( terminated ); 233 verify( kernelTLS.this_processor != &this); 234 } 235 236 pthread_join( kernel_thread, 0p ); 237 free( this.stack ); 238 } 239 240 void ?{}(cluster & this, const char * name, Duration preemption_rate) with( this ) { 241 this.name = name; 242 this.preemption_rate = preemption_rate; 243 ready_queue{}; 244 ready_queue_lock{}; 245 246 procs{ __get }; 247 idles{ __get }; 248 threads{ __get }; 249 250 doregister(this); 251 } 252 253 void ^?{}(cluster & this) { 254 unregister(this); 255 } 116 256 117 257 //============================================================================================= 118 258 // Kernel Scheduling logic 119 259 //============================================================================================= 260 static void runThread(processor * this, thread_desc * dst); 261 static void finishRunning(processor * this); 262 static void halt(processor * this); 263 120 264 //Main of the processor contexts 121 265 void main(processorCtx_t & runner) { 122 266 // Because of a bug, we couldn't initialized the seed on construction 123 267 // Do it here 124 __cfaabi_tls.rand_seed ^= rdtscl(); 125 __cfaabi_tls.ready_rng.fwd_seed = 25214903917_l64u * (rdtscl() ^ (uintptr_t)&runner); 126 __tls_rand_advance_bck(); 268 kernelTLS.rand_seed ^= rdtscl(); 127 269 128 270 processor * this = runner.proc; 129 271 verify(this); 130 272 131 __cfadbg_print_safe(runtime_core, "Kernel : core %p starting\n", this); 132 #if !defined(__CFA_NO_STATISTICS__) 133 if( this->print_halts ) { 134 __cfaabi_bits_print_safe( STDOUT_FILENO, "Processor : %d - %s (%p)\n", this->id, this->name, (void*)this); 135 } 136 #endif 273 __cfaabi_dbg_print_safe("Kernel : core %p starting\n", this); 274 275 doregister(this->cltr, this); 137 276 138 277 { … … 140 279 preemption_scope scope = { this }; 141 280 142 __cfadbg_print_safe(runtime_core, "Kernel : core %p started\n", this); 143 144 $thread * readyThread = 0p; 145 MAIN_LOOP: 146 for() { 147 // Try to get the next thread 148 readyThread = __next_thread( this->cltr ); 149 150 if( !readyThread ) { 151 readyThread = __next_thread_slow( this->cltr ); 281 __cfaabi_dbg_print_safe("Kernel : core %p started\n", this); 282 283 thread_desc * readyThread = 0p; 284 for( unsigned int spin_count = 0; ! __atomic_load_n(&this->do_terminate, __ATOMIC_SEQ_CST); spin_count++ ) { 285 readyThread = nextThread( this->cltr ); 286 287 if(readyThread) { 288 verify( ! kernelTLS.preemption_state.enabled ); 289 290 runThread(this, readyThread); 291 292 verify( ! kernelTLS.preemption_state.enabled ); 293 294 //Some actions need to be taken from the kernel 295 finishRunning(this); 296 297 spin_count = 0; 298 } else { 299 // spin(this, &spin_count); 300 halt(this); 152 301 } 153 154 HALT:155 if( !readyThread ) {156 // Don't block if we are done157 if( __atomic_load_n(&this->do_terminate, __ATOMIC_SEQ_CST) ) break MAIN_LOOP;158 159 #if !defined(__CFA_NO_STATISTICS__)160 __tls_stats()->ready.sleep.halts++;161 #endif162 163 // Push self to idle stack164 push(this->cltr->idles, * this);165 166 // Confirm the ready-queue is empty167 readyThread = __next_thread_slow( this->cltr );168 if( readyThread ) {169 // A thread was found, cancel the halt170 remove(this->cltr->idles, * this);171 172 #if !defined(__CFA_NO_STATISTICS__)173 __tls_stats()->ready.sleep.cancels++;174 #endif175 176 // continue the mai loop177 break HALT;178 }179 180 #if !defined(__CFA_NO_STATISTICS__)181 if(this->print_halts) {182 __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 0\n", this->id, rdtscl());183 }184 #endif185 186 wait( this->idle );187 188 #if !defined(__CFA_NO_STATISTICS__)189 if(this->print_halts) {190 __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 1\n", this->id, rdtscl());191 }192 #endif193 194 // We were woken up, remove self from idle195 remove(this->cltr->idles, * this);196 197 // DON'T just proceed, start looking again198 continue MAIN_LOOP;199 }200 201 /* paranoid */ verify( readyThread );202 203 // We found a thread run it204 __run_thread(this, readyThread);205 206 // Are we done?207 if( __atomic_load_n(&this->do_terminate, __ATOMIC_SEQ_CST) ) break MAIN_LOOP;208 302 } 209 303 210 __cfadbg_print_safe(runtime_core, "Kernel : core %p stopping\n", this); 211 } 304 __cfaabi_dbg_print_safe("Kernel : core %p stopping\n", this); 305 } 306 307 unregister(this->cltr, this); 212 308 213 309 V( this->terminated ); 214 310 215 if(this == mainProcessor) { 216 // HACK : the coroutine context switch expects this_thread to be set 217 // and it make sense for it to be set in all other cases except here 218 // fake it 219 __cfaabi_tls.this_thread = mainThread; 220 } 221 222 __cfadbg_print_safe(runtime_core, "Kernel : core %p terminated\n", this); 311 __cfaabi_dbg_print_safe("Kernel : core %p terminated\n", this); 223 312 } 224 313 … … 229 318 // runThread runs a thread by context switching 230 319 // from the processor coroutine to the target thread 231 static void __run_thread(processor * this, $thread * thrd_dst) { 232 /* paranoid */ verify( ! __preemption_enabled() ); 233 /* paranoid */ verifyf( thrd_dst->state == Ready || thrd_dst->preempted != __NO_PREEMPTION, "state : %d, preempted %d\n", thrd_dst->state, thrd_dst->preempted); 234 /* paranoid */ verifyf( thrd_dst->link.next == 0p, "Expected null got %p", thrd_dst->link.next ); 235 __builtin_prefetch( thrd_dst->context.SP ); 236 237 $coroutine * proc_cor = get_coroutine(this->runner); 238 239 // set state of processor coroutine to inactive 240 verify(proc_cor->state == Active); 241 proc_cor->state = Blocked; 242 243 // Actually run the thread 244 RUNNING: while(true) { 245 thrd_dst->preempted = __NO_PREEMPTION; 246 thrd_dst->state = Active; 247 248 // Update global state 249 kernelTLS().this_thread = thrd_dst; 250 251 /* paranoid */ verify( ! __preemption_enabled() ); 252 /* paranoid */ verify( kernelTLS().this_thread == thrd_dst ); 253 /* paranoid */ verify( thrd_dst->curr_cluster == this->cltr ); 254 /* paranoid */ verify( thrd_dst->context.SP ); 255 /* paranoid */ verify( thrd_dst->state != Halted ); 256 /* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) < ((uintptr_t)__get_stack(thrd_dst->curr_cor)->base ) || thrd_dst->curr_cor == proc_cor, "ERROR : Destination $thread %p has been corrupted.\n StackPointer too small.\n", thrd_dst ); // add escape condition if we are setting up the processor 257 /* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) > ((uintptr_t)__get_stack(thrd_dst->curr_cor)->limit) || thrd_dst->curr_cor == proc_cor, "ERROR : Destination $thread %p has been corrupted.\n StackPointer too large.\n", thrd_dst ); // add escape condition if we are setting up the processor 258 /* paranoid */ verify( 0x0D15EA5E0D15EA5Ep == thrd_dst->canary ); 259 260 261 262 // set context switch to the thread that the processor is executing 263 __cfactx_switch( &proc_cor->context, &thrd_dst->context ); 264 // when __cfactx_switch returns we are back in the processor coroutine 265 266 /* paranoid */ verify( 0x0D15EA5E0D15EA5Ep == thrd_dst->canary ); 267 /* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) > ((uintptr_t)__get_stack(thrd_dst->curr_cor)->limit), "ERROR : Destination $thread %p has been corrupted.\n StackPointer too large.\n", thrd_dst ); 268 /* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) < ((uintptr_t)__get_stack(thrd_dst->curr_cor)->base ), "ERROR : Destination $thread %p has been corrupted.\n StackPointer too small.\n", thrd_dst ); 269 /* paranoid */ verify( thrd_dst->context.SP ); 270 /* paranoid */ verify( thrd_dst->curr_cluster == this->cltr ); 271 /* paranoid */ verify( kernelTLS().this_thread == thrd_dst ); 272 /* paranoid */ verify( ! __preemption_enabled() ); 273 274 // Reset global state 275 kernelTLS().this_thread = 0p; 276 277 // We just finished running a thread, there are a few things that could have happened. 278 // 1 - Regular case : the thread has blocked and now one has scheduled it yet. 279 // 2 - Racy case : the thread has blocked but someone has already tried to schedule it. 280 // 4 - Preempted 281 // In case 1, we may have won a race so we can't write to the state again. 282 // In case 2, we lost the race so we now own the thread. 283 284 if(unlikely(thrd_dst->preempted != __NO_PREEMPTION)) { 285 // The thread was preempted, reschedule it and reset the flag 286 __schedule_thread( thrd_dst ); 287 break RUNNING; 320 static void runThread(processor * this, thread_desc * thrd_dst) { 321 coroutine_desc * proc_cor = get_coroutine(this->runner); 322 323 // Reset the terminating actions here 324 this->finish.action_code = No_Action; 325 326 // Update global state 327 kernelTLS.this_thread = thrd_dst; 328 329 // set state of processor coroutine to inactive and the thread to active 330 proc_cor->state = proc_cor->state == Halted ? Halted : Inactive; 331 thrd_dst->state = Active; 332 333 // set context switch to the thread that the processor is executing 334 verify( thrd_dst->context.SP ); 335 CtxSwitch( &proc_cor->context, &thrd_dst->context ); 336 // when CtxSwitch returns we are back in the processor coroutine 337 338 // set state of processor coroutine to active and the thread to inactive 339 thrd_dst->state = thrd_dst->state == Halted ? Halted : Inactive; 340 proc_cor->state = Active; 341 } 342 343 // KERNEL_ONLY 344 static void returnToKernel() { 345 coroutine_desc * proc_cor = get_coroutine(kernelTLS.this_processor->runner); 346 thread_desc * thrd_src = kernelTLS.this_thread; 347 348 // set state of current coroutine to inactive 349 thrd_src->state = thrd_src->state == Halted ? Halted : Inactive; 350 proc_cor->state = Active; 351 int local_errno = *__volatile_errno(); 352 #if defined( __i386 ) || defined( __x86_64 ) 353 __x87_store; 354 #endif 355 356 // set new coroutine that the processor is executing 357 // and context switch to it 358 verify( proc_cor->context.SP ); 359 CtxSwitch( &thrd_src->context, &proc_cor->context ); 360 361 // set state of new coroutine to active 362 proc_cor->state = proc_cor->state == Halted ? Halted : Inactive; 363 thrd_src->state = Active; 364 365 #if defined( __i386 ) || defined( __x86_64 ) 366 __x87_load; 367 #endif 368 *__volatile_errno() = local_errno; 369 } 370 371 // KERNEL_ONLY 372 // Once a thread has finished running, some of 373 // its final actions must be executed from the kernel 374 static void finishRunning(processor * this) with( this->finish ) { 375 verify( ! kernelTLS.preemption_state.enabled ); 376 choose( action_code ) { 377 case No_Action: 378 break; 379 case Release: 380 unlock( *lock ); 381 case Schedule: 382 ScheduleThread( thrd ); 383 case Release_Schedule: 384 unlock( *lock ); 385 ScheduleThread( thrd ); 386 case Release_Multi: 387 for(int i = 0; i < lock_count; i++) { 388 unlock( *locks[i] ); 288 389 } 289 290 if(unlikely(thrd_dst->state == Halting)) { 291 // The thread has halted, it should never be scheduled/run again 292 // finish the thread 293 __thread_finish( thrd_dst ); 294 break RUNNING; 390 case Release_Multi_Schedule: 391 for(int i = 0; i < lock_count; i++) { 392 unlock( *locks[i] ); 295 393 } 296 297 /* paranoid */ verify( thrd_dst->state == Active ); 298 thrd_dst->state = Blocked; 299 300 // set state of processor coroutine to active and the thread to inactive 301 int old_ticket = __atomic_fetch_sub(&thrd_dst->ticket, 1, __ATOMIC_SEQ_CST); 302 switch(old_ticket) { 303 case TICKET_RUNNING: 304 // This is case 1, the regular case, nothing more is needed 305 break RUNNING; 306 case TICKET_UNBLOCK: 307 // This is case 2, the racy case, someone tried to run this thread before it finished blocking 308 // In this case, just run it again. 309 continue RUNNING; 310 default: 311 // This makes no sense, something is wrong abort 312 abort(); 394 for(int i = 0; i < thrd_count; i++) { 395 ScheduleThread( thrds[i] ); 313 396 } 314 } 315 316 // Just before returning to the processor, set the processor coroutine to active 317 proc_cor->state = Active; 318 319 /* paranoid */ verify( ! __preemption_enabled() ); 397 case Callback: 398 callback(); 399 default: 400 abort("KERNEL ERROR: Unexpected action to run after thread"); 401 } 320 402 } 321 403 322 404 // KERNEL_ONLY 323 void returnToKernel() { 324 /* paranoid */ verify( ! __preemption_enabled() ); 325 $coroutine * proc_cor = get_coroutine(kernelTLS().this_processor->runner); 326 $thread * thrd_src = kernelTLS().this_thread; 327 328 #if !defined(__CFA_NO_STATISTICS__) 329 struct processor * last_proc = kernelTLS().this_processor; 330 #endif 331 332 // Run the thread on this processor 333 { 334 int local_errno = *__volatile_errno(); 335 #if defined( __i386 ) || defined( __x86_64 ) 336 __x87_store; 337 #endif 338 /* paranoid */ verify( proc_cor->context.SP ); 339 /* paranoid */ verify( 0x0D15EA5E0D15EA5Ep == thrd_src->canary ); 340 __cfactx_switch( &thrd_src->context, &proc_cor->context ); 341 /* paranoid */ verify( 0x0D15EA5E0D15EA5Ep == thrd_src->canary ); 342 #if defined( __i386 ) || defined( __x86_64 ) 343 __x87_load; 344 #endif 345 *__volatile_errno() = local_errno; 346 } 347 348 #if !defined(__CFA_NO_STATISTICS__) 349 if(last_proc != kernelTLS().this_processor) { 350 __tls_stats()->ready.threads.migration++; 405 // Context invoker for processors 406 // This is the entry point for processors (kernel threads) 407 // It effectively constructs a coroutine by stealing the pthread stack 408 static void * CtxInvokeProcessor(void * arg) { 409 processor * proc = (processor *) arg; 410 kernelTLS.this_processor = proc; 411 kernelTLS.this_thread = 0p; 412 kernelTLS.preemption_state.[enabled, disable_count] = [false, 1]; 413 // SKULLDUGGERY: We want to create a context for the processor coroutine 414 // which is needed for the 2-step context switch. However, there is no reason 415 // to waste the perfectly valid stack create by pthread. 416 current_stack_info_t info; 417 __stack_t ctx; 418 info.storage = &ctx; 419 (proc->runner){ proc, &info }; 420 421 __cfaabi_dbg_print_safe("Coroutine : created stack %p\n", get_coroutine(proc->runner)->stack.storage); 422 423 //Set global state 424 kernelTLS.this_thread = 0p; 425 426 //We now have a proper context from which to schedule threads 427 __cfaabi_dbg_print_safe("Kernel : core %p created (%p, %p)\n", proc, &proc->runner, &ctx); 428 429 // SKULLDUGGERY: Since the coroutine doesn't have its own stack, we can't 430 // resume it to start it like it normally would, it will just context switch 431 // back to here. Instead directly call the main since we already are on the 432 // appropriate stack. 433 get_coroutine(proc->runner)->state = Active; 434 main( proc->runner ); 435 get_coroutine(proc->runner)->state = Halted; 436 437 // Main routine of the core returned, the core is now fully terminated 438 __cfaabi_dbg_print_safe("Kernel : core %p main ended (%p)\n", proc, &proc->runner); 439 440 return 0p; 441 } 442 443 static void Abort( int ret, const char * func ) { 444 if ( ret ) { // pthread routines return errno values 445 abort( "%s : internal error, error(%d) %s.", func, ret, strerror( ret ) ); 446 } // if 447 } // Abort 448 449 void * create_pthread( pthread_t * pthread, void * (*start)(void *), void * arg ) { 450 pthread_attr_t attr; 451 452 Abort( pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute 453 454 size_t stacksize; 455 // default stack size, normally defined by shell limit 456 Abort( pthread_attr_getstacksize( &attr, &stacksize ), "pthread_attr_getstacksize" ); 457 assert( stacksize >= PTHREAD_STACK_MIN ); 458 459 void * stack; 460 __cfaabi_dbg_debug_do( 461 stack = memalign( __page_size, stacksize + __page_size ); 462 // pthread has no mechanism to create the guard page in user supplied stack. 463 if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) { 464 abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) ); 465 } // if 466 ); 467 __cfaabi_dbg_no_debug_do( 468 stack = malloc( stacksize ); 469 ); 470 471 Abort( pthread_attr_setstack( &attr, stack, stacksize ), "pthread_attr_setstack" ); 472 473 Abort( pthread_create( pthread, &attr, start, arg ), "pthread_create" ); 474 return stack; 475 } 476 477 static void start(processor * this) { 478 __cfaabi_dbg_print_safe("Kernel : Starting core %p\n", this); 479 480 this->stack = create_pthread( &this->kernel_thread, CtxInvokeProcessor, (void *)this ); 481 482 __cfaabi_dbg_print_safe("Kernel : core %p started\n", this); 483 } 484 485 // KERNEL_ONLY 486 void kernel_first_resume( processor * this ) { 487 thread_desc * src = mainThread; 488 coroutine_desc * dst = get_coroutine(this->runner); 489 490 verify( ! kernelTLS.preemption_state.enabled ); 491 492 kernelTLS.this_thread->curr_cor = dst; 493 __stack_prepare( &dst->stack, 65000 ); 494 CtxStart(main, dst, this->runner, CtxInvokeCoroutine); 495 496 verify( ! kernelTLS.preemption_state.enabled ); 497 498 dst->last = &src->self_cor; 499 dst->starter = dst->starter ? dst->starter : &src->self_cor; 500 501 // set state of current coroutine to inactive 502 src->state = src->state == Halted ? Halted : Inactive; 503 504 // context switch to specified coroutine 505 verify( dst->context.SP ); 506 CtxSwitch( &src->context, &dst->context ); 507 // when CtxSwitch returns we are back in the src coroutine 508 509 mainThread->curr_cor = &mainThread->self_cor; 510 511 // set state of new coroutine to active 512 src->state = Active; 513 514 verify( ! kernelTLS.preemption_state.enabled ); 515 } 516 517 // KERNEL_ONLY 518 void kernel_last_resume( processor * this ) { 519 coroutine_desc * src = &mainThread->self_cor; 520 coroutine_desc * dst = get_coroutine(this->runner); 521 522 verify( ! kernelTLS.preemption_state.enabled ); 523 verify( dst->starter == src ); 524 verify( dst->context.SP ); 525 526 // context switch to the processor 527 CtxSwitch( &src->context, &dst->context ); 528 } 529 530 //----------------------------------------------------------------------------- 531 // Scheduler routines 532 533 // KERNEL ONLY 534 void ScheduleThread( thread_desc * thrd ) { 535 verify( thrd ); 536 verify( thrd->state != Halted ); 537 538 verify( ! kernelTLS.preemption_state.enabled ); 539 540 verifyf( thrd->next == 0p, "Expected null got %p", thrd->next ); 541 542 with( *thrd->curr_cluster ) { 543 lock ( ready_queue_lock __cfaabi_dbg_ctx2 ); 544 bool was_empty = !(ready_queue != 0); 545 append( ready_queue, thrd ); 546 unlock( ready_queue_lock ); 547 548 if(was_empty) { 549 lock (proc_list_lock __cfaabi_dbg_ctx2); 550 if(idles) { 551 wake_fast(idles.head); 552 } 553 unlock (proc_list_lock); 351 554 } 352 #endif353 354 /* paranoid */ verify( ! __preemption_enabled() );355 /* paranoid */ verifyf( ((uintptr_t)thrd_src->context.SP) < ((uintptr_t)__get_stack(thrd_src->curr_cor)->base ), "ERROR : Returning $thread %p has been corrupted.\n StackPointer too small.\n", thrd_src ); 356 /* paranoid */ verifyf( ((uintptr_t)thrd_src->context.SP) > ((uintptr_t)__get_stack(thrd_src->curr_cor)->limit), "ERROR : Returning $thread %p has been corrupted.\n StackPointer too large.\n", thrd_src );357 } 358 359 //----------------------------------------------------------------------------- 360 // Scheduler routines 555 else if( struct processor * idle = idles.head ) { 556 wake_fast(idle); 557 } 558 559 } 560 561 verify( ! kernelTLS.preemption_state.enabled ); 562 } 563 361 564 // KERNEL ONLY 362 void __schedule_thread( $thread * thrd ) { 363 /* paranoid */ verify( ! __preemption_enabled() ); 364 /* paranoid */ verify( kernelTLS().this_proc_id ); 365 /* paranoid */ verify( thrd ); 366 /* paranoid */ verify( thrd->state != Halted ); 367 /* paranoid */ verify( thrd->curr_cluster ); 368 /* paranoid */ #if defined( __CFA_WITH_VERIFY__ ) 369 /* paranoid */ if( thrd->state == Blocked || thrd->state == Start ) assertf( thrd->preempted == __NO_PREEMPTION, 370 "Error inactive thread marked as preempted, state %d, preemption %d\n", thrd->state, thrd->preempted ); 371 /* paranoid */ if( thrd->preempted != __NO_PREEMPTION ) assertf(thrd->state == Active, 372 "Error preempted thread marked as not currently running, state %d, preemption %d\n", thrd->state, thrd->preempted ); 373 /* paranoid */ #endif 374 /* paranoid */ verifyf( thrd->link.next == 0p, "Expected null got %p", thrd->link.next ); 375 /* paranoid */ verify( 0x0D15EA5E0D15EA5Ep == thrd->canary ); 376 377 378 if (thrd->preempted == __NO_PREEMPTION) thrd->state = Ready; 379 380 ready_schedule_lock(); 381 // Dereference the thread now because once we push it, there is not guaranteed it's still valid. 382 struct cluster * cl = thrd->curr_cluster; 383 384 // push the thread to the cluster ready-queue 385 push( cl, thrd ); 386 387 // variable thrd is no longer safe to use 388 389 // wake the cluster using the save variable. 390 __wake_one( cl ); 391 ready_schedule_unlock(); 392 393 /* paranoid */ verify( ! __preemption_enabled() ); 565 thread_desc * nextThread(cluster * this) with( *this ) { 566 verify( ! kernelTLS.preemption_state.enabled ); 567 lock( ready_queue_lock __cfaabi_dbg_ctx2 ); 568 thread_desc * head = pop_head( ready_queue ); 569 unlock( ready_queue_lock ); 570 verify( ! kernelTLS.preemption_state.enabled ); 571 return head; 572 } 573 574 void BlockInternal() { 575 disable_interrupts(); 576 verify( ! kernelTLS.preemption_state.enabled ); 577 returnToKernel(); 578 verify( ! kernelTLS.preemption_state.enabled ); 579 enable_interrupts( __cfaabi_dbg_ctx ); 580 } 581 582 void BlockInternal( __spinlock_t * lock ) { 583 disable_interrupts(); 584 with( *kernelTLS.this_processor ) { 585 finish.action_code = Release; 586 finish.lock = lock; 587 } 588 589 verify( ! kernelTLS.preemption_state.enabled ); 590 returnToKernel(); 591 verify( ! kernelTLS.preemption_state.enabled ); 592 593 enable_interrupts( __cfaabi_dbg_ctx ); 594 } 595 596 void BlockInternal( thread_desc * thrd ) { 597 disable_interrupts(); 598 with( * kernelTLS.this_processor ) { 599 finish.action_code = Schedule; 600 finish.thrd = thrd; 601 } 602 603 verify( ! kernelTLS.preemption_state.enabled ); 604 returnToKernel(); 605 verify( ! kernelTLS.preemption_state.enabled ); 606 607 enable_interrupts( __cfaabi_dbg_ctx ); 608 } 609 610 void BlockInternal( __spinlock_t * lock, thread_desc * thrd ) { 611 assert(thrd); 612 disable_interrupts(); 613 with( * kernelTLS.this_processor ) { 614 finish.action_code = Release_Schedule; 615 finish.lock = lock; 616 finish.thrd = thrd; 617 } 618 619 verify( ! kernelTLS.preemption_state.enabled ); 620 returnToKernel(); 621 verify( ! kernelTLS.preemption_state.enabled ); 622 623 enable_interrupts( __cfaabi_dbg_ctx ); 624 } 625 626 void BlockInternal(__spinlock_t * locks [], unsigned short count) { 627 disable_interrupts(); 628 with( * kernelTLS.this_processor ) { 629 finish.action_code = Release_Multi; 630 finish.locks = locks; 631 finish.lock_count = count; 632 } 633 634 verify( ! kernelTLS.preemption_state.enabled ); 635 returnToKernel(); 636 verify( ! kernelTLS.preemption_state.enabled ); 637 638 enable_interrupts( __cfaabi_dbg_ctx ); 639 } 640 641 void BlockInternal(__spinlock_t * locks [], unsigned short lock_count, thread_desc * thrds [], unsigned short thrd_count) { 642 disable_interrupts(); 643 with( *kernelTLS.this_processor ) { 644 finish.action_code = Release_Multi_Schedule; 645 finish.locks = locks; 646 finish.lock_count = lock_count; 647 finish.thrds = thrds; 648 finish.thrd_count = thrd_count; 649 } 650 651 verify( ! kernelTLS.preemption_state.enabled ); 652 returnToKernel(); 653 verify( ! kernelTLS.preemption_state.enabled ); 654 655 enable_interrupts( __cfaabi_dbg_ctx ); 656 } 657 658 void BlockInternal(__finish_callback_fptr_t callback) { 659 disable_interrupts(); 660 with( *kernelTLS.this_processor ) { 661 finish.action_code = Callback; 662 finish.callback = callback; 663 } 664 665 verify( ! kernelTLS.preemption_state.enabled ); 666 returnToKernel(); 667 verify( ! kernelTLS.preemption_state.enabled ); 668 669 enable_interrupts( __cfaabi_dbg_ctx ); 394 670 } 395 671 396 672 // KERNEL ONLY 397 static inline $thread * __next_thread(cluster * this) with( *this ) { 398 /* paranoid */ verify( ! __preemption_enabled() ); 399 /* paranoid */ verify( kernelTLS().this_proc_id ); 400 401 ready_schedule_lock(); 402 $thread * thrd = pop( this ); 403 ready_schedule_unlock(); 404 405 /* paranoid */ verify( kernelTLS().this_proc_id ); 406 /* paranoid */ verify( ! __preemption_enabled() ); 407 return thrd; 408 } 409 410 // KERNEL ONLY 411 static inline $thread * __next_thread_slow(cluster * this) with( *this ) { 412 /* paranoid */ verify( ! __preemption_enabled() ); 413 /* paranoid */ verify( kernelTLS().this_proc_id ); 414 415 ready_schedule_lock(); 416 $thread * thrd = pop_slow( this ); 417 ready_schedule_unlock(); 418 419 /* paranoid */ verify( kernelTLS().this_proc_id ); 420 /* paranoid */ verify( ! __preemption_enabled() ); 421 return thrd; 422 } 423 424 void unpark( $thread * thrd ) { 425 if( !thrd ) return; 426 427 int old_ticket = __atomic_fetch_add(&thrd->ticket, 1, __ATOMIC_SEQ_CST); 428 switch(old_ticket) { 429 case TICKET_RUNNING: 430 // Wake won the race, the thread will reschedule/rerun itself 431 break; 432 case TICKET_BLOCKED: 433 /* paranoid */ verify( ! thrd->preempted != __NO_PREEMPTION ); 434 /* paranoid */ verify( thrd->state == Blocked ); 435 436 { 437 /* paranoid */ verify( publicTLS_get(this_proc_id) ); 438 bool full = publicTLS_get(this_proc_id)->full_proc; 439 if(full) disable_interrupts(); 440 441 /* paranoid */ verify( ! __preemption_enabled() ); 442 443 // Wake lost the race, 444 __schedule_thread( thrd ); 445 446 /* paranoid */ verify( ! __preemption_enabled() ); 447 448 if(full) enable_interrupts( __cfaabi_dbg_ctx ); 449 /* paranoid */ verify( publicTLS_get(this_proc_id) ); 450 } 451 452 break; 453 default: 454 // This makes no sense, something is wrong abort 455 abort("Thread %p (%s) has mismatch park/unpark\n", thrd, thrd->self_cor.name); 456 } 457 } 458 459 void park( void ) { 460 /* paranoid */ verify( __preemption_enabled() ); 673 void LeaveThread(__spinlock_t * lock, thread_desc * thrd) { 674 verify( ! kernelTLS.preemption_state.enabled ); 675 with( * kernelTLS.this_processor ) { 676 finish.action_code = thrd ? Release_Schedule : Release; 677 finish.lock = lock; 678 finish.thrd = thrd; 679 } 680 681 returnToKernel(); 682 } 683 684 //============================================================================================= 685 // Kernel Setup logic 686 //============================================================================================= 687 //----------------------------------------------------------------------------- 688 // Kernel boot procedures 689 static void kernel_startup(void) { 690 verify( ! kernelTLS.preemption_state.enabled ); 691 __cfaabi_dbg_print_safe("Kernel : Starting\n"); 692 693 __page_size = sysconf( _SC_PAGESIZE ); 694 695 __cfa_dbg_global_clusters.list{ __get }; 696 __cfa_dbg_global_clusters.lock{}; 697 698 // Initialize the main cluster 699 mainCluster = (cluster *)&storage_mainCluster; 700 (*mainCluster){"Main Cluster"}; 701 702 __cfaabi_dbg_print_safe("Kernel : Main cluster ready\n"); 703 704 // Start by initializing the main thread 705 // SKULLDUGGERY: the mainThread steals the process main thread 706 // which will then be scheduled by the mainProcessor normally 707 mainThread = (thread_desc *)&storage_mainThread; 708 current_stack_info_t info; 709 info.storage = (__stack_t*)&storage_mainThreadCtx; 710 (*mainThread){ &info }; 711 712 __cfaabi_dbg_print_safe("Kernel : Main thread ready\n"); 713 714 715 716 // Construct the processor context of the main processor 717 void ?{}(processorCtx_t & this, processor * proc) { 718 (this.__cor){ "Processor" }; 719 this.__cor.starter = 0p; 720 this.proc = proc; 721 } 722 723 void ?{}(processor & this) with( this ) { 724 name = "Main Processor"; 725 cltr = mainCluster; 726 terminated{ 0 }; 727 do_terminate = false; 728 preemption_alarm = 0p; 729 pending_preemption = false; 730 kernel_thread = pthread_self(); 731 732 runner{ &this }; 733 __cfaabi_dbg_print_safe("Kernel : constructed main processor context %p\n", &runner); 734 } 735 736 // Initialize the main processor and the main processor ctx 737 // (the coroutine that contains the processing control flow) 738 mainProcessor = (processor *)&storage_mainProcessor; 739 (*mainProcessor){}; 740 741 //initialize the global state variables 742 kernelTLS.this_processor = mainProcessor; 743 kernelTLS.this_thread = mainThread; 744 745 // Enable preemption 746 kernel_start_preemption(); 747 748 // Add the main thread to the ready queue 749 // once resume is called on mainProcessor->runner the mainThread needs to be scheduled like any normal thread 750 ScheduleThread(mainThread); 751 752 // SKULLDUGGERY: Force a context switch to the main processor to set the main thread's context to the current UNIX 753 // context. Hence, the main thread does not begin through CtxInvokeThread, like all other threads. The trick here is that 754 // mainThread is on the ready queue when this call is made. 755 kernel_first_resume( kernelTLS.this_processor ); 756 757 758 759 // THE SYSTEM IS NOW COMPLETELY RUNNING 760 __cfaabi_dbg_print_safe("Kernel : Started\n--------------------------------------------------\n\n"); 761 762 verify( ! kernelTLS.preemption_state.enabled ); 763 enable_interrupts( __cfaabi_dbg_ctx ); 764 verify( TL_GET( preemption_state.enabled ) ); 765 } 766 767 static void kernel_shutdown(void) { 768 __cfaabi_dbg_print_safe("\n--------------------------------------------------\nKernel : Shutting down\n"); 769 770 verify( TL_GET( preemption_state.enabled ) ); 461 771 disable_interrupts(); 462 /* paranoid */ verify( ! __preemption_enabled() ); 463 /* paranoid */ verify( kernelTLS().this_thread->preempted == __NO_PREEMPTION ); 464 465 returnToKernel(); 466 467 /* paranoid */ verify( ! __preemption_enabled() ); 468 enable_interrupts( __cfaabi_dbg_ctx ); 469 /* paranoid */ verify( __preemption_enabled() ); 470 471 } 472 473 extern "C" { 474 // Leave the thread monitor 475 // last routine called by a thread. 476 // Should never return 477 void __cfactx_thrd_leave() { 478 $thread * thrd = active_thread(); 479 $monitor * this = &thrd->self_mon; 480 481 // Lock the monitor now 482 lock( this->lock __cfaabi_dbg_ctx2 ); 483 484 disable_interrupts(); 485 486 /* paranoid */ verify( ! __preemption_enabled() ); 487 /* paranoid */ verify( thrd->state == Active ); 488 /* paranoid */ verify( 0x0D15EA5E0D15EA5Ep == thrd->canary ); 489 /* paranoid */ verify( kernelTLS().this_thread == thrd ); 490 /* paranoid */ verify( thrd->context.SP ); 491 /* paranoid */ verifyf( ((uintptr_t)thrd->context.SP) > ((uintptr_t)__get_stack(thrd->curr_cor)->limit), "ERROR : $thread %p has been corrupted.\n StackPointer too large.\n", thrd ); 492 /* paranoid */ verifyf( ((uintptr_t)thrd->context.SP) < ((uintptr_t)__get_stack(thrd->curr_cor)->base ), "ERROR : $thread %p has been corrupted.\n StackPointer too small.\n", thrd ); 493 494 thrd->state = Halting; 495 if( TICKET_RUNNING != thrd->ticket ) { abort( "Thread terminated with pending unpark" ); } 496 if( thrd != this->owner ) { abort( "Thread internal monitor has incorrect owner" ); } 497 if( this->recursion != 1) { abort( "Thread internal monitor has unbalanced recursion" ); } 498 499 // Leave the thread 500 returnToKernel(); 501 502 // Control flow should never reach here! 503 abort(); 504 } 505 } 506 507 // KERNEL ONLY 508 bool force_yield( __Preemption_Reason reason ) { 509 /* paranoid */ verify( __preemption_enabled() ); 510 disable_interrupts(); 511 /* paranoid */ verify( ! __preemption_enabled() ); 512 513 $thread * thrd = kernelTLS().this_thread; 514 /* paranoid */ verify(thrd->state == Active); 515 516 // SKULLDUGGERY: It is possible that we are preempting this thread just before 517 // it was going to park itself. If that is the case and it is already using the 518 // intrusive fields then we can't use them to preempt the thread 519 // If that is the case, abandon the preemption. 520 bool preempted = false; 521 if(thrd->link.next == 0p) { 522 preempted = true; 523 thrd->preempted = reason; 524 returnToKernel(); 525 } 526 527 /* paranoid */ verify( ! __preemption_enabled() ); 528 enable_interrupts_noPoll(); 529 /* paranoid */ verify( __preemption_enabled() ); 530 531 return preempted; 772 verify( ! kernelTLS.preemption_state.enabled ); 773 774 // SKULLDUGGERY: Notify the mainProcessor it needs to terminates. 775 // When its coroutine terminates, it return control to the mainThread 776 // which is currently here 777 __atomic_store_n(&mainProcessor->do_terminate, true, __ATOMIC_RELEASE); 778 kernel_last_resume( kernelTLS.this_processor ); 779 mainThread->self_cor.state = Halted; 780 781 // THE SYSTEM IS NOW COMPLETELY STOPPED 782 783 // Disable preemption 784 kernel_stop_preemption(); 785 786 // Destroy the main processor and its context in reverse order of construction 787 // These were manually constructed so we need manually destroy them 788 ^(mainProcessor->runner){}; 789 ^(mainProcessor){}; 790 791 // Final step, destroy the main thread since it is no longer needed 792 // Since we provided a stack to this taxk it will not destroy anything 793 ^(mainThread){}; 794 795 ^(__cfa_dbg_global_clusters.list){}; 796 ^(__cfa_dbg_global_clusters.lock){}; 797 798 __cfaabi_dbg_print_safe("Kernel : Shutdown complete\n"); 532 799 } 533 800 534 801 //============================================================================================= 535 // Kernel Idle Sleep802 // Kernel Quiescing 536 803 //============================================================================================= 537 // Wake a thread from the front if there are any 538 static void __wake_one(cluster * this) { 539 /* paranoid */ verify( ! __preemption_enabled() ); 540 /* paranoid */ verify( ready_schedule_islocked() ); 541 542 // Check if there is a sleeping processor 543 processor * p; 544 unsigned idle; 545 unsigned total; 546 [idle, total, p] = query(this->idles); 547 548 // If no one is sleeping, we are done 549 if( idle == 0 ) return; 550 551 // We found a processor, wake it up 552 post( p->idle ); 553 554 #if !defined(__CFA_NO_STATISTICS__) 555 __tls_stats()->ready.sleep.wakes++; 556 #endif 557 558 /* paranoid */ verify( ready_schedule_islocked() ); 559 /* paranoid */ verify( ! __preemption_enabled() ); 560 561 return; 562 } 563 564 // Unconditionnaly wake a thread 565 void __wake_proc(processor * this) { 566 __cfadbg_print_safe(runtime_core, "Kernel : waking Processor %p\n", this); 567 568 disable_interrupts(); 569 /* paranoid */ verify( ! __preemption_enabled() ); 570 post( this->idle ); 571 enable_interrupts( __cfaabi_dbg_ctx ); 572 } 573 574 static void push (__cluster_idles & this, processor & proc) { 575 /* paranoid */ verify( ! __preemption_enabled() ); 576 lock( this ); 577 this.idle++; 578 /* paranoid */ verify( this.idle <= this.total ); 579 580 insert_first(this.list, proc); 581 unlock( this ); 582 /* paranoid */ verify( ! __preemption_enabled() ); 583 } 584 585 static void remove(__cluster_idles & this, processor & proc) { 586 /* paranoid */ verify( ! __preemption_enabled() ); 587 lock( this ); 588 this.idle--; 589 /* paranoid */ verify( this.idle >= 0 ); 590 591 remove(proc); 592 unlock( this ); 593 /* paranoid */ verify( ! __preemption_enabled() ); 594 } 595 596 static [unsigned idle, unsigned total, * processor] query( & __cluster_idles this ) { 597 for() { 598 uint64_t l = __atomic_load_n(&this.lock, __ATOMIC_SEQ_CST); 599 if( 1 == (l % 2) ) { Pause(); continue; } 600 unsigned idle = this.idle; 601 unsigned total = this.total; 602 processor * proc = &this.list`first; 603 // Compiler fence is unnecessary, but gcc-8 and older incorrectly reorder code without it 604 asm volatile("": : :"memory"); 605 if(l != __atomic_load_n(&this.lock, __ATOMIC_SEQ_CST)) { Pause(); continue; } 606 return [idle, total, proc]; 804 static void halt(processor * this) with( *this ) { 805 // verify( ! __atomic_load_n(&do_terminate, __ATOMIC_SEQ_CST) ); 806 807 with( *cltr ) { 808 lock (proc_list_lock __cfaabi_dbg_ctx2); 809 remove (procs, *this); 810 push_front(idles, *this); 811 unlock (proc_list_lock); 812 } 813 814 __cfaabi_dbg_print_safe("Kernel : Processor %p ready to sleep\n", this); 815 816 wait( idleLock ); 817 818 __cfaabi_dbg_print_safe("Kernel : Processor %p woke up and ready to run\n", this); 819 820 with( *cltr ) { 821 lock (proc_list_lock __cfaabi_dbg_ctx2); 822 remove (idles, *this); 823 push_front(procs, *this); 824 unlock (proc_list_lock); 607 825 } 608 826 } … … 619 837 lock( kernel_abort_lock __cfaabi_dbg_ctx2 ); 620 838 621 // disable interrupts, it no longer makes sense to try to interrupt this processor622 disable_interrupts();623 624 839 // first task to abort ? 625 840 if ( kernel_abort_called ) { // not first task to abort ? … … 629 844 sigemptyset( &mask ); 630 845 sigaddset( &mask, SIGALRM ); // block SIGALRM signals 631 sigaddset( &mask, SIGUSR1 ); // block SIGALRM signals 632 sigsuspend( &mask ); // block the processor to prevent further damage during abort 633 _exit( EXIT_FAILURE ); // if processor unblocks before it is killed, terminate it 846 sigsuspend( &mask ); // block the processor to prevent further damage during abort 847 _exit( EXIT_FAILURE ); // if processor unblocks before it is killed, terminate it 634 848 } 635 849 else { … … 638 852 } 639 853 640 return __cfaabi_tls.this_thread;854 return kernelTLS.this_thread; 641 855 } 642 856 643 857 void kernel_abort_msg( void * kernel_data, char * abort_text, int abort_text_size ) { 644 $thread * thrd = ( $thread * )kernel_data;858 thread_desc * thrd = kernel_data; 645 859 646 860 if(thrd) { … … 663 877 664 878 int kernel_abort_lastframe( void ) __attribute__ ((__nothrow__)) { 665 return get_coroutine(kernelTLS ().this_thread) == get_coroutine(mainThread) ? 4 : 2;879 return get_coroutine(kernelTLS.this_thread) == get_coroutine(mainThread) ? 4 : 2; 666 880 } 667 881 … … 690 904 void ^?{}(semaphore & this) {} 691 905 692 boolP(semaphore & this) with( this ){906 void P(semaphore & this) with( this ){ 693 907 lock( lock __cfaabi_dbg_ctx2 ); 694 908 count -= 1; 695 909 if ( count < 0 ) { 696 910 // queue current task 697 append( waiting, active_thread());911 append( waiting, kernelTLS.this_thread ); 698 912 699 913 // atomically release spin lock and block 700 unlock( lock ); 701 park(); 702 return true; 914 BlockInternal( &lock ); 703 915 } 704 916 else { 705 917 unlock( lock ); 706 return false; 707 } 708 } 709 710 bool V(semaphore & this) with( this ) { 711 $thread * thrd = 0p; 918 } 919 } 920 921 void V(semaphore & this) with( this ) { 922 thread_desc * thrd = 0p; 712 923 lock( lock __cfaabi_dbg_ctx2 ); 713 924 count += 1; … … 720 931 721 932 // make new owner 722 unpark( thrd ); 723 724 return thrd != 0p; 725 } 726 727 bool V(semaphore & this, unsigned diff) with( this ) { 728 $thread * thrd = 0p; 729 lock( lock __cfaabi_dbg_ctx2 ); 730 int release = max(-count, (int)diff); 731 count += diff; 732 for(release) { 733 unpark( pop_head( waiting ) ); 734 } 735 736 unlock( lock ); 737 738 return thrd != 0p; 933 WakeThread( thrd ); 934 } 935 936 //----------------------------------------------------------------------------- 937 // Global Queues 938 void doregister( cluster & cltr ) { 939 lock ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2); 940 push_front( __cfa_dbg_global_clusters.list, cltr ); 941 unlock ( __cfa_dbg_global_clusters.lock ); 942 } 943 944 void unregister( cluster & cltr ) { 945 lock ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2); 946 remove( __cfa_dbg_global_clusters.list, cltr ); 947 unlock( __cfa_dbg_global_clusters.lock ); 948 } 949 950 void doregister( cluster * cltr, thread_desc & thrd ) { 951 lock (cltr->thread_list_lock __cfaabi_dbg_ctx2); 952 cltr->nthreads += 1; 953 push_front(cltr->threads, thrd); 954 unlock (cltr->thread_list_lock); 955 } 956 957 void unregister( cluster * cltr, thread_desc & thrd ) { 958 lock (cltr->thread_list_lock __cfaabi_dbg_ctx2); 959 remove(cltr->threads, thrd ); 960 cltr->nthreads -= 1; 961 unlock(cltr->thread_list_lock); 962 } 963 964 void doregister( cluster * cltr, processor * proc ) { 965 lock (cltr->proc_list_lock __cfaabi_dbg_ctx2); 966 cltr->nprocessors += 1; 967 push_front(cltr->procs, *proc); 968 unlock (cltr->proc_list_lock); 969 } 970 971 void unregister( cluster * cltr, processor * proc ) { 972 lock (cltr->proc_list_lock __cfaabi_dbg_ctx2); 973 remove(cltr->procs, *proc ); 974 cltr->nprocessors -= 1; 975 unlock(cltr->proc_list_lock); 739 976 } 740 977 … … 743 980 __cfaabi_dbg_debug_do( 744 981 extern "C" { 745 void __cfaabi_dbg_record _lock(__spinlock_t & this, const char prev_name[]) {982 void __cfaabi_dbg_record(__spinlock_t & this, const char * prev_name) { 746 983 this.prev_name = prev_name; 747 this.prev_thrd = kernelTLS ().this_thread;984 this.prev_thrd = kernelTLS.this_thread; 748 985 } 749 986 } … … 752 989 //----------------------------------------------------------------------------- 753 990 // Debug 754 bool threading_enabled(void) __attribute__((const)){991 bool threading_enabled(void) { 755 992 return true; 756 993 } 757 758 //-----------------------------------------------------------------------------759 // Statistics760 #if !defined(__CFA_NO_STATISTICS__)761 void print_halts( processor & this ) {762 this.print_halts = true;763 }764 765 void print_stats_now( cluster & this, int flags ) {766 __print_stats( this.stats, this.print_stats, "Cluster", this.name, (void*)&this );767 }768 769 extern int __print_alarm_stats;770 void print_alarm_stats() {771 __print_alarm_stats = -1;772 }773 #endif774 994 // Local Variables: // 775 995 // mode: c //
Note:
See TracChangeset
for help on using the changeset viewer.
