Changeset eef8dfb for libcfa/src/concurrency/kernel.cfa
- Timestamp:
- Jan 7, 2021, 2:55:57 PM (5 years ago)
- Branches:
- ADT, arm-eh, ast-experimental, enum, forall-pointer-decay, jacob/cs343-translation, master, new-ast-unique-expr, pthread-emulation, qualifiedEnum
- Children:
- 58fe85a
- Parents:
- bdfc032 (diff), 44e37ef (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the(diff)links above to see all the changes relative to each parent. - File:
-
- 1 edited
-
libcfa/src/concurrency/kernel.cfa (modified) (13 diffs)
Legend:
- Unmodified
- Added
- Removed
-
libcfa/src/concurrency/kernel.cfa
rbdfc032 reef8dfb 10 10 // Created On : Tue Jan 17 12:27:26 2017 11 11 // Last Modified By : Peter A. Buhr 12 // Last Modified On : Thu Jan 30 22:55:50 202013 // Update Count : 5612 // Last Modified On : Mon Aug 31 07:08:20 2020 13 // Update Count : 71 14 14 // 15 15 16 16 #define __cforall_thread__ 17 // #define __CFA_DEBUG_PRINT_RUNTIME_CORE__ 17 18 18 19 //C Includes 19 #include <stddef.h>20 20 #include <errno.h> 21 #include <string.h>22 extern "C" {23 21 #include <stdio.h> 24 #include <fenv.h>25 #include <sys/resource.h>26 22 #include <signal.h> 27 23 #include <unistd.h> 28 #include <limits.h> // PTHREAD_STACK_MIN29 #include <sys/mman.h> // mprotect30 }31 24 32 25 //CFA Includes 33 #include "time.hfa"34 26 #include "kernel_private.hfa" 35 27 #include "preemption.hfa" 36 #include "startup.hfa"37 28 38 29 //Private includes … … 40 31 #include "invoke.h" 41 32 33 42 34 //----------------------------------------------------------------------------- 43 35 // Some assembly required 44 36 #if defined( __i386 ) 45 #define CtxGet( ctx ) \46 __asm__ volatile ( \47 "movl %%esp,%0\n"\48 "movl %%ebp,%1\n"\49 : "=rm" (ctx.SP),\50 "=rm" (ctx.FP) \51 )52 53 37 // mxcr : SSE Status and Control bits (control bits are preserved across function calls) 54 38 // fcw : X87 FPU control word (preserved across function calls) … … 72 56 73 57 #elif defined( __x86_64 ) 74 #define CtxGet( ctx ) \75 __asm__ volatile ( \76 "movq %%rsp,%0\n"\77 "movq %%rbp,%1\n"\78 : "=rm" (ctx.SP),\79 "=rm" (ctx.FP) \80 )81 82 58 #define __x87_store \ 83 59 uint32_t __mxcr; \ … … 98 74 ) 99 75 100 101 #elif defined( __ARM_ARCH ) 102 #define CtxGet( ctx ) __asm__ ( \ 103 "mov %0,%%sp\n" \ 104 "mov %1,%%r11\n" \ 105 : "=rm" (ctx.SP), "=rm" (ctx.FP) ) 76 #elif defined( __arm__ ) 77 #define __x87_store 78 #define __x87_load 79 80 #elif defined( __aarch64__ ) 81 #define __x87_store \ 82 uint32_t __fpcntl[2]; \ 83 __asm__ volatile ( \ 84 "mrs x9, FPCR\n" \ 85 "mrs x10, FPSR\n" \ 86 "stp x9, x10, %0\n" \ 87 : "=m" (__fpcntl) : : "x9", "x10" \ 88 ) 89 90 #define __x87_load \ 91 __asm__ volatile ( \ 92 "ldp x9, x10, %0\n" \ 93 "msr FPSR, x10\n" \ 94 "msr FPCR, x9\n" \ 95 : "=m" (__fpcntl) : : "x9", "x10" \ 96 ) 97 106 98 #else 107 #error un knownhardware architecture99 #error unsupported hardware architecture 108 100 #endif 109 101 102 extern $thread * mainThread; 103 extern processor * mainProcessor; 104 110 105 //----------------------------------------------------------------------------- 111 //Start and stop routine for the kernel, declared first to make sure they run first 112 static void kernel_startup(void) __attribute__(( constructor( STARTUP_PRIORITY_KERNEL ) )); 113 static void kernel_shutdown(void) __attribute__(( destructor ( STARTUP_PRIORITY_KERNEL ) )); 114 115 //----------------------------------------------------------------------------- 116 // Kernel storage 117 KERNEL_STORAGE(cluster, mainCluster); 118 KERNEL_STORAGE(processor, mainProcessor); 119 KERNEL_STORAGE(thread_desc, mainThread); 120 KERNEL_STORAGE(__stack_t, mainThreadCtx); 121 122 cluster * mainCluster; 123 processor * mainProcessor; 124 thread_desc * mainThread; 125 126 extern "C" { 127 struct { __dllist_t(cluster) list; __spinlock_t lock; } __cfa_dbg_global_clusters; 128 } 129 130 size_t __page_size = 0; 131 132 //----------------------------------------------------------------------------- 133 // Global state 134 thread_local struct KernelThreadData kernelTLS __attribute__ ((tls_model ( "initial-exec" ))) = { 135 NULL, // cannot use 0p 136 NULL, 137 { 1, false, false }, 138 6u //this should be seeded better but due to a bug calling rdtsc doesn't work 139 }; 140 141 //----------------------------------------------------------------------------- 142 // Struct to steal stack 143 struct current_stack_info_t { 144 __stack_t * storage; // pointer to stack object 145 void * base; // base of stack 146 void * limit; // stack grows towards stack limit 147 void * context; // address of cfa_context_t 148 }; 149 150 void ?{}( current_stack_info_t & this ) { 151 __stack_context_t ctx; 152 CtxGet( ctx ); 153 this.base = ctx.FP; 154 155 rlimit r; 156 getrlimit( RLIMIT_STACK, &r); 157 size_t size = r.rlim_cur; 158 159 this.limit = (void *)(((intptr_t)this.base) - size); 160 this.context = &storage_mainThreadCtx; 161 } 162 163 //----------------------------------------------------------------------------- 164 // Main thread construction 165 166 void ?{}( coroutine_desc & this, current_stack_info_t * info) with( this ) { 167 stack.storage = info->storage; 168 with(*stack.storage) { 169 limit = info->limit; 170 base = info->base; 171 } 172 __attribute__((may_alias)) intptr_t * istorage = (intptr_t*) &stack.storage; 173 *istorage |= 0x1; 174 name = "Main Thread"; 175 state = Start; 176 starter = 0p; 177 last = 0p; 178 cancellation = 0p; 179 } 180 181 void ?{}( thread_desc & this, current_stack_info_t * info) with( this ) { 182 state = Start; 183 self_cor{ info }; 184 curr_cor = &self_cor; 185 curr_cluster = mainCluster; 186 self_mon.owner = &this; 187 self_mon.recursion = 1; 188 self_mon_p = &self_mon; 189 next = 0p; 190 191 node.next = 0p; 192 node.prev = 0p; 193 doregister(curr_cluster, this); 194 195 monitors{ &self_mon_p, 1, (fptr_t)0 }; 196 } 197 198 //----------------------------------------------------------------------------- 199 // Processor coroutine 200 void ?{}(processorCtx_t & this) { 201 202 } 203 204 // Construct the processor context of non-main processors 205 static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info) { 206 (this.__cor){ info }; 207 this.proc = proc; 208 } 209 210 static void start(processor * this); 211 void ?{}(processor & this, const char * name, cluster & cltr) with( this ) { 212 this.name = name; 213 this.cltr = &cltr; 214 terminated{ 0 }; 215 do_terminate = false; 216 preemption_alarm = 0p; 217 pending_preemption = false; 218 runner.proc = &this; 219 220 idleLock{}; 221 222 start( &this ); 223 } 224 225 void ^?{}(processor & this) with( this ){ 226 if( ! __atomic_load_n(&do_terminate, __ATOMIC_ACQUIRE) ) { 227 __cfaabi_dbg_print_safe("Kernel : core %p signaling termination\n", &this); 228 229 __atomic_store_n(&do_terminate, true, __ATOMIC_RELAXED); 230 wake( &this ); 231 232 P( terminated ); 233 verify( kernelTLS.this_processor != &this); 234 } 235 236 pthread_join( kernel_thread, 0p ); 237 free( this.stack ); 238 } 239 240 void ?{}(cluster & this, const char * name, Duration preemption_rate) with( this ) { 241 this.name = name; 242 this.preemption_rate = preemption_rate; 243 ready_queue{}; 244 ready_queue_lock{}; 245 246 procs{ __get }; 247 idles{ __get }; 248 threads{ __get }; 249 250 doregister(this); 251 } 252 253 void ^?{}(cluster & this) { 254 unregister(this); 255 } 106 // Kernel Scheduling logic 107 static $thread * __next_thread(cluster * this); 108 static $thread * __next_thread_slow(cluster * this); 109 static void __run_thread(processor * this, $thread * dst); 110 static void __wake_one(cluster * cltr); 111 112 static void push (__cluster_idles & idles, processor & proc); 113 static void remove(__cluster_idles & idles, processor & proc); 114 static [unsigned idle, unsigned total, * processor] query( & __cluster_idles idles ); 115 256 116 257 117 //============================================================================================= 258 118 // Kernel Scheduling logic 259 119 //============================================================================================= 260 static void runThread(processor * this, thread_desc * dst);261 static void finishRunning(processor * this);262 static void halt(processor * this);263 264 120 //Main of the processor contexts 265 121 void main(processorCtx_t & runner) { 266 122 // Because of a bug, we couldn't initialized the seed on construction 267 123 // Do it here 268 kernelTLS.rand_seed ^= rdtscl(); 124 __cfaabi_tls.rand_seed ^= rdtscl(); 125 __cfaabi_tls.ready_rng.fwd_seed = 25214903917_l64u * (rdtscl() ^ (uintptr_t)&runner); 126 __tls_rand_advance_bck(); 269 127 270 128 processor * this = runner.proc; 271 129 verify(this); 272 130 273 __cfaabi_dbg_print_safe("Kernel : core %p starting\n", this); 274 275 doregister(this->cltr, this); 131 __cfadbg_print_safe(runtime_core, "Kernel : core %p starting\n", this); 132 #if !defined(__CFA_NO_STATISTICS__) 133 if( this->print_halts ) { 134 __cfaabi_bits_print_safe( STDOUT_FILENO, "Processor : %d - %s (%p)\n", this->id, this->name, (void*)this); 135 } 136 #endif 276 137 277 138 { … … 279 140 preemption_scope scope = { this }; 280 141 281 __cfaabi_dbg_print_safe("Kernel : core %p started\n", this); 282 283 thread_desc * readyThread = 0p; 284 for( unsigned int spin_count = 0; ! __atomic_load_n(&this->do_terminate, __ATOMIC_SEQ_CST); spin_count++ ) { 285 readyThread = nextThread( this->cltr ); 286 287 if(readyThread) { 288 verify( ! kernelTLS.preemption_state.enabled ); 289 290 runThread(this, readyThread); 291 292 verify( ! kernelTLS.preemption_state.enabled ); 293 294 //Some actions need to be taken from the kernel 295 finishRunning(this); 296 297 spin_count = 0; 298 } else { 299 // spin(this, &spin_count); 300 halt(this); 142 __cfadbg_print_safe(runtime_core, "Kernel : core %p started\n", this); 143 144 $thread * readyThread = 0p; 145 MAIN_LOOP: 146 for() { 147 // Try to get the next thread 148 readyThread = __next_thread( this->cltr ); 149 150 if( !readyThread ) { 151 readyThread = __next_thread_slow( this->cltr ); 301 152 } 302 } 303 304 __cfaabi_dbg_print_safe("Kernel : core %p stopping\n", this); 305 } 306 307 unregister(this->cltr, this); 153 154 HALT: 155 if( !readyThread ) { 156 // Don't block if we are done 157 if( __atomic_load_n(&this->do_terminate, __ATOMIC_SEQ_CST) ) break MAIN_LOOP; 158 159 #if !defined(__CFA_NO_STATISTICS__) 160 __tls_stats()->ready.sleep.halts++; 161 #endif 162 163 // Push self to idle stack 164 push(this->cltr->idles, * this); 165 166 // Confirm the ready-queue is empty 167 readyThread = __next_thread_slow( this->cltr ); 168 if( readyThread ) { 169 // A thread was found, cancel the halt 170 remove(this->cltr->idles, * this); 171 172 #if !defined(__CFA_NO_STATISTICS__) 173 __tls_stats()->ready.sleep.cancels++; 174 #endif 175 176 // continue the mai loop 177 break HALT; 178 } 179 180 #if !defined(__CFA_NO_STATISTICS__) 181 if(this->print_halts) { 182 __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 0\n", this->id, rdtscl()); 183 } 184 #endif 185 186 wait( this->idle ); 187 188 #if !defined(__CFA_NO_STATISTICS__) 189 if(this->print_halts) { 190 __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 1\n", this->id, rdtscl()); 191 } 192 #endif 193 194 // We were woken up, remove self from idle 195 remove(this->cltr->idles, * this); 196 197 // DON'T just proceed, start looking again 198 continue MAIN_LOOP; 199 } 200 201 /* paranoid */ verify( readyThread ); 202 203 // We found a thread run it 204 __run_thread(this, readyThread); 205 206 // Are we done? 207 if( __atomic_load_n(&this->do_terminate, __ATOMIC_SEQ_CST) ) break MAIN_LOOP; 208 } 209 210 __cfadbg_print_safe(runtime_core, "Kernel : core %p stopping\n", this); 211 } 308 212 309 213 V( this->terminated ); 310 214 311 __cfaabi_dbg_print_safe("Kernel : core %p terminated\n", this); 215 if(this == mainProcessor) { 216 // HACK : the coroutine context switch expects this_thread to be set 217 // and it make sense for it to be set in all other cases except here 218 // fake it 219 __cfaabi_tls.this_thread = mainThread; 220 } 221 222 __cfadbg_print_safe(runtime_core, "Kernel : core %p terminated\n", this); 312 223 } 313 224 … … 318 229 // runThread runs a thread by context switching 319 230 // from the processor coroutine to the target thread 320 static void runThread(processor * this, thread_desc * thrd_dst) { 321 coroutine_desc * proc_cor = get_coroutine(this->runner); 322 323 // Reset the terminating actions here 324 this->finish.action_code = No_Action; 325 326 // Update global state 327 kernelTLS.this_thread = thrd_dst; 328 329 // set state of processor coroutine to inactive and the thread to active 330 proc_cor->state = proc_cor->state == Halted ? Halted : Inactive; 331 thrd_dst->state = Active; 332 333 // set context switch to the thread that the processor is executing 334 verify( thrd_dst->context.SP ); 335 CtxSwitch( &proc_cor->context, &thrd_dst->context ); 336 // when CtxSwitch returns we are back in the processor coroutine 337 338 // set state of processor coroutine to active and the thread to inactive 339 thrd_dst->state = thrd_dst->state == Halted ? Halted : Inactive; 231 static void __run_thread(processor * this, $thread * thrd_dst) { 232 /* paranoid */ verify( ! __preemption_enabled() ); 233 /* paranoid */ verifyf( thrd_dst->state == Ready || thrd_dst->preempted != __NO_PREEMPTION, "state : %d, preempted %d\n", thrd_dst->state, thrd_dst->preempted); 234 /* paranoid */ verifyf( thrd_dst->link.next == 0p, "Expected null got %p", thrd_dst->link.next ); 235 __builtin_prefetch( thrd_dst->context.SP ); 236 237 $coroutine * proc_cor = get_coroutine(this->runner); 238 239 // set state of processor coroutine to inactive 240 verify(proc_cor->state == Active); 241 proc_cor->state = Blocked; 242 243 // Actually run the thread 244 RUNNING: while(true) { 245 thrd_dst->preempted = __NO_PREEMPTION; 246 thrd_dst->state = Active; 247 248 // Update global state 249 kernelTLS().this_thread = thrd_dst; 250 251 /* paranoid */ verify( ! __preemption_enabled() ); 252 /* paranoid */ verify( kernelTLS().this_thread == thrd_dst ); 253 /* paranoid */ verify( thrd_dst->curr_cluster == this->cltr ); 254 /* paranoid */ verify( thrd_dst->context.SP ); 255 /* paranoid */ verify( thrd_dst->state != Halted ); 256 /* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) < ((uintptr_t)__get_stack(thrd_dst->curr_cor)->base ) || thrd_dst->curr_cor == proc_cor, "ERROR : Destination $thread %p has been corrupted.\n StackPointer too small.\n", thrd_dst ); // add escape condition if we are setting up the processor 257 /* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) > ((uintptr_t)__get_stack(thrd_dst->curr_cor)->limit) || thrd_dst->curr_cor == proc_cor, "ERROR : Destination $thread %p has been corrupted.\n StackPointer too large.\n", thrd_dst ); // add escape condition if we are setting up the processor 258 /* paranoid */ verify( 0x0D15EA5E0D15EA5Ep == thrd_dst->canary ); 259 260 261 262 // set context switch to the thread that the processor is executing 263 __cfactx_switch( &proc_cor->context, &thrd_dst->context ); 264 // when __cfactx_switch returns we are back in the processor coroutine 265 266 /* paranoid */ verify( 0x0D15EA5E0D15EA5Ep == thrd_dst->canary ); 267 /* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) > ((uintptr_t)__get_stack(thrd_dst->curr_cor)->limit), "ERROR : Destination $thread %p has been corrupted.\n StackPointer too large.\n", thrd_dst ); 268 /* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) < ((uintptr_t)__get_stack(thrd_dst->curr_cor)->base ), "ERROR : Destination $thread %p has been corrupted.\n StackPointer too small.\n", thrd_dst ); 269 /* paranoid */ verify( thrd_dst->context.SP ); 270 /* paranoid */ verify( thrd_dst->curr_cluster == this->cltr ); 271 /* paranoid */ verify( kernelTLS().this_thread == thrd_dst ); 272 /* paranoid */ verify( ! __preemption_enabled() ); 273 274 // Reset global state 275 kernelTLS().this_thread = 0p; 276 277 // We just finished running a thread, there are a few things that could have happened. 278 // 1 - Regular case : the thread has blocked and now one has scheduled it yet. 279 // 2 - Racy case : the thread has blocked but someone has already tried to schedule it. 280 // 4 - Preempted 281 // In case 1, we may have won a race so we can't write to the state again. 282 // In case 2, we lost the race so we now own the thread. 283 284 if(unlikely(thrd_dst->preempted != __NO_PREEMPTION)) { 285 // The thread was preempted, reschedule it and reset the flag 286 __schedule_thread( thrd_dst ); 287 break RUNNING; 288 } 289 290 if(unlikely(thrd_dst->state == Halting)) { 291 // The thread has halted, it should never be scheduled/run again 292 // finish the thread 293 __thread_finish( thrd_dst ); 294 break RUNNING; 295 } 296 297 /* paranoid */ verify( thrd_dst->state == Active ); 298 thrd_dst->state = Blocked; 299 300 // set state of processor coroutine to active and the thread to inactive 301 int old_ticket = __atomic_fetch_sub(&thrd_dst->ticket, 1, __ATOMIC_SEQ_CST); 302 switch(old_ticket) { 303 case TICKET_RUNNING: 304 // This is case 1, the regular case, nothing more is needed 305 break RUNNING; 306 case TICKET_UNBLOCK: 307 // This is case 2, the racy case, someone tried to run this thread before it finished blocking 308 // In this case, just run it again. 309 continue RUNNING; 310 default: 311 // This makes no sense, something is wrong abort 312 abort(); 313 } 314 } 315 316 // Just before returning to the processor, set the processor coroutine to active 340 317 proc_cor->state = Active; 318 319 /* paranoid */ verify( ! __preemption_enabled() ); 341 320 } 342 321 343 322 // KERNEL_ONLY 344 static void returnToKernel() { 345 coroutine_desc * proc_cor = get_coroutine(kernelTLS.this_processor->runner); 346 thread_desc * thrd_src = kernelTLS.this_thread; 347 348 // set state of current coroutine to inactive 349 thrd_src->state = thrd_src->state == Halted ? Halted : Inactive; 350 proc_cor->state = Active; 351 int local_errno = *__volatile_errno(); 352 #if defined( __i386 ) || defined( __x86_64 ) 353 __x87_store; 323 void returnToKernel() { 324 /* paranoid */ verify( ! __preemption_enabled() ); 325 $coroutine * proc_cor = get_coroutine(kernelTLS().this_processor->runner); 326 $thread * thrd_src = kernelTLS().this_thread; 327 328 #if !defined(__CFA_NO_STATISTICS__) 329 struct processor * last_proc = kernelTLS().this_processor; 354 330 #endif 355 331 356 // set new coroutine that the processor is executing 357 // and context switch to it 358 verify( proc_cor->context.SP ); 359 CtxSwitch( &thrd_src->context, &proc_cor->context ); 360 361 // set state of new coroutine to active 362 proc_cor->state = proc_cor->state == Halted ? Halted : Inactive; 363 thrd_src->state = Active; 364 365 #if defined( __i386 ) || defined( __x86_64 ) 366 __x87_load; 332 // Run the thread on this processor 333 { 334 int local_errno = *__volatile_errno(); 335 #if defined( __i386 ) || defined( __x86_64 ) 336 __x87_store; 337 #endif 338 /* paranoid */ verify( proc_cor->context.SP ); 339 /* paranoid */ verify( 0x0D15EA5E0D15EA5Ep == thrd_src->canary ); 340 __cfactx_switch( &thrd_src->context, &proc_cor->context ); 341 /* paranoid */ verify( 0x0D15EA5E0D15EA5Ep == thrd_src->canary ); 342 #if defined( __i386 ) || defined( __x86_64 ) 343 __x87_load; 344 #endif 345 *__volatile_errno() = local_errno; 346 } 347 348 #if !defined(__CFA_NO_STATISTICS__) 349 if(last_proc != kernelTLS().this_processor) { 350 __tls_stats()->ready.threads.migration++; 351 } 367 352 #endif 368 *__volatile_errno() = local_errno; 369 } 370 371 // KERNEL_ONLY 372 // Once a thread has finished running, some of 373 // its final actions must be executed from the kernel 374 static void finishRunning(processor * this) with( this->finish ) { 375 verify( ! kernelTLS.preemption_state.enabled ); 376 choose( action_code ) { 377 case No_Action: 378 break; 379 case Release: 380 unlock( *lock ); 381 case Schedule: 382 ScheduleThread( thrd ); 383 case Release_Schedule: 384 unlock( *lock ); 385 ScheduleThread( thrd ); 386 case Release_Multi: 387 for(int i = 0; i < lock_count; i++) { 388 unlock( *locks[i] ); 389 } 390 case Release_Multi_Schedule: 391 for(int i = 0; i < lock_count; i++) { 392 unlock( *locks[i] ); 393 } 394 for(int i = 0; i < thrd_count; i++) { 395 ScheduleThread( thrds[i] ); 396 } 397 case Callback: 398 callback(); 399 default: 400 abort("KERNEL ERROR: Unexpected action to run after thread"); 401 } 402 } 403 404 // KERNEL_ONLY 405 // Context invoker for processors 406 // This is the entry point for processors (kernel threads) 407 // It effectively constructs a coroutine by stealing the pthread stack 408 static void * CtxInvokeProcessor(void * arg) { 409 processor * proc = (processor *) arg; 410 kernelTLS.this_processor = proc; 411 kernelTLS.this_thread = 0p; 412 kernelTLS.preemption_state.[enabled, disable_count] = [false, 1]; 413 // SKULLDUGGERY: We want to create a context for the processor coroutine 414 // which is needed for the 2-step context switch. However, there is no reason 415 // to waste the perfectly valid stack create by pthread. 416 current_stack_info_t info; 417 __stack_t ctx; 418 info.storage = &ctx; 419 (proc->runner){ proc, &info }; 420 421 __cfaabi_dbg_print_safe("Coroutine : created stack %p\n", get_coroutine(proc->runner)->stack.storage); 422 423 //Set global state 424 kernelTLS.this_thread = 0p; 425 426 //We now have a proper context from which to schedule threads 427 __cfaabi_dbg_print_safe("Kernel : core %p created (%p, %p)\n", proc, &proc->runner, &ctx); 428 429 // SKULLDUGGERY: Since the coroutine doesn't have its own stack, we can't 430 // resume it to start it like it normally would, it will just context switch 431 // back to here. Instead directly call the main since we already are on the 432 // appropriate stack. 433 get_coroutine(proc->runner)->state = Active; 434 main( proc->runner ); 435 get_coroutine(proc->runner)->state = Halted; 436 437 // Main routine of the core returned, the core is now fully terminated 438 __cfaabi_dbg_print_safe("Kernel : core %p main ended (%p)\n", proc, &proc->runner); 439 440 return 0p; 441 } 442 443 static void Abort( int ret, const char * func ) { 444 if ( ret ) { // pthread routines return errno values 445 abort( "%s : internal error, error(%d) %s.", func, ret, strerror( ret ) ); 446 } // if 447 } // Abort 448 449 void * create_pthread( pthread_t * pthread, void * (*start)(void *), void * arg ) { 450 pthread_attr_t attr; 451 452 Abort( pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute 453 454 size_t stacksize; 455 // default stack size, normally defined by shell limit 456 Abort( pthread_attr_getstacksize( &attr, &stacksize ), "pthread_attr_getstacksize" ); 457 assert( stacksize >= PTHREAD_STACK_MIN ); 458 459 void * stack; 460 __cfaabi_dbg_debug_do( 461 stack = memalign( __page_size, stacksize + __page_size ); 462 // pthread has no mechanism to create the guard page in user supplied stack. 463 if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) { 464 abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) ); 465 } // if 466 ); 467 __cfaabi_dbg_no_debug_do( 468 stack = malloc( stacksize ); 469 ); 470 471 Abort( pthread_attr_setstack( &attr, stack, stacksize ), "pthread_attr_setstack" ); 472 473 Abort( pthread_create( pthread, &attr, start, arg ), "pthread_create" ); 474 return stack; 475 } 476 477 static void start(processor * this) { 478 __cfaabi_dbg_print_safe("Kernel : Starting core %p\n", this); 479 480 this->stack = create_pthread( &this->kernel_thread, CtxInvokeProcessor, (void *)this ); 481 482 __cfaabi_dbg_print_safe("Kernel : core %p started\n", this); 483 } 484 485 // KERNEL_ONLY 486 void kernel_first_resume( processor * this ) { 487 thread_desc * src = mainThread; 488 coroutine_desc * dst = get_coroutine(this->runner); 489 490 verify( ! kernelTLS.preemption_state.enabled ); 491 492 kernelTLS.this_thread->curr_cor = dst; 493 __stack_prepare( &dst->stack, 65000 ); 494 CtxStart(main, dst, this->runner, CtxInvokeCoroutine); 495 496 verify( ! kernelTLS.preemption_state.enabled ); 497 498 dst->last = &src->self_cor; 499 dst->starter = dst->starter ? dst->starter : &src->self_cor; 500 501 // set state of current coroutine to inactive 502 src->state = src->state == Halted ? Halted : Inactive; 503 504 // context switch to specified coroutine 505 verify( dst->context.SP ); 506 CtxSwitch( &src->context, &dst->context ); 507 // when CtxSwitch returns we are back in the src coroutine 508 509 mainThread->curr_cor = &mainThread->self_cor; 510 511 // set state of new coroutine to active 512 src->state = Active; 513 514 verify( ! kernelTLS.preemption_state.enabled ); 515 } 516 517 // KERNEL_ONLY 518 void kernel_last_resume( processor * this ) { 519 coroutine_desc * src = &mainThread->self_cor; 520 coroutine_desc * dst = get_coroutine(this->runner); 521 522 verify( ! kernelTLS.preemption_state.enabled ); 523 verify( dst->starter == src ); 524 verify( dst->context.SP ); 525 526 // context switch to the processor 527 CtxSwitch( &src->context, &dst->context ); 353 354 /* paranoid */ verify( ! __preemption_enabled() ); 355 /* paranoid */ verifyf( ((uintptr_t)thrd_src->context.SP) < ((uintptr_t)__get_stack(thrd_src->curr_cor)->base ), "ERROR : Returning $thread %p has been corrupted.\n StackPointer too small.\n", thrd_src ); 356 /* paranoid */ verifyf( ((uintptr_t)thrd_src->context.SP) > ((uintptr_t)__get_stack(thrd_src->curr_cor)->limit), "ERROR : Returning $thread %p has been corrupted.\n StackPointer too large.\n", thrd_src ); 528 357 } 529 358 530 359 //----------------------------------------------------------------------------- 531 360 // Scheduler routines 532 533 361 // KERNEL ONLY 534 void ScheduleThread( thread_desc * thrd ) { 535 verify( thrd ); 536 verify( thrd->state != Halted ); 537 538 verify( ! kernelTLS.preemption_state.enabled ); 539 540 verifyf( thrd->next == 0p, "Expected null got %p", thrd->next ); 541 542 with( *thrd->curr_cluster ) { 543 lock ( ready_queue_lock __cfaabi_dbg_ctx2 ); 544 bool was_empty = !(ready_queue != 0); 545 append( ready_queue, thrd ); 546 unlock( ready_queue_lock ); 547 548 if(was_empty) { 549 lock (proc_list_lock __cfaabi_dbg_ctx2); 550 if(idles) { 551 wake_fast(idles.head); 362 void __schedule_thread( $thread * thrd ) { 363 /* paranoid */ verify( ! __preemption_enabled() ); 364 /* paranoid */ verify( kernelTLS().this_proc_id ); 365 /* paranoid */ verify( thrd ); 366 /* paranoid */ verify( thrd->state != Halted ); 367 /* paranoid */ verify( thrd->curr_cluster ); 368 /* paranoid */ #if defined( __CFA_WITH_VERIFY__ ) 369 /* paranoid */ if( thrd->state == Blocked || thrd->state == Start ) assertf( thrd->preempted == __NO_PREEMPTION, 370 "Error inactive thread marked as preempted, state %d, preemption %d\n", thrd->state, thrd->preempted ); 371 /* paranoid */ if( thrd->preempted != __NO_PREEMPTION ) assertf(thrd->state == Active, 372 "Error preempted thread marked as not currently running, state %d, preemption %d\n", thrd->state, thrd->preempted ); 373 /* paranoid */ #endif 374 /* paranoid */ verifyf( thrd->link.next == 0p, "Expected null got %p", thrd->link.next ); 375 /* paranoid */ verify( 0x0D15EA5E0D15EA5Ep == thrd->canary ); 376 377 378 if (thrd->preempted == __NO_PREEMPTION) thrd->state = Ready; 379 380 ready_schedule_lock(); 381 // Dereference the thread now because once we push it, there is not guaranteed it's still valid. 382 struct cluster * cl = thrd->curr_cluster; 383 384 // push the thread to the cluster ready-queue 385 push( cl, thrd ); 386 387 // variable thrd is no longer safe to use 388 389 // wake the cluster using the save variable. 390 __wake_one( cl ); 391 ready_schedule_unlock(); 392 393 /* paranoid */ verify( ! __preemption_enabled() ); 394 } 395 396 // KERNEL ONLY 397 static inline $thread * __next_thread(cluster * this) with( *this ) { 398 /* paranoid */ verify( ! __preemption_enabled() ); 399 /* paranoid */ verify( kernelTLS().this_proc_id ); 400 401 ready_schedule_lock(); 402 $thread * thrd = pop( this ); 403 ready_schedule_unlock(); 404 405 /* paranoid */ verify( kernelTLS().this_proc_id ); 406 /* paranoid */ verify( ! __preemption_enabled() ); 407 return thrd; 408 } 409 410 // KERNEL ONLY 411 static inline $thread * __next_thread_slow(cluster * this) with( *this ) { 412 /* paranoid */ verify( ! __preemption_enabled() ); 413 /* paranoid */ verify( kernelTLS().this_proc_id ); 414 415 ready_schedule_lock(); 416 $thread * thrd = pop_slow( this ); 417 ready_schedule_unlock(); 418 419 /* paranoid */ verify( kernelTLS().this_proc_id ); 420 /* paranoid */ verify( ! __preemption_enabled() ); 421 return thrd; 422 } 423 424 void unpark( $thread * thrd ) { 425 if( !thrd ) return; 426 427 int old_ticket = __atomic_fetch_add(&thrd->ticket, 1, __ATOMIC_SEQ_CST); 428 switch(old_ticket) { 429 case TICKET_RUNNING: 430 // Wake won the race, the thread will reschedule/rerun itself 431 break; 432 case TICKET_BLOCKED: 433 /* paranoid */ verify( ! thrd->preempted != __NO_PREEMPTION ); 434 /* paranoid */ verify( thrd->state == Blocked ); 435 436 { 437 /* paranoid */ verify( publicTLS_get(this_proc_id) ); 438 bool full = publicTLS_get(this_proc_id)->full_proc; 439 if(full) disable_interrupts(); 440 441 /* paranoid */ verify( ! __preemption_enabled() ); 442 443 // Wake lost the race, 444 __schedule_thread( thrd ); 445 446 /* paranoid */ verify( ! __preemption_enabled() ); 447 448 if(full) enable_interrupts( __cfaabi_dbg_ctx ); 449 /* paranoid */ verify( publicTLS_get(this_proc_id) ); 552 450 } 553 unlock (proc_list_lock); 554 } 555 else if( struct processor * idle = idles.head ) { 556 wake_fast(idle); 557 } 558 559 } 560 561 verify( ! kernelTLS.preemption_state.enabled ); 451 452 break; 453 default: 454 // This makes no sense, something is wrong abort 455 abort("Thread %p (%s) has mismatch park/unpark\n", thrd, thrd->self_cor.name); 456 } 457 } 458 459 void park( void ) { 460 /* paranoid */ verify( __preemption_enabled() ); 461 disable_interrupts(); 462 /* paranoid */ verify( ! __preemption_enabled() ); 463 /* paranoid */ verify( kernelTLS().this_thread->preempted == __NO_PREEMPTION ); 464 465 returnToKernel(); 466 467 /* paranoid */ verify( ! __preemption_enabled() ); 468 enable_interrupts( __cfaabi_dbg_ctx ); 469 /* paranoid */ verify( __preemption_enabled() ); 470 471 } 472 473 extern "C" { 474 // Leave the thread monitor 475 // last routine called by a thread. 476 // Should never return 477 void __cfactx_thrd_leave() { 478 $thread * thrd = active_thread(); 479 $monitor * this = &thrd->self_mon; 480 481 // Lock the monitor now 482 lock( this->lock __cfaabi_dbg_ctx2 ); 483 484 disable_interrupts(); 485 486 /* paranoid */ verify( ! __preemption_enabled() ); 487 /* paranoid */ verify( thrd->state == Active ); 488 /* paranoid */ verify( 0x0D15EA5E0D15EA5Ep == thrd->canary ); 489 /* paranoid */ verify( kernelTLS().this_thread == thrd ); 490 /* paranoid */ verify( thrd->context.SP ); 491 /* paranoid */ verifyf( ((uintptr_t)thrd->context.SP) > ((uintptr_t)__get_stack(thrd->curr_cor)->limit), "ERROR : $thread %p has been corrupted.\n StackPointer too large.\n", thrd ); 492 /* paranoid */ verifyf( ((uintptr_t)thrd->context.SP) < ((uintptr_t)__get_stack(thrd->curr_cor)->base ), "ERROR : $thread %p has been corrupted.\n StackPointer too small.\n", thrd ); 493 494 thrd->state = Halting; 495 if( TICKET_RUNNING != thrd->ticket ) { abort( "Thread terminated with pending unpark" ); } 496 if( thrd != this->owner ) { abort( "Thread internal monitor has incorrect owner" ); } 497 if( this->recursion != 1) { abort( "Thread internal monitor has unbalanced recursion" ); } 498 499 // Leave the thread 500 returnToKernel(); 501 502 // Control flow should never reach here! 503 abort(); 504 } 562 505 } 563 506 564 507 // KERNEL ONLY 565 thread_desc * nextThread(cluster * this) with( *this ) { 566 verify( ! kernelTLS.preemption_state.enabled ); 567 lock( ready_queue_lock __cfaabi_dbg_ctx2 ); 568 thread_desc * head = pop_head( ready_queue ); 569 unlock( ready_queue_lock ); 570 verify( ! kernelTLS.preemption_state.enabled ); 571 return head; 572 } 573 574 void BlockInternal() { 508 bool force_yield( __Preemption_Reason reason ) { 509 /* paranoid */ verify( __preemption_enabled() ); 575 510 disable_interrupts(); 576 verify( ! kernelTLS.preemption_state.enabled ); 577 returnToKernel(); 578 verify( ! kernelTLS.preemption_state.enabled ); 511 /* paranoid */ verify( ! __preemption_enabled() ); 512 513 $thread * thrd = kernelTLS().this_thread; 514 /* paranoid */ verify(thrd->state == Active); 515 516 // SKULLDUGGERY: It is possible that we are preempting this thread just before 517 // it was going to park itself. If that is the case and it is already using the 518 // intrusive fields then we can't use them to preempt the thread 519 // If that is the case, abandon the preemption. 520 bool preempted = false; 521 if(thrd->link.next == 0p) { 522 preempted = true; 523 thrd->preempted = reason; 524 returnToKernel(); 525 } 526 527 /* paranoid */ verify( ! __preemption_enabled() ); 528 enable_interrupts_noPoll(); 529 /* paranoid */ verify( __preemption_enabled() ); 530 531 return preempted; 532 } 533 534 //============================================================================================= 535 // Kernel Idle Sleep 536 //============================================================================================= 537 // Wake a thread from the front if there are any 538 static void __wake_one(cluster * this) { 539 /* paranoid */ verify( ! __preemption_enabled() ); 540 /* paranoid */ verify( ready_schedule_islocked() ); 541 542 // Check if there is a sleeping processor 543 processor * p; 544 unsigned idle; 545 unsigned total; 546 [idle, total, p] = query(this->idles); 547 548 // If no one is sleeping, we are done 549 if( idle == 0 ) return; 550 551 // We found a processor, wake it up 552 post( p->idle ); 553 554 #if !defined(__CFA_NO_STATISTICS__) 555 __tls_stats()->ready.sleep.wakes++; 556 #endif 557 558 /* paranoid */ verify( ready_schedule_islocked() ); 559 /* paranoid */ verify( ! __preemption_enabled() ); 560 561 return; 562 } 563 564 // Unconditionnaly wake a thread 565 void __wake_proc(processor * this) { 566 __cfadbg_print_safe(runtime_core, "Kernel : waking Processor %p\n", this); 567 568 disable_interrupts(); 569 /* paranoid */ verify( ! __preemption_enabled() ); 570 post( this->idle ); 579 571 enable_interrupts( __cfaabi_dbg_ctx ); 580 572 } 581 573 582 void BlockInternal( __spinlock_t * lock ) { 583 disable_interrupts(); 584 with( *kernelTLS.this_processor ) { 585 finish.action_code = Release; 586 finish.lock = lock; 587 } 588 589 verify( ! kernelTLS.preemption_state.enabled ); 590 returnToKernel(); 591 verify( ! kernelTLS.preemption_state.enabled ); 592 593 enable_interrupts( __cfaabi_dbg_ctx ); 594 } 595 596 void BlockInternal( thread_desc * thrd ) { 597 disable_interrupts(); 598 with( * kernelTLS.this_processor ) { 599 finish.action_code = Schedule; 600 finish.thrd = thrd; 601 } 602 603 verify( ! kernelTLS.preemption_state.enabled ); 604 returnToKernel(); 605 verify( ! kernelTLS.preemption_state.enabled ); 606 607 enable_interrupts( __cfaabi_dbg_ctx ); 608 } 609 610 void BlockInternal( __spinlock_t * lock, thread_desc * thrd ) { 611 assert(thrd); 612 disable_interrupts(); 613 with( * kernelTLS.this_processor ) { 614 finish.action_code = Release_Schedule; 615 finish.lock = lock; 616 finish.thrd = thrd; 617 } 618 619 verify( ! kernelTLS.preemption_state.enabled ); 620 returnToKernel(); 621 verify( ! kernelTLS.preemption_state.enabled ); 622 623 enable_interrupts( __cfaabi_dbg_ctx ); 624 } 625 626 void BlockInternal(__spinlock_t * locks [], unsigned short count) { 627 disable_interrupts(); 628 with( * kernelTLS.this_processor ) { 629 finish.action_code = Release_Multi; 630 finish.locks = locks; 631 finish.lock_count = count; 632 } 633 634 verify( ! kernelTLS.preemption_state.enabled ); 635 returnToKernel(); 636 verify( ! kernelTLS.preemption_state.enabled ); 637 638 enable_interrupts( __cfaabi_dbg_ctx ); 639 } 640 641 void BlockInternal(__spinlock_t * locks [], unsigned short lock_count, thread_desc * thrds [], unsigned short thrd_count) { 642 disable_interrupts(); 643 with( *kernelTLS.this_processor ) { 644 finish.action_code = Release_Multi_Schedule; 645 finish.locks = locks; 646 finish.lock_count = lock_count; 647 finish.thrds = thrds; 648 finish.thrd_count = thrd_count; 649 } 650 651 verify( ! kernelTLS.preemption_state.enabled ); 652 returnToKernel(); 653 verify( ! kernelTLS.preemption_state.enabled ); 654 655 enable_interrupts( __cfaabi_dbg_ctx ); 656 } 657 658 void BlockInternal(__finish_callback_fptr_t callback) { 659 disable_interrupts(); 660 with( *kernelTLS.this_processor ) { 661 finish.action_code = Callback; 662 finish.callback = callback; 663 } 664 665 verify( ! kernelTLS.preemption_state.enabled ); 666 returnToKernel(); 667 verify( ! kernelTLS.preemption_state.enabled ); 668 669 enable_interrupts( __cfaabi_dbg_ctx ); 670 } 671 672 // KERNEL ONLY 673 void LeaveThread(__spinlock_t * lock, thread_desc * thrd) { 674 verify( ! kernelTLS.preemption_state.enabled ); 675 with( * kernelTLS.this_processor ) { 676 finish.action_code = thrd ? Release_Schedule : Release; 677 finish.lock = lock; 678 finish.thrd = thrd; 679 } 680 681 returnToKernel(); 682 } 683 684 //============================================================================================= 685 // Kernel Setup logic 686 //============================================================================================= 687 //----------------------------------------------------------------------------- 688 // Kernel boot procedures 689 static void kernel_startup(void) { 690 verify( ! kernelTLS.preemption_state.enabled ); 691 __cfaabi_dbg_print_safe("Kernel : Starting\n"); 692 693 __page_size = sysconf( _SC_PAGESIZE ); 694 695 __cfa_dbg_global_clusters.list{ __get }; 696 __cfa_dbg_global_clusters.lock{}; 697 698 // Initialize the main cluster 699 mainCluster = (cluster *)&storage_mainCluster; 700 (*mainCluster){"Main Cluster"}; 701 702 __cfaabi_dbg_print_safe("Kernel : Main cluster ready\n"); 703 704 // Start by initializing the main thread 705 // SKULLDUGGERY: the mainThread steals the process main thread 706 // which will then be scheduled by the mainProcessor normally 707 mainThread = (thread_desc *)&storage_mainThread; 708 current_stack_info_t info; 709 info.storage = (__stack_t*)&storage_mainThreadCtx; 710 (*mainThread){ &info }; 711 712 __cfaabi_dbg_print_safe("Kernel : Main thread ready\n"); 713 714 715 716 // Construct the processor context of the main processor 717 void ?{}(processorCtx_t & this, processor * proc) { 718 (this.__cor){ "Processor" }; 719 this.__cor.starter = 0p; 720 this.proc = proc; 721 } 722 723 void ?{}(processor & this) with( this ) { 724 name = "Main Processor"; 725 cltr = mainCluster; 726 terminated{ 0 }; 727 do_terminate = false; 728 preemption_alarm = 0p; 729 pending_preemption = false; 730 kernel_thread = pthread_self(); 731 732 runner{ &this }; 733 __cfaabi_dbg_print_safe("Kernel : constructed main processor context %p\n", &runner); 734 } 735 736 // Initialize the main processor and the main processor ctx 737 // (the coroutine that contains the processing control flow) 738 mainProcessor = (processor *)&storage_mainProcessor; 739 (*mainProcessor){}; 740 741 //initialize the global state variables 742 kernelTLS.this_processor = mainProcessor; 743 kernelTLS.this_thread = mainThread; 744 745 // Enable preemption 746 kernel_start_preemption(); 747 748 // Add the main thread to the ready queue 749 // once resume is called on mainProcessor->runner the mainThread needs to be scheduled like any normal thread 750 ScheduleThread(mainThread); 751 752 // SKULLDUGGERY: Force a context switch to the main processor to set the main thread's context to the current UNIX 753 // context. Hence, the main thread does not begin through CtxInvokeThread, like all other threads. The trick here is that 754 // mainThread is on the ready queue when this call is made. 755 kernel_first_resume( kernelTLS.this_processor ); 756 757 758 759 // THE SYSTEM IS NOW COMPLETELY RUNNING 760 __cfaabi_dbg_print_safe("Kernel : Started\n--------------------------------------------------\n\n"); 761 762 verify( ! kernelTLS.preemption_state.enabled ); 763 enable_interrupts( __cfaabi_dbg_ctx ); 764 verify( TL_GET( preemption_state.enabled ) ); 765 } 766 767 static void kernel_shutdown(void) { 768 __cfaabi_dbg_print_safe("\n--------------------------------------------------\nKernel : Shutting down\n"); 769 770 verify( TL_GET( preemption_state.enabled ) ); 771 disable_interrupts(); 772 verify( ! kernelTLS.preemption_state.enabled ); 773 774 // SKULLDUGGERY: Notify the mainProcessor it needs to terminates. 775 // When its coroutine terminates, it return control to the mainThread 776 // which is currently here 777 __atomic_store_n(&mainProcessor->do_terminate, true, __ATOMIC_RELEASE); 778 kernel_last_resume( kernelTLS.this_processor ); 779 mainThread->self_cor.state = Halted; 780 781 // THE SYSTEM IS NOW COMPLETELY STOPPED 782 783 // Disable preemption 784 kernel_stop_preemption(); 785 786 // Destroy the main processor and its context in reverse order of construction 787 // These were manually constructed so we need manually destroy them 788 ^(mainProcessor->runner){}; 789 ^(mainProcessor){}; 790 791 // Final step, destroy the main thread since it is no longer needed 792 // Since we provided a stack to this taxk it will not destroy anything 793 ^(mainThread){}; 794 795 ^(__cfa_dbg_global_clusters.list){}; 796 ^(__cfa_dbg_global_clusters.lock){}; 797 798 __cfaabi_dbg_print_safe("Kernel : Shutdown complete\n"); 799 } 800 801 //============================================================================================= 802 // Kernel Quiescing 803 //============================================================================================= 804 static void halt(processor * this) with( *this ) { 805 // verify( ! __atomic_load_n(&do_terminate, __ATOMIC_SEQ_CST) ); 806 807 with( *cltr ) { 808 lock (proc_list_lock __cfaabi_dbg_ctx2); 809 remove (procs, *this); 810 push_front(idles, *this); 811 unlock (proc_list_lock); 812 } 813 814 __cfaabi_dbg_print_safe("Kernel : Processor %p ready to sleep\n", this); 815 816 wait( idleLock ); 817 818 __cfaabi_dbg_print_safe("Kernel : Processor %p woke up and ready to run\n", this); 819 820 with( *cltr ) { 821 lock (proc_list_lock __cfaabi_dbg_ctx2); 822 remove (idles, *this); 823 push_front(procs, *this); 824 unlock (proc_list_lock); 574 static void push (__cluster_idles & this, processor & proc) { 575 /* paranoid */ verify( ! __preemption_enabled() ); 576 lock( this ); 577 this.idle++; 578 /* paranoid */ verify( this.idle <= this.total ); 579 580 insert_first(this.list, proc); 581 unlock( this ); 582 /* paranoid */ verify( ! __preemption_enabled() ); 583 } 584 585 static void remove(__cluster_idles & this, processor & proc) { 586 /* paranoid */ verify( ! __preemption_enabled() ); 587 lock( this ); 588 this.idle--; 589 /* paranoid */ verify( this.idle >= 0 ); 590 591 remove(proc); 592 unlock( this ); 593 /* paranoid */ verify( ! __preemption_enabled() ); 594 } 595 596 static [unsigned idle, unsigned total, * processor] query( & __cluster_idles this ) { 597 for() { 598 uint64_t l = __atomic_load_n(&this.lock, __ATOMIC_SEQ_CST); 599 if( 1 == (l % 2) ) { Pause(); continue; } 600 unsigned idle = this.idle; 601 unsigned total = this.total; 602 processor * proc = &this.list`first; 603 // Compiler fence is unnecessary, but gcc-8 and older incorrectly reorder code without it 604 asm volatile("": : :"memory"); 605 if(l != __atomic_load_n(&this.lock, __ATOMIC_SEQ_CST)) { Pause(); continue; } 606 return [idle, total, proc]; 825 607 } 826 608 } … … 836 618 // the globalAbort flag is true. 837 619 lock( kernel_abort_lock __cfaabi_dbg_ctx2 ); 620 621 // disable interrupts, it no longer makes sense to try to interrupt this processor 622 disable_interrupts(); 838 623 839 624 // first task to abort ? … … 853 638 } 854 639 855 return kernelTLS.this_thread;640 return __cfaabi_tls.this_thread; 856 641 } 857 642 858 643 void kernel_abort_msg( void * kernel_data, char * abort_text, int abort_text_size ) { 859 thread_desc * thrd =kernel_data;644 $thread * thrd = ( $thread * ) kernel_data; 860 645 861 646 if(thrd) { … … 878 663 879 664 int kernel_abort_lastframe( void ) __attribute__ ((__nothrow__)) { 880 return get_coroutine(kernelTLS .this_thread) == get_coroutine(mainThread) ? 4 : 2;665 return get_coroutine(kernelTLS().this_thread) == get_coroutine(mainThread) ? 4 : 2; 881 666 } 882 667 … … 905 690 void ^?{}(semaphore & this) {} 906 691 907 voidP(semaphore & this) with( this ){692 bool P(semaphore & this) with( this ){ 908 693 lock( lock __cfaabi_dbg_ctx2 ); 909 694 count -= 1; 910 695 if ( count < 0 ) { 911 696 // queue current task 912 append( waiting, kernelTLS.this_thread);697 append( waiting, active_thread() ); 913 698 914 699 // atomically release spin lock and block 915 BlockInternal( &lock ); 700 unlock( lock ); 701 park(); 702 return true; 916 703 } 917 704 else { 918 705 unlock( lock ); 919 } 920 } 921 922 void V(semaphore & this) with( this ) { 923 thread_desc * thrd = 0p; 706 return false; 707 } 708 } 709 710 bool V(semaphore & this) with( this ) { 711 $thread * thrd = 0p; 924 712 lock( lock __cfaabi_dbg_ctx2 ); 925 713 count += 1; … … 932 720 933 721 // make new owner 934 WakeThread( thrd ); 935 } 936 937 //----------------------------------------------------------------------------- 938 // Global Queues 939 void doregister( cluster & cltr ) { 940 lock ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2); 941 push_front( __cfa_dbg_global_clusters.list, cltr ); 942 unlock ( __cfa_dbg_global_clusters.lock ); 943 } 944 945 void unregister( cluster & cltr ) { 946 lock ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2); 947 remove( __cfa_dbg_global_clusters.list, cltr ); 948 unlock( __cfa_dbg_global_clusters.lock ); 949 } 950 951 void doregister( cluster * cltr, thread_desc & thrd ) { 952 lock (cltr->thread_list_lock __cfaabi_dbg_ctx2); 953 cltr->nthreads += 1; 954 push_front(cltr->threads, thrd); 955 unlock (cltr->thread_list_lock); 956 } 957 958 void unregister( cluster * cltr, thread_desc & thrd ) { 959 lock (cltr->thread_list_lock __cfaabi_dbg_ctx2); 960 remove(cltr->threads, thrd ); 961 cltr->nthreads -= 1; 962 unlock(cltr->thread_list_lock); 963 } 964 965 void doregister( cluster * cltr, processor * proc ) { 966 lock (cltr->proc_list_lock __cfaabi_dbg_ctx2); 967 cltr->nprocessors += 1; 968 push_front(cltr->procs, *proc); 969 unlock (cltr->proc_list_lock); 970 } 971 972 void unregister( cluster * cltr, processor * proc ) { 973 lock (cltr->proc_list_lock __cfaabi_dbg_ctx2); 974 remove(cltr->procs, *proc ); 975 cltr->nprocessors -= 1; 976 unlock(cltr->proc_list_lock); 722 unpark( thrd ); 723 724 return thrd != 0p; 725 } 726 727 bool V(semaphore & this, unsigned diff) with( this ) { 728 $thread * thrd = 0p; 729 lock( lock __cfaabi_dbg_ctx2 ); 730 int release = max(-count, (int)diff); 731 count += diff; 732 for(release) { 733 unpark( pop_head( waiting ) ); 734 } 735 736 unlock( lock ); 737 738 return thrd != 0p; 977 739 } 978 740 … … 981 743 __cfaabi_dbg_debug_do( 982 744 extern "C" { 983 void __cfaabi_dbg_record (__spinlock_t & this, const char * prev_name) {745 void __cfaabi_dbg_record_lock(__spinlock_t & this, const char prev_name[]) { 984 746 this.prev_name = prev_name; 985 this.prev_thrd = kernelTLS .this_thread;747 this.prev_thrd = kernelTLS().this_thread; 986 748 } 987 749 } … … 990 752 //----------------------------------------------------------------------------- 991 753 // Debug 992 bool threading_enabled(void) {754 bool threading_enabled(void) __attribute__((const)) { 993 755 return true; 994 756 } 757 758 //----------------------------------------------------------------------------- 759 // Statistics 760 #if !defined(__CFA_NO_STATISTICS__) 761 void print_halts( processor & this ) { 762 this.print_halts = true; 763 } 764 765 void print_stats_now( cluster & this, int flags ) { 766 __print_stats( this.stats, this.print_stats, "Cluster", this.name, (void*)&this ); 767 } 768 769 extern int __print_alarm_stats; 770 void print_alarm_stats() { 771 __print_alarm_stats = -1; 772 } 773 #endif 995 774 // Local Variables: // 996 775 // mode: c //
Note:
See TracChangeset
for help on using the changeset viewer.
