Changes in / [231b18f:54dcab1]
- Files:
-
- 15 edited
Legend:
- Unmodified
- Added
- Removed
-
benchmark/readyQ/cycle.cfa
r231b18f r54dcab1 84 84 } 85 85 86 printf("Duration (ms) : %'ld\n", (end - start)` ms);86 printf("Duration (ms) : %'ld\n", (end - start)`dms); 87 87 printf("Number of processors : %'d\n", nprocs); 88 88 printf("Number of threads : %'d\n", tthreads); … … 90 90 printf("Total Operations(ops): %'15llu\n", global_counter); 91 91 printf("Total blocks : %'15llu\n", global_blocks); 92 printf("Ops per second : %'18.2lf\n", ((double)global_counter) / (end - start)` s);93 printf("ns per ops : %'18.2lf\n", ( (double)(end - start)`ns)/ global_counter);92 printf("Ops per second : %'18.2lf\n", ((double)global_counter) / (end - start)`ds); 93 printf("ns per ops : %'18.2lf\n", (end - start)`dns / global_counter); 94 94 printf("Ops per threads : %'15llu\n", global_counter / tthreads); 95 95 printf("Ops per procs : %'15llu\n", global_counter / nprocs); 96 printf("Ops/sec/procs : %'18.2lf\n", (((double)global_counter) / nprocs) / (end - start)` s);97 printf("ns per ops/procs : %'18.2lf\n", ( (double)(end - start)`ns)/ (global_counter / nprocs));96 printf("Ops/sec/procs : %'18.2lf\n", (((double)global_counter) / nprocs) / (end - start)`ds); 97 printf("ns per ops/procs : %'18.2lf\n", (end - start)`dns / (global_counter / nprocs)); 98 98 fflush(stdout); 99 99 } -
benchmark/readyQ/cycle.go
r231b18f r54dcab1 72 72 p.Printf("Cycle size (# thrds) : %d\n", ring_size); 73 73 p.Printf("Total Operations(ops): %15d\n", global_counter) 74 p.Printf(" Yields per second: %18.2f\n", float64(global_counter) / delta.Seconds())74 p.Printf("Ops per second : %18.2f\n", float64(global_counter) / delta.Seconds()) 75 75 p.Printf("ns per ops : %18.2f\n", float64(delta.Nanoseconds()) / float64(global_counter)) 76 76 p.Printf("Ops per threads : %15d\n", global_counter / uint64(tthreads)) -
benchmark/readyQ/rq_bench.hfa
r231b18f r54dcab1 88 88 } 89 89 90 struct bench_sem {90 struct __attribute__((aligned(128))) bench_sem { 91 91 struct $thread * volatile ptr; 92 92 }; -
benchmark/readyQ/rq_bench.hpp
r231b18f r54dcab1 75 75 } 76 76 77 class bench_sem {77 class __attribute__((aligned(128))) bench_sem { 78 78 Fibre * volatile ptr = nullptr; 79 79 public: -
libcfa/src/concurrency/io.cfa
r231b18f r54dcab1 76 76 77 77 static inline bool next( __leaderlock_t & this ) { 78 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled);78 /* paranoid */ verify( ! __preemption_enabled() ); 79 79 struct $thread * nextt; 80 80 for() { … … 168 168 // This is NOT thread-safe 169 169 static [int, bool] __drain_io( & struct __io_data ring ) { 170 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled);170 /* paranoid */ verify( ! __preemption_enabled() ); 171 171 172 172 unsigned to_submit = 0; … … 404 404 return; 405 405 } 406 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled);406 /* paranoid */ verify( ! __preemption_enabled() ); 407 407 __STATS__( true, 408 408 io.submit_q.leader += 1; … … 442 442 443 443 #if defined(LEADER_LOCK) 444 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled);444 /* paranoid */ verify( ! __preemption_enabled() ); 445 445 next(ring.submit_q.submit_lock); 446 446 #else -
libcfa/src/concurrency/io/setup.cfa
r231b18f r54dcab1 149 149 id.full_proc = false; 150 150 id.id = doregister(&id); 151 kernelTLS.this_proc_id = &id;151 __cfaabi_tls.this_proc_id = &id; 152 152 __cfaabi_dbg_print_safe( "Kernel : IO poller thread starting\n" ); 153 153 … … 179 179 __cfadbg_print_safe(io_core, "Kernel I/O : Unparking io poller %p\n", io_ctx); 180 180 #if !defined( __CFA_NO_STATISTICS__ ) 181 kernelTLS.this_stats = io_ctx->self.curr_cluster->stats;181 __cfaabi_tls.this_stats = io_ctx->self.curr_cluster->stats; 182 182 #endif 183 183 post( io_ctx->sem ); -
libcfa/src/concurrency/kernel.cfa
r231b18f r54dcab1 122 122 // Because of a bug, we couldn't initialized the seed on construction 123 123 // Do it here 124 kernelTLS.rand_seed ^= rdtscl();125 kernelTLS.ready_rng.fwd_seed = 25214903917_l64u * (rdtscl() ^ (uintptr_t)&runner);124 __cfaabi_tls.rand_seed ^= rdtscl(); 125 __cfaabi_tls.ready_rng.fwd_seed = 25214903917_l64u * (rdtscl() ^ (uintptr_t)&runner); 126 126 __tls_rand_advance_bck(); 127 127 … … 217 217 // and it make sense for it to be set in all other cases except here 218 218 // fake it 219 kernelTLS.this_thread = mainThread;219 __cfaabi_tls.this_thread = mainThread; 220 220 } 221 221 … … 230 230 // from the processor coroutine to the target thread 231 231 static void __run_thread(processor * this, $thread * thrd_dst) { 232 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled);232 /* paranoid */ verify( ! __preemption_enabled() ); 233 233 /* paranoid */ verifyf( thrd_dst->state == Ready || thrd_dst->preempted != __NO_PREEMPTION, "state : %d, preempted %d\n", thrd_dst->state, thrd_dst->preempted); 234 234 /* paranoid */ verifyf( thrd_dst->link.next == 0p, "Expected null got %p", thrd_dst->link.next ); … … 247 247 248 248 // Update global state 249 kernelTLS .this_thread = thrd_dst;250 251 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled);252 /* paranoid */ verify( kernelTLS .this_thread == thrd_dst );249 kernelTLS().this_thread = thrd_dst; 250 251 /* paranoid */ verify( ! __preemption_enabled() ); 252 /* paranoid */ verify( kernelTLS().this_thread == thrd_dst ); 253 253 /* paranoid */ verify( thrd_dst->context.SP ); 254 254 /* paranoid */ verify( thrd_dst->state != Halted ); … … 267 267 /* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) < ((uintptr_t)__get_stack(thrd_dst->curr_cor)->base ), "ERROR : Destination $thread %p has been corrupted.\n StackPointer too small.\n", thrd_dst ); 268 268 /* paranoid */ verify( thrd_dst->context.SP ); 269 /* paranoid */ verify( kernelTLS .this_thread == thrd_dst );270 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled);269 /* paranoid */ verify( kernelTLS().this_thread == thrd_dst ); 270 /* paranoid */ verify( ! __preemption_enabled() ); 271 271 272 272 // Reset global state 273 kernelTLS .this_thread = 0p;273 kernelTLS().this_thread = 0p; 274 274 275 275 // We just finished running a thread, there are a few things that could have happened. … … 315 315 proc_cor->state = Active; 316 316 317 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled);317 /* paranoid */ verify( ! __preemption_enabled() ); 318 318 } 319 319 320 320 // KERNEL_ONLY 321 321 void returnToKernel() { 322 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled);323 $coroutine * proc_cor = get_coroutine(kernelTLS .this_processor->runner);324 $thread * thrd_src = kernelTLS .this_thread;322 /* paranoid */ verify( ! __preemption_enabled() ); 323 $coroutine * proc_cor = get_coroutine(kernelTLS().this_processor->runner); 324 $thread * thrd_src = kernelTLS().this_thread; 325 325 326 326 #if !defined(__CFA_NO_STATISTICS__) 327 struct processor * last_proc = kernelTLS .this_processor;327 struct processor * last_proc = kernelTLS().this_processor; 328 328 #endif 329 329 … … 345 345 346 346 #if !defined(__CFA_NO_STATISTICS__) 347 if(last_proc != kernelTLS .this_processor) {347 if(last_proc != kernelTLS().this_processor) { 348 348 __tls_stats()->ready.threads.migration++; 349 349 } 350 350 #endif 351 351 352 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled);352 /* paranoid */ verify( ! __preemption_enabled() ); 353 353 /* paranoid */ verifyf( ((uintptr_t)thrd_src->context.SP) < ((uintptr_t)__get_stack(thrd_src->curr_cor)->base ), "ERROR : Returning $thread %p has been corrupted.\n StackPointer too small.\n", thrd_src ); 354 354 /* paranoid */ verifyf( ((uintptr_t)thrd_src->context.SP) > ((uintptr_t)__get_stack(thrd_src->curr_cor)->limit), "ERROR : Returning $thread %p has been corrupted.\n StackPointer too large.\n", thrd_src ); … … 359 359 // KERNEL ONLY 360 360 void __schedule_thread( $thread * thrd ) { 361 /* paranoid */ verify( ! __preemption_enabled() ); 361 362 /* paranoid */ verify( thrd ); 362 363 /* paranoid */ verify( thrd->state != Halted ); 363 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled ); 364 /* paranoid */ verify( kernelTLS.this_proc_id ); 364 /* paranoid */ verify( kernelTLS().this_proc_id ); 365 365 /* paranoid */ #if defined( __CFA_WITH_VERIFY__ ) 366 366 /* paranoid */ if( thrd->state == Blocked || thrd->state == Start ) assertf( thrd->preempted == __NO_PREEMPTION, … … 380 380 ready_schedule_unlock(); 381 381 382 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled);382 /* paranoid */ verify( ! __preemption_enabled() ); 383 383 } 384 384 385 385 // KERNEL ONLY 386 386 static inline $thread * __next_thread(cluster * this) with( *this ) { 387 /* paranoid */ verify( kernelTLS.this_proc_id);388 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );387 /* paranoid */ verify( ! __preemption_enabled() ); 388 /* paranoid */ verify( kernelTLS().this_proc_id ); 389 389 390 390 ready_schedule_lock(); … … 392 392 ready_schedule_unlock(); 393 393 394 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );395 /* paranoid */ verify( kernelTLS.this_proc_id);394 /* paranoid */ verify( kernelTLS().this_proc_id ); 395 /* paranoid */ verify( ! __preemption_enabled() ); 396 396 return thrd; 397 397 } … … 399 399 // KERNEL ONLY 400 400 static inline $thread * __next_thread_slow(cluster * this) with( *this ) { 401 /* paranoid */ verify( kernelTLS.this_proc_id);402 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );401 /* paranoid */ verify( ! __preemption_enabled() ); 402 /* paranoid */ verify( kernelTLS().this_proc_id ); 403 403 404 404 ready_schedule_lock(); … … 406 406 ready_schedule_unlock(); 407 407 408 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );409 /* paranoid */ verify( kernelTLS.this_proc_id);408 /* paranoid */ verify( kernelTLS().this_proc_id ); 409 /* paranoid */ verify( ! __preemption_enabled() ); 410 410 return thrd; 411 411 } … … 414 414 if( !thrd ) return; 415 415 416 /* paranoid */ verify( kernelTLS.this_proc_id );417 bool full = kernelTLS.this_proc_id->full_proc;418 if(full) disable_interrupts();419 420 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );421 416 int old_ticket = __atomic_fetch_add(&thrd->ticket, 1, __ATOMIC_SEQ_CST); 422 417 switch(old_ticket) { … … 428 423 /* paranoid */ verify( thrd->state == Blocked ); 429 424 430 // Wake lost the race, 431 __schedule_thread( thrd ); 425 { 426 /* paranoid */ verify( publicTLS_get(this_proc_id) ); 427 bool full = publicTLS_get(this_proc_id)->full_proc; 428 if(full) disable_interrupts(); 429 430 /* paranoid */ verify( ! __preemption_enabled() ); 431 432 // Wake lost the race, 433 __schedule_thread( thrd ); 434 435 /* paranoid */ verify( ! __preemption_enabled() ); 436 437 if(full) enable_interrupts( __cfaabi_dbg_ctx ); 438 /* paranoid */ verify( publicTLS_get(this_proc_id) ); 439 } 440 432 441 break; 433 442 default: … … 435 444 abort("Thread %p (%s) has mismatch park/unpark\n", thrd, thrd->self_cor.name); 436 445 } 437 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );438 439 if(full) enable_interrupts( __cfaabi_dbg_ctx );440 /* paranoid */ verify( kernelTLS.this_proc_id );441 446 } 442 447 443 448 void park( void ) { 444 /* paranoid */ verify( kernelTLS.preemption_state.enabled);449 /* paranoid */ verify( __preemption_enabled() ); 445 450 disable_interrupts(); 446 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled);447 /* paranoid */ verify( kernelTLS .this_thread->preempted == __NO_PREEMPTION );451 /* paranoid */ verify( ! __preemption_enabled() ); 452 /* paranoid */ verify( kernelTLS().this_thread->preempted == __NO_PREEMPTION ); 448 453 449 454 returnToKernel(); 450 455 451 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled);456 /* paranoid */ verify( ! __preemption_enabled() ); 452 457 enable_interrupts( __cfaabi_dbg_ctx ); 453 /* paranoid */ verify( kernelTLS.preemption_state.enabled);458 /* paranoid */ verify( __preemption_enabled() ); 454 459 455 460 } … … 460 465 // Should never return 461 466 void __cfactx_thrd_leave() { 462 $thread * thrd = TL_GET( this_thread);467 $thread * thrd = active_thread(); 463 468 $monitor * this = &thrd->self_mon; 464 469 … … 473 478 474 479 // Leave the thread 475 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled);480 /* paranoid */ verify( ! __preemption_enabled() ); 476 481 returnToKernel(); 477 482 abort(); … … 483 488 // KERNEL ONLY 484 489 bool force_yield( __Preemption_Reason reason ) { 485 /* paranoid */ verify( kernelTLS.preemption_state.enabled);490 /* paranoid */ verify( __preemption_enabled() ); 486 491 disable_interrupts(); 487 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled);488 489 $thread * thrd = kernelTLS .this_thread;492 /* paranoid */ verify( ! __preemption_enabled() ); 493 494 $thread * thrd = kernelTLS().this_thread; 490 495 /* paranoid */ verify(thrd->state == Active); 491 496 … … 501 506 } 502 507 503 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled);508 /* paranoid */ verify( ! __preemption_enabled() ); 504 509 enable_interrupts_noPoll(); 505 /* paranoid */ verify( kernelTLS.preemption_state.enabled);510 /* paranoid */ verify( __preemption_enabled() ); 506 511 507 512 return preempted; … … 513 518 // Wake a thread from the front if there are any 514 519 static void __wake_one(cluster * this) { 515 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled);520 /* paranoid */ verify( ! __preemption_enabled() ); 516 521 /* paranoid */ verify( ready_schedule_islocked() ); 517 522 … … 533 538 534 539 /* paranoid */ verify( ready_schedule_islocked() ); 535 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled);540 /* paranoid */ verify( ! __preemption_enabled() ); 536 541 537 542 return; … … 543 548 544 549 disable_interrupts(); 545 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled);550 /* paranoid */ verify( ! __preemption_enabled() ); 546 551 post( this->idle ); 547 552 enable_interrupts( __cfaabi_dbg_ctx ); … … 549 554 550 555 static void push (__cluster_idles & this, processor & proc) { 551 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled);556 /* paranoid */ verify( ! __preemption_enabled() ); 552 557 lock( this ); 553 558 this.idle++; … … 556 561 insert_first(this.list, proc); 557 562 unlock( this ); 558 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled);563 /* paranoid */ verify( ! __preemption_enabled() ); 559 564 } 560 565 561 566 static void remove(__cluster_idles & this, processor & proc) { 562 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled);567 /* paranoid */ verify( ! __preemption_enabled() ); 563 568 lock( this ); 564 569 this.idle--; … … 567 572 remove(proc); 568 573 unlock( this ); 569 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled);574 /* paranoid */ verify( ! __preemption_enabled() ); 570 575 } 571 576 … … 611 616 } 612 617 613 return kernelTLS.this_thread;618 return __cfaabi_tls.this_thread; 614 619 } 615 620 … … 636 641 637 642 int kernel_abort_lastframe( void ) __attribute__ ((__nothrow__)) { 638 return get_coroutine(kernelTLS .this_thread) == get_coroutine(mainThread) ? 4 : 2;643 return get_coroutine(kernelTLS().this_thread) == get_coroutine(mainThread) ? 4 : 2; 639 644 } 640 645 … … 668 673 if ( count < 0 ) { 669 674 // queue current task 670 append( waiting, kernelTLS.this_thread);675 append( waiting, active_thread() ); 671 676 672 677 // atomically release spin lock and block … … 718 723 void __cfaabi_dbg_record_lock(__spinlock_t & this, const char prev_name[]) { 719 724 this.prev_name = prev_name; 720 this.prev_thrd = kernelTLS .this_thread;725 this.prev_thrd = kernelTLS().this_thread; 721 726 } 722 727 } -
libcfa/src/concurrency/kernel.hfa
r231b18f r54dcab1 275 275 static inline [cluster *&, cluster *& ] __get( cluster & this ) __attribute__((const)) { return this.node.[next, prev]; } 276 276 277 static inline struct processor * active_processor() { return TL_GET( this_processor ); } // UNSAFE278 static inline struct cluster * active_cluster () { return TL_GET( this_processor )->cltr; }277 static inline struct processor * active_processor() { return publicTLS_get( this_processor ); } // UNSAFE 278 static inline struct cluster * active_cluster () { return publicTLS_get( this_processor )->cltr; } 279 279 280 280 #if !defined(__CFA_NO_STATISTICS__) 281 void print_stats_now( cluster & this, int flags ); 282 281 283 static inline void print_stats_at_exit( cluster & this, int flags ) { 282 284 this.print_stats |= flags; -
libcfa/src/concurrency/kernel/fwd.hfa
r231b18f r54dcab1 55 55 uint64_t bck_seed; 56 56 } ready_rng; 57 } kernelTLS__attribute__ ((tls_model ( "initial-exec" )));57 } __cfaabi_tls __attribute__ ((tls_model ( "initial-exec" ))); 58 58 59 extern bool __preemption_enabled(); 59 60 61 static inline KernelThreadData & kernelTLS( void ) { 62 /* paranoid */ verify( ! __preemption_enabled() ); 63 return __cfaabi_tls; 64 } 65 66 extern uintptr_t __cfatls_get( unsigned long int member ); 67 // #define publicTLS_get( member ) ((typeof(__cfaabi_tls.member))__cfatls_get( __builtin_offsetof(KernelThreadData, member) )) 68 #define publicTLS_get( member ) (__cfaabi_tls.member) 69 // extern forall(otype T) T __cfatls_get( T * member, T value ); 70 // #define publicTLS_set( member, value ) __cfatls_set( (typeof(member)*)__builtin_offsetof(KernelThreadData, member), value ); 60 71 61 72 static inline uint64_t __tls_rand() { 62 73 #if defined(__SIZEOF_INT128__) 63 return __lehmer64( kernelTLS .rand_seed );74 return __lehmer64( kernelTLS().rand_seed ); 64 75 #else 65 return __xorshift64( kernelTLS .rand_seed );76 return __xorshift64( kernelTLS().rand_seed ); 66 77 #endif 67 78 } … … 75 86 static inline unsigned __tls_rand_fwd() { 76 87 77 kernelTLS .ready_rng.fwd_seed = (A * kernelTLS.ready_rng.fwd_seed + C) & (M - 1);78 return kernelTLS .ready_rng.fwd_seed >> D;88 kernelTLS().ready_rng.fwd_seed = (A * kernelTLS().ready_rng.fwd_seed + C) & (M - 1); 89 return kernelTLS().ready_rng.fwd_seed >> D; 79 90 } 80 91 81 92 static inline unsigned __tls_rand_bck() { 82 unsigned int r = kernelTLS .ready_rng.bck_seed >> D;83 kernelTLS .ready_rng.bck_seed = AI * (kernelTLS.ready_rng.bck_seed - C) & (M - 1);93 unsigned int r = kernelTLS().ready_rng.bck_seed >> D; 94 kernelTLS().ready_rng.bck_seed = AI * (kernelTLS().ready_rng.bck_seed - C) & (M - 1); 84 95 return r; 85 96 } … … 92 103 93 104 static inline void __tls_rand_advance_bck(void) { 94 kernelTLS .ready_rng.bck_seed = kernelTLS.ready_rng.fwd_seed;105 kernelTLS().ready_rng.bck_seed = kernelTLS().ready_rng.fwd_seed; 95 106 } 96 107 } 97 108 98 #if 0 // def __ARM_ARCH99 // function prototypes are only really used by these macros on ARM100 void disable_global_interrupts();101 void enable_global_interrupts();102 109 103 #define TL_GET( member ) ( { __typeof__( kernelTLS.member ) target; \104 disable_global_interrupts(); \105 target = kernelTLS.member; \106 enable_global_interrupts(); \107 target; } )108 #define TL_SET( member, value ) disable_global_interrupts(); \109 kernelTLS.member = value; \110 enable_global_interrupts();111 #else112 #define TL_GET( member ) kernelTLS.member113 #define TL_SET( member, value ) kernelTLS.member = value;114 #endif115 110 116 111 extern void disable_interrupts(); … … 121 116 extern void park( void ); 122 117 extern void unpark( struct $thread * this ); 123 static inline struct $thread * active_thread () { return TL_GET( this_thread ); } 118 static inline struct $thread * active_thread () { 119 struct $thread * t = publicTLS_get( this_thread ); 120 /* paranoid */ verify( t ); 121 return t; 122 } 124 123 125 124 extern bool force_yield( enum __Preemption_Reason ); … … 140 139 #if !defined(__CFA_NO_STATISTICS__) 141 140 static inline struct __stats_t * __tls_stats() { 142 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled);143 /* paranoid */ verify( kernelTLS .this_stats );144 return kernelTLS .this_stats;141 /* paranoid */ verify( ! __preemption_enabled() ); 142 /* paranoid */ verify( kernelTLS().this_stats ); 143 return kernelTLS().this_stats; 145 144 } 146 145 -
libcfa/src/concurrency/kernel/startup.cfa
r231b18f r54dcab1 118 118 //----------------------------------------------------------------------------- 119 119 // Global state 120 thread_local struct KernelThreadData kernelTLS__attribute__ ((tls_model ( "initial-exec" ))) @= {120 thread_local struct KernelThreadData __cfaabi_tls __attribute__ ((tls_model ( "initial-exec" ))) @= { 121 121 NULL, // cannot use 0p 122 122 NULL, … … 156 156 // Kernel boot procedures 157 157 static void __kernel_startup(void) { 158 verify( ! kernelTLS.preemption_state.enabled);158 /* paranoid */ verify( ! __preemption_enabled() ); 159 159 __cfadbg_print_safe(runtime_core, "Kernel : Starting\n"); 160 160 … … 212 212 213 213 //initialize the global state variables 214 kernelTLS.this_processor = mainProcessor;215 kernelTLS.this_proc_id = (__processor_id_t*)mainProcessor;216 kernelTLS.this_thread = mainThread;214 __cfaabi_tls.this_processor = mainProcessor; 215 __cfaabi_tls.this_proc_id = (__processor_id_t*)mainProcessor; 216 __cfaabi_tls.this_thread = mainThread; 217 217 218 218 #if !defined( __CFA_NO_STATISTICS__ ) 219 kernelTLS.this_stats = (__stats_t *)& storage_mainProcStats;220 __init_stats( kernelTLS.this_stats );219 __cfaabi_tls.this_stats = (__stats_t *)& storage_mainProcStats; 220 __init_stats( __cfaabi_tls.this_stats ); 221 221 #endif 222 222 … … 234 234 // context. Hence, the main thread does not begin through __cfactx_invoke_thread, like all other threads. The trick here is that 235 235 // mainThread is on the ready queue when this call is made. 236 __kernel_first_resume( kernelTLS.this_processor );236 __kernel_first_resume( __cfaabi_tls.this_processor ); 237 237 238 238 … … 251 251 __cfadbg_print_safe(runtime_core, "Kernel : Started\n--------------------------------------------------\n\n"); 252 252 253 verify( ! kernelTLS.preemption_state.enabled);253 /* paranoid */ verify( ! __preemption_enabled() ); 254 254 enable_interrupts( __cfaabi_dbg_ctx ); 255 verify( TL_GET( preemption_state.enabled ) ); 255 /* paranoid */ verify( __preemption_enabled() ); 256 256 257 } 257 258 … … 262 263 mainCluster->io.ctxs = 0p; 263 264 264 /* paranoid */ verify( TL_GET( preemption_state.enabled) );265 /* paranoid */ verify( __preemption_enabled() ); 265 266 disable_interrupts(); 266 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled);267 /* paranoid */ verify( ! __preemption_enabled() ); 267 268 268 269 __cfadbg_print_safe(runtime_core, "\n--------------------------------------------------\nKernel : Shutting down\n"); … … 272 273 // which is currently here 273 274 __atomic_store_n(&mainProcessor->do_terminate, true, __ATOMIC_RELEASE); 274 __kernel_last_resume( kernelTLS.this_processor );275 __kernel_last_resume( __cfaabi_tls.this_processor ); 275 276 mainThread->self_cor.state = Halted; 276 277 … … 321 322 __stats_t local_stats; 322 323 __init_stats( &local_stats ); 323 kernelTLS.this_stats = &local_stats;324 __cfaabi_tls.this_stats = &local_stats; 324 325 #endif 325 326 326 327 processor * proc = (processor *) arg; 327 kernelTLS.this_processor = proc;328 kernelTLS.this_proc_id = (__processor_id_t*)proc;329 kernelTLS.this_thread = 0p;330 kernelTLS.preemption_state.[enabled, disable_count] = [false, 1];328 __cfaabi_tls.this_processor = proc; 329 __cfaabi_tls.this_proc_id = (__processor_id_t*)proc; 330 __cfaabi_tls.this_thread = 0p; 331 __cfaabi_tls.preemption_state.[enabled, disable_count] = [false, 1]; 331 332 // SKULLDUGGERY: We want to create a context for the processor coroutine 332 333 // which is needed for the 2-step context switch. However, there is no reason … … 340 341 341 342 //Set global state 342 kernelTLS.this_thread = 0p;343 __cfaabi_tls.this_thread = 0p; 343 344 344 345 //We now have a proper context from which to schedule threads … … 370 371 $coroutine * dst = get_coroutine(this->runner); 371 372 372 verify( ! kernelTLS.preemption_state.enabled);373 374 kernelTLS.this_thread->curr_cor = dst;373 /* paranoid */ verify( ! __preemption_enabled() ); 374 375 __cfaabi_tls.this_thread->curr_cor = dst; 375 376 __stack_prepare( &dst->stack, 65000 ); 376 377 __cfactx_start(main, dst, this->runner, __cfactx_invoke_coroutine); 377 378 378 verify( ! kernelTLS.preemption_state.enabled);379 /* paranoid */ verify( ! __preemption_enabled() ); 379 380 380 381 dst->last = &src->self_cor; … … 394 395 /* paranoid */ verify(src->state == Active); 395 396 396 verify( ! kernelTLS.preemption_state.enabled);397 /* paranoid */ verify( ! __preemption_enabled() ); 397 398 } 398 399 … … 402 403 $coroutine * dst = get_coroutine(this->runner); 403 404 404 verify( ! kernelTLS.preemption_state.enabled);405 verify( dst->starter == src );406 verify( dst->context.SP );405 /* paranoid */ verify( ! __preemption_enabled() ); 406 /* paranoid */ verify( dst->starter == src ); 407 /* paranoid */ verify( dst->context.SP ); 407 408 408 409 // SKULLDUGGERY in debug the processors check that the … … 546 547 547 548 P( terminated ); 548 verify( kernelTLS.this_processor!= &this);549 /* paranoid */ verify( active_processor() != &this); 549 550 } 550 551 … … 696 697 #if defined(__CFA_WITH_VERIFY__) 697 698 static bool verify_fwd_bck_rng(void) { 698 kernelTLS.ready_rng.fwd_seed = 25214903917_l64u * (rdtscl() ^ (uintptr_t)&verify_fwd_bck_rng);699 __cfaabi_tls.ready_rng.fwd_seed = 25214903917_l64u * (rdtscl() ^ (uintptr_t)&verify_fwd_bck_rng); 699 700 700 701 unsigned values[10]; -
libcfa/src/concurrency/kernel_private.hfa
r231b18f r54dcab1 38 38 #endif 39 39 ; 40 41 extern bool __preemption_enabled(); 40 42 41 43 //release/wake-up the following resources … … 181 183 // creating/destroying queues 182 184 static inline void ready_schedule_lock(void) with(*__scheduler_lock) { 183 /*paranoid*/ verify( kernelTLS.this_proc_id ); 184 185 unsigned iproc = kernelTLS.this_proc_id->id; 186 /*paranoid*/ verify(data[iproc].handle == kernelTLS.this_proc_id); 185 /* paranoid */ verify( ! __preemption_enabled() ); 186 /* paranoid */ verify( kernelTLS().this_proc_id ); 187 188 unsigned iproc = kernelTLS().this_proc_id->id; 189 /*paranoid*/ verify(data[iproc].handle == kernelTLS().this_proc_id); 187 190 /*paranoid*/ verify(iproc < ready); 188 191 … … 207 210 208 211 static inline void ready_schedule_unlock(void) with(*__scheduler_lock) { 209 /*paranoid*/ verify( kernelTLS.this_proc_id ); 210 211 unsigned iproc = kernelTLS.this_proc_id->id; 212 /*paranoid*/ verify(data[iproc].handle == kernelTLS.this_proc_id); 212 /* paranoid */ verify( ! __preemption_enabled() ); 213 /* paranoid */ verify( kernelTLS().this_proc_id ); 214 215 unsigned iproc = kernelTLS().this_proc_id->id; 216 /*paranoid*/ verify(data[iproc].handle == kernelTLS().this_proc_id); 213 217 /*paranoid*/ verify(iproc < ready); 214 218 /*paranoid*/ verify(data[iproc].lock); … … 223 227 #ifdef __CFA_WITH_VERIFY__ 224 228 static inline bool ready_schedule_islocked(void) { 225 /*paranoid*/ verify( kernelTLS.this_proc_id ); 226 __processor_id_t * proc = kernelTLS.this_proc_id; 229 /* paranoid */ verify( ! __preemption_enabled() ); 230 /*paranoid*/ verify( kernelTLS().this_proc_id ); 231 __processor_id_t * proc = kernelTLS().this_proc_id; 227 232 return __scheduler_lock->data[proc->id].owned; 228 233 } -
libcfa/src/concurrency/monitor.cfa
r231b18f r54dcab1 82 82 // Enter single monitor 83 83 static void __enter( $monitor * this, const __monitor_group_t & group ) { 84 $thread * thrd = active_thread(); 85 84 86 // Lock the monitor spinlock 85 87 lock( this->lock __cfaabi_dbg_ctx2 ); 86 // Interrupts disable inside critical section87 $thread * thrd = kernelTLS.this_thread;88 88 89 89 __cfaabi_dbg_print_safe( "Kernel : %10p Entering mon %p (%p)\n", thrd, this, this->owner); … … 126 126 __cfaabi_dbg_print_safe( "Kernel : %10p Entered mon %p\n", thrd, this); 127 127 128 /* paranoid */ verifyf( kernelTLS.this_thread == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", kernelTLS.this_thread, this->owner, this->recursion, this );128 /* paranoid */ verifyf( active_thread() == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", active_thread(), this->owner, this->recursion, this ); 129 129 return; 130 130 } … … 132 132 __cfaabi_dbg_print_safe( "Kernel : %10p Entered mon %p\n", thrd, this); 133 133 134 /* paranoid */ verifyf( kernelTLS.this_thread == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", kernelTLS.this_thread, this->owner, this->recursion, this );134 /* paranoid */ verifyf( active_thread() == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", active_thread(), this->owner, this->recursion, this ); 135 135 /* paranoid */ verify( this->lock.lock ); 136 136 … … 141 141 142 142 static void __dtor_enter( $monitor * this, fptr_t func, bool join ) { 143 $thread * thrd = active_thread(); 144 143 145 // Lock the monitor spinlock 144 146 lock( this->lock __cfaabi_dbg_ctx2 ); 145 // Interrupts disable inside critical section146 $thread * thrd = kernelTLS.this_thread;147 147 148 148 __cfaabi_dbg_print_safe( "Kernel : %10p Entering dtor for mon %p (%p)\n", thrd, this, this->owner); … … 155 155 __set_owner( this, thrd ); 156 156 157 verifyf( kernelTLS.this_thread == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", kernelTLS.this_thread, this->owner, this->recursion, this );157 verifyf( active_thread() == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", active_thread(), this->owner, this->recursion, this ); 158 158 159 159 unlock( this->lock ); … … 174 174 this->owner = thrd; 175 175 176 verifyf( kernelTLS.this_thread == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", kernelTLS.this_thread, this->owner, this->recursion, this );176 verifyf( active_thread() == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", active_thread(), this->owner, this->recursion, this ); 177 177 178 178 unlock( this->lock ); … … 200 200 201 201 // Release the next thread 202 /* paranoid */ verifyf( urgent->owner->waiting_thread == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", kernelTLS.this_thread, this->owner, this->recursion, this );202 /* paranoid */ verifyf( urgent->owner->waiting_thread == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", active_thread(), this->owner, this->recursion, this ); 203 203 unpark( urgent->owner->waiting_thread ); 204 204 … … 207 207 208 208 // Some one was waiting for us, enter 209 /* paranoid */ verifyf( kernelTLS.this_thread == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", kernelTLS.this_thread, this->owner, this->recursion, this );209 /* paranoid */ verifyf( active_thread() == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", active_thread(), this->owner, this->recursion, this ); 210 210 } 211 211 else { … … 224 224 park(); 225 225 226 /* paranoid */ verifyf( kernelTLS.this_thread == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", kernelTLS.this_thread, this->owner, this->recursion, this );226 /* paranoid */ verifyf( active_thread() == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", active_thread(), this->owner, this->recursion, this ); 227 227 return; 228 228 } … … 237 237 lock( this->lock __cfaabi_dbg_ctx2 ); 238 238 239 __cfaabi_dbg_print_safe( "Kernel : %10p Leaving mon %p (%p)\n", kernelTLS.this_thread, this, this->owner);240 241 /* paranoid */ verifyf( kernelTLS.this_thread == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", kernelTLS.this_thread, this->owner, this->recursion, this );239 __cfaabi_dbg_print_safe( "Kernel : %10p Leaving mon %p (%p)\n", active_thread(), this, this->owner); 240 241 /* paranoid */ verifyf( active_thread() == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", active_thread(), this->owner, this->recursion, this ); 242 242 243 243 // Leaving a recursion level, decrement the counter … … 270 270 void __dtor_leave( $monitor * this, bool join ) { 271 271 __cfaabi_dbg_debug_do( 272 if( TL_GET( this_thread) != this->owner ) {273 abort( "Destroyed monitor %p has inconsistent owner, expected %p got %p.\n", this, TL_GET( this_thread), this->owner);272 if( active_thread() != this->owner ) { 273 abort( "Destroyed monitor %p has inconsistent owner, expected %p got %p.\n", this, active_thread(), this->owner); 274 274 } 275 275 if( this->recursion != 1 && !join ) { … … 287 287 /* paranoid */ verify( this->lock.lock ); 288 288 /* paranoid */ verifyf( thrd == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", thrd, this->owner, this->recursion, this ); 289 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled);289 /* paranoid */ verify( ! __preemption_enabled() ); 290 290 /* paranoid */ verify( thrd->state == Halted ); 291 291 /* paranoid */ verify( this->recursion == 1 ); … … 303 303 // Unpark the next owner if needed 304 304 /* paranoid */ verifyf( !new_owner || new_owner == this->owner, "Expected owner to be %p, got %p (m: %p)", new_owner, this->owner, this ); 305 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled);305 /* paranoid */ verify( ! __preemption_enabled() ); 306 306 /* paranoid */ verify( thrd->state == Halted ); 307 307 unpark( new_owner ); … … 327 327 // Sorts monitors before entering 328 328 void ?{}( monitor_guard_t & this, $monitor * m [], __lock_size_t count, fptr_t func ) { 329 $thread * thrd = TL_GET( this_thread);329 $thread * thrd = active_thread(); 330 330 331 331 // Store current array … … 362 362 363 363 // Restore thread context 364 TL_GET( this_thread)->monitors = this.prev;364 active_thread()->monitors = this.prev; 365 365 } 366 366 … … 369 369 void ?{}( monitor_dtor_guard_t & this, $monitor * m [], fptr_t func, bool join ) { 370 370 // optimization 371 $thread * thrd = TL_GET( this_thread);371 $thread * thrd = active_thread(); 372 372 373 373 // Store current array … … 392 392 393 393 // Restore thread context 394 TL_GET( this_thread)->monitors = this.prev;394 active_thread()->monitors = this.prev; 395 395 } 396 396 … … 432 432 433 433 // Create the node specific to this wait operation 434 wait_ctx( TL_GET( this_thread), user_info );434 wait_ctx( active_thread(), user_info ); 435 435 436 436 // Append the current wait operation to the ones already queued on the condition … … 483 483 //Some more checking in debug 484 484 __cfaabi_dbg_debug_do( 485 $thread * this_thrd = TL_GET( this_thread);485 $thread * this_thrd = active_thread(); 486 486 if ( this.monitor_count != this_thrd->monitors.size ) { 487 487 abort( "Signal on condition %p made with different number of monitor(s), expected %zi got %zi", &this, this.monitor_count, this_thrd->monitors.size ); … … 531 531 532 532 // Create the node specific to this wait operation 533 wait_ctx_primed( kernelTLS.this_thread, 0 )533 wait_ctx_primed( active_thread(), 0 ) 534 534 535 535 //save contexts … … 630 630 631 631 // Create the node specific to this wait operation 632 wait_ctx_primed( kernelTLS.this_thread, 0 );632 wait_ctx_primed( active_thread(), 0 ); 633 633 634 634 // Save monitor states … … 682 682 683 683 // Create the node specific to this wait operation 684 wait_ctx_primed( kernelTLS.this_thread, 0 );684 wait_ctx_primed( active_thread(), 0 ); 685 685 686 686 monitor_save; … … 688 688 689 689 for( __lock_size_t i = 0; i < count; i++) { 690 verify( monitors[i]->owner == kernelTLS.this_thread);690 verify( monitors[i]->owner == active_thread() ); 691 691 } 692 692 … … 724 724 static inline void __set_owner( $monitor * monitors [], __lock_size_t count, $thread * owner ) { 725 725 /* paranoid */ verify ( monitors[0]->lock.lock ); 726 /* paranoid */ verifyf( monitors[0]->owner == kernelTLS.this_thread, "Expected owner to be %p, got %p (r: %i, m: %p)", kernelTLS.this_thread, monitors[0]->owner, monitors[0]->recursion, monitors[0] );726 /* paranoid */ verifyf( monitors[0]->owner == active_thread(), "Expected owner to be %p, got %p (r: %i, m: %p)", active_thread(), monitors[0]->owner, monitors[0]->recursion, monitors[0] ); 727 727 monitors[0]->owner = owner; 728 728 monitors[0]->recursion = 1; 729 729 for( __lock_size_t i = 1; i < count; i++ ) { 730 730 /* paranoid */ verify ( monitors[i]->lock.lock ); 731 /* paranoid */ verifyf( monitors[i]->owner == kernelTLS.this_thread, "Expected owner to be %p, got %p (r: %i, m: %p)", kernelTLS.this_thread, monitors[i]->owner, monitors[i]->recursion, monitors[i] );731 /* paranoid */ verifyf( monitors[i]->owner == active_thread(), "Expected owner to be %p, got %p (r: %i, m: %p)", active_thread(), monitors[i]->owner, monitors[i]->recursion, monitors[i] ); 732 732 monitors[i]->owner = owner; 733 733 monitors[i]->recursion = 0; … … 755 755 //regardless of if we are ready to baton pass, 756 756 //we need to set the monitor as in use 757 /* paranoid */ verifyf( !this->owner || kernelTLS.this_thread == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", kernelTLS.this_thread, this->owner, this->recursion, this );757 /* paranoid */ verifyf( !this->owner || active_thread() == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", active_thread(), this->owner, this->recursion, this ); 758 758 __set_owner( this, urgent->owner->waiting_thread ); 759 759 … … 764 764 // Get the next thread in the entry_queue 765 765 $thread * new_owner = pop_head( this->entry_queue ); 766 /* paranoid */ verifyf( !this->owner || kernelTLS.this_thread == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", kernelTLS.this_thread, this->owner, this->recursion, this );766 /* paranoid */ verifyf( !this->owner || active_thread() == this->owner, "Expected owner to be %p, got %p (r: %i, m: %p)", active_thread(), this->owner, this->recursion, this ); 767 767 /* paranoid */ verify( !new_owner || new_owner->link.next == 0p ); 768 768 __set_owner( this, new_owner ); … … 892 892 893 893 static inline void brand_condition( condition & this ) { 894 $thread * thrd = TL_GET( this_thread);894 $thread * thrd = active_thread(); 895 895 if( !this.monitors ) { 896 896 // __cfaabi_dbg_print_safe( "Branding\n" ); -
libcfa/src/concurrency/preemption.cfa
r231b18f r54dcab1 163 163 // Kernel Signal Tools 164 164 //============================================================================================= 165 166 165 // In a user-level threading system, there are handful of thread-local variables where this problem occurs on the ARM. 167 // 166 // 168 167 // For each kernel thread running user-level threads, there is a flag variable to indicate if interrupts are 169 168 // enabled/disabled for that kernel thread. Therefore, this variable is made thread local. 170 // 169 // 171 170 // For example, this code fragment sets the state of the "interrupt" variable in thread-local memory. 172 // 171 // 173 172 // _Thread_local volatile int interrupts; 174 173 // int main() { 175 174 // interrupts = 0; // disable interrupts } 176 // 175 // 177 176 // which generates the following code on the ARM 178 // 177 // 179 178 // (gdb) disassemble main 180 179 // Dump of assembler code for function main: … … 185 184 // 0x0000000000000620 <+16>: str wzr, [x1] 186 185 // 0x0000000000000624 <+20>: ret 187 // 186 // 188 187 // The mrs moves a pointer from coprocessor register tpidr_el0 into register x1. Register w0 is set to 0. The two adds 189 188 // increase the TLS pointer with the displacement (offset) 0x10, which is the location in the TSL of variable … … 193 192 // the user-level ready-queue it is run on kernel thread M. It now stores 0 into "interrupts" back on kernel thread N, 194 193 // turning off interrupt on the wrong kernel thread. 195 // 194 // 196 195 // On the x86, the following code is generated for the same code fragment. 197 // 196 // 198 197 // (gdb) disassemble main 199 198 // Dump of assembler code for function main: 200 199 // 0x0000000000400420 <+0>: movl $0x0,%fs:0xfffffffffffffffc 201 200 // 0x000000000040042c <+12>: xor %eax,%eax 202 // 0x000000000040042e <+14>: retq 203 // 201 // 0x000000000040042e <+14>: retq 202 // 204 203 // and there is base-displacement addressing used to atomically reset variable "interrupts" off of the TSL pointer in 205 204 // register "fs". 206 // 205 // 207 206 // Hence, the ARM has base-displacement address for the general purpose registers, BUT not to the coprocessor 208 207 // registers. As a result, generating the address for the write into variable "interrupts" is no longer atomic. 209 // 208 // 210 209 // Note this problem does NOT occur when just using multiple kernel threads because the preemption ALWAYS restarts the 211 210 // thread on the same kernel thread. 212 // 211 // 213 212 // The obvious question is why does ARM use a coprocessor register to store the TSL pointer given that coprocessor 214 213 // registers are second-class registers with respect to the instruction set. One possible answer is that they did not … … 216 215 // available. 217 216 218 __cfaabi_dbg_debug_do( static thread_local void * last_interrupt = 0; ) 217 //---------- 218 // special case for preemption since used often 219 bool __preemption_enabled() { 220 // create a assembler label before 221 // marked as clobber all to avoid movement 222 asm volatile("__cfaasm_check_before:":::"memory"); 223 224 // access tls as normal 225 bool enabled = __cfaabi_tls.preemption_state.enabled; 226 227 // create a assembler label after 228 // marked as clobber all to avoid movement 229 asm volatile("__cfaasm_check_after:":::"memory"); 230 return enabled; 231 } 232 233 //---------- 234 // Get data from the TLS block 235 uintptr_t __cfatls_get( unsigned long int offset ) __attribute__((__noinline__)); //no inline to avoid problems 236 uintptr_t __cfatls_get( unsigned long int offset ) { 237 // create a assembler label before 238 // marked as clobber all to avoid movement 239 asm volatile("__cfaasm_get_before:":::"memory"); 240 241 // access tls as normal (except for pointer arithmetic) 242 uintptr_t val = *(uintptr_t*)((uintptr_t)&__cfaabi_tls + offset); 243 244 // create a assembler label after 245 // marked as clobber all to avoid movement 246 asm volatile("__cfaasm_get_after:":::"memory"); 247 return val; 248 } 219 249 220 250 extern "C" { 221 251 // Disable interrupts by incrementing the counter 222 252 void disable_interrupts() { 223 with( kernelTLS.preemption_state ) { 253 // create a assembler label before 254 // marked as clobber all to avoid movement 255 asm volatile("__cfaasm_disable_before:":::"memory"); 256 257 with( __cfaabi_tls.preemption_state ) { 224 258 #if GCC_VERSION > 50000 225 259 static_assert(__atomic_always_lock_free(sizeof(enabled), &enabled), "Must be lock-free"); … … 238 272 verify( new_val < 65_000u ); // If this triggers someone is disabling interrupts without enabling them 239 273 } 274 275 // create a assembler label after 276 // marked as clobber all to avoid movement 277 asm volatile("__cfaasm_disable_after:":::"memory"); 240 278 } 241 279 … … 243 281 // If counter reaches 0, execute any pending __cfactx_switch 244 282 void enable_interrupts( __cfaabi_dbg_ctx_param ) { 245 processor * proc = kernelTLS.this_processor; // Cache the processor now since interrupts can start happening after the atomic store 283 // create a assembler label before 284 // marked as clobber all to avoid movement 285 asm volatile("__cfaasm_enable_before:":::"memory"); 286 287 processor * proc = __cfaabi_tls.this_processor; // Cache the processor now since interrupts can start happening after the atomic store 246 288 /* paranoid */ verify( proc ); 247 289 248 with( kernelTLS.preemption_state ){290 with( __cfaabi_tls.preemption_state ){ 249 291 unsigned short prev = disable_count; 250 292 disable_count -= 1; … … 273 315 // For debugging purposes : keep track of the last person to enable the interrupts 274 316 __cfaabi_dbg_debug_do( proc->last_enable = caller; ) 317 318 // create a assembler label after 319 // marked as clobber all to avoid movement 320 asm volatile("__cfaasm_enable_after:":::"memory"); 275 321 } 276 322 … … 278 324 // Don't execute any pending __cfactx_switch even if counter reaches 0 279 325 void enable_interrupts_noPoll() { 280 unsigned short prev = kernelTLS.preemption_state.disable_count; 281 kernelTLS.preemption_state.disable_count -= 1; 326 // create a assembler label before 327 // marked as clobber all to avoid movement 328 asm volatile("__cfaasm_nopoll_before:":::"memory"); 329 330 unsigned short prev = __cfaabi_tls.preemption_state.disable_count; 331 __cfaabi_tls.preemption_state.disable_count -= 1; 282 332 verifyf( prev != 0u, "Incremented from %u\n", prev ); // If this triggers someone is enabled already enabled interrupts 283 333 if( prev == 1 ) { 284 334 #if GCC_VERSION > 50000 285 static_assert(__atomic_always_lock_free(sizeof( kernelTLS.preemption_state.enabled), &kernelTLS.preemption_state.enabled), "Must be lock-free");335 static_assert(__atomic_always_lock_free(sizeof(__cfaabi_tls.preemption_state.enabled), &__cfaabi_tls.preemption_state.enabled), "Must be lock-free"); 286 336 #endif 287 337 // Set enabled flag to true 288 338 // should be atomic to avoid preemption in the middle of the operation. 289 339 // use memory order RELAXED since there is no inter-thread on this variable requirements 290 __atomic_store_n(& kernelTLS.preemption_state.enabled, true, __ATOMIC_RELAXED);340 __atomic_store_n(&__cfaabi_tls.preemption_state.enabled, true, __ATOMIC_RELAXED); 291 341 292 342 // Signal the compiler that a fence is needed but only for signal handlers 293 343 __atomic_signal_fence(__ATOMIC_RELEASE); 294 344 } 345 346 // create a assembler label after 347 // marked as clobber all to avoid movement 348 asm volatile("__cfaasm_nopoll_after:":::"memory"); 295 349 } 296 350 } … … 327 381 static void timeout( $thread * this ) { 328 382 #if !defined( __CFA_NO_STATISTICS__ ) 329 kernelTLS .this_stats = this->curr_cluster->stats;383 kernelTLS().this_stats = this->curr_cluster->stats; 330 384 #endif 331 385 unpark( this ); … … 338 392 static inline bool preemption_ready() { 339 393 // Check if preemption is safe 340 bool ready = kernelTLS.preemption_state.enabled && ! kernelTLS.preemption_state.in_progress;394 bool ready = __cfaabi_tls.preemption_state.enabled && ! __cfaabi_tls.preemption_state.in_progress; 341 395 342 396 // Adjust the pending flag accordingly 343 kernelTLS.this_processor->pending_preemption = !ready;397 __cfaabi_tls.this_processor->pending_preemption = !ready; 344 398 return ready; 345 399 } … … 355 409 356 410 // Start with preemption disabled until ready 357 kernelTLS.preemption_state.enabled = false;358 kernelTLS.preemption_state.disable_count = 1;411 __cfaabi_tls.preemption_state.enabled = false; 412 __cfaabi_tls.preemption_state.disable_count = 1; 359 413 360 414 // Initialize the event kernel … … 414 468 // Kernel Signal Handlers 415 469 //============================================================================================= 470 struct asm_region { 471 void * before; 472 void * after; 473 }; 474 475 //----------------------------------------------------------------------------- 476 // Some assembly required 477 #if defined( __i386 ) 478 #define __cfaasm_label( label ) \ 479 ({ \ 480 struct asm_region region; \ 481 asm( \ 482 "movl $__cfaasm_" #label "_before, %[vb]\n\t" \ 483 "movl $__cfaasm_" #label "_after , %[va]\n\t" \ 484 : [vb]"=r"(region.before), [vb]"=r"(region.before) \ 485 ); \ 486 region; \ 487 }); 488 #elif defined( __x86_64 ) 489 #ifdef __PIC__ 490 #define PLT "@PLT" 491 #else 492 #define PLT "" 493 #endif 494 #define __cfaasm_label( label ) \ 495 ({ \ 496 struct asm_region region; \ 497 asm( \ 498 "movq $__cfaasm_" #label "_before" PLT ", %[vb]\n\t" \ 499 "movq $__cfaasm_" #label "_after" PLT ", %[va]\n\t" \ 500 : [vb]"=r"(region.before), [va]"=r"(region.after) \ 501 ); \ 502 region; \ 503 }); 504 #elif defined( __aarch64__ ) 505 #error __cfaasm_label undefined for arm 506 #else 507 #error unknown hardware architecture 508 #endif 509 510 __cfaabi_dbg_debug_do( static thread_local void * last_interrupt = 0; ) 416 511 417 512 // Context switch signal handler 418 513 // Receives SIGUSR1 signal and causes the current thread to yield 419 514 static void sigHandler_ctxSwitch( __CFA_SIGPARMS__ ) { 420 __cfaabi_dbg_debug_do( last_interrupt = (void *)(cxt->uc_mcontext.CFA_REG_IP); ) 515 void * ip = (void *)(cxt->uc_mcontext.CFA_REG_IP); 516 __cfaabi_dbg_debug_do( last_interrupt = ip; ) 421 517 422 518 // SKULLDUGGERY: if a thread creates a processor and the immediately deletes it, … … 424 520 // before the kernel thread has even started running. When that happens, an interrupt 425 521 // with a null 'this_processor' will be caught, just ignore it. 426 if(! kernelTLS.this_processor ) return;522 if(! __cfaabi_tls.this_processor ) return; 427 523 428 524 choose(sfp->si_value.sival_int) { 429 525 case PREEMPT_NORMAL : ;// Normal case, nothing to do here 430 case PREEMPT_TERMINATE: verify( __atomic_load_n( & kernelTLS.this_processor->do_terminate, __ATOMIC_SEQ_CST ) );526 case PREEMPT_TERMINATE: verify( __atomic_load_n( &__cfaabi_tls.this_processor->do_terminate, __ATOMIC_SEQ_CST ) ); 431 527 default: 432 528 abort( "internal error, signal value is %d", sfp->si_value.sival_int ); … … 436 532 if( !preemption_ready() ) { return; } 437 533 438 __cfaabi_dbg_print_buffer_decl( " KERNEL: preempting core %p (%p @ %p).\n", kernelTLS.this_processor, kernelTLS.this_thread, (void *)(cxt->uc_mcontext.CFA_REG_IP) ); 534 struct asm_region region; 535 region = __cfaasm_label( get ); if( ip >= region.before && ip <= region.after ) return; 536 region = __cfaasm_label( check ); if( ip >= region.before && ip <= region.after ) return; 537 region = __cfaasm_label( disable ); if( ip >= region.before && ip <= region.after ) return; 538 region = __cfaasm_label( enable ); if( ip >= region.before && ip <= region.after ) return; 539 region = __cfaasm_label( nopoll ); if( ip >= region.before && ip <= region.after ) return; 540 541 __cfaabi_dbg_print_buffer_decl( " KERNEL: preempting core %p (%p @ %p).\n", __cfaabi_tls.this_processor, __cfaabi_tls.this_thread, (void *)(cxt->uc_mcontext.CFA_REG_IP) ); 439 542 440 543 // Sync flag : prevent recursive calls to the signal handler 441 kernelTLS.preemption_state.in_progress = true;544 __cfaabi_tls.preemption_state.in_progress = true; 442 545 443 546 // Clear sighandler mask before context switching. … … 449 552 } 450 553 451 // TODO: this should go in finish action452 554 // Clear the in progress flag 453 kernelTLS.preemption_state.in_progress = false;555 __cfaabi_tls.preemption_state.in_progress = false; 454 556 455 557 // Preemption can occur here … … 468 570 id.full_proc = false; 469 571 id.id = doregister(&id); 470 kernelTLS.this_proc_id = &id;572 __cfaabi_tls.this_proc_id = &id; 471 573 472 574 // Block sigalrms to control when they arrive … … 536 638 537 639 void __cfaabi_check_preemption() { 538 bool ready = kernelTLS.preemption_state.enabled;640 bool ready = __preemption_enabled(); 539 641 if(!ready) { abort("Preemption should be ready"); } 540 642 … … 559 661 #ifdef __CFA_WITH_VERIFY__ 560 662 bool __cfaabi_dbg_in_kernel() { 561 return ! kernelTLS.preemption_state.enabled;663 return !__preemption_enabled(); 562 664 } 563 665 #endif -
libcfa/src/concurrency/ready_queue.cfa
r231b18f r54dcab1 150 150 // queues or removing them. 151 151 uint_fast32_t ready_mutate_lock( void ) with(*__scheduler_lock) { 152 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled);152 /* paranoid */ verify( ! __preemption_enabled() ); 153 153 154 154 // Step 1 : lock global lock … … 166 166 } 167 167 168 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled);168 /* paranoid */ verify( ! __preemption_enabled() ); 169 169 return s; 170 170 } 171 171 172 172 void ready_mutate_unlock( uint_fast32_t last_s ) with(*__scheduler_lock) { 173 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled);173 /* paranoid */ verify( ! __preemption_enabled() ); 174 174 175 175 // Step 1 : release local locks … … 188 188 __atomic_store_n(&lock, (bool)false, __ATOMIC_RELEASE); 189 189 190 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled);190 /* paranoid */ verify( ! __preemption_enabled() ); 191 191 } 192 192 … … 252 252 preferred = 253 253 //* 254 kernelTLS .this_processor ? kernelTLS.this_processor->id * 4 : -1;254 kernelTLS().this_processor ? kernelTLS().this_processor->id * 4 : -1; 255 255 /*/ 256 256 thrd->link.preferred * 4; … … 331 331 // Don't bother trying locally too much 332 332 int local_tries = 8; 333 preferred = kernelTLS .this_processor->id * 4;333 preferred = kernelTLS().this_processor->id * 4; 334 334 #endif 335 335 -
tools/stat.py
r231b18f r54dcab1 1 #!/usr/bin/python 1 #!/usr/bin/python3 2 2 3 3 import sys … … 17 17 avg = numpy.mean (content) 18 18 std = numpy.std (content) 19 print "median {0:.1f} avg {1:.1f} stddev {2:.1f}".format( med, avg, std)19 print("median {0:.1f} avg {1:.1f} stddev {2:.1f}".format( med, avg, std )) 20 20 21 21
Note: See TracChangeset
for help on using the changeset viewer.