source: libcfa/src/concurrency/kernel/startup.cfa@ ea593a3

ADT ast-experimental enum forall-pointer-decay jacob/cs343-translation new-ast-unique-expr pthread-emulation qualifiedEnum
Last change on this file since ea593a3 was ef94ae7, checked in by Thierry Delisle <tdelisle@…>, 4 years ago

Changed ready-queue to use -1 for empty ts.
  • Property mode set to 100644
File size: 24.5 KB
Line 
1//
2// Cforall Version 1.0.0 Copyright (C) 2020 University of Waterloo
3//
4// The contents of this file are covered under the licence agreement in the
5// file "LICENCE" distributed with Cforall.
6//
7// kernel/startup.cfa --
8//
9// Author : Thierry Delisle
10// Created On : Thu Jul 30 15:12:54 2020
11// Last Modified By :
12// Last Modified On :
13// Update Count :
14//
15
16#define __cforall_thread__
17#define _GNU_SOURCE
18
19// C Includes
20#include <errno.h> // errno
21#include <signal.h>
22#include <string.h> // strerror
23#include <unistd.h> // sysconf
24
25extern "C" {
26 #include <limits.h> // PTHREAD_STACK_MIN
27 #include <unistd.h> // syscall
28 #include <sys/eventfd.h> // eventfd
29 #include <sys/mman.h> // mprotect
30 #include <sys/resource.h> // getrlimit
31}
32
33// CFA Includes
34#include "kernel_private.hfa"
35#include "startup.hfa" // STARTUP_PRIORITY_XXX
36#include "math.hfa"
37
38#define CFA_PROCESSOR_USE_MMAP 0
39
40//-----------------------------------------------------------------------------
41// Some assembly required
42#if defined( __i386 )
43 #define CtxGet( ctx ) __asm__ volatile ( \
44 "movl %%esp,%0\n" \
45 "movl %%ebp,%1\n" \
46 : "=rm" (ctx.SP), \
47 "=rm" (ctx.FP) \
48 )
49#elif defined( __x86_64 )
50 #define CtxGet( ctx ) __asm__ volatile ( \
51 "movq %%rsp,%0\n" \
52 "movq %%rbp,%1\n" \
53 : "=rm" (ctx.SP), \
54 "=rm" (ctx.FP) \
55 )
56#elif defined( __aarch64__ )
57 #define CtxGet( ctx ) __asm__ volatile ( \
58 "mov %0, sp\n" \
59 "mov %1, fp\n" \
60 : "=rm" (ctx.SP), \
61 "=rm" (ctx.FP) \
62 )
63#else
64 #error unknown hardware architecture
65#endif
66
67//-----------------------------------------------------------------------------
68// Start and stop routine for the kernel, declared first to make sure they run first
69static void __kernel_startup (void) __attribute__(( constructor( STARTUP_PRIORITY_KERNEL ) ));
70static void __kernel_shutdown(void) __attribute__(( destructor ( STARTUP_PRIORITY_KERNEL ) ));
71
72//-----------------------------------------------------------------------------
73// Static Forward Declarations
74struct current_stack_info_t;
75
76static void * __invoke_processor(void * arg);
77static void __kernel_first_resume( processor * this );
78static void __kernel_last_resume ( processor * this );
79static void init(processor & this, const char name[], cluster & _cltr, $thread * initT);
80static void deinit(processor & this);
81static void doregister( struct cluster & cltr );
82static void unregister( struct cluster & cltr );
83static void register_tls( processor * this );
84static void unregister_tls( processor * this );
85static void ?{}( $coroutine & this, current_stack_info_t * info);
86static void ?{}( $thread & this, current_stack_info_t * info);
87static void ?{}(processorCtx_t & this) {}
88static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info);
89
90#if defined(__CFA_WITH_VERIFY__)
91 static bool verify_fwd_bck_rng(void);
92#endif
93
94//-----------------------------------------------------------------------------
95// Forward Declarations for other modules
96extern void __kernel_alarm_startup(void);
97extern void __kernel_alarm_shutdown(void);
98
99//-----------------------------------------------------------------------------
100// Other Forward Declarations
101extern void __wake_proc(processor *);
102
103//-----------------------------------------------------------------------------
104// Kernel storage
105KERNEL_STORAGE(cluster, mainCluster);
106KERNEL_STORAGE(processor, mainProcessor);
107KERNEL_STORAGE($thread, mainThread);
108KERNEL_STORAGE(__stack_t, mainThreadCtx);
109KERNEL_STORAGE(__scheduler_RWLock_t, __scheduler_lock);
110#if !defined(__CFA_NO_STATISTICS__)
111KERNEL_STORAGE(__stats_t, mainProcStats);
112#endif
113
114cluster * mainCluster;
115processor * mainProcessor;
116$thread * mainThread;
117__scheduler_RWLock_t * __scheduler_lock;
118
119extern "C" {
120 struct { __dllist_t(cluster) list; __spinlock_t lock; } __cfa_dbg_global_clusters;
121}
122
123extern size_t __page_size;
124extern int __map_prot;
125
126//-----------------------------------------------------------------------------
127// Global state
128thread_local struct KernelThreadData __cfaabi_tls __attribute__ ((tls_model ( "initial-exec" ))) @= {
129 NULL, // cannot use 0p
130 NULL,
131 false,
132 { 1, false, false },
133 0,
134 { 0, 0 },
135 NULL,
136 #ifdef __CFA_WITH_VERIFY__
137 false,
138 0,
139 #endif
140};
141
142#if defined(CFA_HAVE_LINUX_LIBRSEQ)
143 // No data needed
144#elif defined(CFA_HAVE_LINUX_RSEQ_H)
145 extern "Cforall" {
146 __attribute__((aligned(128))) thread_local volatile struct rseq __cfaabi_rseq @= {
147 .cpu_id : RSEQ_CPU_ID_UNINITIALIZED,
148 };
149 }
150#else
151 // No data needed
152#endif
153
154//-----------------------------------------------------------------------------
155// Struct to steal stack
156struct current_stack_info_t {
157 __stack_t * storage; // pointer to stack object
158 void * base; // base of stack
159 void * limit; // stack grows towards stack limit
160 void * context; // address of cfa_context_t
161};
162
163void ?{}( current_stack_info_t & this ) {
164 __stack_context_t ctx;
165 CtxGet( ctx );
166 this.base = ctx.FP;
167
168 rlimit r;
169 getrlimit( RLIMIT_STACK, &r);
170 size_t size = r.rlim_cur;
171
172 this.limit = (void *)(((intptr_t)this.base) - size);
173 this.context = &storage_mainThreadCtx;
174}
175
176
177
178//=============================================================================================
179// Kernel Setup logic
180//=============================================================================================
181//-----------------------------------------------------------------------------
182// Kernel boot procedures
183static void __kernel_startup(void) {
184 /* paranoid */ verify( ! __preemption_enabled() );
185 __cfadbg_print_safe(runtime_core, "Kernel : Starting\n");
186
187 __cfa_dbg_global_clusters.list{ __get };
188 __cfa_dbg_global_clusters.lock{};
189
190 /* paranoid */ verify( verify_fwd_bck_rng() );
191
192 // Initialize the global scheduler lock
193 __scheduler_lock = (__scheduler_RWLock_t*)&storage___scheduler_lock;
194 (*__scheduler_lock){};
195
196 // Initialize the main cluster
197 mainCluster = (cluster *)&storage_mainCluster;
198 (*mainCluster){"Main Cluster", 0};
199
200 __cfadbg_print_safe(runtime_core, "Kernel : Main cluster ready\n");
201
202 // Start by initializing the main thread
203 // SKULLDUGGERY: the mainThread steals the process main thread
204 // which will then be scheduled by the mainProcessor normally
205 mainThread = ($thread *)&storage_mainThread;
206 current_stack_info_t info;
207 info.storage = (__stack_t*)&storage_mainThreadCtx;
208 (*mainThread){ &info };
209
210 __cfadbg_print_safe(runtime_core, "Kernel : Main thread ready\n");
211
212
213
214 // Construct the processor context of the main processor
215 void ?{}(processorCtx_t & this, processor * proc) {
216 (this.__cor){ "Processor" };
217 this.__cor.starter = 0p;
218 this.proc = proc;
219 }
220
221 void ?{}(processor & this) with( this ) {
222 ( this.terminated ){};
223 ( this.runner ){};
224 init( this, "Main Processor", *mainCluster, 0p );
225 kernel_thread = pthread_self();
226
227 runner{ &this };
228 __cfadbg_print_safe(runtime_core, "Kernel : constructed main processor context %p\n", &runner);
229 }
230
231 // Initialize the main processor and the main processor ctx
232 // (the coroutine that contains the processing control flow)
233 mainProcessor = (processor *)&storage_mainProcessor;
234 (*mainProcessor){};
235
236 register_tls( mainProcessor );
237
238 //initialize the global state variables
239 __cfaabi_tls.this_processor = mainProcessor;
240 __cfaabi_tls.this_thread = mainThread;
241
242 #if !defined( __CFA_NO_STATISTICS__ )
243 __cfaabi_tls.this_stats = (__stats_t *)& storage_mainProcStats;
244 __init_stats( __cfaabi_tls.this_stats );
245 #endif
246 mainProcessor->local_data = &__cfaabi_tls;
247
248 // Enable preemption
249 __kernel_alarm_startup();
250
251 // Add the main thread to the ready queue
252 // once resume is called on mainProcessor->runner the mainThread needs to be scheduled like any normal thread
253 schedule_thread$(mainThread);
254
255 // SKULLDUGGERY: Force a context switch to the main processor to set the main thread's context to the current UNIX
256 // context. Hence, the main thread does not begin through __cfactx_invoke_thread, like all other threads. The trick here is that
257 // mainThread is on the ready queue when this call is made.
258 __kernel_first_resume( __cfaabi_tls.this_processor );
259
260
261 // THE SYSTEM IS NOW COMPLETELY RUNNING
262
263 __cfadbg_print_safe(runtime_core, "Kernel : Started\n--------------------------------------------------\n\n");
264
265 /* paranoid */ verify( ! __preemption_enabled() );
266 enable_interrupts();
267 /* paranoid */ verify( __preemption_enabled() );
268
269}
270
271static void __kernel_shutdown(void) {
272 /* paranoid */ verify( __preemption_enabled() );
273 disable_interrupts();
274 /* paranoid */ verify( ! __preemption_enabled() );
275
276 __cfadbg_print_safe(runtime_core, "\n--------------------------------------------------\nKernel : Shutting down\n");
277
278 // SKULLDUGGERY: Notify the mainProcessor it needs to terminates.
279 // When its coroutine terminates, it return control to the mainThread
280 // which is currently here
281 __atomic_store_n(&mainProcessor->do_terminate, true, __ATOMIC_RELEASE);
282 __kernel_last_resume( __cfaabi_tls.this_processor );
283 mainThread->self_cor.state = Halted;
284
285 // THE SYSTEM IS NOW COMPLETELY STOPPED
286
287 // Disable preemption
288 __kernel_alarm_shutdown();
289
290 #if !defined( __CFA_NO_STATISTICS__ )
291 __stats_t * st = (__stats_t *)& storage_mainProcStats;
292 __tally_stats(mainCluster->stats, st);
293 if( 0 != mainProcessor->print_stats ) {
294 __print_stats( st, mainProcessor->print_stats, "Processor ", mainProcessor->name, (void*)mainProcessor );
295 }
296 #if defined(CFA_STATS_ARRAY)
297 __flush_stat( st, "Processor", mainProcessor );
298 #endif
299 #endif
300
301 mainProcessor->local_data = 0p;
302
303 unregister_tls( mainProcessor );
304
305 // Destroy the main processor and its context in reverse order of construction
306 // These were manually constructed so we need manually destroy them
307 void ^?{}(processor & this) with( this ){
308 deinit( this );
309
310 /* paranoid */ verify( this.do_terminate == true );
311 __cfaabi_dbg_print_safe("Kernel : destroyed main processor context %p\n", &runner);
312 }
313
314 ^(*mainProcessor){};
315
316 // Final step, destroy the main thread since it is no longer needed
317
318 // Since we provided a stack to this taxk it will not destroy anything
319 /* paranoid */ verify(mainThread->self_cor.stack.storage == (__stack_t*)(((uintptr_t)&storage_mainThreadCtx)| 0x1));
320 ^(*mainThread){};
321
322 ^(*mainCluster){};
323
324 ^(*__scheduler_lock){};
325
326 ^(__cfa_dbg_global_clusters.list){};
327 ^(__cfa_dbg_global_clusters.lock){};
328
329 __cfadbg_print_safe(runtime_core, "Kernel : Shutdown complete\n");
330}
331
332//=============================================================================================
333// Kernel Initial Scheduling logic
334//=============================================================================================
335
336// Context invoker for processors
337// This is the entry point for processors (kernel threads) *except* for the main processor
338// It effectively constructs a coroutine by stealing the pthread stack
339static void * __invoke_processor(void * arg) {
340 #if !defined( __CFA_NO_STATISTICS__ )
341 __stats_t local_stats;
342 __init_stats( &local_stats );
343 __cfaabi_tls.this_stats = &local_stats;
344 #endif
345
346 processor * proc = (processor *) arg;
347 __cfaabi_tls.this_processor = proc;
348 __cfaabi_tls.this_thread = 0p;
349 __cfaabi_tls.preemption_state.[enabled, disable_count] = [false, 1];
350 proc->local_data = &__cfaabi_tls;
351
352 register_tls( proc );
353
354 // SKULLDUGGERY: We want to create a context for the processor coroutine
355 // which is needed for the 2-step context switch. However, there is no reason
356 // to waste the perfectly valid stack create by pthread.
357 current_stack_info_t info;
358 __stack_t ctx;
359 info.storage = &ctx;
360 (proc->runner){ proc, &info };
361
362 __cfaabi_dbg_print_safe("Coroutine : created stack %p\n", get_coroutine(proc->runner)->stack.storage);
363
364 //Set global state
365 __cfaabi_tls.this_thread = 0p;
366
367 //We now have a proper context from which to schedule threads
368 __cfadbg_print_safe(runtime_core, "Kernel : core %p created (%p, %p)\n", proc, &proc->runner, &ctx);
369
370 // SKULLDUGGERY: Since the coroutine doesn't have its own stack, we can't
371 // resume it to start it like it normally would, it will just context switch
372 // back to here. Instead directly call the main since we already are on the
373 // appropriate stack.
374 get_coroutine(proc->runner)->state = Active;
375 main( proc->runner );
376 get_coroutine(proc->runner)->state = Halted;
377
378 // Main routine of the core returned, the core is now fully terminated
379 __cfadbg_print_safe(runtime_core, "Kernel : core %p main ended (%p)\n", proc, &proc->runner);
380
381 #if !defined(__CFA_NO_STATISTICS__)
382 __tally_stats(proc->cltr->stats, &local_stats);
383 if( 0 != proc->print_stats ) {
384 __print_stats( &local_stats, proc->print_stats, "Processor ", proc->name, (void*)proc );
385 }
386 #if defined(CFA_STATS_ARRAY)
387 __flush_stat( &local_stats, "Processor", proc );
388 #endif
389 #endif
390
391 proc->local_data = 0p;
392
393 unregister_tls( proc );
394
395 return 0p;
396}
397
398static void __kernel_first_resume( processor * this ) {
399 $thread * src = mainThread;
400 $coroutine * dst = get_coroutine(this->runner);
401
402 /* paranoid */ verify( ! __preemption_enabled() );
403
404 __cfaabi_tls.this_thread->curr_cor = dst;
405 __stack_prepare( &dst->stack, 65000 );
406 __cfactx_start(main, dst, this->runner, __cfactx_invoke_coroutine);
407
408 /* paranoid */ verify( ! __preemption_enabled() );
409
410 dst->last = &src->self_cor;
411 dst->starter = dst->starter ? dst->starter : &src->self_cor;
412
413 // make sure the current state is still correct
414 /* paranoid */ verify(src->state == Ready);
415 src->corctx_flag = true;
416
417 // context switch to specified coroutine
418 verify( dst->context.SP );
419 __cfactx_switch( &src->context, &dst->context );
420 // when __cfactx_switch returns we are back in the src coroutine
421
422 mainThread->curr_cor = &mainThread->self_cor;
423
424 // make sure the current state has been update
425 /* paranoid */ verify(src->state == Active);
426
427 /* paranoid */ verify( ! __preemption_enabled() );
428}
429
430// KERNEL_ONLY
431static void __kernel_last_resume( processor * this ) {
432 $coroutine * src = &mainThread->self_cor;
433 $coroutine * dst = get_coroutine(this->runner);
434
435 /* paranoid */ verify( ! __preemption_enabled() );
436 /* paranoid */ verify( dst->starter == src );
437 /* paranoid */ verify( dst->context.SP );
438
439 // SKULLDUGGERY in debug the processors check that the
440 // stack is still within the limit of the stack limits after running a thread.
441 // that check doesn't make sense if we context switch to the processor using the
442 // coroutine semantics. Since this is a special case, use the current context
443 // info to populate these fields.
444 __cfaabi_dbg_debug_do(
445 __stack_context_t ctx;
446 CtxGet( ctx );
447 mainThread->context.SP = ctx.SP;
448 mainThread->context.FP = ctx.FP;
449 )
450
451 // context switch to the processor
452 __cfactx_switch( &src->context, &dst->context );
453}
454
455
456//=============================================================================================
457// Kernel Object Constructors logic
458//=============================================================================================
459//-----------------------------------------------------------------------------
460// Main thread construction
461static void ?{}( $coroutine & this, current_stack_info_t * info) with( this ) {
462 stack.storage = info->storage;
463 with(*stack.storage) {
464 limit = info->limit;
465 base = info->base;
466 }
467 __attribute__((may_alias)) intptr_t * istorage = (intptr_t*) &stack.storage;
468 *istorage |= 0x1;
469 name = "Main Thread";
470 state = Start;
471 starter = 0p;
472 last = 0p;
473 cancellation = 0p;
474}
475
476static void ?{}( $thread & this, current_stack_info_t * info) with( this ) {
477 ticket = TICKET_RUNNING;
478 state = Start;
479 self_cor{ info };
480 curr_cor = &self_cor;
481 curr_cluster = mainCluster;
482 self_mon.owner = &this;
483 self_mon.recursion = 1;
484 self_mon_p = &self_mon;
485 link.next = 0p;
486 link.ts = -1llu;
487 preferred = -1u;
488 last_proc = 0p;
489 #if defined( __CFA_WITH_VERIFY__ )
490 canary = 0x0D15EA5E0D15EA5Ep;
491 #endif
492
493 node.next = 0p;
494 node.prev = 0p;
495 doregister(curr_cluster, this);
496
497 monitors{ &self_mon_p, 1, (fptr_t)0 };
498}
499
500//-----------------------------------------------------------------------------
501// Processor
502// Construct the processor context of non-main processors
503static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info) {
504 (this.__cor){ info };
505 this.proc = proc;
506}
507
508static void init(processor & this, const char name[], cluster & _cltr, $thread * initT) with( this ) {
509 this.name = name;
510 this.cltr = &_cltr;
511 this.rdq.its = 0;
512 this.rdq.itr = 0;
513 this.rdq.id = -1u;
514 this.rdq.target = -1u;
515 this.rdq.last = -1u;
516 this.rdq.cutoff = 0ull;
517 do_terminate = false;
518 preemption_alarm = 0p;
519 pending_preemption = false;
520
521 this.io.ctx = 0p;
522 this.io.pending = false;
523 this.io.dirty = false;
524
525 this.init.thrd = initT;
526
527 this.local_data = 0p;
528
529 this.idle = eventfd(0, 0);
530 if (idle < 0) {
531 abort("KERNEL ERROR: PROCESSOR EVENTFD - %s\n", strerror(errno));
532 }
533
534 #if !defined(__CFA_NO_STATISTICS__)
535 print_stats = 0;
536 print_halts = false;
537 #endif
538
539 __cfadbg_print_safe(runtime_core, "Kernel : core %p created\n", &this);
540}
541
542// Not a ctor, it just preps the destruction but should not destroy members
543static void deinit(processor & this) {
544 close(this.idle);
545}
546
547void ?{}(processor & this, const char name[], cluster & _cltr, $thread * initT) {
548 ( this.terminated ){};
549 ( this.runner ){};
550
551 disable_interrupts();
552 init( this, name, _cltr, initT );
553 enable_interrupts();
554
555 __cfadbg_print_safe(runtime_core, "Kernel : Starting core %p\n", &this);
556
557 this.stack = __create_pthread( &this.kernel_thread, __invoke_processor, (void *)&this );
558}
559
560void ?{}(processor & this, const char name[], cluster & _cltr) {
561 (this){name, _cltr, 0p};
562}
563
564extern size_t __page_size;
565void ^?{}(processor & this) with( this ){
566 if( ! __atomic_load_n(&do_terminate, __ATOMIC_ACQUIRE) ) {
567 __cfadbg_print_safe(runtime_core, "Kernel : core %p signaling termination\n", &this);
568
569 __atomic_store_n(&do_terminate, true, __ATOMIC_RELAXED);
570 __wake_proc( &this );
571
572 wait( terminated );
573 /* paranoid */ verify( active_processor() != &this);
574 }
575
576 __destroy_pthread( kernel_thread, this.stack, 0p );
577
578 disable_interrupts();
579 deinit( this );
580 enable_interrupts();
581}
582
583//-----------------------------------------------------------------------------
584// Cluster
585static void ?{}(__cluster_proc_list & this) {
586 this.lock = 0;
587 this.idle = 0;
588 this.total = 0;
589}
590
591void ?{}(cluster & this, const char name[], Duration preemption_rate, unsigned num_io, const io_context_params & io_params) with( this ) {
592 this.name = name;
593 this.preemption_rate = preemption_rate;
594 ready_queue{};
595
596 #if !defined(__CFA_NO_STATISTICS__)
597 print_stats = 0;
598 stats = alloc();
599 __init_stats( stats );
600 #endif
601
602 threads{ __get };
603
604 io.arbiter = create();
605 io.params = io_params;
606
607 doregister(this);
608
609 // Lock the RWlock so no-one pushes/pops while we are changing the queue
610 disable_interrupts();
611 uint_fast32_t last_size = ready_mutate_lock();
612
613 // Adjust the ready queue size
614 ready_queue_grow( &this );
615
616 // Unlock the RWlock
617 ready_mutate_unlock( last_size );
618 enable_interrupts( false ); // Don't poll, could be in main cluster
619}
620
621void ^?{}(cluster & this) {
622 destroy(this.io.arbiter);
623
624 // Lock the RWlock so no-one pushes/pops while we are changing the queue
625 disable_interrupts();
626 uint_fast32_t last_size = ready_mutate_lock();
627
628 // Adjust the ready queue size
629 ready_queue_shrink( &this );
630
631 // Unlock the RWlock
632 ready_mutate_unlock( last_size );
633 enable_interrupts( false ); // Don't poll, could be in main cluster
634
635 #if !defined(__CFA_NO_STATISTICS__)
636 if( 0 != this.print_stats ) {
637 __print_stats( this.stats, this.print_stats, "Cluster", this.name, (void*)&this );
638 }
639 #if defined(CFA_STATS_ARRAY)
640 __flush_stat( this.stats, "Cluster", &this );
641 #endif
642 free( this.stats );
643 #endif
644
645 unregister(this);
646}
647
648//=============================================================================================
649// Miscellaneous Initialization
650//=============================================================================================
651//-----------------------------------------------------------------------------
652// Global Queues
653static void doregister( cluster & cltr ) {
654 lock ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
655 push_front( __cfa_dbg_global_clusters.list, cltr );
656 unlock ( __cfa_dbg_global_clusters.lock );
657}
658
659static void unregister( cluster & cltr ) {
660 lock ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
661 remove( __cfa_dbg_global_clusters.list, cltr );
662 unlock( __cfa_dbg_global_clusters.lock );
663}
664
665void doregister( cluster * cltr, $thread & thrd ) {
666 lock (cltr->thread_list_lock __cfaabi_dbg_ctx2);
667 cltr->nthreads += 1;
668 push_front(cltr->threads, thrd);
669 unlock (cltr->thread_list_lock);
670}
671
672void unregister( cluster * cltr, $thread & thrd ) {
673 lock (cltr->thread_list_lock __cfaabi_dbg_ctx2);
674 remove(cltr->threads, thrd );
675 cltr->nthreads -= 1;
676 unlock(cltr->thread_list_lock);
677}
678
679static void register_tls( processor * this ) {
680 // Register and Lock the RWlock so no-one pushes/pops while we are changing the queue
681 uint_fast32_t last_size;
682 [this->unique_id, last_size] = ready_mutate_register();
683
684 this->cltr->procs.total += 1u;
685 insert_last(this->cltr->procs.actives, *this);
686
687 // Adjust the ready queue size
688 ready_queue_grow( this->cltr );
689
690 // Unlock the RWlock
691 ready_mutate_unlock( last_size );
692}
693
694
695static void unregister_tls( processor * this ) {
696 // Lock the RWlock so no-one pushes/pops while we are changing the queue
697 uint_fast32_t last_size = ready_mutate_lock();
698 this->cltr->procs.total -= 1u;
699 remove(*this);
700
701 // clear the cluster so nothing gets pushed to local queues
702 cluster * cltr = this->cltr;
703 this->cltr = 0p;
704
705 // Adjust the ready queue size
706 ready_queue_shrink( cltr );
707
708 // Unlock the RWlock and unregister: we don't need the read_lock any more
709 ready_mutate_unregister( this->unique_id, last_size );
710}
711
712static void check( int ret, const char func[] ) {
713 if ( ret ) { // pthread routines return errno values
714 abort( "%s : internal error, error(%d) %s.", func, ret, strerror( ret ) );
715 } // if
716} // Abort
717
718void * __create_pthread( pthread_t * pthread, void * (*start)(void *), void * arg ) {
719 pthread_attr_t attr;
720
721 check( pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
722
723 size_t stacksize;
724 // default stack size, normally defined by shell limit
725 check( pthread_attr_getstacksize( &attr, &stacksize ), "pthread_attr_getstacksize" );
726 assert( stacksize >= PTHREAD_STACK_MIN );
727
728 void * stack;
729 #if CFA_PROCESSOR_USE_MMAP
730 stacksize = ceiling( stacksize, __page_size ) + __page_size;
731 stack = mmap(0p, stacksize, __map_prot, MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
732 if(stack == ((void*)-1)) {
733 abort( "pthread stack creation : internal error, mmap failure, error(%d) %s.", errno, strerror( errno ) );
734 }
735 if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
736 abort( "pthread stack creation : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
737 } // if
738 #else
739 __cfaabi_dbg_debug_do(
740 stack = memalign( __page_size, stacksize + __page_size );
741 // pthread has no mechanism to create the guard page in user supplied stack.
742 if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
743 abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
744 } // if
745 );
746 __cfaabi_dbg_no_debug_do(
747 stack = malloc( stacksize );
748 );
749 #endif
750
751
752 check( pthread_attr_setstack( &attr, stack, stacksize ), "pthread_attr_setstack" );
753
754 check( pthread_create( pthread, &attr, start, arg ), "pthread_create" );
755 return stack;
756}
757
758void __destroy_pthread( pthread_t pthread, void * stack, void ** retval ) {
759 int err = pthread_join( pthread, retval );
760 if( err != 0 ) abort("KERNEL ERROR: joining pthread %p caused error %s\n", (void*)pthread, strerror(err));
761
762 #if CFA_PROCESSOR_USE_MMAP
763 pthread_attr_t attr;
764
765 check( pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
766
767 size_t stacksize;
768 // default stack size, normally defined by shell limit
769 check( pthread_attr_getstacksize( &attr, &stacksize ), "pthread_attr_getstacksize" );
770 assert( stacksize >= PTHREAD_STACK_MIN );
771 stacksize += __page_size;
772
773 if(munmap(stack, stacksize) == -1) {
774 abort( "pthread stack destruction : internal error, munmap failure, error(%d) %s.", errno, strerror( errno ) );
775 }
776 #else
777 __cfaabi_dbg_debug_do(
778 // pthread has no mechanism to create the guard page in user supplied stack.
779 if ( mprotect( stack, __page_size, __map_prot ) == -1 ) {
780 abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
781 } // if
782 );
783 free( stack );
784 #endif
785}
786
787#if defined(__CFA_WITH_VERIFY__)
788static bool verify_fwd_bck_rng(void) {
789 __cfaabi_tls.ready_rng.fwd_seed = 25214903917_l64u * (rdtscl() ^ (uintptr_t)&verify_fwd_bck_rng);
790
791 unsigned values[10];
792 for(i; 10) {
793 values[i] = __tls_rand_fwd();
794 }
795
796 __tls_rand_advance_bck();
797
798 for ( i; 9 -~= 0 ) {
799 if(values[i] != __tls_rand_bck()) {
800 return false;
801 }
802 }
803
804 return true;
805}
806#endif
Note: See TracBrowser for help on using the repository browser.