source: libcfa/src/concurrency/kernel/startup.cfa@ d96becd

ADT ast-experimental enum forall-pointer-decay jacob/cs343-translation new-ast-unique-expr pthread-emulation qualifiedEnum
Last change on this file since d96becd was 3814957, checked in by Thierry Delisle <tdelisle@…>, 4 years ago

Commented out broken code
  • Property mode set to 100644
File size: 24.4 KB
Line 
1//
2// Cforall Version 1.0.0 Copyright (C) 2020 University of Waterloo
3//
4// The contents of this file are covered under the licence agreement in the
5// file "LICENCE" distributed with Cforall.
6//
7// kernel/startup.cfa --
8//
9// Author : Thierry Delisle
10// Created On : Thu Jul 30 15:12:54 2020
11// Last Modified By :
12// Last Modified On :
13// Update Count :
14//
15
16#define __cforall_thread__
17#define _GNU_SOURCE
18
19// C Includes
20#include <errno.h> // errno
21#include <string.h> // strerror
22#include <unistd.h> // sysconf
23extern "C" {
24 #include <limits.h> // PTHREAD_STACK_MIN
25 #include <sys/eventfd.h> // eventfd
26 #include <sys/mman.h> // mprotect
27 #include <sys/resource.h> // getrlimit
28}
29
30// CFA Includes
31#include "kernel_private.hfa"
32#include "startup.hfa" // STARTUP_PRIORITY_XXX
33#include "math.hfa"
34
35#define CFA_PROCESSOR_USE_MMAP 0
36
37//-----------------------------------------------------------------------------
38// Some assembly required
39#if defined( __i386 )
40 #define CtxGet( ctx ) __asm__ volatile ( \
41 "movl %%esp,%0\n" \
42 "movl %%ebp,%1\n" \
43 : "=rm" (ctx.SP), \
44 "=rm" (ctx.FP) \
45 )
46#elif defined( __x86_64 )
47 #define CtxGet( ctx ) __asm__ volatile ( \
48 "movq %%rsp,%0\n" \
49 "movq %%rbp,%1\n" \
50 : "=rm" (ctx.SP), \
51 "=rm" (ctx.FP) \
52 )
53#elif defined( __aarch64__ )
54 #define CtxGet( ctx ) __asm__ volatile ( \
55 "mov %0, sp\n" \
56 "mov %1, fp\n" \
57 : "=rm" (ctx.SP), \
58 "=rm" (ctx.FP) \
59 )
60#else
61 #error unknown hardware architecture
62#endif
63
64//-----------------------------------------------------------------------------
65// Start and stop routine for the kernel, declared first to make sure they run first
66static void __kernel_startup (void) __attribute__(( constructor( STARTUP_PRIORITY_KERNEL ) ));
67static void __kernel_shutdown(void) __attribute__(( destructor ( STARTUP_PRIORITY_KERNEL ) ));
68
69//-----------------------------------------------------------------------------
70// Static Forward Declarations
71struct current_stack_info_t;
72
73static void * __invoke_processor(void * arg);
74static void __kernel_first_resume( processor * this );
75static void __kernel_last_resume ( processor * this );
76static void init(processor & this, const char name[], cluster & _cltr, $thread * initT);
77static void deinit(processor & this);
78static void doregister( struct cluster & cltr );
79static void unregister( struct cluster & cltr );
80static void register_tls( processor * this );
81static void unregister_tls( processor * this );
82static void ?{}( $coroutine & this, current_stack_info_t * info);
83static void ?{}( $thread & this, current_stack_info_t * info);
84static void ?{}(processorCtx_t & this) {}
85static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info);
86
87#if defined(__CFA_WITH_VERIFY__)
88 static bool verify_fwd_bck_rng(void);
89#endif
90
91//-----------------------------------------------------------------------------
92// Forward Declarations for other modules
93extern void __kernel_alarm_startup(void);
94extern void __kernel_alarm_shutdown(void);
95
96//-----------------------------------------------------------------------------
97// Other Forward Declarations
98extern void __wake_proc(processor *);
99
100//-----------------------------------------------------------------------------
101// Kernel storage
102KERNEL_STORAGE(cluster, mainCluster);
103KERNEL_STORAGE(processor, mainProcessor);
104KERNEL_STORAGE($thread, mainThread);
105KERNEL_STORAGE(__stack_t, mainThreadCtx);
106KERNEL_STORAGE(__scheduler_RWLock_t, __scheduler_lock);
107#if !defined(__CFA_NO_STATISTICS__)
108KERNEL_STORAGE(__stats_t, mainProcStats);
109#endif
110
111cluster * mainCluster;
112processor * mainProcessor;
113$thread * mainThread;
114__scheduler_RWLock_t * __scheduler_lock;
115
116extern "C" {
117 struct { __dllist_t(cluster) list; __spinlock_t lock; } __cfa_dbg_global_clusters;
118}
119
120extern size_t __page_size;
121extern int __map_prot;
122
123//-----------------------------------------------------------------------------
124// Global state
125thread_local struct KernelThreadData __cfaabi_tls __attribute__ ((tls_model ( "initial-exec" ))) @= {
126 NULL, // cannot use 0p
127 NULL,
128 false,
129 { 1, false, false },
130 0,
131 { 0, 0 },
132 NULL,
133 #ifdef __CFA_WITH_VERIFY__
134 false,
135 0,
136 #endif
137};
138
139#if defined(CFA_HAVE_LINUX_LIBRSEQ)
140 // No data needed
141#elif defined(CFA_HAVE_LINUX_RSEQ_H)
142 extern "Cforall" {
143 // thread_local volatile struct rseq __cfaabi_rseq;
144 }
145#else
146 // No data needed
147#endif
148
149//-----------------------------------------------------------------------------
150// Struct to steal stack
151struct current_stack_info_t {
152 __stack_t * storage; // pointer to stack object
153 void * base; // base of stack
154 void * limit; // stack grows towards stack limit
155 void * context; // address of cfa_context_t
156};
157
158void ?{}( current_stack_info_t & this ) {
159 __stack_context_t ctx;
160 CtxGet( ctx );
161 this.base = ctx.FP;
162
163 rlimit r;
164 getrlimit( RLIMIT_STACK, &r);
165 size_t size = r.rlim_cur;
166
167 this.limit = (void *)(((intptr_t)this.base) - size);
168 this.context = &storage_mainThreadCtx;
169}
170
171
172
173//=============================================================================================
174// Kernel Setup logic
175//=============================================================================================
176//-----------------------------------------------------------------------------
177// Kernel boot procedures
178static void __kernel_startup(void) {
179 /* paranoid */ verify( ! __preemption_enabled() );
180 __cfadbg_print_safe(runtime_core, "Kernel : Starting\n");
181
182 __cfa_dbg_global_clusters.list{ __get };
183 __cfa_dbg_global_clusters.lock{};
184
185 /* paranoid */ verify( verify_fwd_bck_rng() );
186
187 // Initialize the global scheduler lock
188 __scheduler_lock = (__scheduler_RWLock_t*)&storage___scheduler_lock;
189 (*__scheduler_lock){};
190
191 // Initialize the main cluster
192 mainCluster = (cluster *)&storage_mainCluster;
193 (*mainCluster){"Main Cluster", 0};
194
195 __cfadbg_print_safe(runtime_core, "Kernel : Main cluster ready\n");
196
197 // Start by initializing the main thread
198 // SKULLDUGGERY: the mainThread steals the process main thread
199 // which will then be scheduled by the mainProcessor normally
200 mainThread = ($thread *)&storage_mainThread;
201 current_stack_info_t info;
202 info.storage = (__stack_t*)&storage_mainThreadCtx;
203 (*mainThread){ &info };
204
205 __cfadbg_print_safe(runtime_core, "Kernel : Main thread ready\n");
206
207
208
209 // Construct the processor context of the main processor
210 void ?{}(processorCtx_t & this, processor * proc) {
211 (this.__cor){ "Processor" };
212 this.__cor.starter = 0p;
213 this.proc = proc;
214 }
215
216 void ?{}(processor & this) with( this ) {
217 ( this.terminated ){};
218 ( this.runner ){};
219 init( this, "Main Processor", *mainCluster, 0p );
220 kernel_thread = pthread_self();
221
222 runner{ &this };
223 __cfadbg_print_safe(runtime_core, "Kernel : constructed main processor context %p\n", &runner);
224 }
225
226 // Initialize the main processor and the main processor ctx
227 // (the coroutine that contains the processing control flow)
228 mainProcessor = (processor *)&storage_mainProcessor;
229 (*mainProcessor){};
230
231 register_tls( mainProcessor );
232
233 //initialize the global state variables
234 __cfaabi_tls.this_processor = mainProcessor;
235 __cfaabi_tls.this_thread = mainThread;
236
237 #if !defined( __CFA_NO_STATISTICS__ )
238 __cfaabi_tls.this_stats = (__stats_t *)& storage_mainProcStats;
239 __init_stats( __cfaabi_tls.this_stats );
240 #endif
241 mainProcessor->local_data = &__cfaabi_tls;
242
243 // Enable preemption
244 __kernel_alarm_startup();
245
246 // Add the main thread to the ready queue
247 // once resume is called on mainProcessor->runner the mainThread needs to be scheduled like any normal thread
248 schedule_thread$(mainThread);
249
250 // SKULLDUGGERY: Force a context switch to the main processor to set the main thread's context to the current UNIX
251 // context. Hence, the main thread does not begin through __cfactx_invoke_thread, like all other threads. The trick here is that
252 // mainThread is on the ready queue when this call is made.
253 __kernel_first_resume( __cfaabi_tls.this_processor );
254
255
256 // THE SYSTEM IS NOW COMPLETELY RUNNING
257
258 __cfadbg_print_safe(runtime_core, "Kernel : Started\n--------------------------------------------------\n\n");
259
260 /* paranoid */ verify( ! __preemption_enabled() );
261 enable_interrupts();
262 /* paranoid */ verify( __preemption_enabled() );
263
264}
265
266static void __kernel_shutdown(void) {
267 /* paranoid */ verify( __preemption_enabled() );
268 disable_interrupts();
269 /* paranoid */ verify( ! __preemption_enabled() );
270
271 __cfadbg_print_safe(runtime_core, "\n--------------------------------------------------\nKernel : Shutting down\n");
272
273 // SKULLDUGGERY: Notify the mainProcessor it needs to terminates.
274 // When its coroutine terminates, it return control to the mainThread
275 // which is currently here
276 __atomic_store_n(&mainProcessor->do_terminate, true, __ATOMIC_RELEASE);
277 __kernel_last_resume( __cfaabi_tls.this_processor );
278 mainThread->self_cor.state = Halted;
279
280 // THE SYSTEM IS NOW COMPLETELY STOPPED
281
282 // Disable preemption
283 __kernel_alarm_shutdown();
284
285 #if !defined( __CFA_NO_STATISTICS__ )
286 __stats_t * st = (__stats_t *)& storage_mainProcStats;
287 __tally_stats(mainCluster->stats, st);
288 if( 0 != mainProcessor->print_stats ) {
289 __print_stats( st, mainProcessor->print_stats, "Processor ", mainProcessor->name, (void*)mainProcessor );
290 }
291 #if defined(CFA_STATS_ARRAY)
292 __flush_stat( st, "Processor", mainProcessor );
293 #endif
294 #endif
295
296 mainProcessor->local_data = 0p;
297
298 unregister_tls( mainProcessor );
299
300 // Destroy the main processor and its context in reverse order of construction
301 // These were manually constructed so we need manually destroy them
302 void ^?{}(processor & this) with( this ){
303 deinit( this );
304
305 /* paranoid */ verify( this.do_terminate == true );
306 __cfaabi_dbg_print_safe("Kernel : destroyed main processor context %p\n", &runner);
307 }
308
309 ^(*mainProcessor){};
310
311 // Final step, destroy the main thread since it is no longer needed
312
313 // Since we provided a stack to this taxk it will not destroy anything
314 /* paranoid */ verify(mainThread->self_cor.stack.storage == (__stack_t*)(((uintptr_t)&storage_mainThreadCtx)| 0x1));
315 ^(*mainThread){};
316
317 ^(*mainCluster){};
318
319 ^(*__scheduler_lock){};
320
321 ^(__cfa_dbg_global_clusters.list){};
322 ^(__cfa_dbg_global_clusters.lock){};
323
324 __cfadbg_print_safe(runtime_core, "Kernel : Shutdown complete\n");
325}
326
327//=============================================================================================
328// Kernel Initial Scheduling logic
329//=============================================================================================
330
331// Context invoker for processors
332// This is the entry point for processors (kernel threads) *except* for the main processor
333// It effectively constructs a coroutine by stealing the pthread stack
334static void * __invoke_processor(void * arg) {
335 #if !defined( __CFA_NO_STATISTICS__ )
336 __stats_t local_stats;
337 __init_stats( &local_stats );
338 __cfaabi_tls.this_stats = &local_stats;
339 #endif
340
341 processor * proc = (processor *) arg;
342 __cfaabi_tls.this_processor = proc;
343 __cfaabi_tls.this_thread = 0p;
344 __cfaabi_tls.preemption_state.[enabled, disable_count] = [false, 1];
345 proc->local_data = &__cfaabi_tls;
346
347 register_tls( proc );
348
349 #if defined(CFA_HAVE_LINUX_LIBRSEQ)
350 rseq_register_current_thread();
351 #endif
352
353 // SKULLDUGGERY: We want to create a context for the processor coroutine
354 // which is needed for the 2-step context switch. However, there is no reason
355 // to waste the perfectly valid stack create by pthread.
356 current_stack_info_t info;
357 __stack_t ctx;
358 info.storage = &ctx;
359 (proc->runner){ proc, &info };
360
361 __cfaabi_dbg_print_safe("Coroutine : created stack %p\n", get_coroutine(proc->runner)->stack.storage);
362
363 //Set global state
364 __cfaabi_tls.this_thread = 0p;
365
366 //We now have a proper context from which to schedule threads
367 __cfadbg_print_safe(runtime_core, "Kernel : core %p created (%p, %p)\n", proc, &proc->runner, &ctx);
368
369 // SKULLDUGGERY: Since the coroutine doesn't have its own stack, we can't
370 // resume it to start it like it normally would, it will just context switch
371 // back to here. Instead directly call the main since we already are on the
372 // appropriate stack.
373 get_coroutine(proc->runner)->state = Active;
374 main( proc->runner );
375 get_coroutine(proc->runner)->state = Halted;
376
377 // Main routine of the core returned, the core is now fully terminated
378 __cfadbg_print_safe(runtime_core, "Kernel : core %p main ended (%p)\n", proc, &proc->runner);
379
380 #if !defined(__CFA_NO_STATISTICS__)
381 __tally_stats(proc->cltr->stats, &local_stats);
382 if( 0 != proc->print_stats ) {
383 __print_stats( &local_stats, proc->print_stats, "Processor ", proc->name, (void*)proc );
384 }
385 #if defined(CFA_STATS_ARRAY)
386 __flush_stat( &local_stats, "Processor", proc );
387 #endif
388 #endif
389
390 proc->local_data = 0p;
391
392 unregister_tls( proc );
393
394 return 0p;
395}
396
397static void __kernel_first_resume( processor * this ) {
398 $thread * src = mainThread;
399 $coroutine * dst = get_coroutine(this->runner);
400
401 /* paranoid */ verify( ! __preemption_enabled() );
402
403 __cfaabi_tls.this_thread->curr_cor = dst;
404 __stack_prepare( &dst->stack, 65000 );
405 __cfactx_start(main, dst, this->runner, __cfactx_invoke_coroutine);
406
407 /* paranoid */ verify( ! __preemption_enabled() );
408
409 dst->last = &src->self_cor;
410 dst->starter = dst->starter ? dst->starter : &src->self_cor;
411
412 // make sure the current state is still correct
413 /* paranoid */ verify(src->state == Ready);
414 src->corctx_flag = true;
415
416 // context switch to specified coroutine
417 verify( dst->context.SP );
418 __cfactx_switch( &src->context, &dst->context );
419 // when __cfactx_switch returns we are back in the src coroutine
420
421 mainThread->curr_cor = &mainThread->self_cor;
422
423 // make sure the current state has been update
424 /* paranoid */ verify(src->state == Active);
425
426 /* paranoid */ verify( ! __preemption_enabled() );
427}
428
429// KERNEL_ONLY
430static void __kernel_last_resume( processor * this ) {
431 $coroutine * src = &mainThread->self_cor;
432 $coroutine * dst = get_coroutine(this->runner);
433
434 /* paranoid */ verify( ! __preemption_enabled() );
435 /* paranoid */ verify( dst->starter == src );
436 /* paranoid */ verify( dst->context.SP );
437
438 // SKULLDUGGERY in debug the processors check that the
439 // stack is still within the limit of the stack limits after running a thread.
440 // that check doesn't make sense if we context switch to the processor using the
441 // coroutine semantics. Since this is a special case, use the current context
442 // info to populate these fields.
443 __cfaabi_dbg_debug_do(
444 __stack_context_t ctx;
445 CtxGet( ctx );
446 mainThread->context.SP = ctx.SP;
447 mainThread->context.FP = ctx.FP;
448 )
449
450 // context switch to the processor
451 __cfactx_switch( &src->context, &dst->context );
452}
453
454
455//=============================================================================================
456// Kernel Object Constructors logic
457//=============================================================================================
458//-----------------------------------------------------------------------------
459// Main thread construction
460static void ?{}( $coroutine & this, current_stack_info_t * info) with( this ) {
461 stack.storage = info->storage;
462 with(*stack.storage) {
463 limit = info->limit;
464 base = info->base;
465 }
466 __attribute__((may_alias)) intptr_t * istorage = (intptr_t*) &stack.storage;
467 *istorage |= 0x1;
468 name = "Main Thread";
469 state = Start;
470 starter = 0p;
471 last = 0p;
472 cancellation = 0p;
473}
474
475static void ?{}( $thread & this, current_stack_info_t * info) with( this ) {
476 ticket = TICKET_RUNNING;
477 state = Start;
478 self_cor{ info };
479 curr_cor = &self_cor;
480 curr_cluster = mainCluster;
481 self_mon.owner = &this;
482 self_mon.recursion = 1;
483 self_mon_p = &self_mon;
484 link.next = 0p;
485 link.ts = 0;
486 preferred = -1u;
487 last_proc = 0p;
488 #if defined( __CFA_WITH_VERIFY__ )
489 canary = 0x0D15EA5E0D15EA5Ep;
490 #endif
491
492 node.next = 0p;
493 node.prev = 0p;
494 doregister(curr_cluster, this);
495
496 monitors{ &self_mon_p, 1, (fptr_t)0 };
497}
498
499//-----------------------------------------------------------------------------
500// Processor
501// Construct the processor context of non-main processors
502static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info) {
503 (this.__cor){ info };
504 this.proc = proc;
505}
506
507static void init(processor & this, const char name[], cluster & _cltr, $thread * initT) with( this ) {
508 this.name = name;
509 this.cltr = &_cltr;
510 this.rdq.its = 0;
511 this.rdq.itr = 0;
512 this.rdq.id = -1u;
513 this.rdq.target = -1u;
514 this.rdq.cutoff = 0ull;
515 do_terminate = false;
516 preemption_alarm = 0p;
517 pending_preemption = false;
518
519 this.io.ctx = 0p;
520 this.io.pending = false;
521 this.io.dirty = false;
522
523 this.init.thrd = initT;
524
525 this.local_data = 0p;
526
527 this.idle = eventfd(0, 0);
528 if (idle < 0) {
529 abort("KERNEL ERROR: PROCESSOR EVENTFD - %s\n", strerror(errno));
530 }
531
532 #if !defined(__CFA_NO_STATISTICS__)
533 print_stats = 0;
534 print_halts = false;
535 #endif
536
537 __cfadbg_print_safe(runtime_core, "Kernel : core %p created\n", &this);
538}
539
540// Not a ctor, it just preps the destruction but should not destroy members
541static void deinit(processor & this) {
542 close(this.idle);
543}
544
545void ?{}(processor & this, const char name[], cluster & _cltr, $thread * initT) {
546 ( this.terminated ){};
547 ( this.runner ){};
548
549 disable_interrupts();
550 init( this, name, _cltr, initT );
551 enable_interrupts();
552
553 __cfadbg_print_safe(runtime_core, "Kernel : Starting core %p\n", &this);
554
555 this.stack = __create_pthread( &this.kernel_thread, __invoke_processor, (void *)&this );
556}
557
558void ?{}(processor & this, const char name[], cluster & _cltr) {
559 (this){name, _cltr, 0p};
560}
561
562extern size_t __page_size;
563void ^?{}(processor & this) with( this ){
564 if( ! __atomic_load_n(&do_terminate, __ATOMIC_ACQUIRE) ) {
565 __cfadbg_print_safe(runtime_core, "Kernel : core %p signaling termination\n", &this);
566
567 __atomic_store_n(&do_terminate, true, __ATOMIC_RELAXED);
568 __wake_proc( &this );
569
570 wait( terminated );
571 /* paranoid */ verify( active_processor() != &this);
572 }
573
574 __destroy_pthread( kernel_thread, this.stack, 0p );
575
576 disable_interrupts();
577 deinit( this );
578 enable_interrupts();
579}
580
581//-----------------------------------------------------------------------------
582// Cluster
583static void ?{}(__cluster_proc_list & this) {
584 this.lock = 0;
585 this.idle = 0;
586 this.total = 0;
587}
588
589void ?{}(cluster & this, const char name[], Duration preemption_rate, unsigned num_io, const io_context_params & io_params) with( this ) {
590 this.name = name;
591 this.preemption_rate = preemption_rate;
592 ready_queue{};
593
594 #if !defined(__CFA_NO_STATISTICS__)
595 print_stats = 0;
596 stats = alloc();
597 __init_stats( stats );
598 #endif
599
600 threads{ __get };
601
602 io.arbiter = create();
603 io.params = io_params;
604
605 doregister(this);
606
607 // Lock the RWlock so no-one pushes/pops while we are changing the queue
608 disable_interrupts();
609 uint_fast32_t last_size = ready_mutate_lock();
610
611 // Adjust the ready queue size
612 ready_queue_grow( &this );
613
614 // Unlock the RWlock
615 ready_mutate_unlock( last_size );
616 enable_interrupts( false ); // Don't poll, could be in main cluster
617}
618
619void ^?{}(cluster & this) {
620 destroy(this.io.arbiter);
621
622 // Lock the RWlock so no-one pushes/pops while we are changing the queue
623 disable_interrupts();
624 uint_fast32_t last_size = ready_mutate_lock();
625
626 // Adjust the ready queue size
627 ready_queue_shrink( &this );
628
629 // Unlock the RWlock
630 ready_mutate_unlock( last_size );
631 enable_interrupts( false ); // Don't poll, could be in main cluster
632
633 #if !defined(__CFA_NO_STATISTICS__)
634 if( 0 != this.print_stats ) {
635 __print_stats( this.stats, this.print_stats, "Cluster", this.name, (void*)&this );
636 }
637 #if defined(CFA_STATS_ARRAY)
638 __flush_stat( this.stats, "Cluster", &this );
639 #endif
640 free( this.stats );
641 #endif
642
643 unregister(this);
644}
645
646//=============================================================================================
647// Miscellaneous Initialization
648//=============================================================================================
649//-----------------------------------------------------------------------------
650// Global Queues
651static void doregister( cluster & cltr ) {
652 lock ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
653 push_front( __cfa_dbg_global_clusters.list, cltr );
654 unlock ( __cfa_dbg_global_clusters.lock );
655}
656
657static void unregister( cluster & cltr ) {
658 lock ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
659 remove( __cfa_dbg_global_clusters.list, cltr );
660 unlock( __cfa_dbg_global_clusters.lock );
661}
662
663void doregister( cluster * cltr, $thread & thrd ) {
664 lock (cltr->thread_list_lock __cfaabi_dbg_ctx2);
665 cltr->nthreads += 1;
666 push_front(cltr->threads, thrd);
667 unlock (cltr->thread_list_lock);
668}
669
670void unregister( cluster * cltr, $thread & thrd ) {
671 lock (cltr->thread_list_lock __cfaabi_dbg_ctx2);
672 remove(cltr->threads, thrd );
673 cltr->nthreads -= 1;
674 unlock(cltr->thread_list_lock);
675}
676
677static void register_tls( processor * this ) {
678 // Register and Lock the RWlock so no-one pushes/pops while we are changing the queue
679 uint_fast32_t last_size;
680 [this->unique_id, last_size] = ready_mutate_register();
681
682 this->cltr->procs.total += 1u;
683 insert_last(this->cltr->procs.actives, *this);
684
685 // Adjust the ready queue size
686 ready_queue_grow( this->cltr );
687
688 // Unlock the RWlock
689 ready_mutate_unlock( last_size );
690}
691
692
693static void unregister_tls( processor * this ) {
694 // Lock the RWlock so no-one pushes/pops while we are changing the queue
695 uint_fast32_t last_size = ready_mutate_lock();
696 this->cltr->procs.total -= 1u;
697 remove(*this);
698
699 // clear the cluster so nothing gets pushed to local queues
700 cluster * cltr = this->cltr;
701 this->cltr = 0p;
702
703 // Adjust the ready queue size
704 ready_queue_shrink( cltr );
705
706 // Unlock the RWlock and unregister: we don't need the read_lock any more
707 ready_mutate_unregister( this->unique_id, last_size );
708}
709
710static void check( int ret, const char func[] ) {
711 if ( ret ) { // pthread routines return errno values
712 abort( "%s : internal error, error(%d) %s.", func, ret, strerror( ret ) );
713 } // if
714} // Abort
715
716void * __create_pthread( pthread_t * pthread, void * (*start)(void *), void * arg ) {
717 pthread_attr_t attr;
718
719 check( pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
720
721 size_t stacksize;
722 // default stack size, normally defined by shell limit
723 check( pthread_attr_getstacksize( &attr, &stacksize ), "pthread_attr_getstacksize" );
724 assert( stacksize >= PTHREAD_STACK_MIN );
725
726 void * stack;
727 #if CFA_PROCESSOR_USE_MMAP
728 stacksize = ceiling( stacksize, __page_size ) + __page_size;
729 stack = mmap(0p, stacksize, __map_prot, MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
730 if(stack == ((void*)-1)) {
731 abort( "pthread stack creation : internal error, mmap failure, error(%d) %s.", errno, strerror( errno ) );
732 }
733 if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
734 abort( "pthread stack creation : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
735 } // if
736 #else
737 __cfaabi_dbg_debug_do(
738 stack = memalign( __page_size, stacksize + __page_size );
739 // pthread has no mechanism to create the guard page in user supplied stack.
740 if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
741 abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
742 } // if
743 );
744 __cfaabi_dbg_no_debug_do(
745 stack = malloc( stacksize );
746 );
747 #endif
748
749
750 check( pthread_attr_setstack( &attr, stack, stacksize ), "pthread_attr_setstack" );
751
752 check( pthread_create( pthread, &attr, start, arg ), "pthread_create" );
753 return stack;
754}
755
756void __destroy_pthread( pthread_t pthread, void * stack, void ** retval ) {
757 int err = pthread_join( pthread, retval );
758 if( err != 0 ) abort("KERNEL ERROR: joining pthread %p caused error %s\n", (void*)pthread, strerror(err));
759
760 #if CFA_PROCESSOR_USE_MMAP
761 pthread_attr_t attr;
762
763 check( pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
764
765 size_t stacksize;
766 // default stack size, normally defined by shell limit
767 check( pthread_attr_getstacksize( &attr, &stacksize ), "pthread_attr_getstacksize" );
768 assert( stacksize >= PTHREAD_STACK_MIN );
769 stacksize += __page_size;
770
771 if(munmap(stack, stacksize) == -1) {
772 abort( "pthread stack destruction : internal error, munmap failure, error(%d) %s.", errno, strerror( errno ) );
773 }
774 #else
775 __cfaabi_dbg_debug_do(
776 // pthread has no mechanism to create the guard page in user supplied stack.
777 if ( mprotect( stack, __page_size, __map_prot ) == -1 ) {
778 abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
779 } // if
780 );
781 free( stack );
782 #endif
783}
784
785#if defined(__CFA_WITH_VERIFY__)
786static bool verify_fwd_bck_rng(void) {
787 __cfaabi_tls.ready_rng.fwd_seed = 25214903917_l64u * (rdtscl() ^ (uintptr_t)&verify_fwd_bck_rng);
788
789 unsigned values[10];
790 for(i; 10) {
791 values[i] = __tls_rand_fwd();
792 }
793
794 __tls_rand_advance_bck();
795
796 for ( i; 9 -~= 0 ) {
797 if(values[i] != __tls_rand_bck()) {
798 return false;
799 }
800 }
801
802 return true;
803}
804#endif
Note: See TracBrowser for help on using the repository browser.