source: libcfa/src/concurrency/kernel/startup.cfa@ 0bcd707

ADT ast-experimental
Last change on this file since 0bcd707 was 56bb2e1, checked in by Peter A. Buhr <pabuhr@…>, 3 years ago

clean up #include files
  • Property mode set to 100644
File size: 27.6 KB
Line 
1//
2// Cforall Version 1.0.0 Copyright (C) 2020 University of Waterloo
3//
4// The contents of this file are covered under the licence agreement in the
5// file "LICENCE" distributed with Cforall.
6//
7// kernel/startup.cfa --
8//
9// Author : Thierry Delisle
10// Created On : Thu Jul 30 15:12:54 2020
11// Last Modified By :
12// Last Modified On :
13// Update Count :
14//
15
16#define __cforall_thread__
17
18// #define __CFA_DEBUG_PRINT_RUNTIME_CORE__
19
20// C Includes
21#include <errno.h> // errno
22#include <signal.h>
23#include <string.h> // strerror
24#include <unistd.h>
25#include <limits.h> // PTHREAD_STACK_MIN
26extern "C" {
27 #include <sys/eventfd.h> // eventfd
28 #include <sys/mman.h> // mprotect
29 #include <sys/resource.h> // getrlimit
30}
31
32// CFA Includes
33#include "kernel/private.hfa"
34#include "iofwd.hfa"
35#include "startup.hfa" // STARTUP_PRIORITY_XXX
36#include "limits.hfa"
37#include "math.hfa"
38#include "bits/random.hfa" // prng
39
40#define CFA_PROCESSOR_USE_MMAP 0
41
42//-----------------------------------------------------------------------------
43// Some assembly required
44#if defined( __i386 )
45 #define CtxGet( ctx ) __asm__ volatile ( \
46 "movl %%esp,%0\n" \
47 "movl %%ebp,%1\n" \
48 : "=rm" (ctx.SP), \
49 "=rm" (ctx.FP) \
50 )
51#elif defined( __x86_64 )
52 #define CtxGet( ctx ) __asm__ volatile ( \
53 "movq %%rsp,%0\n" \
54 "movq %%rbp,%1\n" \
55 : "=rm" (ctx.SP), \
56 "=rm" (ctx.FP) \
57 )
58#elif defined( __aarch64__ )
59 #define CtxGet( ctx ) __asm__ volatile ( \
60 "mov %0, sp\n" \
61 "mov %1, fp\n" \
62 : "=rm" (ctx.SP), \
63 "=rm" (ctx.FP) \
64 )
65#else
66 #error unknown hardware architecture
67#endif
68
69//-----------------------------------------------------------------------------
70// Start and stop routine for the kernel, declared first to make sure they run first
71static void __kernel_startup (void) __attribute__(( constructor( STARTUP_PRIORITY_KERNEL ) ));
72static void __kernel_shutdown(void) __attribute__(( destructor ( STARTUP_PRIORITY_KERNEL ) ));
73
74//-----------------------------------------------------------------------------
75// Static Forward Declarations
76struct current_stack_info_t;
77
78static void * __invoke_processor(void * arg);
79static void __kernel_first_resume( processor * this );
80static void __kernel_last_resume ( processor * this );
81static void init(processor & this, const char name[], cluster & _cltr, thread$ * initT);
82static void deinit(processor & this);
83static void doregister( struct cluster & cltr );
84static void unregister( struct cluster & cltr );
85static void register_tls( processor * this );
86static void unregister_tls( processor * this );
87static void ?{}( coroutine$ & this, current_stack_info_t * info);
88static void ?{}( thread$ & this, current_stack_info_t * info);
89static void ?{}(processorCtx_t & this) {}
90static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info);
91
92#if defined(__CFA_WITH_VERIFY__)
93 static bool verify_fwd_bck_rng(void);
94#endif
95
96//-----------------------------------------------------------------------------
97// Forward Declarations for other modules
98extern void __kernel_alarm_startup(void);
99extern void __kernel_alarm_shutdown(void);
100extern void __cfa_io_start( processor * );
101extern void __cfa_io_stop ( processor * );
102
103//-----------------------------------------------------------------------------
104// Other Forward Declarations
105extern void __wake_proc(processor *);
106extern int cfa_main_returned; // from interpose.cfa
107size_t __global_random_prime = 4_294_967_291u;
108bool __global_random_mask = false;
109
110//-----------------------------------------------------------------------------
111// Kernel storage
112KERNEL_STORAGE(cluster, mainCluster);
113KERNEL_STORAGE(processor, mainProcessor);
114KERNEL_STORAGE(thread$, mainThread);
115KERNEL_STORAGE(__stack_t, mainThreadCtx);
116#if !defined(__CFA_NO_STATISTICS__)
117KERNEL_STORAGE(__stats_t, mainProcStats);
118#endif
119
120cluster * mainCluster libcfa_public;
121processor * mainProcessor;
122thread$ * mainThread;
123
124extern "C" {
125 struct { __dllist_t(cluster) list; __spinlock_t lock; } __cfa_dbg_global_clusters;
126}
127
128extern size_t __page_size;
129extern int __map_prot;
130
131//-----------------------------------------------------------------------------
132// Global state
133__thread struct KernelThreadData __cfaabi_tls __attribute__ ((tls_model ( "initial-exec" ))) @= {
134 .this_thread : NULL, // cannot use 0p
135 .this_processor : NULL,
136 .sched_lock : false,
137 .preemption_state : { .disable_count : 1, .enabled : false, .in_progress : false },
138 // random_state uninitialized
139 .ready_rng : { .fwd_seed : 0, .bck_seed : 0 },
140 .this_stats : NULL,
141 #ifdef __CFA_WITH_VERIFY__
142 .in_sched_lock : false,
143 .sched_id : 0,
144 #endif
145};
146
147__scheduler_RWLock_t __scheduler_lock @= { 0 };
148
149#if defined(CFA_HAVE_LINUX_LIBRSEQ)
150 // No data needed
151#elif defined(CFA_HAVE_LINUX_RSEQ_H)
152 extern "Cforall" {
153 __attribute__((aligned(64))) __thread volatile struct rseq __cfaabi_rseq @= {
154 .cpu_id : RSEQ_CPU_ID_UNINITIALIZED,
155 };
156 }
157#else
158 // No data needed
159#endif
160
161//-----------------------------------------------------------------------------
162// Struct to steal stack
163struct current_stack_info_t {
164 __stack_t * storage; // pointer to stack object
165 void * base; // base of stack
166 void * limit; // stack grows towards stack limit
167 void * context; // address of cfa_context_t
168};
169
170static void ?{}( current_stack_info_t & this ) {
171 __stack_context_t ctx;
172 CtxGet( ctx );
173 this.base = ctx.FP;
174
175 rlimit r;
176 getrlimit( RLIMIT_STACK, &r);
177 size_t size = r.rlim_cur;
178
179 this.limit = (void *)(((intptr_t)this.base) - size);
180 this.context = &storage_mainThreadCtx;
181}
182
183
184extern void heapManagerCtor();
185extern void heapManagerDtor();
186
187//=============================================================================================
188// Kernel Setup logic
189//=============================================================================================
190//-----------------------------------------------------------------------------
191// Kernel boot procedures
192static void __kernel_startup(void) {
193 /* paranoid */ verify( ! __preemption_enabled() );
194 __cfadbg_print_safe(runtime_core, "Kernel : Starting\n");
195
196 __cfa_dbg_global_clusters.list{ __get };
197 __cfa_dbg_global_clusters.lock{};
198
199 /* paranoid */ verify( verify_fwd_bck_rng() );
200
201 // Initialize the global scheduler lock
202 // __scheduler_lock = (__scheduler_RWLock_t*)&storage___scheduler_lock;
203 (__scheduler_lock){};
204
205 // Initialize the main cluster
206 mainCluster = (cluster *)&storage_mainCluster;
207 (*mainCluster){"Main Cluster", 0};
208
209 __cfadbg_print_safe(runtime_core, "Kernel : Main cluster ready\n");
210
211 // Construct the processor context of the main processor
212 void ?{}(processorCtx_t & this, processor * proc) {
213 (this.self){ "Processor" };
214 this.self.starter = 0p;
215 this.proc = proc;
216 }
217
218 void ?{}(processor & this) with( this ) {
219 ( this.terminated ){};
220 ( this.runner ){};
221 init( this, "Main Processor", *mainCluster, 0p );
222 kernel_thread = __cfaabi_pthread_self();
223
224 runner{ &this };
225 __cfadbg_print_safe(runtime_core, "Kernel : constructed main processor context %p\n", &runner);
226 }
227
228 // Initialize the main processor and the main processor ctx
229 // (the coroutine that contains the processing control flow)
230 mainProcessor = (processor *)&storage_mainProcessor;
231 (*mainProcessor){};
232
233 __cfa_io_start( mainProcessor );
234 register_tls( mainProcessor );
235
236 // Start by initializing the main thread
237 // SKULLDUGGERY: the mainThread steals the process main thread
238 // which will then be scheduled by the mainProcessor normally
239 mainThread = (thread$ *)&storage_mainThread;
240 current_stack_info_t info;
241 info.storage = (__stack_t*)&storage_mainThreadCtx;
242 (*mainThread){ &info };
243
244 __cfadbg_print_safe(runtime_core, "Kernel : Main thread ready\n");
245
246 //initialize the global state variables
247 __cfaabi_tls.this_processor = mainProcessor;
248 __cfaabi_tls.this_thread = mainThread;
249
250 #if !defined( __CFA_NO_STATISTICS__ )
251 __cfaabi_tls.this_stats = (__stats_t *)& storage_mainProcStats;
252 __init_stats( __cfaabi_tls.this_stats );
253 #endif
254 mainProcessor->local_data = &__cfaabi_tls;
255
256 // Enable preemption
257 __kernel_alarm_startup();
258
259 // Add the main thread to the ready queue
260 // once resume is called on mainProcessor->runner the mainThread needs to be scheduled like any normal thread
261 schedule_thread$(mainThread, UNPARK_LOCAL);
262
263 // SKULLDUGGERY: Force a context switch to the main processor to set the main thread's context to the current UNIX
264 // context. Hence, the main thread does not begin through __cfactx_invoke_thread, like all other threads. The trick here is that
265 // mainThread is on the ready queue when this call is made.
266 __kernel_first_resume( __cfaabi_tls.this_processor );
267
268
269 // THE SYSTEM IS NOW COMPLETELY RUNNING
270
271 __cfadbg_print_safe(runtime_core, "Kernel : Started\n--------------------------------------------------\n\n");
272
273 /* paranoid */ verify( ! __preemption_enabled() );
274 enable_interrupts();
275 /* paranoid */ verify( __preemption_enabled() );
276
277}
278
279extern "C"{
280 void pthread_delete_kernel_threads_();
281}
282
283
284static void __kernel_shutdown(void) {
285 if(!cfa_main_returned) return;
286
287 //delete kernel threads for pthread_concurrency
288 pthread_delete_kernel_threads_();
289
290 /* paranoid */ verify( __preemption_enabled() );
291 disable_interrupts();
292 /* paranoid */ verify( ! __preemption_enabled() );
293
294 __cfadbg_print_safe(runtime_core, "\n--------------------------------------------------\nKernel : Shutting down\n");
295
296 // SKULLDUGGERY: Notify the mainProcessor it needs to terminates.
297 // When its coroutine terminates, it return control to the mainThread
298 // which is currently here
299 /* paranoid */ verify( !__atomic_load_n(&mainProcessor->do_terminate, __ATOMIC_ACQUIRE) );
300 __atomic_store_n(&mainProcessor->do_terminate, true, __ATOMIC_RELEASE);
301 __wake_proc( mainProcessor );
302 __kernel_last_resume( __cfaabi_tls.this_processor );
303 mainThread->self_cor.state = Halted;
304
305 // THE SYSTEM IS NOW COMPLETELY STOPPED
306
307 // Disable preemption
308 __kernel_alarm_shutdown();
309
310 #if !defined( __CFA_NO_STATISTICS__ )
311 __stats_t * st = (__stats_t *)& storage_mainProcStats;
312 __tally_stats(mainCluster->stats, st);
313 if( 0 != mainProcessor->print_stats ) {
314 __print_stats( st, mainProcessor->print_stats, "Processor ", mainProcessor->name, (void*)mainProcessor );
315 }
316 #if defined(CFA_STATS_ARRAY)
317 __flush_stat( st, "Processor", mainProcessor );
318 #endif
319 #endif
320
321 mainProcessor->local_data = 0p;
322
323 unregister_tls( mainProcessor );
324 __cfa_io_stop( mainProcessor );
325
326 // Destroy the main processor and its context in reverse order of construction
327 // These were manually constructed so we need manually destroy them
328 void ^?{}(processor & this) with( this ){
329 deinit( this );
330
331 /* paranoid */ verify( this.do_terminate == true );
332 __cfadbg_print_safe(runtime_core, "Kernel : destroyed main processor context %p\n", &runner);
333 }
334
335 ^(*mainProcessor){};
336
337 // Final step, destroy the main thread since it is no longer needed
338
339 // Since we provided a stack to this taxk it will not destroy anything
340 /* paranoid */ verify(mainThread->self_cor.stack.storage == (__stack_t*)(((uintptr_t)&storage_mainThreadCtx)| 0x1));
341 ^(*mainThread){};
342
343 ^(*mainCluster){};
344
345 ^(__scheduler_lock){};
346
347 ^(__cfa_dbg_global_clusters.list){};
348 ^(__cfa_dbg_global_clusters.lock){};
349
350 __cfadbg_print_safe(runtime_core, "Kernel : Shutdown complete\n");
351}
352
353//=============================================================================================
354// Kernel Initial Scheduling logic
355//=============================================================================================
356
357// Context invoker for processors
358// This is the entry point for processors (kernel threads) *except* for the main processor
359// It effectively constructs a coroutine by stealing the pthread stack
360static void * __invoke_processor(void * arg) {
361 #if !defined( __CFA_NO_STATISTICS__ )
362 __stats_t local_stats;
363 __init_stats( &local_stats );
364 __cfaabi_tls.this_stats = &local_stats;
365 #endif
366
367 processor * proc = (processor *) arg;
368 __cfaabi_tls.this_processor = proc;
369 __cfaabi_tls.this_thread = 0p;
370 __cfaabi_tls.preemption_state.[enabled, disable_count] = [false, 1];
371 proc->local_data = &__cfaabi_tls;
372
373 heapManagerCtor(); // initialize heap
374
375 __cfa_io_start( proc );
376 register_tls( proc );
377
378 // SKULLDUGGERY: We want to create a context for the processor coroutine
379 // which is needed for the 2-step context switch. However, there is no reason
380 // to waste the perfectly valid stack create by pthread.
381 current_stack_info_t info;
382 __stack_t ctx;
383 info.storage = &ctx;
384 (proc->runner){ proc, &info };
385
386 __cfadbg_print_safe(runtime_core, "Coroutine : created stack %p\n", get_coroutine(proc->runner)->stack.storage);
387
388 //Set global state
389 __cfaabi_tls.this_thread = 0p;
390
391 //We now have a proper context from which to schedule threads
392 __cfadbg_print_safe(runtime_core, "Kernel : core %p created (%p, %p)\n", proc, &proc->runner, &ctx);
393
394 // SKULLDUGGERY: Since the coroutine doesn't have its own stack, we can't
395 // resume it to start it like it normally would, it will just context switch
396 // back to here. Instead directly call the main since we already are on the
397 // appropriate stack.
398 get_coroutine(proc->runner)->state = Active;
399 main( proc->runner );
400 get_coroutine(proc->runner)->state = Halted;
401
402 // Main routine of the core returned, the core is now fully terminated
403 __cfadbg_print_safe(runtime_core, "Kernel : core %p main ended (%p)\n", proc, &proc->runner);
404
405 #if !defined(__CFA_NO_STATISTICS__)
406 __tally_stats(proc->cltr->stats, &local_stats);
407 if( 0 != proc->print_stats ) {
408 __print_stats( &local_stats, proc->print_stats, "Processor ", proc->name, (void*)proc );
409 }
410 #if defined(CFA_STATS_ARRAY)
411 __flush_stat( &local_stats, "Processor", proc );
412 #endif
413 #endif
414
415 proc->local_data = 0p;
416
417 unregister_tls( proc );
418 __cfa_io_stop( proc );
419
420 heapManagerDtor(); // de-initialize heap
421
422 return 0p;
423}
424
425static void __kernel_first_resume( processor * this ) {
426 thread$ * src = mainThread;
427 coroutine$ * dst = get_coroutine(this->runner);
428
429 /* paranoid */ verify( ! __preemption_enabled() );
430
431 __cfaabi_tls.this_thread->curr_cor = dst;
432 __stack_prepare( &dst->stack, DEFAULT_STACK_SIZE );
433 __cfactx_start(main, dst, this->runner, __cfactx_invoke_coroutine);
434
435 /* paranoid */ verify( ! __preemption_enabled() );
436
437 dst->last = &src->self_cor;
438 dst->starter = dst->starter ? dst->starter : &src->self_cor;
439
440 // make sure the current state is still correct
441 /* paranoid */ verify(src->state == Ready);
442 src->corctx_flag = true;
443
444 // context switch to specified coroutine
445 verify( dst->context.SP );
446 __cfactx_switch( &src->context, &dst->context );
447 // when __cfactx_switch returns we are back in the src coroutine
448
449 mainThread->curr_cor = &mainThread->self_cor;
450
451 // make sure the current state has been update
452 /* paranoid */ verify(src->state == Active);
453
454 /* paranoid */ verify( ! __preemption_enabled() );
455}
456
457// KERNEL_ONLY
458static void __kernel_last_resume( processor * this ) {
459 coroutine$ * src = &mainThread->self_cor;
460 coroutine$ * dst = get_coroutine(this->runner);
461
462 /* paranoid */ verify( ! __preemption_enabled() );
463 /* paranoid */ verify( dst->starter == src );
464 /* paranoid */ verify( dst->context.SP );
465
466 // SKULLDUGGERY in debug the processors check that the
467 // stack is still within the limit of the stack limits after running a thread.
468 // that check doesn't make sense if we context switch to the processor using the
469 // coroutine semantics. Since this is a special case, use the current context
470 // info to populate these fields.
471 __cfaabi_dbg_debug_do(
472 __stack_context_t ctx;
473 CtxGet( ctx );
474 mainThread->context.SP = ctx.SP;
475 mainThread->context.FP = ctx.FP;
476 )
477
478 // context switch to the processor
479 __cfactx_switch( &src->context, &dst->context );
480}
481
482
483//=============================================================================================
484// Kernel Object Constructors logic
485//=============================================================================================
486//-----------------------------------------------------------------------------
487// Main thread construction
488static void ?{}( coroutine$ & this, current_stack_info_t * info) with( this ) {
489 stack.storage = info->storage;
490 with(*stack.storage) {
491 limit = info->limit;
492 base = info->base;
493 }
494 __attribute__((may_alias)) intptr_t * istorage = (intptr_t*) &stack.storage;
495 *istorage |= 0x1;
496 name = "Main Thread";
497 state = Start;
498 starter = 0p;
499 last = 0p;
500 cancellation = 0p;
501}
502
503static void ?{}( thread$ & this, current_stack_info_t * info) with( this ) {
504 ticket = TICKET_RUNNING;
505 state = Start;
506 self_cor{ info };
507 curr_cor = &self_cor;
508 curr_cluster = mainCluster;
509 self_mon.owner = &this;
510 self_mon.recursion = 1;
511 self_mon_p = &self_mon;
512 rdy_link.next = 0p;
513 rdy_link.ts = MAX;
514 user_link.next = 0p;
515 user_link.prev = 0p;
516 cltr_link.next = 0p;
517 cltr_link.prev = 0p;
518 preferred = ready_queue_new_preferred();
519 last_proc = 0p;
520 PRNG_SET_SEED( random_state, __global_random_mask ? __global_random_prime : __global_random_prime ^ rdtscl() );
521 #if defined( __CFA_WITH_VERIFY__ )
522 executing = 0p;
523 canary = 0x0D15EA5E0D15EA5Ep;
524 #endif
525
526 doregister(curr_cluster, this);
527
528 monitors{ &self_mon_p, 1, (fptr_t)0 };
529}
530
531//-----------------------------------------------------------------------------
532// Processor
533// Construct the processor context of non-main processors
534static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info) {
535 (this.self){ info };
536 this.proc = proc;
537}
538
539static void init(processor & this, const char name[], cluster & _cltr, thread$ * initT) with( this ) {
540 this.name = name;
541 this.cltr = &_cltr;
542 this.rdq.its = 0;
543 this.rdq.itr = 0;
544 this.rdq.id = 0;
545 this.rdq.target = MAX;
546 this.rdq.last = MAX;
547 this.rdq.cpu = 0;
548 // this.rdq.cutoff = 0ull;
549 do_terminate = false;
550 preemption_alarm = 0p;
551 pending_preemption = false;
552
553 this.io.ctx = 0p;
554 this.io.pending = false;
555 this.io.dirty = false;
556
557 this.init.thrd = initT;
558
559 this.local_data = 0p;
560
561 idle_wctx.evfd = eventfd(0, 0);
562 if (idle_wctx.evfd < 0) {
563 abort("KERNEL ERROR: PROCESSOR EVENTFD - %s\n", strerror(errno));
564 }
565
566 idle_wctx.sem = 0;
567 idle_wctx.wake__time = 0;
568
569 // I'm assuming these two are reserved for standard input and output
570 // so I'm using them as sentinels with idle_wctx.
571 /* paranoid */ verify( idle_wctx.evfd != 0 );
572 /* paranoid */ verify( idle_wctx.evfd != 1 );
573
574 #if !defined(__CFA_NO_STATISTICS__)
575 print_stats = 0;
576 print_halts = false;
577 #endif
578
579 __cfadbg_print_safe(runtime_core, "Kernel : core %p created\n", &this);
580}
581
582// Not a ctor, it just preps the destruction but should not destroy members
583static void deinit(processor & this) {
584 close(this.idle_wctx.evfd);
585}
586
587void ?{}(processor & this, const char name[], cluster & _cltr, thread$ * initT) libcfa_public {
588 ( this.terminated ){};
589 ( this.runner ){};
590
591 disable_interrupts();
592 init( this, name, _cltr, initT );
593 enable_interrupts();
594
595 __cfadbg_print_safe(runtime_core, "Kernel : Starting core %p\n", &this);
596
597 this.stack = __create_pthread( &this.kernel_thread, __invoke_processor, (void *)&this );
598}
599
600void ?{}(processor & this, const char name[], cluster & _cltr) libcfa_public {
601 (this){name, _cltr, 0p};
602}
603
604extern size_t __page_size;
605void ^?{}(processor & this) libcfa_public with( this ) {
606 /* paranoid */ verify( !__atomic_load_n(&do_terminate, __ATOMIC_ACQUIRE) );
607 __cfadbg_print_safe(runtime_core, "Kernel : core %p signaling termination\n", &this);
608
609 __atomic_store_n(&do_terminate, true, __ATOMIC_RELAXED);
610 __disable_interrupts_checked();
611 __wake_proc( &this );
612 __enable_interrupts_checked();
613
614 wait( terminated );
615 /* paranoid */ verify( active_processor() != &this);
616
617 __destroy_pthread( kernel_thread, this.stack, 0p );
618
619 disable_interrupts();
620 deinit( this );
621 enable_interrupts();
622}
623
624//-----------------------------------------------------------------------------
625// Cluster
626static void ?{}(__cluster_proc_list & this) {
627 this.fdw = 0p;
628 this.idle = 0;
629 this.total = 0;
630}
631
632void ?{}(cluster & this, const char name[], Duration preemption_rate, unsigned num_io, const io_context_params & io_params) libcfa_public with( this ) {
633 this.name = name;
634 this.preemption_rate = preemption_rate;
635 this.sched.readyQ.data = 0p;
636 this.sched.readyQ.tscs = 0p;
637 this.sched.readyQ.count = 0;
638 this.sched.io.tscs = 0p;
639 this.sched.io.data = 0p;
640 this.sched.caches = 0p;
641
642 #if !defined(__CFA_NO_STATISTICS__)
643 print_stats = 0;
644 stats = alloc();
645 __init_stats( stats );
646 #endif
647
648 threads{};
649
650 io.arbiter = create();
651 io.params = io_params;
652
653 managed.procs = 0p;
654 managed.cnt = 0;
655
656 doregister(this);
657
658 // Lock the RWlock so no-one pushes/pops while we are changing the queue
659 disable_interrupts();
660 uint_fast32_t last_size = ready_mutate_lock();
661
662 // Adjust the ready queue size
663 ready_queue_grow( &this );
664
665 // Unlock the RWlock
666 ready_mutate_unlock( last_size );
667 enable_interrupts( false ); // Don't poll, could be in main cluster
668}
669
670void ^?{}(cluster & this) libcfa_public {
671 set_concurrency( this, 0 );
672
673 destroy(this.io.arbiter);
674
675 // Lock the RWlock so no-one pushes/pops while we are changing the queue
676 disable_interrupts();
677 uint_fast32_t last_size = ready_mutate_lock();
678
679 // Adjust the ready queue size
680 ready_queue_shrink( &this );
681
682 // Unlock the RWlock
683 ready_mutate_unlock( last_size );
684
685 ready_queue_close( &this );
686 /* paranoid */ verify( this.sched.readyQ.data == 0p );
687 /* paranoid */ verify( this.sched.readyQ.tscs == 0p );
688 /* paranoid */ verify( this.sched.readyQ.count == 0 );
689 /* paranoid */ verify( this.sched.io.tscs == 0p );
690 /* paranoid */ verify( this.sched.caches == 0p );
691
692 enable_interrupts( false ); // Don't poll, could be in main cluster
693
694
695 #if !defined(__CFA_NO_STATISTICS__)
696 if( 0 != this.print_stats ) {
697 __print_stats( this.stats, this.print_stats, "Cluster", this.name, (void*)&this );
698 }
699 #if defined(CFA_STATS_ARRAY)
700 __flush_stat( this.stats, "Cluster", &this );
701 #endif
702 free( this.stats );
703 #endif
704
705 unregister(this);
706}
707
708//=============================================================================================
709// Miscellaneous Initialization
710//=============================================================================================
711//-----------------------------------------------------------------------------
712// Global Queues
713static void doregister( cluster & cltr ) {
714 lock ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
715 push_front( __cfa_dbg_global_clusters.list, cltr );
716 unlock ( __cfa_dbg_global_clusters.lock );
717}
718
719static void unregister( cluster & cltr ) {
720 lock ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
721 remove( __cfa_dbg_global_clusters.list, cltr );
722 unlock( __cfa_dbg_global_clusters.lock );
723}
724
725void doregister( cluster * cltr, thread$ & thrd ) {
726 lock (cltr->thread_list_lock __cfaabi_dbg_ctx2);
727 cltr->nthreads += 1;
728 insert_first(cltr->threads, thrd);
729 unlock (cltr->thread_list_lock);
730}
731
732void unregister( cluster * cltr, thread$ & thrd ) {
733 lock (cltr->thread_list_lock __cfaabi_dbg_ctx2);
734 {
735 tytagref( dlink(thread$), dlink(thread$) ) ?`inner( thread$ & this ) = void;
736 with( DLINK_VIA( thread$, struct __thread_user_link ) )
737 remove( thrd );
738 cltr->nthreads -= 1;
739 }
740 unlock(cltr->thread_list_lock);
741}
742
743static void register_tls( processor * this ) {
744 // Register and Lock the RWlock so no-one pushes/pops while we are changing the queue
745 uint_fast32_t last_size;
746 [this->unique_id, last_size] = ready_mutate_register();
747
748 this->rdq.cpu = __kernel_getcpu();
749
750 this->cltr->procs.total += 1u;
751 insert_last(this->cltr->procs.actives, *this);
752
753 // Adjust the ready queue size
754 ready_queue_grow( this->cltr );
755
756 // Unlock the RWlock
757 ready_mutate_unlock( last_size );
758}
759
760
761static void unregister_tls( processor * this ) {
762 // Lock the RWlock so no-one pushes/pops while we are changing the queue
763 uint_fast32_t last_size = ready_mutate_lock();
764 this->cltr->procs.total -= 1u;
765 remove(*this);
766
767 // clear the cluster so nothing gets pushed to local queues
768 cluster * cltr = this->cltr;
769 this->cltr = 0p;
770
771 // Adjust the ready queue size
772 ready_queue_shrink( cltr );
773
774 // Unlock the RWlock and unregister: we don't need the read_lock any more
775 ready_mutate_unregister( this->unique_id, last_size );
776}
777
778static void check( int ret, const char func[] ) {
779 if ( ret ) { // pthread routines return errno values
780 abort( "%s : internal error, error(%d) %s.", func, ret, strerror( ret ) );
781 } // if
782} // Abort
783
784void * __create_pthread( pthread_t * pthread, void * (*start)(void *), void * arg ) {
785 pthread_attr_t attr;
786
787 check( __cfaabi_pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
788
789 size_t stacksize = max( PTHREAD_STACK_MIN, DEFAULT_STACK_SIZE );
790
791 void * stack;
792 #if CFA_PROCESSOR_USE_MMAP
793 stacksize = ceiling( stacksize, __page_size ) + __page_size;
794 stack = mmap(0p, stacksize, __map_prot, MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
795 if(stack == ((void*)-1)) {
796 abort( "pthread stack creation : internal error, mmap failure, error(%d) %s.", errno, strerror( errno ) );
797 }
798 if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
799 abort( "pthread stack creation : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
800 } // if
801 #else
802 __cfaabi_dbg_debug_do(
803 stack = memalign( __page_size, stacksize + __page_size );
804 // pthread has no mechanism to create the guard page in user supplied stack.
805 if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
806 abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
807 } // if
808 );
809 __cfaabi_dbg_no_debug_do(
810 stack = malloc( stacksize );
811 );
812 #endif
813
814 check( __cfaabi_pthread_attr_setstack( &attr, stack, stacksize ), "pthread_attr_setstack" );
815 check( __cfaabi_pthread_create( pthread, &attr, start, arg ), "pthread_create" );
816 return stack;
817}
818
819void __destroy_pthread( pthread_t pthread, void * stack, void ** retval ) {
820 int err = __cfaabi_pthread_join( pthread, retval );
821 if( err != 0 ) abort("KERNEL ERROR: joining pthread %p caused error %s\n", (void*)pthread, strerror(err));
822
823 #if CFA_PROCESSOR_USE_MMAP
824 pthread_attr_t attr;
825
826 check( __cfaabi_pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
827
828 size_t stacksize;
829 // default stack size, normally defined by shell limit
830 check( __cfaabi_pthread_attr_getstacksize( &attr, &stacksize ), "pthread_attr_getstacksize" );
831 assert( stacksize >= PTHREAD_STACK_MIN );
832 stacksize += __page_size;
833
834 if(munmap(stack, stacksize) == -1) {
835 abort( "pthread stack destruction : internal error, munmap failure, error(%d) %s.", errno, strerror( errno ) );
836 }
837 #else
838 __cfaabi_dbg_debug_do(
839 // pthread has no mechanism to create the guard page in user supplied stack.
840 if ( mprotect( stack, __page_size, __map_prot ) == -1 ) {
841 abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
842 } // if
843 );
844 free( stack );
845 #endif
846}
847
848unsigned set_concurrency( cluster & this, unsigned new ) libcfa_public {
849 unsigned old = this.managed.cnt;
850
851 __cfadbg_print_safe(runtime_core, "Kernel : resizing cluster from %u to %u\n", old, (unsigned)new);
852
853 // Delete all the old unneeded procs
854 if(old > new) for(i; (unsigned)new ~ old) {
855 __cfadbg_print_safe(runtime_core, "Kernel : destroying %u\n", i);
856 delete( this.managed.procs[i] );
857 }
858
859 // Allocate new array (uses realloc and memcpies the data)
860 this.managed.procs = alloc( new, this.managed.procs`realloc );
861 this.managed.cnt = new;
862
863 // Create the desired new procs
864 if(old < new) for(i; old ~ new) {
865 __cfadbg_print_safe(runtime_core, "Kernel : constructing %u\n", i);
866 (*(this.managed.procs[i] = alloc())){ this };
867 }
868
869 // return the old count
870 return old;
871}
872
873#if defined(__CFA_WITH_VERIFY__)
874static bool verify_fwd_bck_rng(void) {
875 __cfaabi_tls.ready_rng.fwd_seed = 25214903917_l64u * (rdtscl() ^ (uintptr_t)&verify_fwd_bck_rng);
876
877 unsigned values[10];
878 for(i; 10) {
879 values[i] = __tls_rand_fwd();
880 }
881
882 __tls_rand_advance_bck();
883
884 for ( i; 9 -~= 0 ) {
885 if(values[i] != __tls_rand_bck()) {
886 return false;
887 }
888 }
889
890 return true;
891}
892#endif
Note: See TracBrowser for help on using the repository browser.