source: libcfa/src/concurrency/kernel/startup.cfa@ 7bc84b8

ADT ast-experimental stuck-waitfor-destruct
Last change on this file since 7bc84b8 was d2ad151, checked in by Peter A. Buhr <pabuhr@…>, 4 years ago

major update of PRNG
  • Property mode set to 100644
File size: 27.3 KB
Line 
1//
2// Cforall Version 1.0.0 Copyright (C) 2020 University of Waterloo
3//
4// The contents of this file are covered under the licence agreement in the
5// file "LICENCE" distributed with Cforall.
6//
7// kernel/startup.cfa --
8//
9// Author : Thierry Delisle
10// Created On : Thu Jul 30 15:12:54 2020
11// Last Modified By :
12// Last Modified On :
13// Update Count :
14//
15
16#define __cforall_thread__
17#define _GNU_SOURCE
18
19// #define __CFA_DEBUG_PRINT_RUNTIME_CORE__
20
21// C Includes
22#include <errno.h> // errno
23#include <signal.h>
24#include <string.h> // strerror
25#include <unistd.h> // sysconf
26
27extern "C" {
28 #include <limits.h> // PTHREAD_STACK_MIN
29 #include <unistd.h> // syscall
30 #include <sys/eventfd.h> // eventfd
31 #include <sys/mman.h> // mprotect
32 #include <sys/resource.h> // getrlimit
33}
34
35// CFA Includes
36#include "kernel/private.hfa"
37#include "iofwd.hfa"
38#include "startup.hfa" // STARTUP_PRIORITY_XXX
39#include "limits.hfa"
40#include "math.hfa"
41#include "bits/random.hfa" // prng
42
43#define CFA_PROCESSOR_USE_MMAP 0
44
45//-----------------------------------------------------------------------------
46// Some assembly required
47#if defined( __i386 )
48 #define CtxGet( ctx ) __asm__ volatile ( \
49 "movl %%esp,%0\n" \
50 "movl %%ebp,%1\n" \
51 : "=rm" (ctx.SP), \
52 "=rm" (ctx.FP) \
53 )
54#elif defined( __x86_64 )
55 #define CtxGet( ctx ) __asm__ volatile ( \
56 "movq %%rsp,%0\n" \
57 "movq %%rbp,%1\n" \
58 : "=rm" (ctx.SP), \
59 "=rm" (ctx.FP) \
60 )
61#elif defined( __aarch64__ )
62 #define CtxGet( ctx ) __asm__ volatile ( \
63 "mov %0, sp\n" \
64 "mov %1, fp\n" \
65 : "=rm" (ctx.SP), \
66 "=rm" (ctx.FP) \
67 )
68#else
69 #error unknown hardware architecture
70#endif
71
72//-----------------------------------------------------------------------------
73// Start and stop routine for the kernel, declared first to make sure they run first
74static void __kernel_startup (void) __attribute__(( constructor( STARTUP_PRIORITY_KERNEL ) ));
75static void __kernel_shutdown(void) __attribute__(( destructor ( STARTUP_PRIORITY_KERNEL ) ));
76
77//-----------------------------------------------------------------------------
78// Static Forward Declarations
79struct current_stack_info_t;
80
81static void * __invoke_processor(void * arg);
82static void __kernel_first_resume( processor * this );
83static void __kernel_last_resume ( processor * this );
84static void init(processor & this, const char name[], cluster & _cltr, thread$ * initT);
85static void deinit(processor & this);
86static void doregister( struct cluster & cltr );
87static void unregister( struct cluster & cltr );
88static void register_tls( processor * this );
89static void unregister_tls( processor * this );
90static void ?{}( coroutine$ & this, current_stack_info_t * info);
91static void ?{}( thread$ & this, current_stack_info_t * info);
92static void ?{}(processorCtx_t & this) {}
93static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info);
94
95#if defined(__CFA_WITH_VERIFY__)
96 static bool verify_fwd_bck_rng(void);
97#endif
98
99//-----------------------------------------------------------------------------
100// Forward Declarations for other modules
101extern void __kernel_alarm_startup(void);
102extern void __kernel_alarm_shutdown(void);
103extern void __cfa_io_start( processor * );
104extern void __cfa_io_stop ( processor * );
105
106//-----------------------------------------------------------------------------
107// Other Forward Declarations
108extern void __wake_proc(processor *);
109extern int cfa_main_returned; // from interpose.cfa
110PRNG_ARG_T __global_random_prime = 4_294_967_291u;
111bool __global_random_mask = false;
112
113//-----------------------------------------------------------------------------
114// Kernel storage
115KERNEL_STORAGE(cluster, mainCluster);
116KERNEL_STORAGE(processor, mainProcessor);
117KERNEL_STORAGE(thread$, mainThread);
118KERNEL_STORAGE(__stack_t, mainThreadCtx);
119#if !defined(__CFA_NO_STATISTICS__)
120KERNEL_STORAGE(__stats_t, mainProcStats);
121#endif
122
123cluster * mainCluster libcfa_public;
124processor * mainProcessor;
125thread$ * mainThread;
126
127extern "C" {
128 struct { __dllist_t(cluster) list; __spinlock_t lock; } __cfa_dbg_global_clusters;
129}
130
131extern size_t __page_size;
132extern int __map_prot;
133
134//-----------------------------------------------------------------------------
135// Global state
136__thread struct KernelThreadData __cfaabi_tls __attribute__ ((tls_model ( "initial-exec" ))) @= {
137 NULL, // cannot use 0p
138 NULL,
139 false,
140 { 1, false, false },
141 0,
142 { 0, 0 },
143 NULL,
144 #ifdef __CFA_WITH_VERIFY__
145 false,
146 0,
147 #endif
148};
149
150__scheduler_RWLock_t __scheduler_lock @= { 0 };
151
152#if defined(CFA_HAVE_LINUX_LIBRSEQ)
153 // No data needed
154#elif defined(CFA_HAVE_LINUX_RSEQ_H)
155 extern "Cforall" {
156 __attribute__((aligned(64))) __thread volatile struct rseq __cfaabi_rseq @= {
157 .cpu_id : RSEQ_CPU_ID_UNINITIALIZED,
158 };
159 }
160#else
161 // No data needed
162#endif
163
164//-----------------------------------------------------------------------------
165// Struct to steal stack
166struct current_stack_info_t {
167 __stack_t * storage; // pointer to stack object
168 void * base; // base of stack
169 void * limit; // stack grows towards stack limit
170 void * context; // address of cfa_context_t
171};
172
173static void ?{}( current_stack_info_t & this ) {
174 __stack_context_t ctx;
175 CtxGet( ctx );
176 this.base = ctx.FP;
177
178 rlimit r;
179 getrlimit( RLIMIT_STACK, &r);
180 size_t size = r.rlim_cur;
181
182 this.limit = (void *)(((intptr_t)this.base) - size);
183 this.context = &storage_mainThreadCtx;
184}
185
186
187extern void heapManagerCtor();
188extern void heapManagerDtor();
189
190//=============================================================================================
191// Kernel Setup logic
192//=============================================================================================
193//-----------------------------------------------------------------------------
194// Kernel boot procedures
195static void __kernel_startup(void) {
196 /* paranoid */ verify( ! __preemption_enabled() );
197 __cfadbg_print_safe(runtime_core, "Kernel : Starting\n");
198
199 __cfa_dbg_global_clusters.list{ __get };
200 __cfa_dbg_global_clusters.lock{};
201
202 /* paranoid */ verify( verify_fwd_bck_rng() );
203
204 // Initialize the global scheduler lock
205 // __scheduler_lock = (__scheduler_RWLock_t*)&storage___scheduler_lock;
206 (__scheduler_lock){};
207
208 // Initialize the main cluster
209 mainCluster = (cluster *)&storage_mainCluster;
210 (*mainCluster){"Main Cluster", 0};
211
212 __cfadbg_print_safe(runtime_core, "Kernel : Main cluster ready\n");
213
214 // Construct the processor context of the main processor
215 void ?{}(processorCtx_t & this, processor * proc) {
216 (this.self){ "Processor" };
217 this.self.starter = 0p;
218 this.proc = proc;
219 }
220
221 void ?{}(processor & this) with( this ) {
222 ( this.terminated ){};
223 ( this.runner ){};
224 init( this, "Main Processor", *mainCluster, 0p );
225 kernel_thread = __cfaabi_pthread_self();
226
227 runner{ &this };
228 __cfadbg_print_safe(runtime_core, "Kernel : constructed main processor context %p\n", &runner);
229 }
230
231 // Initialize the main processor and the main processor ctx
232 // (the coroutine that contains the processing control flow)
233 mainProcessor = (processor *)&storage_mainProcessor;
234 (*mainProcessor){};
235
236 __cfa_io_start( mainProcessor );
237 register_tls( mainProcessor );
238
239 // Start by initializing the main thread
240 // SKULLDUGGERY: the mainThread steals the process main thread
241 // which will then be scheduled by the mainProcessor normally
242 mainThread = (thread$ *)&storage_mainThread;
243 current_stack_info_t info;
244 info.storage = (__stack_t*)&storage_mainThreadCtx;
245 (*mainThread){ &info };
246
247 __cfadbg_print_safe(runtime_core, "Kernel : Main thread ready\n");
248
249 //initialize the global state variables
250 __cfaabi_tls.this_processor = mainProcessor;
251 __cfaabi_tls.this_thread = mainThread;
252
253 #if !defined( __CFA_NO_STATISTICS__ )
254 __cfaabi_tls.this_stats = (__stats_t *)& storage_mainProcStats;
255 __init_stats( __cfaabi_tls.this_stats );
256 #endif
257 mainProcessor->local_data = &__cfaabi_tls;
258
259 // Enable preemption
260 __kernel_alarm_startup();
261
262 // Add the main thread to the ready queue
263 // once resume is called on mainProcessor->runner the mainThread needs to be scheduled like any normal thread
264 schedule_thread$(mainThread, UNPARK_LOCAL);
265
266 // SKULLDUGGERY: Force a context switch to the main processor to set the main thread's context to the current UNIX
267 // context. Hence, the main thread does not begin through __cfactx_invoke_thread, like all other threads. The trick here is that
268 // mainThread is on the ready queue when this call is made.
269 __kernel_first_resume( __cfaabi_tls.this_processor );
270
271
272 // THE SYSTEM IS NOW COMPLETELY RUNNING
273
274 __cfadbg_print_safe(runtime_core, "Kernel : Started\n--------------------------------------------------\n\n");
275
276 /* paranoid */ verify( ! __preemption_enabled() );
277 enable_interrupts();
278 /* paranoid */ verify( __preemption_enabled() );
279
280}
281
282extern "C"{
283 void pthread_delete_kernel_threads_();
284}
285
286
287static void __kernel_shutdown(void) {
288 if(!cfa_main_returned) return;
289
290 //delete kernel threads for pthread_concurrency
291 pthread_delete_kernel_threads_();
292
293 /* paranoid */ verify( __preemption_enabled() );
294 disable_interrupts();
295 /* paranoid */ verify( ! __preemption_enabled() );
296
297 __cfadbg_print_safe(runtime_core, "\n--------------------------------------------------\nKernel : Shutting down\n");
298
299 // SKULLDUGGERY: Notify the mainProcessor it needs to terminates.
300 // When its coroutine terminates, it return control to the mainThread
301 // which is currently here
302 /* paranoid */ verify( !__atomic_load_n(&mainProcessor->do_terminate, __ATOMIC_ACQUIRE) );
303 __atomic_store_n(&mainProcessor->do_terminate, true, __ATOMIC_RELEASE);
304 __wake_proc( mainProcessor );
305 __kernel_last_resume( __cfaabi_tls.this_processor );
306 mainThread->self_cor.state = Halted;
307
308 // THE SYSTEM IS NOW COMPLETELY STOPPED
309
310 // Disable preemption
311 __kernel_alarm_shutdown();
312
313 #if !defined( __CFA_NO_STATISTICS__ )
314 __stats_t * st = (__stats_t *)& storage_mainProcStats;
315 __tally_stats(mainCluster->stats, st);
316 if( 0 != mainProcessor->print_stats ) {
317 __print_stats( st, mainProcessor->print_stats, "Processor ", mainProcessor->name, (void*)mainProcessor );
318 }
319 #if defined(CFA_STATS_ARRAY)
320 __flush_stat( st, "Processor", mainProcessor );
321 #endif
322 #endif
323
324 mainProcessor->local_data = 0p;
325
326 unregister_tls( mainProcessor );
327 __cfa_io_stop( mainProcessor );
328
329 // Destroy the main processor and its context in reverse order of construction
330 // These were manually constructed so we need manually destroy them
331 void ^?{}(processor & this) with( this ){
332 deinit( this );
333
334 /* paranoid */ verify( this.do_terminate == true );
335 __cfadbg_print_safe(runtime_core, "Kernel : destroyed main processor context %p\n", &runner);
336 }
337
338 ^(*mainProcessor){};
339
340 // Final step, destroy the main thread since it is no longer needed
341
342 // Since we provided a stack to this taxk it will not destroy anything
343 /* paranoid */ verify(mainThread->self_cor.stack.storage == (__stack_t*)(((uintptr_t)&storage_mainThreadCtx)| 0x1));
344 ^(*mainThread){};
345
346 ^(*mainCluster){};
347
348 ^(__scheduler_lock){};
349
350 ^(__cfa_dbg_global_clusters.list){};
351 ^(__cfa_dbg_global_clusters.lock){};
352
353 __cfadbg_print_safe(runtime_core, "Kernel : Shutdown complete\n");
354}
355
356//=============================================================================================
357// Kernel Initial Scheduling logic
358//=============================================================================================
359
360// Context invoker for processors
361// This is the entry point for processors (kernel threads) *except* for the main processor
362// It effectively constructs a coroutine by stealing the pthread stack
363static void * __invoke_processor(void * arg) {
364 #if !defined( __CFA_NO_STATISTICS__ )
365 __stats_t local_stats;
366 __init_stats( &local_stats );
367 __cfaabi_tls.this_stats = &local_stats;
368 #endif
369
370 processor * proc = (processor *) arg;
371 __cfaabi_tls.this_processor = proc;
372 __cfaabi_tls.this_thread = 0p;
373 __cfaabi_tls.preemption_state.[enabled, disable_count] = [false, 1];
374 proc->local_data = &__cfaabi_tls;
375
376 heapManagerCtor(); // initialize heap
377
378 __cfa_io_start( proc );
379 register_tls( proc );
380
381 // SKULLDUGGERY: We want to create a context for the processor coroutine
382 // which is needed for the 2-step context switch. However, there is no reason
383 // to waste the perfectly valid stack create by pthread.
384 current_stack_info_t info;
385 __stack_t ctx;
386 info.storage = &ctx;
387 (proc->runner){ proc, &info };
388
389 __cfadbg_print_safe(runtime_core, "Coroutine : created stack %p\n", get_coroutine(proc->runner)->stack.storage);
390
391 //Set global state
392 __cfaabi_tls.this_thread = 0p;
393
394 //We now have a proper context from which to schedule threads
395 __cfadbg_print_safe(runtime_core, "Kernel : core %p created (%p, %p)\n", proc, &proc->runner, &ctx);
396
397 // SKULLDUGGERY: Since the coroutine doesn't have its own stack, we can't
398 // resume it to start it like it normally would, it will just context switch
399 // back to here. Instead directly call the main since we already are on the
400 // appropriate stack.
401 get_coroutine(proc->runner)->state = Active;
402 main( proc->runner );
403 get_coroutine(proc->runner)->state = Halted;
404
405 // Main routine of the core returned, the core is now fully terminated
406 __cfadbg_print_safe(runtime_core, "Kernel : core %p main ended (%p)\n", proc, &proc->runner);
407
408 #if !defined(__CFA_NO_STATISTICS__)
409 __tally_stats(proc->cltr->stats, &local_stats);
410 if( 0 != proc->print_stats ) {
411 __print_stats( &local_stats, proc->print_stats, "Processor ", proc->name, (void*)proc );
412 }
413 #if defined(CFA_STATS_ARRAY)
414 __flush_stat( &local_stats, "Processor", proc );
415 #endif
416 #endif
417
418 proc->local_data = 0p;
419
420 unregister_tls( proc );
421 __cfa_io_stop( proc );
422
423 heapManagerDtor(); // de-initialize heap
424
425 return 0p;
426}
427
428static void __kernel_first_resume( processor * this ) {
429 thread$ * src = mainThread;
430 coroutine$ * dst = get_coroutine(this->runner);
431
432 /* paranoid */ verify( ! __preemption_enabled() );
433
434 __cfaabi_tls.this_thread->curr_cor = dst;
435 __stack_prepare( &dst->stack, DEFAULT_STACK_SIZE );
436 __cfactx_start(main, dst, this->runner, __cfactx_invoke_coroutine);
437
438 /* paranoid */ verify( ! __preemption_enabled() );
439
440 dst->last = &src->self_cor;
441 dst->starter = dst->starter ? dst->starter : &src->self_cor;
442
443 // make sure the current state is still correct
444 /* paranoid */ verify(src->state == Ready);
445 src->corctx_flag = true;
446
447 // context switch to specified coroutine
448 verify( dst->context.SP );
449 __cfactx_switch( &src->context, &dst->context );
450 // when __cfactx_switch returns we are back in the src coroutine
451
452 mainThread->curr_cor = &mainThread->self_cor;
453
454 // make sure the current state has been update
455 /* paranoid */ verify(src->state == Active);
456
457 /* paranoid */ verify( ! __preemption_enabled() );
458}
459
460// KERNEL_ONLY
461static void __kernel_last_resume( processor * this ) {
462 coroutine$ * src = &mainThread->self_cor;
463 coroutine$ * dst = get_coroutine(this->runner);
464
465 /* paranoid */ verify( ! __preemption_enabled() );
466 /* paranoid */ verify( dst->starter == src );
467 /* paranoid */ verify( dst->context.SP );
468
469 // SKULLDUGGERY in debug the processors check that the
470 // stack is still within the limit of the stack limits after running a thread.
471 // that check doesn't make sense if we context switch to the processor using the
472 // coroutine semantics. Since this is a special case, use the current context
473 // info to populate these fields.
474 __cfaabi_dbg_debug_do(
475 __stack_context_t ctx;
476 CtxGet( ctx );
477 mainThread->context.SP = ctx.SP;
478 mainThread->context.FP = ctx.FP;
479 )
480
481 // context switch to the processor
482 __cfactx_switch( &src->context, &dst->context );
483}
484
485
486//=============================================================================================
487// Kernel Object Constructors logic
488//=============================================================================================
489//-----------------------------------------------------------------------------
490// Main thread construction
491static void ?{}( coroutine$ & this, current_stack_info_t * info) with( this ) {
492 stack.storage = info->storage;
493 with(*stack.storage) {
494 limit = info->limit;
495 base = info->base;
496 }
497 __attribute__((may_alias)) intptr_t * istorage = (intptr_t*) &stack.storage;
498 *istorage |= 0x1;
499 name = "Main Thread";
500 state = Start;
501 starter = 0p;
502 last = 0p;
503 cancellation = 0p;
504}
505
506static void ?{}( thread$ & this, current_stack_info_t * info) with( this ) {
507 ticket = TICKET_RUNNING;
508 state = Start;
509 self_cor{ info };
510 curr_cor = &self_cor;
511 curr_cluster = mainCluster;
512 self_mon.owner = &this;
513 self_mon.recursion = 1;
514 self_mon_p = &self_mon;
515 rdy_link.next = 0p;
516 rdy_link.ts = MAX;
517 preferred = ready_queue_new_preferred();
518 last_proc = 0p;
519 random_state = __global_random_mask ? __global_random_prime : __global_random_prime ^ rdtscl();
520 #if defined( __CFA_WITH_VERIFY__ )
521 executing = 0p;
522 canary = 0x0D15EA5E0D15EA5Ep;
523 #endif
524
525 doregister(curr_cluster, this);
526
527 monitors{ &self_mon_p, 1, (fptr_t)0 };
528}
529
530//-----------------------------------------------------------------------------
531// Processor
532// Construct the processor context of non-main processors
533static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info) {
534 (this.self){ info };
535 this.proc = proc;
536}
537
538static void init(processor & this, const char name[], cluster & _cltr, thread$ * initT) with( this ) {
539 this.name = name;
540 this.cltr = &_cltr;
541 this.rdq.its = 0;
542 this.rdq.itr = 0;
543 this.rdq.id = 0;
544 this.rdq.target = MAX;
545 this.rdq.last = MAX;
546 this.rdq.cpu = 0;
547 // this.rdq.cutoff = 0ull;
548 do_terminate = false;
549 preemption_alarm = 0p;
550 pending_preemption = false;
551
552 this.io.ctx = 0p;
553 this.io.pending = false;
554 this.io.dirty = false;
555
556 this.init.thrd = initT;
557
558 this.local_data = 0p;
559
560 idle_wctx.evfd = eventfd(0, 0);
561 if (idle_wctx.evfd < 0) {
562 abort("KERNEL ERROR: PROCESSOR EVENTFD - %s\n", strerror(errno));
563 }
564
565 idle_wctx.sem = 0;
566 idle_wctx.wake__time = 0;
567
568 // I'm assuming these two are reserved for standard input and output
569 // so I'm using them as sentinels with idle_wctx.
570 /* paranoid */ verify( idle_wctx.evfd != 0 );
571 /* paranoid */ verify( idle_wctx.evfd != 1 );
572
573 #if !defined(__CFA_NO_STATISTICS__)
574 print_stats = 0;
575 print_halts = false;
576 #endif
577
578 __cfadbg_print_safe(runtime_core, "Kernel : core %p created\n", &this);
579}
580
581// Not a ctor, it just preps the destruction but should not destroy members
582static void deinit(processor & this) {
583 close(this.idle_wctx.evfd);
584}
585
586void ?{}(processor & this, const char name[], cluster & _cltr, thread$ * initT) libcfa_public {
587 ( this.terminated ){};
588 ( this.runner ){};
589
590 disable_interrupts();
591 init( this, name, _cltr, initT );
592 enable_interrupts();
593
594 __cfadbg_print_safe(runtime_core, "Kernel : Starting core %p\n", &this);
595
596 this.stack = __create_pthread( &this.kernel_thread, __invoke_processor, (void *)&this );
597}
598
599void ?{}(processor & this, const char name[], cluster & _cltr) libcfa_public {
600 (this){name, _cltr, 0p};
601}
602
603extern size_t __page_size;
604void ^?{}(processor & this) libcfa_public with( this ) {
605 /* paranoid */ verify( !__atomic_load_n(&do_terminate, __ATOMIC_ACQUIRE) );
606 __cfadbg_print_safe(runtime_core, "Kernel : core %p signaling termination\n", &this);
607
608 __atomic_store_n(&do_terminate, true, __ATOMIC_RELAXED);
609 __disable_interrupts_checked();
610 __wake_proc( &this );
611 __enable_interrupts_checked();
612
613 wait( terminated );
614 /* paranoid */ verify( active_processor() != &this);
615
616 __destroy_pthread( kernel_thread, this.stack, 0p );
617
618 disable_interrupts();
619 deinit( this );
620 enable_interrupts();
621}
622
623//-----------------------------------------------------------------------------
624// Cluster
625static void ?{}(__cluster_proc_list & this) {
626 this.fdw = 0p;
627 this.idle = 0;
628 this.total = 0;
629}
630
631void ?{}(cluster & this, const char name[], Duration preemption_rate, unsigned num_io, const io_context_params & io_params) libcfa_public with( this ) {
632 this.name = name;
633 this.preemption_rate = preemption_rate;
634 this.sched.readyQ.data = 0p;
635 this.sched.readyQ.tscs = 0p;
636 this.sched.readyQ.count = 0;
637 this.sched.io.tscs = 0p;
638 this.sched.io.data = 0p;
639 this.sched.caches = 0p;
640
641 #if !defined(__CFA_NO_STATISTICS__)
642 print_stats = 0;
643 stats = alloc();
644 __init_stats( stats );
645 #endif
646
647 threads{};
648
649 io.arbiter = create();
650 io.params = io_params;
651
652 managed.procs = 0p;
653 managed.cnt = 0;
654
655 doregister(this);
656
657 // Lock the RWlock so no-one pushes/pops while we are changing the queue
658 disable_interrupts();
659 uint_fast32_t last_size = ready_mutate_lock();
660
661 // Adjust the ready queue size
662 ready_queue_grow( &this );
663
664 // Unlock the RWlock
665 ready_mutate_unlock( last_size );
666 enable_interrupts( false ); // Don't poll, could be in main cluster
667}
668
669void ^?{}(cluster & this) libcfa_public {
670 set_concurrency( this, 0 );
671
672 destroy(this.io.arbiter);
673
674 // Lock the RWlock so no-one pushes/pops while we are changing the queue
675 disable_interrupts();
676 uint_fast32_t last_size = ready_mutate_lock();
677
678 // Adjust the ready queue size
679 ready_queue_shrink( &this );
680
681 // Unlock the RWlock
682 ready_mutate_unlock( last_size );
683
684 ready_queue_close( &this );
685 /* paranoid */ verify( this.sched.readyQ.data == 0p );
686 /* paranoid */ verify( this.sched.readyQ.tscs == 0p );
687 /* paranoid */ verify( this.sched.readyQ.count == 0 );
688 /* paranoid */ verify( this.sched.io.tscs == 0p );
689 /* paranoid */ verify( this.sched.caches == 0p );
690
691 enable_interrupts( false ); // Don't poll, could be in main cluster
692
693
694 #if !defined(__CFA_NO_STATISTICS__)
695 if( 0 != this.print_stats ) {
696 __print_stats( this.stats, this.print_stats, "Cluster", this.name, (void*)&this );
697 }
698 #if defined(CFA_STATS_ARRAY)
699 __flush_stat( this.stats, "Cluster", &this );
700 #endif
701 free( this.stats );
702 #endif
703
704 unregister(this);
705}
706
707//=============================================================================================
708// Miscellaneous Initialization
709//=============================================================================================
710//-----------------------------------------------------------------------------
711// Global Queues
712static void doregister( cluster & cltr ) {
713 lock ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
714 push_front( __cfa_dbg_global_clusters.list, cltr );
715 unlock ( __cfa_dbg_global_clusters.lock );
716}
717
718static void unregister( cluster & cltr ) {
719 lock ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
720 remove( __cfa_dbg_global_clusters.list, cltr );
721 unlock( __cfa_dbg_global_clusters.lock );
722}
723
724void doregister( cluster * cltr, thread$ & thrd ) {
725 lock (cltr->thread_list_lock __cfaabi_dbg_ctx2);
726 cltr->nthreads += 1;
727 insert_first(cltr->threads, thrd);
728 unlock (cltr->thread_list_lock);
729}
730
731void unregister( cluster * cltr, thread$ & thrd ) {
732 lock (cltr->thread_list_lock __cfaabi_dbg_ctx2);
733 {
734 tytagref( dlink(thread$), dlink(thread$) ) ?`inner( thread$ & this ) = void;
735 with( DLINK_VIA( thread$, struct __thread_user_link ) )
736 remove( thrd );
737 cltr->nthreads -= 1;
738 }
739 unlock(cltr->thread_list_lock);
740}
741
742static void register_tls( processor * this ) {
743 // Register and Lock the RWlock so no-one pushes/pops while we are changing the queue
744 uint_fast32_t last_size;
745 [this->unique_id, last_size] = ready_mutate_register();
746
747 this->rdq.cpu = __kernel_getcpu();
748
749 this->cltr->procs.total += 1u;
750 insert_last(this->cltr->procs.actives, *this);
751
752 // Adjust the ready queue size
753 ready_queue_grow( this->cltr );
754
755 // Unlock the RWlock
756 ready_mutate_unlock( last_size );
757}
758
759
760static void unregister_tls( processor * this ) {
761 // Lock the RWlock so no-one pushes/pops while we are changing the queue
762 uint_fast32_t last_size = ready_mutate_lock();
763 this->cltr->procs.total -= 1u;
764 remove(*this);
765
766 // clear the cluster so nothing gets pushed to local queues
767 cluster * cltr = this->cltr;
768 this->cltr = 0p;
769
770 // Adjust the ready queue size
771 ready_queue_shrink( cltr );
772
773 // Unlock the RWlock and unregister: we don't need the read_lock any more
774 ready_mutate_unregister( this->unique_id, last_size );
775}
776
777static void check( int ret, const char func[] ) {
778 if ( ret ) { // pthread routines return errno values
779 abort( "%s : internal error, error(%d) %s.", func, ret, strerror( ret ) );
780 } // if
781} // Abort
782
783void * __create_pthread( pthread_t * pthread, void * (*start)(void *), void * arg ) {
784 pthread_attr_t attr;
785
786 check( __cfaabi_pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
787
788 size_t stacksize = max( PTHREAD_STACK_MIN, DEFAULT_STACK_SIZE );
789
790 void * stack;
791 #if CFA_PROCESSOR_USE_MMAP
792 stacksize = ceiling( stacksize, __page_size ) + __page_size;
793 stack = mmap(0p, stacksize, __map_prot, MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
794 if(stack == ((void*)-1)) {
795 abort( "pthread stack creation : internal error, mmap failure, error(%d) %s.", errno, strerror( errno ) );
796 }
797 if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
798 abort( "pthread stack creation : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
799 } // if
800 #else
801 __cfaabi_dbg_debug_do(
802 stack = memalign( __page_size, stacksize + __page_size );
803 // pthread has no mechanism to create the guard page in user supplied stack.
804 if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
805 abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
806 } // if
807 );
808 __cfaabi_dbg_no_debug_do(
809 stack = malloc( stacksize );
810 );
811 #endif
812
813 check( __cfaabi_pthread_attr_setstack( &attr, stack, stacksize ), "pthread_attr_setstack" );
814 check( __cfaabi_pthread_create( pthread, &attr, start, arg ), "pthread_create" );
815 return stack;
816}
817
818void __destroy_pthread( pthread_t pthread, void * stack, void ** retval ) {
819 int err = __cfaabi_pthread_join( pthread, retval );
820 if( err != 0 ) abort("KERNEL ERROR: joining pthread %p caused error %s\n", (void*)pthread, strerror(err));
821
822 #if CFA_PROCESSOR_USE_MMAP
823 pthread_attr_t attr;
824
825 check( __cfaabi_pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
826
827 size_t stacksize;
828 // default stack size, normally defined by shell limit
829 check( __cfaabi_pthread_attr_getstacksize( &attr, &stacksize ), "pthread_attr_getstacksize" );
830 assert( stacksize >= PTHREAD_STACK_MIN );
831 stacksize += __page_size;
832
833 if(munmap(stack, stacksize) == -1) {
834 abort( "pthread stack destruction : internal error, munmap failure, error(%d) %s.", errno, strerror( errno ) );
835 }
836 #else
837 __cfaabi_dbg_debug_do(
838 // pthread has no mechanism to create the guard page in user supplied stack.
839 if ( mprotect( stack, __page_size, __map_prot ) == -1 ) {
840 abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
841 } // if
842 );
843 free( stack );
844 #endif
845}
846
847unsigned set_concurrency( cluster & this, unsigned new ) libcfa_public {
848 unsigned old = this.managed.cnt;
849
850 __cfadbg_print_safe(runtime_core, "Kernel : resizing cluster from %u to %u\n", old, (unsigned)new);
851
852 // Delete all the old unneeded procs
853 if(old > new) for(i; (unsigned)new ~ old) {
854 __cfadbg_print_safe(runtime_core, "Kernel : destroying %u\n", i);
855 delete( this.managed.procs[i] );
856 }
857
858 // Allocate new array (uses realloc and memcpies the data)
859 this.managed.procs = alloc( new, this.managed.procs`realloc );
860 this.managed.cnt = new;
861
862 // Create the desired new procs
863 if(old < new) for(i; old ~ new) {
864 __cfadbg_print_safe(runtime_core, "Kernel : constructing %u\n", i);
865 (*(this.managed.procs[i] = alloc())){ this };
866 }
867
868 // return the old count
869 return old;
870}
871
872#if defined(__CFA_WITH_VERIFY__)
873static bool verify_fwd_bck_rng(void) {
874 __cfaabi_tls.ready_rng.fwd_seed = 25214903917_l64u * (rdtscl() ^ (uintptr_t)&verify_fwd_bck_rng);
875
876 unsigned values[10];
877 for(i; 10) {
878 values[i] = __tls_rand_fwd();
879 }
880
881 __tls_rand_advance_bck();
882
883 for ( i; 9 -~= 0 ) {
884 if(values[i] != __tls_rand_bck()) {
885 return false;
886 }
887 }
888
889 return true;
890}
891#endif
Note: See TracBrowser for help on using the repository browser.