source: libcfa/src/concurrency/kernel/startup.cfa@ 692c1cc

ADT ast-experimental
Last change on this file since 692c1cc was f5f2768, checked in by Peter A. Buhr <pabuhr@…>, 3 years ago

make _GNU_SOURCE default, change IO to use SOCKADDR_ARG and CONST_SOCKADDR_ARG, move sys/socket.h to first include because of anonymous naming problem
  • Property mode set to 100644
File size: 27.6 KB
Line 
1//
2// Cforall Version 1.0.0 Copyright (C) 2020 University of Waterloo
3//
4// The contents of this file are covered under the licence agreement in the
5// file "LICENCE" distributed with Cforall.
6//
7// kernel/startup.cfa --
8//
9// Author : Thierry Delisle
10// Created On : Thu Jul 30 15:12:54 2020
11// Last Modified By :
12// Last Modified On :
13// Update Count :
14//
15
16#define __cforall_thread__
17
18// #define __CFA_DEBUG_PRINT_RUNTIME_CORE__
19
20// C Includes
21#include <errno.h> // errno
22#include <signal.h>
23#include <string.h> // strerror
24#include <unistd.h> // sysconf
25
26extern "C" {
27 #include <limits.h> // PTHREAD_STACK_MIN
28 #include <unistd.h> // syscall
29 #include <sys/eventfd.h> // eventfd
30 #include <sys/mman.h> // mprotect
31 #include <sys/resource.h> // getrlimit
32}
33
34// CFA Includes
35#include "kernel/private.hfa"
36#include "iofwd.hfa"
37#include "startup.hfa" // STARTUP_PRIORITY_XXX
38#include "limits.hfa"
39#include "math.hfa"
40#include "bits/random.hfa" // prng
41
42#define CFA_PROCESSOR_USE_MMAP 0
43
44//-----------------------------------------------------------------------------
45// Some assembly required
46#if defined( __i386 )
47 #define CtxGet( ctx ) __asm__ volatile ( \
48 "movl %%esp,%0\n" \
49 "movl %%ebp,%1\n" \
50 : "=rm" (ctx.SP), \
51 "=rm" (ctx.FP) \
52 )
53#elif defined( __x86_64 )
54 #define CtxGet( ctx ) __asm__ volatile ( \
55 "movq %%rsp,%0\n" \
56 "movq %%rbp,%1\n" \
57 : "=rm" (ctx.SP), \
58 "=rm" (ctx.FP) \
59 )
60#elif defined( __aarch64__ )
61 #define CtxGet( ctx ) __asm__ volatile ( \
62 "mov %0, sp\n" \
63 "mov %1, fp\n" \
64 : "=rm" (ctx.SP), \
65 "=rm" (ctx.FP) \
66 )
67#else
68 #error unknown hardware architecture
69#endif
70
71//-----------------------------------------------------------------------------
72// Start and stop routine for the kernel, declared first to make sure they run first
73static void __kernel_startup (void) __attribute__(( constructor( STARTUP_PRIORITY_KERNEL ) ));
74static void __kernel_shutdown(void) __attribute__(( destructor ( STARTUP_PRIORITY_KERNEL ) ));
75
76//-----------------------------------------------------------------------------
77// Static Forward Declarations
78struct current_stack_info_t;
79
80static void * __invoke_processor(void * arg);
81static void __kernel_first_resume( processor * this );
82static void __kernel_last_resume ( processor * this );
83static void init(processor & this, const char name[], cluster & _cltr, thread$ * initT);
84static void deinit(processor & this);
85static void doregister( struct cluster & cltr );
86static void unregister( struct cluster & cltr );
87static void register_tls( processor * this );
88static void unregister_tls( processor * this );
89static void ?{}( coroutine$ & this, current_stack_info_t * info);
90static void ?{}( thread$ & this, current_stack_info_t * info);
91static void ?{}(processorCtx_t & this) {}
92static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info);
93
94#if defined(__CFA_WITH_VERIFY__)
95 static bool verify_fwd_bck_rng(void);
96#endif
97
98//-----------------------------------------------------------------------------
99// Forward Declarations for other modules
100extern void __kernel_alarm_startup(void);
101extern void __kernel_alarm_shutdown(void);
102extern void __cfa_io_start( processor * );
103extern void __cfa_io_stop ( processor * );
104
105//-----------------------------------------------------------------------------
106// Other Forward Declarations
107extern void __wake_proc(processor *);
108extern int cfa_main_returned; // from interpose.cfa
109size_t __global_random_prime = 4_294_967_291u;
110bool __global_random_mask = false;
111
112//-----------------------------------------------------------------------------
113// Kernel storage
114KERNEL_STORAGE(cluster, mainCluster);
115KERNEL_STORAGE(processor, mainProcessor);
116KERNEL_STORAGE(thread$, mainThread);
117KERNEL_STORAGE(__stack_t, mainThreadCtx);
118#if !defined(__CFA_NO_STATISTICS__)
119KERNEL_STORAGE(__stats_t, mainProcStats);
120#endif
121
122cluster * mainCluster libcfa_public;
123processor * mainProcessor;
124thread$ * mainThread;
125
126extern "C" {
127 struct { __dllist_t(cluster) list; __spinlock_t lock; } __cfa_dbg_global_clusters;
128}
129
130extern size_t __page_size;
131extern int __map_prot;
132
133//-----------------------------------------------------------------------------
134// Global state
135__thread struct KernelThreadData __cfaabi_tls __attribute__ ((tls_model ( "initial-exec" ))) @= {
136 .this_thread : NULL, // cannot use 0p
137 .this_processor : NULL,
138 .sched_lock : false,
139 .preemption_state : { .disable_count : 1, .enabled : false, .in_progress : false },
140 // random_state uninitialized
141 .ready_rng : { .fwd_seed : 0, .bck_seed : 0 },
142 .this_stats : NULL,
143 #ifdef __CFA_WITH_VERIFY__
144 .in_sched_lock : false,
145 .sched_id : 0,
146 #endif
147};
148
149__scheduler_RWLock_t __scheduler_lock @= { 0 };
150
151#if defined(CFA_HAVE_LINUX_LIBRSEQ)
152 // No data needed
153#elif defined(CFA_HAVE_LINUX_RSEQ_H)
154 extern "Cforall" {
155 __attribute__((aligned(64))) __thread volatile struct rseq __cfaabi_rseq @= {
156 .cpu_id : RSEQ_CPU_ID_UNINITIALIZED,
157 };
158 }
159#else
160 // No data needed
161#endif
162
163//-----------------------------------------------------------------------------
164// Struct to steal stack
165struct current_stack_info_t {
166 __stack_t * storage; // pointer to stack object
167 void * base; // base of stack
168 void * limit; // stack grows towards stack limit
169 void * context; // address of cfa_context_t
170};
171
172static void ?{}( current_stack_info_t & this ) {
173 __stack_context_t ctx;
174 CtxGet( ctx );
175 this.base = ctx.FP;
176
177 rlimit r;
178 getrlimit( RLIMIT_STACK, &r);
179 size_t size = r.rlim_cur;
180
181 this.limit = (void *)(((intptr_t)this.base) - size);
182 this.context = &storage_mainThreadCtx;
183}
184
185
186extern void heapManagerCtor();
187extern void heapManagerDtor();
188
189//=============================================================================================
190// Kernel Setup logic
191//=============================================================================================
192//-----------------------------------------------------------------------------
193// Kernel boot procedures
194static void __kernel_startup(void) {
195 /* paranoid */ verify( ! __preemption_enabled() );
196 __cfadbg_print_safe(runtime_core, "Kernel : Starting\n");
197
198 __cfa_dbg_global_clusters.list{ __get };
199 __cfa_dbg_global_clusters.lock{};
200
201 /* paranoid */ verify( verify_fwd_bck_rng() );
202
203 // Initialize the global scheduler lock
204 // __scheduler_lock = (__scheduler_RWLock_t*)&storage___scheduler_lock;
205 (__scheduler_lock){};
206
207 // Initialize the main cluster
208 mainCluster = (cluster *)&storage_mainCluster;
209 (*mainCluster){"Main Cluster", 0};
210
211 __cfadbg_print_safe(runtime_core, "Kernel : Main cluster ready\n");
212
213 // Construct the processor context of the main processor
214 void ?{}(processorCtx_t & this, processor * proc) {
215 (this.self){ "Processor" };
216 this.self.starter = 0p;
217 this.proc = proc;
218 }
219
220 void ?{}(processor & this) with( this ) {
221 ( this.terminated ){};
222 ( this.runner ){};
223 init( this, "Main Processor", *mainCluster, 0p );
224 kernel_thread = __cfaabi_pthread_self();
225
226 runner{ &this };
227 __cfadbg_print_safe(runtime_core, "Kernel : constructed main processor context %p\n", &runner);
228 }
229
230 // Initialize the main processor and the main processor ctx
231 // (the coroutine that contains the processing control flow)
232 mainProcessor = (processor *)&storage_mainProcessor;
233 (*mainProcessor){};
234
235 __cfa_io_start( mainProcessor );
236 register_tls( mainProcessor );
237
238 // Start by initializing the main thread
239 // SKULLDUGGERY: the mainThread steals the process main thread
240 // which will then be scheduled by the mainProcessor normally
241 mainThread = (thread$ *)&storage_mainThread;
242 current_stack_info_t info;
243 info.storage = (__stack_t*)&storage_mainThreadCtx;
244 (*mainThread){ &info };
245
246 __cfadbg_print_safe(runtime_core, "Kernel : Main thread ready\n");
247
248 //initialize the global state variables
249 __cfaabi_tls.this_processor = mainProcessor;
250 __cfaabi_tls.this_thread = mainThread;
251
252 #if !defined( __CFA_NO_STATISTICS__ )
253 __cfaabi_tls.this_stats = (__stats_t *)& storage_mainProcStats;
254 __init_stats( __cfaabi_tls.this_stats );
255 #endif
256 mainProcessor->local_data = &__cfaabi_tls;
257
258 // Enable preemption
259 __kernel_alarm_startup();
260
261 // Add the main thread to the ready queue
262 // once resume is called on mainProcessor->runner the mainThread needs to be scheduled like any normal thread
263 schedule_thread$(mainThread, UNPARK_LOCAL);
264
265 // SKULLDUGGERY: Force a context switch to the main processor to set the main thread's context to the current UNIX
266 // context. Hence, the main thread does not begin through __cfactx_invoke_thread, like all other threads. The trick here is that
267 // mainThread is on the ready queue when this call is made.
268 __kernel_first_resume( __cfaabi_tls.this_processor );
269
270
271 // THE SYSTEM IS NOW COMPLETELY RUNNING
272
273 __cfadbg_print_safe(runtime_core, "Kernel : Started\n--------------------------------------------------\n\n");
274
275 /* paranoid */ verify( ! __preemption_enabled() );
276 enable_interrupts();
277 /* paranoid */ verify( __preemption_enabled() );
278
279}
280
281extern "C"{
282 void pthread_delete_kernel_threads_();
283}
284
285
286static void __kernel_shutdown(void) {
287 if(!cfa_main_returned) return;
288
289 //delete kernel threads for pthread_concurrency
290 pthread_delete_kernel_threads_();
291
292 /* paranoid */ verify( __preemption_enabled() );
293 disable_interrupts();
294 /* paranoid */ verify( ! __preemption_enabled() );
295
296 __cfadbg_print_safe(runtime_core, "\n--------------------------------------------------\nKernel : Shutting down\n");
297
298 // SKULLDUGGERY: Notify the mainProcessor it needs to terminates.
299 // When its coroutine terminates, it return control to the mainThread
300 // which is currently here
301 /* paranoid */ verify( !__atomic_load_n(&mainProcessor->do_terminate, __ATOMIC_ACQUIRE) );
302 __atomic_store_n(&mainProcessor->do_terminate, true, __ATOMIC_RELEASE);
303 __wake_proc( mainProcessor );
304 __kernel_last_resume( __cfaabi_tls.this_processor );
305 mainThread->self_cor.state = Halted;
306
307 // THE SYSTEM IS NOW COMPLETELY STOPPED
308
309 // Disable preemption
310 __kernel_alarm_shutdown();
311
312 #if !defined( __CFA_NO_STATISTICS__ )
313 __stats_t * st = (__stats_t *)& storage_mainProcStats;
314 __tally_stats(mainCluster->stats, st);
315 if( 0 != mainProcessor->print_stats ) {
316 __print_stats( st, mainProcessor->print_stats, "Processor ", mainProcessor->name, (void*)mainProcessor );
317 }
318 #if defined(CFA_STATS_ARRAY)
319 __flush_stat( st, "Processor", mainProcessor );
320 #endif
321 #endif
322
323 mainProcessor->local_data = 0p;
324
325 unregister_tls( mainProcessor );
326 __cfa_io_stop( mainProcessor );
327
328 // Destroy the main processor and its context in reverse order of construction
329 // These were manually constructed so we need manually destroy them
330 void ^?{}(processor & this) with( this ){
331 deinit( this );
332
333 /* paranoid */ verify( this.do_terminate == true );
334 __cfadbg_print_safe(runtime_core, "Kernel : destroyed main processor context %p\n", &runner);
335 }
336
337 ^(*mainProcessor){};
338
339 // Final step, destroy the main thread since it is no longer needed
340
341 // Since we provided a stack to this taxk it will not destroy anything
342 /* paranoid */ verify(mainThread->self_cor.stack.storage == (__stack_t*)(((uintptr_t)&storage_mainThreadCtx)| 0x1));
343 ^(*mainThread){};
344
345 ^(*mainCluster){};
346
347 ^(__scheduler_lock){};
348
349 ^(__cfa_dbg_global_clusters.list){};
350 ^(__cfa_dbg_global_clusters.lock){};
351
352 __cfadbg_print_safe(runtime_core, "Kernel : Shutdown complete\n");
353}
354
355//=============================================================================================
356// Kernel Initial Scheduling logic
357//=============================================================================================
358
359// Context invoker for processors
360// This is the entry point for processors (kernel threads) *except* for the main processor
361// It effectively constructs a coroutine by stealing the pthread stack
362static void * __invoke_processor(void * arg) {
363 #if !defined( __CFA_NO_STATISTICS__ )
364 __stats_t local_stats;
365 __init_stats( &local_stats );
366 __cfaabi_tls.this_stats = &local_stats;
367 #endif
368
369 processor * proc = (processor *) arg;
370 __cfaabi_tls.this_processor = proc;
371 __cfaabi_tls.this_thread = 0p;
372 __cfaabi_tls.preemption_state.[enabled, disable_count] = [false, 1];
373 proc->local_data = &__cfaabi_tls;
374
375 heapManagerCtor(); // initialize heap
376
377 __cfa_io_start( proc );
378 register_tls( proc );
379
380 // SKULLDUGGERY: We want to create a context for the processor coroutine
381 // which is needed for the 2-step context switch. However, there is no reason
382 // to waste the perfectly valid stack create by pthread.
383 current_stack_info_t info;
384 __stack_t ctx;
385 info.storage = &ctx;
386 (proc->runner){ proc, &info };
387
388 __cfadbg_print_safe(runtime_core, "Coroutine : created stack %p\n", get_coroutine(proc->runner)->stack.storage);
389
390 //Set global state
391 __cfaabi_tls.this_thread = 0p;
392
393 //We now have a proper context from which to schedule threads
394 __cfadbg_print_safe(runtime_core, "Kernel : core %p created (%p, %p)\n", proc, &proc->runner, &ctx);
395
396 // SKULLDUGGERY: Since the coroutine doesn't have its own stack, we can't
397 // resume it to start it like it normally would, it will just context switch
398 // back to here. Instead directly call the main since we already are on the
399 // appropriate stack.
400 get_coroutine(proc->runner)->state = Active;
401 main( proc->runner );
402 get_coroutine(proc->runner)->state = Halted;
403
404 // Main routine of the core returned, the core is now fully terminated
405 __cfadbg_print_safe(runtime_core, "Kernel : core %p main ended (%p)\n", proc, &proc->runner);
406
407 #if !defined(__CFA_NO_STATISTICS__)
408 __tally_stats(proc->cltr->stats, &local_stats);
409 if( 0 != proc->print_stats ) {
410 __print_stats( &local_stats, proc->print_stats, "Processor ", proc->name, (void*)proc );
411 }
412 #if defined(CFA_STATS_ARRAY)
413 __flush_stat( &local_stats, "Processor", proc );
414 #endif
415 #endif
416
417 proc->local_data = 0p;
418
419 unregister_tls( proc );
420 __cfa_io_stop( proc );
421
422 heapManagerDtor(); // de-initialize heap
423
424 return 0p;
425}
426
427static void __kernel_first_resume( processor * this ) {
428 thread$ * src = mainThread;
429 coroutine$ * dst = get_coroutine(this->runner);
430
431 /* paranoid */ verify( ! __preemption_enabled() );
432
433 __cfaabi_tls.this_thread->curr_cor = dst;
434 __stack_prepare( &dst->stack, DEFAULT_STACK_SIZE );
435 __cfactx_start(main, dst, this->runner, __cfactx_invoke_coroutine);
436
437 /* paranoid */ verify( ! __preemption_enabled() );
438
439 dst->last = &src->self_cor;
440 dst->starter = dst->starter ? dst->starter : &src->self_cor;
441
442 // make sure the current state is still correct
443 /* paranoid */ verify(src->state == Ready);
444 src->corctx_flag = true;
445
446 // context switch to specified coroutine
447 verify( dst->context.SP );
448 __cfactx_switch( &src->context, &dst->context );
449 // when __cfactx_switch returns we are back in the src coroutine
450
451 mainThread->curr_cor = &mainThread->self_cor;
452
453 // make sure the current state has been update
454 /* paranoid */ verify(src->state == Active);
455
456 /* paranoid */ verify( ! __preemption_enabled() );
457}
458
459// KERNEL_ONLY
460static void __kernel_last_resume( processor * this ) {
461 coroutine$ * src = &mainThread->self_cor;
462 coroutine$ * dst = get_coroutine(this->runner);
463
464 /* paranoid */ verify( ! __preemption_enabled() );
465 /* paranoid */ verify( dst->starter == src );
466 /* paranoid */ verify( dst->context.SP );
467
468 // SKULLDUGGERY in debug the processors check that the
469 // stack is still within the limit of the stack limits after running a thread.
470 // that check doesn't make sense if we context switch to the processor using the
471 // coroutine semantics. Since this is a special case, use the current context
472 // info to populate these fields.
473 __cfaabi_dbg_debug_do(
474 __stack_context_t ctx;
475 CtxGet( ctx );
476 mainThread->context.SP = ctx.SP;
477 mainThread->context.FP = ctx.FP;
478 )
479
480 // context switch to the processor
481 __cfactx_switch( &src->context, &dst->context );
482}
483
484
485//=============================================================================================
486// Kernel Object Constructors logic
487//=============================================================================================
488//-----------------------------------------------------------------------------
489// Main thread construction
490static void ?{}( coroutine$ & this, current_stack_info_t * info) with( this ) {
491 stack.storage = info->storage;
492 with(*stack.storage) {
493 limit = info->limit;
494 base = info->base;
495 }
496 __attribute__((may_alias)) intptr_t * istorage = (intptr_t*) &stack.storage;
497 *istorage |= 0x1;
498 name = "Main Thread";
499 state = Start;
500 starter = 0p;
501 last = 0p;
502 cancellation = 0p;
503}
504
505static void ?{}( thread$ & this, current_stack_info_t * info) with( this ) {
506 ticket = TICKET_RUNNING;
507 state = Start;
508 self_cor{ info };
509 curr_cor = &self_cor;
510 curr_cluster = mainCluster;
511 self_mon.owner = &this;
512 self_mon.recursion = 1;
513 self_mon_p = &self_mon;
514 rdy_link.next = 0p;
515 rdy_link.ts = MAX;
516 user_link.next = 0p;
517 user_link.prev = 0p;
518 cltr_link.next = 0p;
519 cltr_link.prev = 0p;
520 preferred = ready_queue_new_preferred();
521 last_proc = 0p;
522 PRNG_SET_SEED( random_state, __global_random_mask ? __global_random_prime : __global_random_prime ^ rdtscl() );
523 #if defined( __CFA_WITH_VERIFY__ )
524 executing = 0p;
525 canary = 0x0D15EA5E0D15EA5Ep;
526 #endif
527
528 doregister(curr_cluster, this);
529
530 monitors{ &self_mon_p, 1, (fptr_t)0 };
531}
532
533//-----------------------------------------------------------------------------
534// Processor
535// Construct the processor context of non-main processors
536static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info) {
537 (this.self){ info };
538 this.proc = proc;
539}
540
541static void init(processor & this, const char name[], cluster & _cltr, thread$ * initT) with( this ) {
542 this.name = name;
543 this.cltr = &_cltr;
544 this.rdq.its = 0;
545 this.rdq.itr = 0;
546 this.rdq.id = 0;
547 this.rdq.target = MAX;
548 this.rdq.last = MAX;
549 this.rdq.cpu = 0;
550 // this.rdq.cutoff = 0ull;
551 do_terminate = false;
552 preemption_alarm = 0p;
553 pending_preemption = false;
554
555 this.io.ctx = 0p;
556 this.io.pending = false;
557 this.io.dirty = false;
558
559 this.init.thrd = initT;
560
561 this.local_data = 0p;
562
563 idle_wctx.evfd = eventfd(0, 0);
564 if (idle_wctx.evfd < 0) {
565 abort("KERNEL ERROR: PROCESSOR EVENTFD - %s\n", strerror(errno));
566 }
567
568 idle_wctx.sem = 0;
569 idle_wctx.wake__time = 0;
570
571 // I'm assuming these two are reserved for standard input and output
572 // so I'm using them as sentinels with idle_wctx.
573 /* paranoid */ verify( idle_wctx.evfd != 0 );
574 /* paranoid */ verify( idle_wctx.evfd != 1 );
575
576 #if !defined(__CFA_NO_STATISTICS__)
577 print_stats = 0;
578 print_halts = false;
579 #endif
580
581 __cfadbg_print_safe(runtime_core, "Kernel : core %p created\n", &this);
582}
583
584// Not a ctor, it just preps the destruction but should not destroy members
585static void deinit(processor & this) {
586 close(this.idle_wctx.evfd);
587}
588
589void ?{}(processor & this, const char name[], cluster & _cltr, thread$ * initT) libcfa_public {
590 ( this.terminated ){};
591 ( this.runner ){};
592
593 disable_interrupts();
594 init( this, name, _cltr, initT );
595 enable_interrupts();
596
597 __cfadbg_print_safe(runtime_core, "Kernel : Starting core %p\n", &this);
598
599 this.stack = __create_pthread( &this.kernel_thread, __invoke_processor, (void *)&this );
600}
601
602void ?{}(processor & this, const char name[], cluster & _cltr) libcfa_public {
603 (this){name, _cltr, 0p};
604}
605
606extern size_t __page_size;
607void ^?{}(processor & this) libcfa_public with( this ) {
608 /* paranoid */ verify( !__atomic_load_n(&do_terminate, __ATOMIC_ACQUIRE) );
609 __cfadbg_print_safe(runtime_core, "Kernel : core %p signaling termination\n", &this);
610
611 __atomic_store_n(&do_terminate, true, __ATOMIC_RELAXED);
612 __disable_interrupts_checked();
613 __wake_proc( &this );
614 __enable_interrupts_checked();
615
616 wait( terminated );
617 /* paranoid */ verify( active_processor() != &this);
618
619 __destroy_pthread( kernel_thread, this.stack, 0p );
620
621 disable_interrupts();
622 deinit( this );
623 enable_interrupts();
624}
625
626//-----------------------------------------------------------------------------
627// Cluster
628static void ?{}(__cluster_proc_list & this) {
629 this.fdw = 0p;
630 this.idle = 0;
631 this.total = 0;
632}
633
634void ?{}(cluster & this, const char name[], Duration preemption_rate, unsigned num_io, const io_context_params & io_params) libcfa_public with( this ) {
635 this.name = name;
636 this.preemption_rate = preemption_rate;
637 this.sched.readyQ.data = 0p;
638 this.sched.readyQ.tscs = 0p;
639 this.sched.readyQ.count = 0;
640 this.sched.io.tscs = 0p;
641 this.sched.io.data = 0p;
642 this.sched.caches = 0p;
643
644 #if !defined(__CFA_NO_STATISTICS__)
645 print_stats = 0;
646 stats = alloc();
647 __init_stats( stats );
648 #endif
649
650 threads{};
651
652 io.arbiter = create();
653 io.params = io_params;
654
655 managed.procs = 0p;
656 managed.cnt = 0;
657
658 doregister(this);
659
660 // Lock the RWlock so no-one pushes/pops while we are changing the queue
661 disable_interrupts();
662 uint_fast32_t last_size = ready_mutate_lock();
663
664 // Adjust the ready queue size
665 ready_queue_grow( &this );
666
667 // Unlock the RWlock
668 ready_mutate_unlock( last_size );
669 enable_interrupts( false ); // Don't poll, could be in main cluster
670}
671
672void ^?{}(cluster & this) libcfa_public {
673 set_concurrency( this, 0 );
674
675 destroy(this.io.arbiter);
676
677 // Lock the RWlock so no-one pushes/pops while we are changing the queue
678 disable_interrupts();
679 uint_fast32_t last_size = ready_mutate_lock();
680
681 // Adjust the ready queue size
682 ready_queue_shrink( &this );
683
684 // Unlock the RWlock
685 ready_mutate_unlock( last_size );
686
687 ready_queue_close( &this );
688 /* paranoid */ verify( this.sched.readyQ.data == 0p );
689 /* paranoid */ verify( this.sched.readyQ.tscs == 0p );
690 /* paranoid */ verify( this.sched.readyQ.count == 0 );
691 /* paranoid */ verify( this.sched.io.tscs == 0p );
692 /* paranoid */ verify( this.sched.caches == 0p );
693
694 enable_interrupts( false ); // Don't poll, could be in main cluster
695
696
697 #if !defined(__CFA_NO_STATISTICS__)
698 if( 0 != this.print_stats ) {
699 __print_stats( this.stats, this.print_stats, "Cluster", this.name, (void*)&this );
700 }
701 #if defined(CFA_STATS_ARRAY)
702 __flush_stat( this.stats, "Cluster", &this );
703 #endif
704 free( this.stats );
705 #endif
706
707 unregister(this);
708}
709
710//=============================================================================================
711// Miscellaneous Initialization
712//=============================================================================================
713//-----------------------------------------------------------------------------
714// Global Queues
715static void doregister( cluster & cltr ) {
716 lock ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
717 push_front( __cfa_dbg_global_clusters.list, cltr );
718 unlock ( __cfa_dbg_global_clusters.lock );
719}
720
721static void unregister( cluster & cltr ) {
722 lock ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
723 remove( __cfa_dbg_global_clusters.list, cltr );
724 unlock( __cfa_dbg_global_clusters.lock );
725}
726
727void doregister( cluster * cltr, thread$ & thrd ) {
728 lock (cltr->thread_list_lock __cfaabi_dbg_ctx2);
729 cltr->nthreads += 1;
730 insert_first(cltr->threads, thrd);
731 unlock (cltr->thread_list_lock);
732}
733
734void unregister( cluster * cltr, thread$ & thrd ) {
735 lock (cltr->thread_list_lock __cfaabi_dbg_ctx2);
736 {
737 tytagref( dlink(thread$), dlink(thread$) ) ?`inner( thread$ & this ) = void;
738 with( DLINK_VIA( thread$, struct __thread_user_link ) )
739 remove( thrd );
740 cltr->nthreads -= 1;
741 }
742 unlock(cltr->thread_list_lock);
743}
744
745static void register_tls( processor * this ) {
746 // Register and Lock the RWlock so no-one pushes/pops while we are changing the queue
747 uint_fast32_t last_size;
748 [this->unique_id, last_size] = ready_mutate_register();
749
750 this->rdq.cpu = __kernel_getcpu();
751
752 this->cltr->procs.total += 1u;
753 insert_last(this->cltr->procs.actives, *this);
754
755 // Adjust the ready queue size
756 ready_queue_grow( this->cltr );
757
758 // Unlock the RWlock
759 ready_mutate_unlock( last_size );
760}
761
762
763static void unregister_tls( processor * this ) {
764 // Lock the RWlock so no-one pushes/pops while we are changing the queue
765 uint_fast32_t last_size = ready_mutate_lock();
766 this->cltr->procs.total -= 1u;
767 remove(*this);
768
769 // clear the cluster so nothing gets pushed to local queues
770 cluster * cltr = this->cltr;
771 this->cltr = 0p;
772
773 // Adjust the ready queue size
774 ready_queue_shrink( cltr );
775
776 // Unlock the RWlock and unregister: we don't need the read_lock any more
777 ready_mutate_unregister( this->unique_id, last_size );
778}
779
780static void check( int ret, const char func[] ) {
781 if ( ret ) { // pthread routines return errno values
782 abort( "%s : internal error, error(%d) %s.", func, ret, strerror( ret ) );
783 } // if
784} // Abort
785
786void * __create_pthread( pthread_t * pthread, void * (*start)(void *), void * arg ) {
787 pthread_attr_t attr;
788
789 check( __cfaabi_pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
790
791 size_t stacksize = max( PTHREAD_STACK_MIN, DEFAULT_STACK_SIZE );
792
793 void * stack;
794 #if CFA_PROCESSOR_USE_MMAP
795 stacksize = ceiling( stacksize, __page_size ) + __page_size;
796 stack = mmap(0p, stacksize, __map_prot, MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
797 if(stack == ((void*)-1)) {
798 abort( "pthread stack creation : internal error, mmap failure, error(%d) %s.", errno, strerror( errno ) );
799 }
800 if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
801 abort( "pthread stack creation : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
802 } // if
803 #else
804 __cfaabi_dbg_debug_do(
805 stack = memalign( __page_size, stacksize + __page_size );
806 // pthread has no mechanism to create the guard page in user supplied stack.
807 if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
808 abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
809 } // if
810 );
811 __cfaabi_dbg_no_debug_do(
812 stack = malloc( stacksize );
813 );
814 #endif
815
816 check( __cfaabi_pthread_attr_setstack( &attr, stack, stacksize ), "pthread_attr_setstack" );
817 check( __cfaabi_pthread_create( pthread, &attr, start, arg ), "pthread_create" );
818 return stack;
819}
820
821void __destroy_pthread( pthread_t pthread, void * stack, void ** retval ) {
822 int err = __cfaabi_pthread_join( pthread, retval );
823 if( err != 0 ) abort("KERNEL ERROR: joining pthread %p caused error %s\n", (void*)pthread, strerror(err));
824
825 #if CFA_PROCESSOR_USE_MMAP
826 pthread_attr_t attr;
827
828 check( __cfaabi_pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
829
830 size_t stacksize;
831 // default stack size, normally defined by shell limit
832 check( __cfaabi_pthread_attr_getstacksize( &attr, &stacksize ), "pthread_attr_getstacksize" );
833 assert( stacksize >= PTHREAD_STACK_MIN );
834 stacksize += __page_size;
835
836 if(munmap(stack, stacksize) == -1) {
837 abort( "pthread stack destruction : internal error, munmap failure, error(%d) %s.", errno, strerror( errno ) );
838 }
839 #else
840 __cfaabi_dbg_debug_do(
841 // pthread has no mechanism to create the guard page in user supplied stack.
842 if ( mprotect( stack, __page_size, __map_prot ) == -1 ) {
843 abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
844 } // if
845 );
846 free( stack );
847 #endif
848}
849
850unsigned set_concurrency( cluster & this, unsigned new ) libcfa_public {
851 unsigned old = this.managed.cnt;
852
853 __cfadbg_print_safe(runtime_core, "Kernel : resizing cluster from %u to %u\n", old, (unsigned)new);
854
855 // Delete all the old unneeded procs
856 if(old > new) for(i; (unsigned)new ~ old) {
857 __cfadbg_print_safe(runtime_core, "Kernel : destroying %u\n", i);
858 delete( this.managed.procs[i] );
859 }
860
861 // Allocate new array (uses realloc and memcpies the data)
862 this.managed.procs = alloc( new, this.managed.procs`realloc );
863 this.managed.cnt = new;
864
865 // Create the desired new procs
866 if(old < new) for(i; old ~ new) {
867 __cfadbg_print_safe(runtime_core, "Kernel : constructing %u\n", i);
868 (*(this.managed.procs[i] = alloc())){ this };
869 }
870
871 // return the old count
872 return old;
873}
874
875#if defined(__CFA_WITH_VERIFY__)
876static bool verify_fwd_bck_rng(void) {
877 __cfaabi_tls.ready_rng.fwd_seed = 25214903917_l64u * (rdtscl() ^ (uintptr_t)&verify_fwd_bck_rng);
878
879 unsigned values[10];
880 for(i; 10) {
881 values[i] = __tls_rand_fwd();
882 }
883
884 __tls_rand_advance_bck();
885
886 for ( i; 9 -~= 0 ) {
887 if(values[i] != __tls_rand_bck()) {
888 return false;
889 }
890 }
891
892 return true;
893}
894#endif
Note: See TracBrowser for help on using the repository browser.