source: libcfa/src/concurrency/kernel/startup.cfa@ 9e042d8

ADT ast-experimental
Last change on this file since 9e042d8 was df6cc9d, checked in by Thierry Delisle <tdelisle@…>, 3 years ago

Merge branch 'master' into pthread-emulation
  • Property mode set to 100644
File size: 26.8 KB
Line 
1//
2// Cforall Version 1.0.0 Copyright (C) 2020 University of Waterloo
3//
4// The contents of this file are covered under the licence agreement in the
5// file "LICENCE" distributed with Cforall.
6//
7// kernel/startup.cfa --
8//
9// Author : Thierry Delisle
10// Created On : Thu Jul 30 15:12:54 2020
11// Last Modified By :
12// Last Modified On :
13// Update Count :
14//
15
16#define __cforall_thread__
17#define _GNU_SOURCE
18
19// C Includes
20#include <errno.h> // errno
21#include <signal.h>
22#include <string.h> // strerror
23#include <unistd.h> // sysconf
24
25extern "C" {
26 #include <limits.h> // PTHREAD_STACK_MIN
27 #include <unistd.h> // syscall
28 #include <sys/eventfd.h> // eventfd
29 #include <sys/mman.h> // mprotect
30 #include <sys/resource.h> // getrlimit
31}
32
33// CFA Includes
34#include "kernel/private.hfa"
35#include "iofwd.hfa"
36#include "startup.hfa" // STARTUP_PRIORITY_XXX
37#include "limits.hfa"
38#include "math.hfa"
39
40#define CFA_PROCESSOR_USE_MMAP 0
41
42//-----------------------------------------------------------------------------
43// Some assembly required
44#if defined( __i386 )
45 #define CtxGet( ctx ) __asm__ volatile ( \
46 "movl %%esp,%0\n" \
47 "movl %%ebp,%1\n" \
48 : "=rm" (ctx.SP), \
49 "=rm" (ctx.FP) \
50 )
51#elif defined( __x86_64 )
52 #define CtxGet( ctx ) __asm__ volatile ( \
53 "movq %%rsp,%0\n" \
54 "movq %%rbp,%1\n" \
55 : "=rm" (ctx.SP), \
56 "=rm" (ctx.FP) \
57 )
58#elif defined( __aarch64__ )
59 #define CtxGet( ctx ) __asm__ volatile ( \
60 "mov %0, sp\n" \
61 "mov %1, fp\n" \
62 : "=rm" (ctx.SP), \
63 "=rm" (ctx.FP) \
64 )
65#else
66 #error unknown hardware architecture
67#endif
68
69//-----------------------------------------------------------------------------
70// Start and stop routine for the kernel, declared first to make sure they run first
71static void __kernel_startup (void) __attribute__(( constructor( STARTUP_PRIORITY_KERNEL ) ));
72static void __kernel_shutdown(void) __attribute__(( destructor ( STARTUP_PRIORITY_KERNEL ) ));
73
74//-----------------------------------------------------------------------------
75// Static Forward Declarations
76struct current_stack_info_t;
77
78static void * __invoke_processor(void * arg);
79static void __kernel_first_resume( processor * this );
80static void __kernel_last_resume ( processor * this );
81static void init(processor & this, const char name[], cluster & _cltr, thread$ * initT);
82static void deinit(processor & this);
83static void doregister( struct cluster & cltr );
84static void unregister( struct cluster & cltr );
85static void register_tls( processor * this );
86static void unregister_tls( processor * this );
87static void ?{}( coroutine$ & this, current_stack_info_t * info);
88static void ?{}( thread$ & this, current_stack_info_t * info);
89static void ?{}(processorCtx_t & this) {}
90static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info);
91
92#if defined(__CFA_WITH_VERIFY__)
93 static bool verify_fwd_bck_rng(void);
94#endif
95
96//-----------------------------------------------------------------------------
97// Forward Declarations for other modules
98extern void __kernel_alarm_startup(void);
99extern void __kernel_alarm_shutdown(void);
100extern void __cfa_io_start( processor * );
101extern void __cfa_io_stop ( processor * );
102
103//-----------------------------------------------------------------------------
104// Other Forward Declarations
105extern void __wake_proc(processor *);
106extern int cfa_main_returned; // from interpose.cfa
107uint32_t __global_random_prime = 4_294_967_291u, __global_random_mask = false;
108
109//-----------------------------------------------------------------------------
110// Kernel storage
111KERNEL_STORAGE(cluster, mainCluster);
112KERNEL_STORAGE(processor, mainProcessor);
113KERNEL_STORAGE(thread$, mainThread);
114KERNEL_STORAGE(__stack_t, mainThreadCtx);
115// KERNEL_STORAGE(__scheduler_RWLock_t, __scheduler_lock);
116KERNEL_STORAGE(eventfd_t, mainIdleEventFd);
117KERNEL_STORAGE(io_future_t, mainIdleFuture);
118#if !defined(__CFA_NO_STATISTICS__)
119KERNEL_STORAGE(__stats_t, mainProcStats);
120#endif
121
122cluster * mainCluster libcfa_public;
123processor * mainProcessor;
124thread$ * mainThread;
125
126extern "C" {
127 struct { __dllist_t(cluster) list; __spinlock_t lock; } __cfa_dbg_global_clusters;
128}
129
130extern size_t __page_size;
131extern int __map_prot;
132
133//-----------------------------------------------------------------------------
134// Global state
135__thread struct KernelThreadData __cfaabi_tls __attribute__ ((tls_model ( "initial-exec" ))) @= {
136 NULL, // cannot use 0p
137 NULL,
138 false,
139 { 1, false, false },
140 0,
141 { 0, 0 },
142 NULL,
143 #ifdef __CFA_WITH_VERIFY__
144 false,
145 0,
146 #endif
147};
148
149__scheduler_RWLock_t __scheduler_lock @= { 0 };
150
151#if defined(CFA_HAVE_LINUX_LIBRSEQ)
152 // No data needed
153#elif defined(CFA_HAVE_LINUX_RSEQ_H)
154 extern "Cforall" {
155 __attribute__((aligned(64))) __thread volatile struct rseq __cfaabi_rseq @= {
156 .cpu_id : RSEQ_CPU_ID_UNINITIALIZED,
157 };
158 }
159#else
160 // No data needed
161#endif
162
163//-----------------------------------------------------------------------------
164// Struct to steal stack
165struct current_stack_info_t {
166 __stack_t * storage; // pointer to stack object
167 void * base; // base of stack
168 void * limit; // stack grows towards stack limit
169 void * context; // address of cfa_context_t
170};
171
172static void ?{}( current_stack_info_t & this ) {
173 __stack_context_t ctx;
174 CtxGet( ctx );
175 this.base = ctx.FP;
176
177 rlimit r;
178 getrlimit( RLIMIT_STACK, &r);
179 size_t size = r.rlim_cur;
180
181 this.limit = (void *)(((intptr_t)this.base) - size);
182 this.context = &storage_mainThreadCtx;
183}
184
185
186extern void heapManagerCtor();
187extern void heapManagerDtor();
188
189//=============================================================================================
190// Kernel Setup logic
191//=============================================================================================
192//-----------------------------------------------------------------------------
193// Kernel boot procedures
194static void __kernel_startup(void) {
195 /* paranoid */ verify( ! __preemption_enabled() );
196 __cfadbg_print_safe(runtime_core, "Kernel : Starting\n");
197
198 __cfa_dbg_global_clusters.list{ __get };
199 __cfa_dbg_global_clusters.lock{};
200
201 /* paranoid */ verify( verify_fwd_bck_rng() );
202
203 // Initialize the global scheduler lock
204 // __scheduler_lock = (__scheduler_RWLock_t*)&storage___scheduler_lock;
205 (__scheduler_lock){};
206
207 // Initialize the main cluster
208 mainCluster = (cluster *)&storage_mainCluster;
209 (*mainCluster){"Main Cluster", 0};
210
211 __cfadbg_print_safe(runtime_core, "Kernel : Main cluster ready\n");
212
213 // Construct the processor context of the main processor
214 void ?{}(processorCtx_t & this, processor * proc) {
215 (this.self){ "Processor" };
216 this.self.starter = 0p;
217 this.proc = proc;
218 }
219
220 void ?{}(processor & this) with( this ) {
221 ( this.terminated ){};
222 ( this.runner ){};
223 init( this, "Main Processor", *mainCluster, 0p );
224 kernel_thread = __cfaabi_pthread_self();
225
226 runner{ &this };
227 __cfadbg_print_safe(runtime_core, "Kernel : constructed main processor context %p\n", &runner);
228 }
229
230 // Initialize the main processor and the main processor ctx
231 // (the coroutine that contains the processing control flow)
232 mainProcessor = (processor *)&storage_mainProcessor;
233 (*mainProcessor){};
234
235 mainProcessor->idle_wctx.rdbuf = &storage_mainIdleEventFd;
236 mainProcessor->idle_wctx.ftr = (io_future_t*)&storage_mainIdleFuture;
237 /* paranoid */ verify( sizeof(storage_mainIdleEventFd) == sizeof(eventfd_t) );
238
239 __cfa_io_start( mainProcessor );
240 register_tls( mainProcessor );
241
242 // Start by initializing the main thread
243 // SKULLDUGGERY: the mainThread steals the process main thread
244 // which will then be scheduled by the mainProcessor normally
245 mainThread = (thread$ *)&storage_mainThread;
246 current_stack_info_t info;
247 info.storage = (__stack_t*)&storage_mainThreadCtx;
248 (*mainThread){ &info };
249
250 __cfadbg_print_safe(runtime_core, "Kernel : Main thread ready\n");
251
252 //initialize the global state variables
253 __cfaabi_tls.this_processor = mainProcessor;
254 __cfaabi_tls.this_thread = mainThread;
255
256 #if !defined( __CFA_NO_STATISTICS__ )
257 __cfaabi_tls.this_stats = (__stats_t *)& storage_mainProcStats;
258 __init_stats( __cfaabi_tls.this_stats );
259 #endif
260 mainProcessor->local_data = &__cfaabi_tls;
261
262 // Enable preemption
263 __kernel_alarm_startup();
264
265 // Add the main thread to the ready queue
266 // once resume is called on mainProcessor->runner the mainThread needs to be scheduled like any normal thread
267 schedule_thread$(mainThread, UNPARK_LOCAL);
268
269 // SKULLDUGGERY: Force a context switch to the main processor to set the main thread's context to the current UNIX
270 // context. Hence, the main thread does not begin through __cfactx_invoke_thread, like all other threads. The trick here is that
271 // mainThread is on the ready queue when this call is made.
272 __kernel_first_resume( __cfaabi_tls.this_processor );
273
274
275 // THE SYSTEM IS NOW COMPLETELY RUNNING
276
277 __cfadbg_print_safe(runtime_core, "Kernel : Started\n--------------------------------------------------\n\n");
278
279 /* paranoid */ verify( ! __preemption_enabled() );
280 enable_interrupts();
281 /* paranoid */ verify( __preemption_enabled() );
282
283}
284
285extern "C"{
286 void pthread_delete_kernel_threads_();
287}
288
289
290static void __kernel_shutdown(void) {
291 if(!cfa_main_returned) return;
292
293 //delete kernel threads for pthread_concurrency
294 pthread_delete_kernel_threads_();
295
296 /* paranoid */ verify( __preemption_enabled() );
297 disable_interrupts();
298 /* paranoid */ verify( ! __preemption_enabled() );
299
300 __cfadbg_print_safe(runtime_core, "\n--------------------------------------------------\nKernel : Shutting down\n");
301
302 // SKULLDUGGERY: Notify the mainProcessor it needs to terminates.
303 // When its coroutine terminates, it return control to the mainThread
304 // which is currently here
305 /* paranoid */ verify( !__atomic_load_n(&mainProcessor->do_terminate, __ATOMIC_ACQUIRE) );
306 __atomic_store_n(&mainProcessor->do_terminate, true, __ATOMIC_RELEASE);
307 __wake_proc( mainProcessor );
308 __kernel_last_resume( __cfaabi_tls.this_processor );
309 mainThread->self_cor.state = Halted;
310
311 // THE SYSTEM IS NOW COMPLETELY STOPPED
312
313 // Disable preemption
314 __kernel_alarm_shutdown();
315
316 #if !defined( __CFA_NO_STATISTICS__ )
317 __stats_t * st = (__stats_t *)& storage_mainProcStats;
318 __tally_stats(mainCluster->stats, st);
319 if( 0 != mainProcessor->print_stats ) {
320 __print_stats( st, mainProcessor->print_stats, "Processor ", mainProcessor->name, (void*)mainProcessor );
321 }
322 #if defined(CFA_STATS_ARRAY)
323 __flush_stat( st, "Processor", mainProcessor );
324 #endif
325 #endif
326
327 mainProcessor->local_data = 0p;
328
329 unregister_tls( mainProcessor );
330 __cfa_io_stop( mainProcessor );
331
332 // Destroy the main processor and its context in reverse order of construction
333 // These were manually constructed so we need manually destroy them
334 void ^?{}(processor & this) with( this ){
335 deinit( this );
336
337 /* paranoid */ verify( this.do_terminate == true );
338 __cfaabi_dbg_print_safe("Kernel : destroyed main processor context %p\n", &runner);
339 }
340
341 ^(*mainProcessor){};
342
343 // Final step, destroy the main thread since it is no longer needed
344
345 // Since we provided a stack to this taxk it will not destroy anything
346 /* paranoid */ verify(mainThread->self_cor.stack.storage == (__stack_t*)(((uintptr_t)&storage_mainThreadCtx)| 0x1));
347 ^(*mainThread){};
348
349 ^(*mainCluster){};
350
351 ^(__scheduler_lock){};
352
353 ^(__cfa_dbg_global_clusters.list){};
354 ^(__cfa_dbg_global_clusters.lock){};
355
356 __cfadbg_print_safe(runtime_core, "Kernel : Shutdown complete\n");
357}
358
359//=============================================================================================
360// Kernel Initial Scheduling logic
361//=============================================================================================
362
363// Context invoker for processors
364// This is the entry point for processors (kernel threads) *except* for the main processor
365// It effectively constructs a coroutine by stealing the pthread stack
366static void * __invoke_processor(void * arg) {
367 #if !defined( __CFA_NO_STATISTICS__ )
368 __stats_t local_stats;
369 __init_stats( &local_stats );
370 __cfaabi_tls.this_stats = &local_stats;
371 #endif
372
373 processor * proc = (processor *) arg;
374 __cfaabi_tls.this_processor = proc;
375 __cfaabi_tls.this_thread = 0p;
376 __cfaabi_tls.preemption_state.[enabled, disable_count] = [false, 1];
377 proc->local_data = &__cfaabi_tls;
378
379 heapManagerCtor(); // initialize heap
380
381 __cfa_io_start( proc );
382 register_tls( proc );
383
384 // used for idle sleep when io_uring is present
385 io_future_t future;
386 eventfd_t idle_buf;
387 proc->idle_wctx.ftr = &future;
388 proc->idle_wctx.rdbuf = &idle_buf;
389
390
391 // SKULLDUGGERY: We want to create a context for the processor coroutine
392 // which is needed for the 2-step context switch. However, there is no reason
393 // to waste the perfectly valid stack create by pthread.
394 current_stack_info_t info;
395 __stack_t ctx;
396 info.storage = &ctx;
397 (proc->runner){ proc, &info };
398
399 __cfaabi_dbg_print_safe("Coroutine : created stack %p\n", get_coroutine(proc->runner)->stack.storage);
400
401 //Set global state
402 __cfaabi_tls.this_thread = 0p;
403
404 //We now have a proper context from which to schedule threads
405 __cfadbg_print_safe(runtime_core, "Kernel : core %p created (%p, %p)\n", proc, &proc->runner, &ctx);
406
407 // SKULLDUGGERY: Since the coroutine doesn't have its own stack, we can't
408 // resume it to start it like it normally would, it will just context switch
409 // back to here. Instead directly call the main since we already are on the
410 // appropriate stack.
411 get_coroutine(proc->runner)->state = Active;
412 main( proc->runner );
413 get_coroutine(proc->runner)->state = Halted;
414
415 // Main routine of the core returned, the core is now fully terminated
416 __cfadbg_print_safe(runtime_core, "Kernel : core %p main ended (%p)\n", proc, &proc->runner);
417
418 #if !defined(__CFA_NO_STATISTICS__)
419 __tally_stats(proc->cltr->stats, &local_stats);
420 if( 0 != proc->print_stats ) {
421 __print_stats( &local_stats, proc->print_stats, "Processor ", proc->name, (void*)proc );
422 }
423 #if defined(CFA_STATS_ARRAY)
424 __flush_stat( &local_stats, "Processor", proc );
425 #endif
426 #endif
427
428 proc->local_data = 0p;
429
430 unregister_tls( proc );
431 __cfa_io_stop( proc );
432
433 heapManagerDtor(); // de-initialize heap
434
435 return 0p;
436}
437
438static void __kernel_first_resume( processor * this ) {
439 thread$ * src = mainThread;
440 coroutine$ * dst = get_coroutine(this->runner);
441
442 /* paranoid */ verify( ! __preemption_enabled() );
443
444 __cfaabi_tls.this_thread->curr_cor = dst;
445 __stack_prepare( &dst->stack, DEFAULT_STACK_SIZE );
446 __cfactx_start(main, dst, this->runner, __cfactx_invoke_coroutine);
447
448 /* paranoid */ verify( ! __preemption_enabled() );
449
450 dst->last = &src->self_cor;
451 dst->starter = dst->starter ? dst->starter : &src->self_cor;
452
453 // make sure the current state is still correct
454 /* paranoid */ verify(src->state == Ready);
455 src->corctx_flag = true;
456
457 // context switch to specified coroutine
458 verify( dst->context.SP );
459 __cfactx_switch( &src->context, &dst->context );
460 // when __cfactx_switch returns we are back in the src coroutine
461
462 mainThread->curr_cor = &mainThread->self_cor;
463
464 // make sure the current state has been update
465 /* paranoid */ verify(src->state == Active);
466
467 /* paranoid */ verify( ! __preemption_enabled() );
468}
469
470// KERNEL_ONLY
471static void __kernel_last_resume( processor * this ) {
472 coroutine$ * src = &mainThread->self_cor;
473 coroutine$ * dst = get_coroutine(this->runner);
474
475 /* paranoid */ verify( ! __preemption_enabled() );
476 /* paranoid */ verify( dst->starter == src );
477 /* paranoid */ verify( dst->context.SP );
478
479 // SKULLDUGGERY in debug the processors check that the
480 // stack is still within the limit of the stack limits after running a thread.
481 // that check doesn't make sense if we context switch to the processor using the
482 // coroutine semantics. Since this is a special case, use the current context
483 // info to populate these fields.
484 __cfaabi_dbg_debug_do(
485 __stack_context_t ctx;
486 CtxGet( ctx );
487 mainThread->context.SP = ctx.SP;
488 mainThread->context.FP = ctx.FP;
489 )
490
491 // context switch to the processor
492 __cfactx_switch( &src->context, &dst->context );
493}
494
495
496//=============================================================================================
497// Kernel Object Constructors logic
498//=============================================================================================
499//-----------------------------------------------------------------------------
500// Main thread construction
501static void ?{}( coroutine$ & this, current_stack_info_t * info) with( this ) {
502 stack.storage = info->storage;
503 with(*stack.storage) {
504 limit = info->limit;
505 base = info->base;
506 }
507 __attribute__((may_alias)) intptr_t * istorage = (intptr_t*) &stack.storage;
508 *istorage |= 0x1;
509 name = "Main Thread";
510 state = Start;
511 starter = 0p;
512 last = 0p;
513 cancellation = 0p;
514}
515
516static void ?{}( thread$ & this, current_stack_info_t * info) with( this ) {
517 ticket = TICKET_RUNNING;
518 state = Start;
519 self_cor{ info };
520 curr_cor = &self_cor;
521 curr_cluster = mainCluster;
522 self_mon.owner = &this;
523 self_mon.recursion = 1;
524 self_mon_p = &self_mon;
525 link.next = 0p;
526 link.ts = MAX;
527 preferred = ready_queue_new_preferred();
528 last_proc = 0p;
529 random_state = __global_random_mask ? __global_random_prime : __global_random_prime ^ rdtscl();
530 #if defined( __CFA_WITH_VERIFY__ )
531 canary = 0x0D15EA5E0D15EA5Ep;
532 #endif
533
534 node.next = 0p;
535 node.prev = 0p;
536 doregister(curr_cluster, this);
537
538 monitors{ &self_mon_p, 1, (fptr_t)0 };
539}
540
541//-----------------------------------------------------------------------------
542// Processor
543// Construct the processor context of non-main processors
544static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info) {
545 (this.self){ info };
546 this.proc = proc;
547}
548
549static void init(processor & this, const char name[], cluster & _cltr, thread$ * initT) with( this ) {
550 this.name = name;
551 this.cltr = &_cltr;
552 this.rdq.its = 0;
553 this.rdq.itr = 0;
554 this.rdq.id = 0;
555 this.rdq.target = MAX;
556 this.rdq.last = MAX;
557 this.rdq.cpu = 0;
558 // this.rdq.cutoff = 0ull;
559 do_terminate = false;
560 preemption_alarm = 0p;
561 pending_preemption = false;
562
563 this.io.ctx = 0p;
564 this.io.pending = false;
565 this.io.dirty = false;
566
567 this.init.thrd = initT;
568
569 this.local_data = 0p;
570
571 idle_wctx.evfd = eventfd(0, 0);
572 if (idle_wctx.evfd < 0) {
573 abort("KERNEL ERROR: PROCESSOR EVENTFD - %s\n", strerror(errno));
574 }
575
576 idle_wctx.sem = 0;
577 idle_wctx.wake__time = 0;
578
579 // I'm assuming these two are reserved for standard input and output
580 // so I'm using them as sentinels with idle_wctx.
581 /* paranoid */ verify( idle_wctx.evfd != 0 );
582 /* paranoid */ verify( idle_wctx.evfd != 1 );
583
584 #if !defined(__CFA_NO_STATISTICS__)
585 print_stats = 0;
586 print_halts = false;
587 #endif
588
589 __cfadbg_print_safe(runtime_core, "Kernel : core %p created\n", &this);
590}
591
592// Not a ctor, it just preps the destruction but should not destroy members
593static void deinit(processor & this) {
594 close(this.idle_wctx.evfd);
595}
596
597void ?{}(processor & this, const char name[], cluster & _cltr, thread$ * initT) libcfa_public {
598 ( this.terminated ){};
599 ( this.runner ){};
600
601 disable_interrupts();
602 init( this, name, _cltr, initT );
603 enable_interrupts();
604
605 __cfadbg_print_safe(runtime_core, "Kernel : Starting core %p\n", &this);
606
607 this.stack = __create_pthread( &this.kernel_thread, __invoke_processor, (void *)&this );
608}
609
610void ?{}(processor & this, const char name[], cluster & _cltr) libcfa_public {
611 (this){name, _cltr, 0p};
612}
613
614extern size_t __page_size;
615void ^?{}(processor & this) libcfa_public with( this ) {
616 /* paranoid */ verify( !__atomic_load_n(&do_terminate, __ATOMIC_ACQUIRE) );
617 __cfadbg_print_safe(runtime_core, "Kernel : core %p signaling termination\n", &this);
618
619 __atomic_store_n(&do_terminate, true, __ATOMIC_RELAXED);
620 __disable_interrupts_checked();
621 __wake_proc( &this );
622 __enable_interrupts_checked();
623
624 wait( terminated );
625 /* paranoid */ verify( active_processor() != &this);
626
627 __destroy_pthread( kernel_thread, this.stack, 0p );
628
629 disable_interrupts();
630 deinit( this );
631 enable_interrupts();
632}
633
634//-----------------------------------------------------------------------------
635// Cluster
636static void ?{}(__cluster_proc_list & this) {
637 this.fdw = 0p;
638 this.idle = 0;
639 this.total = 0;
640}
641
642void ?{}(cluster & this, const char name[], Duration preemption_rate, unsigned num_io, const io_context_params & io_params) libcfa_public with( this ) {
643 this.name = name;
644 this.preemption_rate = preemption_rate;
645 this.sched.readyQ.data = 0p;
646 this.sched.readyQ.tscs = 0p;
647 this.sched.readyQ.count = 0;
648 this.sched.io.tscs = 0p;
649 this.sched.io.data = 0p;
650 this.sched.caches = 0p;
651
652 #if !defined(__CFA_NO_STATISTICS__)
653 print_stats = 0;
654 stats = alloc();
655 __init_stats( stats );
656 #endif
657
658 threads{ __get };
659
660 io.arbiter = create();
661 io.params = io_params;
662
663 doregister(this);
664
665 // Lock the RWlock so no-one pushes/pops while we are changing the queue
666 disable_interrupts();
667 uint_fast32_t last_size = ready_mutate_lock();
668
669 // Adjust the ready queue size
670 ready_queue_grow( &this );
671
672 // Unlock the RWlock
673 ready_mutate_unlock( last_size );
674 enable_interrupts( false ); // Don't poll, could be in main cluster
675}
676
677void ^?{}(cluster & this) libcfa_public {
678 destroy(this.io.arbiter);
679
680 // Lock the RWlock so no-one pushes/pops while we are changing the queue
681 disable_interrupts();
682 uint_fast32_t last_size = ready_mutate_lock();
683
684 // Adjust the ready queue size
685 ready_queue_shrink( &this );
686
687 // Unlock the RWlock
688 ready_mutate_unlock( last_size );
689
690 ready_queue_close( &this );
691 /* paranoid */ verify( this.sched.readyQ.data == 0p );
692 /* paranoid */ verify( this.sched.readyQ.tscs == 0p );
693 /* paranoid */ verify( this.sched.readyQ.count == 0 );
694 /* paranoid */ verify( this.sched.io.tscs == 0p );
695 /* paranoid */ verify( this.sched.caches == 0p );
696
697 enable_interrupts( false ); // Don't poll, could be in main cluster
698
699
700 #if !defined(__CFA_NO_STATISTICS__)
701 if( 0 != this.print_stats ) {
702 __print_stats( this.stats, this.print_stats, "Cluster", this.name, (void*)&this );
703 }
704 #if defined(CFA_STATS_ARRAY)
705 __flush_stat( this.stats, "Cluster", &this );
706 #endif
707 free( this.stats );
708 #endif
709
710 unregister(this);
711}
712
713//=============================================================================================
714// Miscellaneous Initialization
715//=============================================================================================
716//-----------------------------------------------------------------------------
717// Global Queues
718static void doregister( cluster & cltr ) {
719 lock ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
720 push_front( __cfa_dbg_global_clusters.list, cltr );
721 unlock ( __cfa_dbg_global_clusters.lock );
722}
723
724static void unregister( cluster & cltr ) {
725 lock ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
726 remove( __cfa_dbg_global_clusters.list, cltr );
727 unlock( __cfa_dbg_global_clusters.lock );
728}
729
730void doregister( cluster * cltr, thread$ & thrd ) {
731 lock (cltr->thread_list_lock __cfaabi_dbg_ctx2);
732 cltr->nthreads += 1;
733 push_front(cltr->threads, thrd);
734 unlock (cltr->thread_list_lock);
735}
736
737void unregister( cluster * cltr, thread$ & thrd ) {
738 lock (cltr->thread_list_lock __cfaabi_dbg_ctx2);
739 remove(cltr->threads, thrd );
740 cltr->nthreads -= 1;
741 unlock(cltr->thread_list_lock);
742}
743
744static void register_tls( processor * this ) {
745 // Register and Lock the RWlock so no-one pushes/pops while we are changing the queue
746 uint_fast32_t last_size;
747 [this->unique_id, last_size] = ready_mutate_register();
748
749 this->rdq.cpu = __kernel_getcpu();
750
751 this->cltr->procs.total += 1u;
752 insert_last(this->cltr->procs.actives, *this);
753
754 // Adjust the ready queue size
755 ready_queue_grow( this->cltr );
756
757 // Unlock the RWlock
758 ready_mutate_unlock( last_size );
759}
760
761
762static void unregister_tls( processor * this ) {
763 // Lock the RWlock so no-one pushes/pops while we are changing the queue
764 uint_fast32_t last_size = ready_mutate_lock();
765 this->cltr->procs.total -= 1u;
766 remove(*this);
767
768 // clear the cluster so nothing gets pushed to local queues
769 cluster * cltr = this->cltr;
770 this->cltr = 0p;
771
772 // Adjust the ready queue size
773 ready_queue_shrink( cltr );
774
775 // Unlock the RWlock and unregister: we don't need the read_lock any more
776 ready_mutate_unregister( this->unique_id, last_size );
777}
778
779static void check( int ret, const char func[] ) {
780 if ( ret ) { // pthread routines return errno values
781 abort( "%s : internal error, error(%d) %s.", func, ret, strerror( ret ) );
782 } // if
783} // Abort
784
785void * __create_pthread( pthread_t * pthread, void * (*start)(void *), void * arg ) {
786 pthread_attr_t attr;
787
788 check( __cfaabi_pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
789
790 size_t stacksize = max( PTHREAD_STACK_MIN, DEFAULT_STACK_SIZE );
791
792 void * stack;
793 #if CFA_PROCESSOR_USE_MMAP
794 stacksize = ceiling( stacksize, __page_size ) + __page_size;
795 stack = mmap(0p, stacksize, __map_prot, MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
796 if(stack == ((void*)-1)) {
797 abort( "pthread stack creation : internal error, mmap failure, error(%d) %s.", errno, strerror( errno ) );
798 }
799 if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
800 abort( "pthread stack creation : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
801 } // if
802 #else
803 __cfaabi_dbg_debug_do(
804 stack = memalign( __page_size, stacksize + __page_size );
805 // pthread has no mechanism to create the guard page in user supplied stack.
806 if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
807 abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
808 } // if
809 );
810 __cfaabi_dbg_no_debug_do(
811 stack = malloc( stacksize );
812 );
813 #endif
814
815 check( __cfaabi_pthread_attr_setstack( &attr, stack, stacksize ), "pthread_attr_setstack" );
816 check( __cfaabi_pthread_create( pthread, &attr, start, arg ), "pthread_create" );
817 return stack;
818}
819
820void __destroy_pthread( pthread_t pthread, void * stack, void ** retval ) {
821 int err = __cfaabi_pthread_join( pthread, retval );
822 if( err != 0 ) abort("KERNEL ERROR: joining pthread %p caused error %s\n", (void*)pthread, strerror(err));
823
824 #if CFA_PROCESSOR_USE_MMAP
825 pthread_attr_t attr;
826
827 check( __cfaabi_pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
828
829 size_t stacksize;
830 // default stack size, normally defined by shell limit
831 check( __cfaabi_pthread_attr_getstacksize( &attr, &stacksize ), "pthread_attr_getstacksize" );
832 assert( stacksize >= PTHREAD_STACK_MIN );
833 stacksize += __page_size;
834
835 if(munmap(stack, stacksize) == -1) {
836 abort( "pthread stack destruction : internal error, munmap failure, error(%d) %s.", errno, strerror( errno ) );
837 }
838 #else
839 __cfaabi_dbg_debug_do(
840 // pthread has no mechanism to create the guard page in user supplied stack.
841 if ( mprotect( stack, __page_size, __map_prot ) == -1 ) {
842 abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
843 } // if
844 );
845 free( stack );
846 #endif
847}
848
849#if defined(__CFA_WITH_VERIFY__)
850static bool verify_fwd_bck_rng(void) {
851 __cfaabi_tls.ready_rng.fwd_seed = 25214903917_l64u * (rdtscl() ^ (uintptr_t)&verify_fwd_bck_rng);
852
853 unsigned values[10];
854 for(i; 10) {
855 values[i] = __tls_rand_fwd();
856 }
857
858 __tls_rand_advance_bck();
859
860 for ( i; 9 -~= 0 ) {
861 if(values[i] != __tls_rand_bck()) {
862 return false;
863 }
864 }
865
866 return true;
867}
868#endif
Note: See TracBrowser for help on using the repository browser.