source: libcfa/src/concurrency/kernel/startup.cfa@ 428adbc

ADT ast-experimental pthread-emulation
Last change on this file since 428adbc was 20be782, checked in by z277zhu <z277zhu@…>, 3 years ago

add pthread
  • Property mode set to 100644
File size: 26.6 KB
Line 
1//
2// Cforall Version 1.0.0 Copyright (C) 2020 University of Waterloo
3//
4// The contents of this file are covered under the licence agreement in the
5// file "LICENCE" distributed with Cforall.
6//
7// kernel/startup.cfa --
8//
9// Author : Thierry Delisle
10// Created On : Thu Jul 30 15:12:54 2020
11// Last Modified By :
12// Last Modified On :
13// Update Count :
14//
15
16#define __cforall_thread__
17#define _GNU_SOURCE
18
19// C Includes
20#include <errno.h> // errno
21#include <signal.h>
22#include <string.h> // strerror
23#include <unistd.h> // sysconf
24
25extern "C" {
26 #include <limits.h> // PTHREAD_STACK_MIN
27 #include <unistd.h> // syscall
28 #include <sys/eventfd.h> // eventfd
29 #include <sys/mman.h> // mprotect
30 #include <sys/resource.h> // getrlimit
31}
32
33// CFA Includes
34#include "kernel/private.hfa"
35#include "iofwd.hfa"
36#include "startup.hfa" // STARTUP_PRIORITY_XXX
37#include "limits.hfa"
38#include "math.hfa"
39
40#define CFA_PROCESSOR_USE_MMAP 0
41
42//-----------------------------------------------------------------------------
43// Some assembly required
44#if defined( __i386 )
45 #define CtxGet( ctx ) __asm__ volatile ( \
46 "movl %%esp,%0\n" \
47 "movl %%ebp,%1\n" \
48 : "=rm" (ctx.SP), \
49 "=rm" (ctx.FP) \
50 )
51#elif defined( __x86_64 )
52 #define CtxGet( ctx ) __asm__ volatile ( \
53 "movq %%rsp,%0\n" \
54 "movq %%rbp,%1\n" \
55 : "=rm" (ctx.SP), \
56 "=rm" (ctx.FP) \
57 )
58#elif defined( __aarch64__ )
59 #define CtxGet( ctx ) __asm__ volatile ( \
60 "mov %0, sp\n" \
61 "mov %1, fp\n" \
62 : "=rm" (ctx.SP), \
63 "=rm" (ctx.FP) \
64 )
65#else
66 #error unknown hardware architecture
67#endif
68
69//-----------------------------------------------------------------------------
70// Start and stop routine for the kernel, declared first to make sure they run first
71static void __kernel_startup (void) __attribute__(( constructor( STARTUP_PRIORITY_KERNEL ) ));
72static void __kernel_shutdown(void) __attribute__(( destructor ( STARTUP_PRIORITY_KERNEL ) ));
73
74//-----------------------------------------------------------------------------
75// Static Forward Declarations
76struct current_stack_info_t;
77
78static void * __invoke_processor(void * arg);
79static void __kernel_first_resume( processor * this );
80static void __kernel_last_resume ( processor * this );
81static void init(processor & this, const char name[], cluster & _cltr, thread$ * initT);
82static void deinit(processor & this);
83static void doregister( struct cluster & cltr );
84static void unregister( struct cluster & cltr );
85static void register_tls( processor * this );
86static void unregister_tls( processor * this );
87static void ?{}( coroutine$ & this, current_stack_info_t * info);
88static void ?{}( thread$ & this, current_stack_info_t * info);
89static void ?{}(processorCtx_t & this) {}
90static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info);
91
92#if defined(__CFA_WITH_VERIFY__)
93 static bool verify_fwd_bck_rng(void);
94#endif
95
96//-----------------------------------------------------------------------------
97// Forward Declarations for other modules
98extern void __kernel_alarm_startup(void);
99extern void __kernel_alarm_shutdown(void);
100extern void __cfa_io_start( processor * );
101extern void __cfa_io_stop ( processor * );
102
103//-----------------------------------------------------------------------------
104// Other Forward Declarations
105extern void __wake_proc(processor *);
106extern int cfa_main_returned; // from interpose.cfa
107uint32_t __global_random_prime = 4_294_967_291u, __global_random_mask = false;
108
109//-----------------------------------------------------------------------------
110// Kernel storage
111KERNEL_STORAGE(cluster, mainCluster);
112KERNEL_STORAGE(processor, mainProcessor);
113KERNEL_STORAGE(thread$, mainThread);
114KERNEL_STORAGE(__stack_t, mainThreadCtx);
115KERNEL_STORAGE(__scheduler_RWLock_t, __scheduler_lock);
116KERNEL_STORAGE(eventfd_t, mainIdleEventFd);
117KERNEL_STORAGE(io_future_t, mainIdleFuture);
118#if !defined(__CFA_NO_STATISTICS__)
119KERNEL_STORAGE(__stats_t, mainProcStats);
120#endif
121
122cluster * mainCluster libcfa_public;
123processor * mainProcessor;
124thread$ * mainThread;
125__scheduler_RWLock_t * __scheduler_lock;
126
127extern "C" {
128 struct { __dllist_t(cluster) list; __spinlock_t lock; } __cfa_dbg_global_clusters;
129}
130
131extern size_t __page_size;
132extern int __map_prot;
133
134//-----------------------------------------------------------------------------
135// Global state
136thread_local struct KernelThreadData __cfaabi_tls __attribute__ ((tls_model ( "initial-exec" ))) @= {
137 NULL, // cannot use 0p
138 NULL,
139 false,
140 { 1, false, false },
141 0,
142 { 0, 0 },
143 NULL,
144 #ifdef __CFA_WITH_VERIFY__
145 false,
146 0,
147 #endif
148};
149
150#if defined(CFA_HAVE_LINUX_LIBRSEQ)
151 // No data needed
152#elif defined(CFA_HAVE_LINUX_RSEQ_H)
153 extern "Cforall" {
154 __attribute__((aligned(64))) thread_local volatile struct rseq __cfaabi_rseq @= {
155 .cpu_id : RSEQ_CPU_ID_UNINITIALIZED,
156 };
157 }
158#else
159 // No data needed
160#endif
161
162//-----------------------------------------------------------------------------
163// Struct to steal stack
164struct current_stack_info_t {
165 __stack_t * storage; // pointer to stack object
166 void * base; // base of stack
167 void * limit; // stack grows towards stack limit
168 void * context; // address of cfa_context_t
169};
170
171static void ?{}( current_stack_info_t & this ) {
172 __stack_context_t ctx;
173 CtxGet( ctx );
174 this.base = ctx.FP;
175
176 rlimit r;
177 getrlimit( RLIMIT_STACK, &r);
178 size_t size = r.rlim_cur;
179
180 this.limit = (void *)(((intptr_t)this.base) - size);
181 this.context = &storage_mainThreadCtx;
182}
183
184
185//=============================================================================================
186// Kernel Setup logic
187//=============================================================================================
188//-----------------------------------------------------------------------------
189// Kernel boot procedures
190static void __kernel_startup(void) {
191 /* paranoid */ verify( ! __preemption_enabled() );
192 __cfadbg_print_safe(runtime_core, "Kernel : Starting\n");
193
194 __cfa_dbg_global_clusters.list{ __get };
195 __cfa_dbg_global_clusters.lock{};
196
197 /* paranoid */ verify( verify_fwd_bck_rng() );
198
199 // Initialize the global scheduler lock
200 __scheduler_lock = (__scheduler_RWLock_t*)&storage___scheduler_lock;
201 (*__scheduler_lock){};
202
203 // Initialize the main cluster
204 mainCluster = (cluster *)&storage_mainCluster;
205 (*mainCluster){"Main Cluster", 0};
206
207 __cfadbg_print_safe(runtime_core, "Kernel : Main cluster ready\n");
208
209 // Construct the processor context of the main processor
210 void ?{}(processorCtx_t & this, processor * proc) {
211 (this.self){ "Processor" };
212 this.self.starter = 0p;
213 this.proc = proc;
214 }
215
216 void ?{}(processor & this) with( this ) {
217 ( this.terminated ){};
218 ( this.runner ){};
219 init( this, "Main Processor", *mainCluster, 0p );
220 kernel_thread = real_pthread_self();
221
222 runner{ &this };
223 __cfadbg_print_safe(runtime_core, "Kernel : constructed main processor context %p\n", &runner);
224 }
225
226 // Initialize the main processor and the main processor ctx
227 // (the coroutine that contains the processing control flow)
228 mainProcessor = (processor *)&storage_mainProcessor;
229 (*mainProcessor){};
230
231 mainProcessor->idle_wctx.rdbuf = &storage_mainIdleEventFd;
232 mainProcessor->idle_wctx.ftr = (io_future_t*)&storage_mainIdleFuture;
233 /* paranoid */ verify( sizeof(storage_mainIdleEventFd) == sizeof(eventfd_t) );
234
235 __cfa_io_start( mainProcessor );
236 register_tls( mainProcessor );
237
238 // Start by initializing the main thread
239 // SKULLDUGGERY: the mainThread steals the process main thread
240 // which will then be scheduled by the mainProcessor normally
241 mainThread = (thread$ *)&storage_mainThread;
242 current_stack_info_t info;
243 info.storage = (__stack_t*)&storage_mainThreadCtx;
244 (*mainThread){ &info };
245
246 __cfadbg_print_safe(runtime_core, "Kernel : Main thread ready\n");
247
248 //initialize the global state variables
249 __cfaabi_tls.this_processor = mainProcessor;
250 __cfaabi_tls.this_thread = mainThread;
251
252 #if !defined( __CFA_NO_STATISTICS__ )
253 __cfaabi_tls.this_stats = (__stats_t *)& storage_mainProcStats;
254 __init_stats( __cfaabi_tls.this_stats );
255 #endif
256 mainProcessor->local_data = &__cfaabi_tls;
257
258 // Enable preemption
259 __kernel_alarm_startup();
260
261 // Add the main thread to the ready queue
262 // once resume is called on mainProcessor->runner the mainThread needs to be scheduled like any normal thread
263 schedule_thread$(mainThread, UNPARK_LOCAL);
264
265 // SKULLDUGGERY: Force a context switch to the main processor to set the main thread's context to the current UNIX
266 // context. Hence, the main thread does not begin through __cfactx_invoke_thread, like all other threads. The trick here is that
267 // mainThread is on the ready queue when this call is made.
268 __kernel_first_resume( __cfaabi_tls.this_processor );
269
270
271 // THE SYSTEM IS NOW COMPLETELY RUNNING
272
273 __cfadbg_print_safe(runtime_core, "Kernel : Started\n--------------------------------------------------\n\n");
274
275 /* paranoid */ verify( ! __preemption_enabled() );
276 enable_interrupts();
277 /* paranoid */ verify( __preemption_enabled() );
278
279}
280
281extern "C"{
282 void pthread_delete_kernel_threads_();
283}
284
285
286static void __kernel_shutdown(void) {
287 if(!cfa_main_returned) return;
288
289 //delete kernel threads for pthread_concurrency
290 pthread_delete_kernel_threads_();
291
292 /* paranoid */ verify( __preemption_enabled() );
293 disable_interrupts();
294 /* paranoid */ verify( ! __preemption_enabled() );
295
296 __cfadbg_print_safe(runtime_core, "\n--------------------------------------------------\nKernel : Shutting down\n");
297
298 // SKULLDUGGERY: Notify the mainProcessor it needs to terminates.
299 // When its coroutine terminates, it return control to the mainThread
300 // which is currently here
301 /* paranoid */ verify( !__atomic_load_n(&mainProcessor->do_terminate, __ATOMIC_ACQUIRE) );
302 __atomic_store_n(&mainProcessor->do_terminate, true, __ATOMIC_RELEASE);
303 __wake_proc( mainProcessor );
304 __kernel_last_resume( __cfaabi_tls.this_processor );
305 mainThread->self_cor.state = Halted;
306
307 // THE SYSTEM IS NOW COMPLETELY STOPPED
308
309 // Disable preemption
310 __kernel_alarm_shutdown();
311
312 #if !defined( __CFA_NO_STATISTICS__ )
313 __stats_t * st = (__stats_t *)& storage_mainProcStats;
314 __tally_stats(mainCluster->stats, st);
315 if( 0 != mainProcessor->print_stats ) {
316 __print_stats( st, mainProcessor->print_stats, "Processor ", mainProcessor->name, (void*)mainProcessor );
317 }
318 #if defined(CFA_STATS_ARRAY)
319 __flush_stat( st, "Processor", mainProcessor );
320 #endif
321 #endif
322
323 mainProcessor->local_data = 0p;
324
325 unregister_tls( mainProcessor );
326 __cfa_io_stop( mainProcessor );
327
328 // Destroy the main processor and its context in reverse order of construction
329 // These were manually constructed so we need manually destroy them
330 void ^?{}(processor & this) with( this ){
331 deinit( this );
332
333 /* paranoid */ verify( this.do_terminate == true );
334 __cfaabi_dbg_print_safe("Kernel : destroyed main processor context %p\n", &runner);
335 }
336
337 ^(*mainProcessor){};
338
339 // Final step, destroy the main thread since it is no longer needed
340
341 // Since we provided a stack to this taxk it will not destroy anything
342 /* paranoid */ verify(mainThread->self_cor.stack.storage == (__stack_t*)(((uintptr_t)&storage_mainThreadCtx)| 0x1));
343 ^(*mainThread){};
344
345 ^(*mainCluster){};
346
347 ^(*__scheduler_lock){};
348
349 ^(__cfa_dbg_global_clusters.list){};
350 ^(__cfa_dbg_global_clusters.lock){};
351
352 __cfadbg_print_safe(runtime_core, "Kernel : Shutdown complete\n");
353}
354
355//=============================================================================================
356// Kernel Initial Scheduling logic
357//=============================================================================================
358
359// Context invoker for processors
360// This is the entry point for processors (kernel threads) *except* for the main processor
361// It effectively constructs a coroutine by stealing the pthread stack
362static void * __invoke_processor(void * arg) {
363 #if !defined( __CFA_NO_STATISTICS__ )
364 __stats_t local_stats;
365 __init_stats( &local_stats );
366 __cfaabi_tls.this_stats = &local_stats;
367 #endif
368
369 processor * proc = (processor *) arg;
370 __cfaabi_tls.this_processor = proc;
371 __cfaabi_tls.this_thread = 0p;
372 __cfaabi_tls.preemption_state.[enabled, disable_count] = [false, 1];
373 proc->local_data = &__cfaabi_tls;
374
375 __cfa_io_start( proc );
376 register_tls( proc );
377
378 // used for idle sleep when io_uring is present
379 io_future_t future;
380 eventfd_t idle_buf;
381 proc->idle_wctx.ftr = &future;
382 proc->idle_wctx.rdbuf = &idle_buf;
383
384
385 // SKULLDUGGERY: We want to create a context for the processor coroutine
386 // which is needed for the 2-step context switch. However, there is no reason
387 // to waste the perfectly valid stack create by pthread.
388 current_stack_info_t info;
389 __stack_t ctx;
390 info.storage = &ctx;
391 (proc->runner){ proc, &info };
392
393 __cfaabi_dbg_print_safe("Coroutine : created stack %p\n", get_coroutine(proc->runner)->stack.storage);
394
395 //Set global state
396 __cfaabi_tls.this_thread = 0p;
397
398 //We now have a proper context from which to schedule threads
399 __cfadbg_print_safe(runtime_core, "Kernel : core %p created (%p, %p)\n", proc, &proc->runner, &ctx);
400
401 // SKULLDUGGERY: Since the coroutine doesn't have its own stack, we can't
402 // resume it to start it like it normally would, it will just context switch
403 // back to here. Instead directly call the main since we already are on the
404 // appropriate stack.
405 get_coroutine(proc->runner)->state = Active;
406 main( proc->runner );
407 get_coroutine(proc->runner)->state = Halted;
408
409 // Main routine of the core returned, the core is now fully terminated
410 __cfadbg_print_safe(runtime_core, "Kernel : core %p main ended (%p)\n", proc, &proc->runner);
411
412 #if !defined(__CFA_NO_STATISTICS__)
413 __tally_stats(proc->cltr->stats, &local_stats);
414 if( 0 != proc->print_stats ) {
415 __print_stats( &local_stats, proc->print_stats, "Processor ", proc->name, (void*)proc );
416 }
417 #if defined(CFA_STATS_ARRAY)
418 __flush_stat( &local_stats, "Processor", proc );
419 #endif
420 #endif
421
422 proc->local_data = 0p;
423
424 unregister_tls( proc );
425 __cfa_io_stop( proc );
426
427 return 0p;
428}
429
430static void __kernel_first_resume( processor * this ) {
431 thread$ * src = mainThread;
432 coroutine$ * dst = get_coroutine(this->runner);
433
434 /* paranoid */ verify( ! __preemption_enabled() );
435
436 __cfaabi_tls.this_thread->curr_cor = dst;
437 __stack_prepare( &dst->stack, DEFAULT_STACK_SIZE );
438 __cfactx_start(main, dst, this->runner, __cfactx_invoke_coroutine);
439
440 /* paranoid */ verify( ! __preemption_enabled() );
441
442 dst->last = &src->self_cor;
443 dst->starter = dst->starter ? dst->starter : &src->self_cor;
444
445 // make sure the current state is still correct
446 /* paranoid */ verify(src->state == Ready);
447 src->corctx_flag = true;
448
449 // context switch to specified coroutine
450 verify( dst->context.SP );
451 __cfactx_switch( &src->context, &dst->context );
452 // when __cfactx_switch returns we are back in the src coroutine
453
454 mainThread->curr_cor = &mainThread->self_cor;
455
456 // make sure the current state has been update
457 /* paranoid */ verify(src->state == Active);
458
459 /* paranoid */ verify( ! __preemption_enabled() );
460}
461
462// KERNEL_ONLY
463static void __kernel_last_resume( processor * this ) {
464 coroutine$ * src = &mainThread->self_cor;
465 coroutine$ * dst = get_coroutine(this->runner);
466
467 /* paranoid */ verify( ! __preemption_enabled() );
468 /* paranoid */ verify( dst->starter == src );
469 /* paranoid */ verify( dst->context.SP );
470
471 // SKULLDUGGERY in debug the processors check that the
472 // stack is still within the limit of the stack limits after running a thread.
473 // that check doesn't make sense if we context switch to the processor using the
474 // coroutine semantics. Since this is a special case, use the current context
475 // info to populate these fields.
476 __cfaabi_dbg_debug_do(
477 __stack_context_t ctx;
478 CtxGet( ctx );
479 mainThread->context.SP = ctx.SP;
480 mainThread->context.FP = ctx.FP;
481 )
482
483 // context switch to the processor
484 __cfactx_switch( &src->context, &dst->context );
485}
486
487
488//=============================================================================================
489// Kernel Object Constructors logic
490//=============================================================================================
491//-----------------------------------------------------------------------------
492// Main thread construction
493static void ?{}( coroutine$ & this, current_stack_info_t * info) with( this ) {
494 stack.storage = info->storage;
495 with(*stack.storage) {
496 limit = info->limit;
497 base = info->base;
498 }
499 __attribute__((may_alias)) intptr_t * istorage = (intptr_t*) &stack.storage;
500 *istorage |= 0x1;
501 name = "Main Thread";
502 state = Start;
503 starter = 0p;
504 last = 0p;
505 cancellation = 0p;
506}
507
508static void ?{}( thread$ & this, current_stack_info_t * info) with( this ) {
509 ticket = TICKET_RUNNING;
510 state = Start;
511 self_cor{ info };
512 curr_cor = &self_cor;
513 curr_cluster = mainCluster;
514 self_mon.owner = &this;
515 self_mon.recursion = 1;
516 self_mon_p = &self_mon;
517 link.next = 0p;
518 link.ts = MAX;
519 preferred = ready_queue_new_preferred();
520 last_proc = 0p;
521 random_state = __global_random_mask ? __global_random_prime : __global_random_prime ^ rdtscl();
522 #if defined( __CFA_WITH_VERIFY__ )
523 canary = 0x0D15EA5E0D15EA5Ep;
524 #endif
525
526 node.next = 0p;
527 node.prev = 0p;
528 doregister(curr_cluster, this);
529
530 monitors{ &self_mon_p, 1, (fptr_t)0 };
531}
532
533//-----------------------------------------------------------------------------
534// Processor
535// Construct the processor context of non-main processors
536static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info) {
537 (this.self){ info };
538 this.proc = proc;
539}
540
541static void init(processor & this, const char name[], cluster & _cltr, thread$ * initT) with( this ) {
542 this.name = name;
543 this.cltr = &_cltr;
544 this.rdq.its = 0;
545 this.rdq.itr = 0;
546 this.rdq.id = 0;
547 this.rdq.target = MAX;
548 this.rdq.last = MAX;
549 this.rdq.cpu = 0;
550 // this.rdq.cutoff = 0ull;
551 do_terminate = false;
552 preemption_alarm = 0p;
553 pending_preemption = false;
554
555 this.io.ctx = 0p;
556 this.io.pending = false;
557 this.io.dirty = false;
558
559 this.init.thrd = initT;
560
561 this.local_data = 0p;
562
563 idle_wctx.evfd = eventfd(0, 0);
564 if (idle_wctx.evfd < 0) {
565 abort("KERNEL ERROR: PROCESSOR EVENTFD - %s\n", strerror(errno));
566 }
567
568 idle_wctx.sem = 0;
569 idle_wctx.wake__time = 0;
570
571 // I'm assuming these two are reserved for standard input and output
572 // so I'm using them as sentinels with idle_wctx.
573 /* paranoid */ verify( idle_wctx.evfd != 0 );
574 /* paranoid */ verify( idle_wctx.evfd != 1 );
575
576 #if !defined(__CFA_NO_STATISTICS__)
577 print_stats = 0;
578 print_halts = false;
579 #endif
580
581 __cfadbg_print_safe(runtime_core, "Kernel : core %p created\n", &this);
582}
583
584// Not a ctor, it just preps the destruction but should not destroy members
585static void deinit(processor & this) {
586 close(this.idle_wctx.evfd);
587}
588
589void ?{}(processor & this, const char name[], cluster & _cltr, thread$ * initT) libcfa_public {
590 ( this.terminated ){};
591 ( this.runner ){};
592
593 disable_interrupts();
594 init( this, name, _cltr, initT );
595 enable_interrupts();
596
597 __cfadbg_print_safe(runtime_core, "Kernel : Starting core %p\n", &this);
598
599 this.stack = __create_pthread( &this.kernel_thread, __invoke_processor, (void *)&this );
600}
601
602void ?{}(processor & this, const char name[], cluster & _cltr) libcfa_public {
603 (this){name, _cltr, 0p};
604}
605
606extern size_t __page_size;
607void ^?{}(processor & this) libcfa_public with( this ) {
608 /* paranoid */ verify( !__atomic_load_n(&do_terminate, __ATOMIC_ACQUIRE) );
609 __cfadbg_print_safe(runtime_core, "Kernel : core %p signaling termination\n", &this);
610
611 __atomic_store_n(&do_terminate, true, __ATOMIC_RELAXED);
612 __disable_interrupts_checked();
613 __wake_proc( &this );
614 __enable_interrupts_checked();
615
616 wait( terminated );
617 /* paranoid */ verify( active_processor() != &this);
618
619 __destroy_pthread( kernel_thread, this.stack, 0p );
620
621 disable_interrupts();
622 deinit( this );
623 enable_interrupts();
624}
625
626//-----------------------------------------------------------------------------
627// Cluster
628static void ?{}(__cluster_proc_list & this) {
629 this.fdw = 0p;
630 this.idle = 0;
631 this.total = 0;
632}
633
634void ?{}(cluster & this, const char name[], Duration preemption_rate, unsigned num_io, const io_context_params & io_params) libcfa_public with( this ) {
635 this.name = name;
636 this.preemption_rate = preemption_rate;
637 this.sched.readyQ.data = 0p;
638 this.sched.readyQ.tscs = 0p;
639 this.sched.readyQ.count = 0;
640 this.sched.io.tscs = 0p;
641 this.sched.io.data = 0p;
642 this.sched.caches = 0p;
643
644 #if !defined(__CFA_NO_STATISTICS__)
645 print_stats = 0;
646 stats = alloc();
647 __init_stats( stats );
648 #endif
649
650 threads{ __get };
651
652 io.arbiter = create();
653 io.params = io_params;
654
655 doregister(this);
656
657 // Lock the RWlock so no-one pushes/pops while we are changing the queue
658 disable_interrupts();
659 uint_fast32_t last_size = ready_mutate_lock();
660
661 // Adjust the ready queue size
662 ready_queue_grow( &this );
663
664 // Unlock the RWlock
665 ready_mutate_unlock( last_size );
666 enable_interrupts( false ); // Don't poll, could be in main cluster
667}
668
669void ^?{}(cluster & this) libcfa_public {
670 destroy(this.io.arbiter);
671
672 // Lock the RWlock so no-one pushes/pops while we are changing the queue
673 disable_interrupts();
674 uint_fast32_t last_size = ready_mutate_lock();
675
676 // Adjust the ready queue size
677 ready_queue_shrink( &this );
678
679 // Unlock the RWlock
680 ready_mutate_unlock( last_size );
681
682 ready_queue_close( &this );
683 /* paranoid */ verify( this.sched.readyQ.data == 0p );
684 /* paranoid */ verify( this.sched.readyQ.tscs == 0p );
685 /* paranoid */ verify( this.sched.readyQ.count == 0 );
686 /* paranoid */ verify( this.sched.io.tscs == 0p );
687 /* paranoid */ verify( this.sched.caches == 0p );
688
689 enable_interrupts( false ); // Don't poll, could be in main cluster
690
691
692 #if !defined(__CFA_NO_STATISTICS__)
693 if( 0 != this.print_stats ) {
694 __print_stats( this.stats, this.print_stats, "Cluster", this.name, (void*)&this );
695 }
696 #if defined(CFA_STATS_ARRAY)
697 __flush_stat( this.stats, "Cluster", &this );
698 #endif
699 free( this.stats );
700 #endif
701
702 unregister(this);
703}
704
705//=============================================================================================
706// Miscellaneous Initialization
707//=============================================================================================
708//-----------------------------------------------------------------------------
709// Global Queues
710static void doregister( cluster & cltr ) {
711 lock ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
712 push_front( __cfa_dbg_global_clusters.list, cltr );
713 unlock ( __cfa_dbg_global_clusters.lock );
714}
715
716static void unregister( cluster & cltr ) {
717 lock ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
718 remove( __cfa_dbg_global_clusters.list, cltr );
719 unlock( __cfa_dbg_global_clusters.lock );
720}
721
722void doregister( cluster * cltr, thread$ & thrd ) {
723 lock (cltr->thread_list_lock __cfaabi_dbg_ctx2);
724 cltr->nthreads += 1;
725 push_front(cltr->threads, thrd);
726 unlock (cltr->thread_list_lock);
727}
728
729void unregister( cluster * cltr, thread$ & thrd ) {
730 lock (cltr->thread_list_lock __cfaabi_dbg_ctx2);
731 remove(cltr->threads, thrd );
732 cltr->nthreads -= 1;
733 unlock(cltr->thread_list_lock);
734}
735
736static void register_tls( processor * this ) {
737 // Register and Lock the RWlock so no-one pushes/pops while we are changing the queue
738 uint_fast32_t last_size;
739 [this->unique_id, last_size] = ready_mutate_register();
740
741 this->rdq.cpu = __kernel_getcpu();
742
743 this->cltr->procs.total += 1u;
744 insert_last(this->cltr->procs.actives, *this);
745
746 // Adjust the ready queue size
747 ready_queue_grow( this->cltr );
748
749 // Unlock the RWlock
750 ready_mutate_unlock( last_size );
751}
752
753
754static void unregister_tls( processor * this ) {
755 // Lock the RWlock so no-one pushes/pops while we are changing the queue
756 uint_fast32_t last_size = ready_mutate_lock();
757 this->cltr->procs.total -= 1u;
758 remove(*this);
759
760 // clear the cluster so nothing gets pushed to local queues
761 cluster * cltr = this->cltr;
762 this->cltr = 0p;
763
764 // Adjust the ready queue size
765 ready_queue_shrink( cltr );
766
767 // Unlock the RWlock and unregister: we don't need the read_lock any more
768 ready_mutate_unregister( this->unique_id, last_size );
769}
770
771static void check( int ret, const char func[] ) {
772 if ( ret ) { // pthread routines return errno values
773 abort( "%s : internal error, error(%d) %s.", func, ret, strerror( ret ) );
774 } // if
775} // Abort
776
777void * __create_pthread( pthread_t * pthread, void * (*start)(void *), void * arg ) {
778 pthread_attr_t attr;
779
780 check( real_pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
781
782 size_t stacksize = max( PTHREAD_STACK_MIN, DEFAULT_STACK_SIZE );
783
784 void * stack;
785 #if CFA_PROCESSOR_USE_MMAP
786 stacksize = ceiling( stacksize, __page_size ) + __page_size;
787 stack = mmap(0p, stacksize, __map_prot, MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
788 if(stack == ((void*)-1)) {
789 abort( "pthread stack creation : internal error, mmap failure, error(%d) %s.", errno, strerror( errno ) );
790 }
791 if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
792 abort( "pthread stack creation : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
793 } // if
794 #else
795 __cfaabi_dbg_debug_do(
796 stack = memalign( __page_size, stacksize + __page_size );
797 // pthread has no mechanism to create the guard page in user supplied stack.
798 if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
799 abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
800 } // if
801 );
802 __cfaabi_dbg_no_debug_do(
803 stack = malloc( stacksize );
804 );
805 #endif
806
807 check( real_pthread_attr_setstack( &attr, stack, stacksize ), "pthread_attr_setstack" );
808 check( real_pthread_create( pthread, &attr, start, arg ), "pthread_create" );
809 return stack;
810}
811
812void __destroy_pthread( pthread_t pthread, void * stack, void ** retval ) {
813 int err = real_pthread_join( pthread, retval );
814 if( err != 0 ) abort("KERNEL ERROR: joining pthread %p caused error %s\n", (void*)pthread, strerror(err));
815
816 #if CFA_PROCESSOR_USE_MMAP
817 pthread_attr_t attr;
818
819 check( real_pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
820
821 size_t stacksize;
822 // default stack size, normally defined by shell limit
823 check( real_pthread_attr_getstacksize( &attr, &stacksize ), "pthread_attr_getstacksize" );
824 assert( stacksize >= PTHREAD_STACK_MIN );
825 stacksize += __page_size;
826
827 if(munmap(stack, stacksize) == -1) {
828 abort( "pthread stack destruction : internal error, munmap failure, error(%d) %s.", errno, strerror( errno ) );
829 }
830 #else
831 __cfaabi_dbg_debug_do(
832 // pthread has no mechanism to create the guard page in user supplied stack.
833 if ( mprotect( stack, __page_size, __map_prot ) == -1 ) {
834 abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
835 } // if
836 );
837 free( stack );
838 #endif
839}
840
841#if defined(__CFA_WITH_VERIFY__)
842static bool verify_fwd_bck_rng(void) {
843 __cfaabi_tls.ready_rng.fwd_seed = 25214903917_l64u * (rdtscl() ^ (uintptr_t)&verify_fwd_bck_rng);
844
845 unsigned values[10];
846 for(i; 10) {
847 values[i] = __tls_rand_fwd();
848 }
849
850 __tls_rand_advance_bck();
851
852 for ( i; 9 -~= 0 ) {
853 if(values[i] != __tls_rand_bck()) {
854 return false;
855 }
856 }
857
858 return true;
859}
860#endif
Note: See TracBrowser for help on using the repository browser.