source: libcfa/src/concurrency/kernel/startup.cfa@ fa2e183

ADT ast-experimental
Last change on this file since fa2e183 was 8b74fa7, checked in by Thierry Delisle <tdelisle@…>, 3 years ago

cluster now support 'set_concurrency' which addes/removes processors internal to clusters
  • Property mode set to 100644
File size: 27.7 KB
Line 
1//
2// Cforall Version 1.0.0 Copyright (C) 2020 University of Waterloo
3//
4// The contents of this file are covered under the licence agreement in the
5// file "LICENCE" distributed with Cforall.
6//
7// kernel/startup.cfa --
8//
9// Author : Thierry Delisle
10// Created On : Thu Jul 30 15:12:54 2020
11// Last Modified By :
12// Last Modified On :
13// Update Count :
14//
15
16#define __cforall_thread__
17#define _GNU_SOURCE
18
19// #define __CFA_DEBUG_PRINT_RUNTIME_CORE__
20
21// C Includes
22#include <errno.h> // errno
23#include <signal.h>
24#include <string.h> // strerror
25#include <unistd.h> // sysconf
26
27extern "C" {
28 #include <limits.h> // PTHREAD_STACK_MIN
29 #include <unistd.h> // syscall
30 #include <sys/eventfd.h> // eventfd
31 #include <sys/mman.h> // mprotect
32 #include <sys/resource.h> // getrlimit
33}
34
35// CFA Includes
36#include "kernel/private.hfa"
37#include "iofwd.hfa"
38#include "startup.hfa" // STARTUP_PRIORITY_XXX
39#include "limits.hfa"
40#include "math.hfa"
41
42#define CFA_PROCESSOR_USE_MMAP 0
43
44//-----------------------------------------------------------------------------
45// Some assembly required
46#if defined( __i386 )
47 #define CtxGet( ctx ) __asm__ volatile ( \
48 "movl %%esp,%0\n" \
49 "movl %%ebp,%1\n" \
50 : "=rm" (ctx.SP), \
51 "=rm" (ctx.FP) \
52 )
53#elif defined( __x86_64 )
54 #define CtxGet( ctx ) __asm__ volatile ( \
55 "movq %%rsp,%0\n" \
56 "movq %%rbp,%1\n" \
57 : "=rm" (ctx.SP), \
58 "=rm" (ctx.FP) \
59 )
60#elif defined( __aarch64__ )
61 #define CtxGet( ctx ) __asm__ volatile ( \
62 "mov %0, sp\n" \
63 "mov %1, fp\n" \
64 : "=rm" (ctx.SP), \
65 "=rm" (ctx.FP) \
66 )
67#else
68 #error unknown hardware architecture
69#endif
70
71//-----------------------------------------------------------------------------
72// Start and stop routine for the kernel, declared first to make sure they run first
73static void __kernel_startup (void) __attribute__(( constructor( STARTUP_PRIORITY_KERNEL ) ));
74static void __kernel_shutdown(void) __attribute__(( destructor ( STARTUP_PRIORITY_KERNEL ) ));
75
76//-----------------------------------------------------------------------------
77// Static Forward Declarations
78struct current_stack_info_t;
79
80static void * __invoke_processor(void * arg);
81static void __kernel_first_resume( processor * this );
82static void __kernel_last_resume ( processor * this );
83static void init(processor & this, const char name[], cluster & _cltr, thread$ * initT);
84static void deinit(processor & this);
85static void doregister( struct cluster & cltr );
86static void unregister( struct cluster & cltr );
87static void register_tls( processor * this );
88static void unregister_tls( processor * this );
89static void ?{}( coroutine$ & this, current_stack_info_t * info);
90static void ?{}( thread$ & this, current_stack_info_t * info);
91static void ?{}(processorCtx_t & this) {}
92static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info);
93
94#if defined(__CFA_WITH_VERIFY__)
95 static bool verify_fwd_bck_rng(void);
96#endif
97
98//-----------------------------------------------------------------------------
99// Forward Declarations for other modules
100extern void __kernel_alarm_startup(void);
101extern void __kernel_alarm_shutdown(void);
102extern void __cfa_io_start( processor * );
103extern void __cfa_io_stop ( processor * );
104
105//-----------------------------------------------------------------------------
106// Other Forward Declarations
107extern void __wake_proc(processor *);
108extern int cfa_main_returned; // from interpose.cfa
109uint32_t __global_random_prime = 4_294_967_291u, __global_random_mask = false;
110
111//-----------------------------------------------------------------------------
112// Kernel storage
113KERNEL_STORAGE(cluster, mainCluster);
114KERNEL_STORAGE(processor, mainProcessor);
115KERNEL_STORAGE(thread$, mainThread);
116KERNEL_STORAGE(__stack_t, mainThreadCtx);
117// KERNEL_STORAGE(__scheduler_RWLock_t, __scheduler_lock);
118KERNEL_STORAGE(eventfd_t, mainIdleEventFd);
119KERNEL_STORAGE(io_future_t, mainIdleFuture);
120#if !defined(__CFA_NO_STATISTICS__)
121KERNEL_STORAGE(__stats_t, mainProcStats);
122#endif
123
124cluster * mainCluster libcfa_public;
125processor * mainProcessor;
126thread$ * mainThread;
127
128extern "C" {
129 struct { __dllist_t(cluster) list; __spinlock_t lock; } __cfa_dbg_global_clusters;
130}
131
132extern size_t __page_size;
133extern int __map_prot;
134
135//-----------------------------------------------------------------------------
136// Global state
137__thread struct KernelThreadData __cfaabi_tls __attribute__ ((tls_model ( "initial-exec" ))) @= {
138 NULL, // cannot use 0p
139 NULL,
140 false,
141 { 1, false, false },
142 0,
143 { 0, 0 },
144 NULL,
145 #ifdef __CFA_WITH_VERIFY__
146 false,
147 0,
148 #endif
149};
150
151__scheduler_RWLock_t __scheduler_lock @= { 0 };
152
153#if defined(CFA_HAVE_LINUX_LIBRSEQ)
154 // No data needed
155#elif defined(CFA_HAVE_LINUX_RSEQ_H)
156 extern "Cforall" {
157 __attribute__((aligned(64))) __thread volatile struct rseq __cfaabi_rseq @= {
158 .cpu_id : RSEQ_CPU_ID_UNINITIALIZED,
159 };
160 }
161#else
162 // No data needed
163#endif
164
165//-----------------------------------------------------------------------------
166// Struct to steal stack
167struct current_stack_info_t {
168 __stack_t * storage; // pointer to stack object
169 void * base; // base of stack
170 void * limit; // stack grows towards stack limit
171 void * context; // address of cfa_context_t
172};
173
174static void ?{}( current_stack_info_t & this ) {
175 __stack_context_t ctx;
176 CtxGet( ctx );
177 this.base = ctx.FP;
178
179 rlimit r;
180 getrlimit( RLIMIT_STACK, &r);
181 size_t size = r.rlim_cur;
182
183 this.limit = (void *)(((intptr_t)this.base) - size);
184 this.context = &storage_mainThreadCtx;
185}
186
187
188extern void heapManagerCtor();
189extern void heapManagerDtor();
190
191//=============================================================================================
192// Kernel Setup logic
193//=============================================================================================
194//-----------------------------------------------------------------------------
195// Kernel boot procedures
196static void __kernel_startup(void) {
197 /* paranoid */ verify( ! __preemption_enabled() );
198 __cfadbg_print_safe(runtime_core, "Kernel : Starting\n");
199
200 __cfa_dbg_global_clusters.list{ __get };
201 __cfa_dbg_global_clusters.lock{};
202
203 /* paranoid */ verify( verify_fwd_bck_rng() );
204
205 // Initialize the global scheduler lock
206 // __scheduler_lock = (__scheduler_RWLock_t*)&storage___scheduler_lock;
207 (__scheduler_lock){};
208
209 // Initialize the main cluster
210 mainCluster = (cluster *)&storage_mainCluster;
211 (*mainCluster){"Main Cluster", 0};
212
213 __cfadbg_print_safe(runtime_core, "Kernel : Main cluster ready\n");
214
215 // Construct the processor context of the main processor
216 void ?{}(processorCtx_t & this, processor * proc) {
217 (this.self){ "Processor" };
218 this.self.starter = 0p;
219 this.proc = proc;
220 }
221
222 void ?{}(processor & this) with( this ) {
223 ( this.terminated ){};
224 ( this.runner ){};
225 init( this, "Main Processor", *mainCluster, 0p );
226 kernel_thread = __cfaabi_pthread_self();
227
228 runner{ &this };
229 __cfadbg_print_safe(runtime_core, "Kernel : constructed main processor context %p\n", &runner);
230 }
231
232 // Initialize the main processor and the main processor ctx
233 // (the coroutine that contains the processing control flow)
234 mainProcessor = (processor *)&storage_mainProcessor;
235 (*mainProcessor){};
236
237 mainProcessor->idle_wctx.rdbuf = &storage_mainIdleEventFd;
238 mainProcessor->idle_wctx.ftr = (io_future_t*)&storage_mainIdleFuture;
239 /* paranoid */ verify( sizeof(storage_mainIdleEventFd) == sizeof(eventfd_t) );
240
241 __cfa_io_start( mainProcessor );
242 register_tls( mainProcessor );
243
244 // Start by initializing the main thread
245 // SKULLDUGGERY: the mainThread steals the process main thread
246 // which will then be scheduled by the mainProcessor normally
247 mainThread = (thread$ *)&storage_mainThread;
248 current_stack_info_t info;
249 info.storage = (__stack_t*)&storage_mainThreadCtx;
250 (*mainThread){ &info };
251
252 __cfadbg_print_safe(runtime_core, "Kernel : Main thread ready\n");
253
254 //initialize the global state variables
255 __cfaabi_tls.this_processor = mainProcessor;
256 __cfaabi_tls.this_thread = mainThread;
257
258 #if !defined( __CFA_NO_STATISTICS__ )
259 __cfaabi_tls.this_stats = (__stats_t *)& storage_mainProcStats;
260 __init_stats( __cfaabi_tls.this_stats );
261 #endif
262 mainProcessor->local_data = &__cfaabi_tls;
263
264 // Enable preemption
265 __kernel_alarm_startup();
266
267 // Add the main thread to the ready queue
268 // once resume is called on mainProcessor->runner the mainThread needs to be scheduled like any normal thread
269 schedule_thread$(mainThread, UNPARK_LOCAL);
270
271 // SKULLDUGGERY: Force a context switch to the main processor to set the main thread's context to the current UNIX
272 // context. Hence, the main thread does not begin through __cfactx_invoke_thread, like all other threads. The trick here is that
273 // mainThread is on the ready queue when this call is made.
274 __kernel_first_resume( __cfaabi_tls.this_processor );
275
276
277 // THE SYSTEM IS NOW COMPLETELY RUNNING
278
279 __cfadbg_print_safe(runtime_core, "Kernel : Started\n--------------------------------------------------\n\n");
280
281 /* paranoid */ verify( ! __preemption_enabled() );
282 enable_interrupts();
283 /* paranoid */ verify( __preemption_enabled() );
284
285}
286
287extern "C"{
288 void pthread_delete_kernel_threads_();
289}
290
291
292static void __kernel_shutdown(void) {
293 if(!cfa_main_returned) return;
294
295 //delete kernel threads for pthread_concurrency
296 pthread_delete_kernel_threads_();
297
298 /* paranoid */ verify( __preemption_enabled() );
299 disable_interrupts();
300 /* paranoid */ verify( ! __preemption_enabled() );
301
302 __cfadbg_print_safe(runtime_core, "\n--------------------------------------------------\nKernel : Shutting down\n");
303
304 // SKULLDUGGERY: Notify the mainProcessor it needs to terminates.
305 // When its coroutine terminates, it return control to the mainThread
306 // which is currently here
307 /* paranoid */ verify( !__atomic_load_n(&mainProcessor->do_terminate, __ATOMIC_ACQUIRE) );
308 __atomic_store_n(&mainProcessor->do_terminate, true, __ATOMIC_RELEASE);
309 __wake_proc( mainProcessor );
310 __kernel_last_resume( __cfaabi_tls.this_processor );
311 mainThread->self_cor.state = Halted;
312
313 // THE SYSTEM IS NOW COMPLETELY STOPPED
314
315 // Disable preemption
316 __kernel_alarm_shutdown();
317
318 #if !defined( __CFA_NO_STATISTICS__ )
319 __stats_t * st = (__stats_t *)& storage_mainProcStats;
320 __tally_stats(mainCluster->stats, st);
321 if( 0 != mainProcessor->print_stats ) {
322 __print_stats( st, mainProcessor->print_stats, "Processor ", mainProcessor->name, (void*)mainProcessor );
323 }
324 #if defined(CFA_STATS_ARRAY)
325 __flush_stat( st, "Processor", mainProcessor );
326 #endif
327 #endif
328
329 mainProcessor->local_data = 0p;
330
331 unregister_tls( mainProcessor );
332 __cfa_io_stop( mainProcessor );
333
334 // Destroy the main processor and its context in reverse order of construction
335 // These were manually constructed so we need manually destroy them
336 void ^?{}(processor & this) with( this ){
337 deinit( this );
338
339 /* paranoid */ verify( this.do_terminate == true );
340 __cfadbg_print_safe(runtime_core, "Kernel : destroyed main processor context %p\n", &runner);
341 }
342
343 ^(*mainProcessor){};
344
345 // Final step, destroy the main thread since it is no longer needed
346
347 // Since we provided a stack to this taxk it will not destroy anything
348 /* paranoid */ verify(mainThread->self_cor.stack.storage == (__stack_t*)(((uintptr_t)&storage_mainThreadCtx)| 0x1));
349 ^(*mainThread){};
350
351 ^(*mainCluster){};
352
353 ^(__scheduler_lock){};
354
355 ^(__cfa_dbg_global_clusters.list){};
356 ^(__cfa_dbg_global_clusters.lock){};
357
358 __cfadbg_print_safe(runtime_core, "Kernel : Shutdown complete\n");
359}
360
361//=============================================================================================
362// Kernel Initial Scheduling logic
363//=============================================================================================
364
365// Context invoker for processors
366// This is the entry point for processors (kernel threads) *except* for the main processor
367// It effectively constructs a coroutine by stealing the pthread stack
368static void * __invoke_processor(void * arg) {
369 #if !defined( __CFA_NO_STATISTICS__ )
370 __stats_t local_stats;
371 __init_stats( &local_stats );
372 __cfaabi_tls.this_stats = &local_stats;
373 #endif
374
375 processor * proc = (processor *) arg;
376 __cfaabi_tls.this_processor = proc;
377 __cfaabi_tls.this_thread = 0p;
378 __cfaabi_tls.preemption_state.[enabled, disable_count] = [false, 1];
379 proc->local_data = &__cfaabi_tls;
380
381 heapManagerCtor(); // initialize heap
382
383 __cfa_io_start( proc );
384 register_tls( proc );
385
386 // used for idle sleep when io_uring is present
387 io_future_t future;
388 eventfd_t idle_buf;
389 proc->idle_wctx.ftr = &future;
390 proc->idle_wctx.rdbuf = &idle_buf;
391
392
393 // SKULLDUGGERY: We want to create a context for the processor coroutine
394 // which is needed for the 2-step context switch. However, there is no reason
395 // to waste the perfectly valid stack create by pthread.
396 current_stack_info_t info;
397 __stack_t ctx;
398 info.storage = &ctx;
399 (proc->runner){ proc, &info };
400
401 __cfadbg_print_safe(runtime_core, "Coroutine : created stack %p\n", get_coroutine(proc->runner)->stack.storage);
402
403 //Set global state
404 __cfaabi_tls.this_thread = 0p;
405
406 //We now have a proper context from which to schedule threads
407 __cfadbg_print_safe(runtime_core, "Kernel : core %p created (%p, %p)\n", proc, &proc->runner, &ctx);
408
409 // SKULLDUGGERY: Since the coroutine doesn't have its own stack, we can't
410 // resume it to start it like it normally would, it will just context switch
411 // back to here. Instead directly call the main since we already are on the
412 // appropriate stack.
413 get_coroutine(proc->runner)->state = Active;
414 main( proc->runner );
415 get_coroutine(proc->runner)->state = Halted;
416
417 // Main routine of the core returned, the core is now fully terminated
418 __cfadbg_print_safe(runtime_core, "Kernel : core %p main ended (%p)\n", proc, &proc->runner);
419
420 #if !defined(__CFA_NO_STATISTICS__)
421 __tally_stats(proc->cltr->stats, &local_stats);
422 if( 0 != proc->print_stats ) {
423 __print_stats( &local_stats, proc->print_stats, "Processor ", proc->name, (void*)proc );
424 }
425 #if defined(CFA_STATS_ARRAY)
426 __flush_stat( &local_stats, "Processor", proc );
427 #endif
428 #endif
429
430 proc->local_data = 0p;
431
432 unregister_tls( proc );
433 __cfa_io_stop( proc );
434
435 heapManagerDtor(); // de-initialize heap
436
437 return 0p;
438}
439
440static void __kernel_first_resume( processor * this ) {
441 thread$ * src = mainThread;
442 coroutine$ * dst = get_coroutine(this->runner);
443
444 /* paranoid */ verify( ! __preemption_enabled() );
445
446 __cfaabi_tls.this_thread->curr_cor = dst;
447 __stack_prepare( &dst->stack, DEFAULT_STACK_SIZE );
448 __cfactx_start(main, dst, this->runner, __cfactx_invoke_coroutine);
449
450 /* paranoid */ verify( ! __preemption_enabled() );
451
452 dst->last = &src->self_cor;
453 dst->starter = dst->starter ? dst->starter : &src->self_cor;
454
455 // make sure the current state is still correct
456 /* paranoid */ verify(src->state == Ready);
457 src->corctx_flag = true;
458
459 // context switch to specified coroutine
460 verify( dst->context.SP );
461 __cfactx_switch( &src->context, &dst->context );
462 // when __cfactx_switch returns we are back in the src coroutine
463
464 mainThread->curr_cor = &mainThread->self_cor;
465
466 // make sure the current state has been update
467 /* paranoid */ verify(src->state == Active);
468
469 /* paranoid */ verify( ! __preemption_enabled() );
470}
471
472// KERNEL_ONLY
473static void __kernel_last_resume( processor * this ) {
474 coroutine$ * src = &mainThread->self_cor;
475 coroutine$ * dst = get_coroutine(this->runner);
476
477 /* paranoid */ verify( ! __preemption_enabled() );
478 /* paranoid */ verify( dst->starter == src );
479 /* paranoid */ verify( dst->context.SP );
480
481 // SKULLDUGGERY in debug the processors check that the
482 // stack is still within the limit of the stack limits after running a thread.
483 // that check doesn't make sense if we context switch to the processor using the
484 // coroutine semantics. Since this is a special case, use the current context
485 // info to populate these fields.
486 __cfaabi_dbg_debug_do(
487 __stack_context_t ctx;
488 CtxGet( ctx );
489 mainThread->context.SP = ctx.SP;
490 mainThread->context.FP = ctx.FP;
491 )
492
493 // context switch to the processor
494 __cfactx_switch( &src->context, &dst->context );
495}
496
497
498//=============================================================================================
499// Kernel Object Constructors logic
500//=============================================================================================
501//-----------------------------------------------------------------------------
502// Main thread construction
503static void ?{}( coroutine$ & this, current_stack_info_t * info) with( this ) {
504 stack.storage = info->storage;
505 with(*stack.storage) {
506 limit = info->limit;
507 base = info->base;
508 }
509 __attribute__((may_alias)) intptr_t * istorage = (intptr_t*) &stack.storage;
510 *istorage |= 0x1;
511 name = "Main Thread";
512 state = Start;
513 starter = 0p;
514 last = 0p;
515 cancellation = 0p;
516}
517
518static void ?{}( thread$ & this, current_stack_info_t * info) with( this ) {
519 ticket = TICKET_RUNNING;
520 state = Start;
521 self_cor{ info };
522 curr_cor = &self_cor;
523 curr_cluster = mainCluster;
524 self_mon.owner = &this;
525 self_mon.recursion = 1;
526 self_mon_p = &self_mon;
527 link.next = 0p;
528 link.ts = MAX;
529 preferred = ready_queue_new_preferred();
530 last_proc = 0p;
531 random_state = __global_random_mask ? __global_random_prime : __global_random_prime ^ rdtscl();
532 #if defined( __CFA_WITH_VERIFY__ )
533 executing = 0p;
534 canary = 0x0D15EA5E0D15EA5Ep;
535 #endif
536
537 node.next = 0p;
538 node.prev = 0p;
539 doregister(curr_cluster, this);
540
541 monitors{ &self_mon_p, 1, (fptr_t)0 };
542}
543
544//-----------------------------------------------------------------------------
545// Processor
546// Construct the processor context of non-main processors
547static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info) {
548 (this.self){ info };
549 this.proc = proc;
550}
551
552static void init(processor & this, const char name[], cluster & _cltr, thread$ * initT) with( this ) {
553 this.name = name;
554 this.cltr = &_cltr;
555 this.rdq.its = 0;
556 this.rdq.itr = 0;
557 this.rdq.id = 0;
558 this.rdq.target = MAX;
559 this.rdq.last = MAX;
560 this.rdq.cpu = 0;
561 // this.rdq.cutoff = 0ull;
562 do_terminate = false;
563 preemption_alarm = 0p;
564 pending_preemption = false;
565
566 this.io.ctx = 0p;
567 this.io.pending = false;
568 this.io.dirty = false;
569
570 this.init.thrd = initT;
571
572 this.local_data = 0p;
573
574 idle_wctx.evfd = eventfd(0, 0);
575 if (idle_wctx.evfd < 0) {
576 abort("KERNEL ERROR: PROCESSOR EVENTFD - %s\n", strerror(errno));
577 }
578
579 idle_wctx.sem = 0;
580 idle_wctx.wake__time = 0;
581
582 // I'm assuming these two are reserved for standard input and output
583 // so I'm using them as sentinels with idle_wctx.
584 /* paranoid */ verify( idle_wctx.evfd != 0 );
585 /* paranoid */ verify( idle_wctx.evfd != 1 );
586
587 #if !defined(__CFA_NO_STATISTICS__)
588 print_stats = 0;
589 print_halts = false;
590 #endif
591
592 __cfadbg_print_safe(runtime_core, "Kernel : core %p created\n", &this);
593}
594
595// Not a ctor, it just preps the destruction but should not destroy members
596static void deinit(processor & this) {
597 close(this.idle_wctx.evfd);
598}
599
600void ?{}(processor & this, const char name[], cluster & _cltr, thread$ * initT) libcfa_public {
601 ( this.terminated ){};
602 ( this.runner ){};
603
604 disable_interrupts();
605 init( this, name, _cltr, initT );
606 enable_interrupts();
607
608 __cfadbg_print_safe(runtime_core, "Kernel : Starting core %p\n", &this);
609
610 this.stack = __create_pthread( &this.kernel_thread, __invoke_processor, (void *)&this );
611}
612
613void ?{}(processor & this, const char name[], cluster & _cltr) libcfa_public {
614 (this){name, _cltr, 0p};
615}
616
617extern size_t __page_size;
618void ^?{}(processor & this) libcfa_public with( this ) {
619 /* paranoid */ verify( !__atomic_load_n(&do_terminate, __ATOMIC_ACQUIRE) );
620 __cfadbg_print_safe(runtime_core, "Kernel : core %p signaling termination\n", &this);
621
622 __atomic_store_n(&do_terminate, true, __ATOMIC_RELAXED);
623 __disable_interrupts_checked();
624 __wake_proc( &this );
625 __enable_interrupts_checked();
626
627 wait( terminated );
628 /* paranoid */ verify( active_processor() != &this);
629
630 __destroy_pthread( kernel_thread, this.stack, 0p );
631
632 disable_interrupts();
633 deinit( this );
634 enable_interrupts();
635}
636
637//-----------------------------------------------------------------------------
638// Cluster
639static void ?{}(__cluster_proc_list & this) {
640 this.fdw = 0p;
641 this.idle = 0;
642 this.total = 0;
643}
644
645void ?{}(cluster & this, const char name[], Duration preemption_rate, unsigned num_io, const io_context_params & io_params) libcfa_public with( this ) {
646 this.name = name;
647 this.preemption_rate = preemption_rate;
648 this.sched.readyQ.data = 0p;
649 this.sched.readyQ.tscs = 0p;
650 this.sched.readyQ.count = 0;
651 this.sched.io.tscs = 0p;
652 this.sched.io.data = 0p;
653 this.sched.caches = 0p;
654
655 #if !defined(__CFA_NO_STATISTICS__)
656 print_stats = 0;
657 stats = alloc();
658 __init_stats( stats );
659 #endif
660
661 threads{ __get };
662
663 io.arbiter = create();
664 io.params = io_params;
665
666 managed.procs = 0p;
667 managed.cnt = 0;
668
669 doregister(this);
670
671 // Lock the RWlock so no-one pushes/pops while we are changing the queue
672 disable_interrupts();
673 uint_fast32_t last_size = ready_mutate_lock();
674
675 // Adjust the ready queue size
676 ready_queue_grow( &this );
677
678 // Unlock the RWlock
679 ready_mutate_unlock( last_size );
680 enable_interrupts( false ); // Don't poll, could be in main cluster
681}
682
683void ^?{}(cluster & this) libcfa_public {
684 set_concurrency( this, 0 );
685
686 destroy(this.io.arbiter);
687
688 // Lock the RWlock so no-one pushes/pops while we are changing the queue
689 disable_interrupts();
690 uint_fast32_t last_size = ready_mutate_lock();
691
692 // Adjust the ready queue size
693 ready_queue_shrink( &this );
694
695 // Unlock the RWlock
696 ready_mutate_unlock( last_size );
697
698 ready_queue_close( &this );
699 /* paranoid */ verify( this.sched.readyQ.data == 0p );
700 /* paranoid */ verify( this.sched.readyQ.tscs == 0p );
701 /* paranoid */ verify( this.sched.readyQ.count == 0 );
702 /* paranoid */ verify( this.sched.io.tscs == 0p );
703 /* paranoid */ verify( this.sched.caches == 0p );
704
705 enable_interrupts( false ); // Don't poll, could be in main cluster
706
707
708 #if !defined(__CFA_NO_STATISTICS__)
709 if( 0 != this.print_stats ) {
710 __print_stats( this.stats, this.print_stats, "Cluster", this.name, (void*)&this );
711 }
712 #if defined(CFA_STATS_ARRAY)
713 __flush_stat( this.stats, "Cluster", &this );
714 #endif
715 free( this.stats );
716 #endif
717
718 unregister(this);
719}
720
721//=============================================================================================
722// Miscellaneous Initialization
723//=============================================================================================
724//-----------------------------------------------------------------------------
725// Global Queues
726static void doregister( cluster & cltr ) {
727 lock ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
728 push_front( __cfa_dbg_global_clusters.list, cltr );
729 unlock ( __cfa_dbg_global_clusters.lock );
730}
731
732static void unregister( cluster & cltr ) {
733 lock ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
734 remove( __cfa_dbg_global_clusters.list, cltr );
735 unlock( __cfa_dbg_global_clusters.lock );
736}
737
738void doregister( cluster * cltr, thread$ & thrd ) {
739 lock (cltr->thread_list_lock __cfaabi_dbg_ctx2);
740 cltr->nthreads += 1;
741 push_front(cltr->threads, thrd);
742 unlock (cltr->thread_list_lock);
743}
744
745void unregister( cluster * cltr, thread$ & thrd ) {
746 lock (cltr->thread_list_lock __cfaabi_dbg_ctx2);
747 remove(cltr->threads, thrd );
748 cltr->nthreads -= 1;
749 unlock(cltr->thread_list_lock);
750}
751
752static void register_tls( processor * this ) {
753 // Register and Lock the RWlock so no-one pushes/pops while we are changing the queue
754 uint_fast32_t last_size;
755 [this->unique_id, last_size] = ready_mutate_register();
756
757 this->rdq.cpu = __kernel_getcpu();
758
759 this->cltr->procs.total += 1u;
760 insert_last(this->cltr->procs.actives, *this);
761
762 // Adjust the ready queue size
763 ready_queue_grow( this->cltr );
764
765 // Unlock the RWlock
766 ready_mutate_unlock( last_size );
767}
768
769
770static void unregister_tls( processor * this ) {
771 // Lock the RWlock so no-one pushes/pops while we are changing the queue
772 uint_fast32_t last_size = ready_mutate_lock();
773 this->cltr->procs.total -= 1u;
774 remove(*this);
775
776 // clear the cluster so nothing gets pushed to local queues
777 cluster * cltr = this->cltr;
778 this->cltr = 0p;
779
780 // Adjust the ready queue size
781 ready_queue_shrink( cltr );
782
783 // Unlock the RWlock and unregister: we don't need the read_lock any more
784 ready_mutate_unregister( this->unique_id, last_size );
785}
786
787static void check( int ret, const char func[] ) {
788 if ( ret ) { // pthread routines return errno values
789 abort( "%s : internal error, error(%d) %s.", func, ret, strerror( ret ) );
790 } // if
791} // Abort
792
793void * __create_pthread( pthread_t * pthread, void * (*start)(void *), void * arg ) {
794 pthread_attr_t attr;
795
796 check( __cfaabi_pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
797
798 size_t stacksize = max( PTHREAD_STACK_MIN, DEFAULT_STACK_SIZE );
799
800 void * stack;
801 #if CFA_PROCESSOR_USE_MMAP
802 stacksize = ceiling( stacksize, __page_size ) + __page_size;
803 stack = mmap(0p, stacksize, __map_prot, MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
804 if(stack == ((void*)-1)) {
805 abort( "pthread stack creation : internal error, mmap failure, error(%d) %s.", errno, strerror( errno ) );
806 }
807 if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
808 abort( "pthread stack creation : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
809 } // if
810 #else
811 __cfaabi_dbg_debug_do(
812 stack = memalign( __page_size, stacksize + __page_size );
813 // pthread has no mechanism to create the guard page in user supplied stack.
814 if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
815 abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
816 } // if
817 );
818 __cfaabi_dbg_no_debug_do(
819 stack = malloc( stacksize );
820 );
821 #endif
822
823 check( __cfaabi_pthread_attr_setstack( &attr, stack, stacksize ), "pthread_attr_setstack" );
824 check( __cfaabi_pthread_create( pthread, &attr, start, arg ), "pthread_create" );
825 return stack;
826}
827
828void __destroy_pthread( pthread_t pthread, void * stack, void ** retval ) {
829 int err = __cfaabi_pthread_join( pthread, retval );
830 if( err != 0 ) abort("KERNEL ERROR: joining pthread %p caused error %s\n", (void*)pthread, strerror(err));
831
832 #if CFA_PROCESSOR_USE_MMAP
833 pthread_attr_t attr;
834
835 check( __cfaabi_pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
836
837 size_t stacksize;
838 // default stack size, normally defined by shell limit
839 check( __cfaabi_pthread_attr_getstacksize( &attr, &stacksize ), "pthread_attr_getstacksize" );
840 assert( stacksize >= PTHREAD_STACK_MIN );
841 stacksize += __page_size;
842
843 if(munmap(stack, stacksize) == -1) {
844 abort( "pthread stack destruction : internal error, munmap failure, error(%d) %s.", errno, strerror( errno ) );
845 }
846 #else
847 __cfaabi_dbg_debug_do(
848 // pthread has no mechanism to create the guard page in user supplied stack.
849 if ( mprotect( stack, __page_size, __map_prot ) == -1 ) {
850 abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
851 } // if
852 );
853 free( stack );
854 #endif
855}
856
857unsigned set_concurrency( cluster & this, unsigned new ) libcfa_public {
858 unsigned old = this.managed.cnt;
859
860 __cfadbg_print_safe(runtime_core, "Kernel : resizing cluster from %u to %u\n", old, (unsigned)new);
861
862 // Delete all the old unneeded procs
863 if(old > new) for(i; (unsigned)new ~ old) {
864 __cfadbg_print_safe(runtime_core, "Kernel : destroying %u\n", i);
865 delete( this.managed.procs[i] );
866 }
867
868 // Allocate new array (uses realloc and memcpies the data)
869 this.managed.procs = alloc( new, this.managed.procs`realloc );
870 this.managed.cnt = new;
871
872 // Create the desired new procs
873 if(old < new) for(i; old ~ new) {
874 __cfadbg_print_safe(runtime_core, "Kernel : constructing %u\n", i);
875 (*(this.managed.procs[i] = alloc())){ this };
876 }
877
878 // return the old count
879 return old;
880}
881
882#if defined(__CFA_WITH_VERIFY__)
883static bool verify_fwd_bck_rng(void) {
884 __cfaabi_tls.ready_rng.fwd_seed = 25214903917_l64u * (rdtscl() ^ (uintptr_t)&verify_fwd_bck_rng);
885
886 unsigned values[10];
887 for(i; 10) {
888 values[i] = __tls_rand_fwd();
889 }
890
891 __tls_rand_advance_bck();
892
893 for ( i; 9 -~= 0 ) {
894 if(values[i] != __tls_rand_bck()) {
895 return false;
896 }
897 }
898
899 return true;
900}
901#endif
Note: See TracBrowser for help on using the repository browser.