source: libcfa/src/concurrency/kernel/startup.cfa @ c34bb1f

Last change on this file since c34bb1f was c34bb1f, checked in by caparsons <caparson@…>, 11 months ago

fixed nonlocal exception edge case for program main and added poll() routine with no args that polls the currrent active coroutine

  • Property mode set to 100644
File size: 27.6 KB
Line 
1//
2// Cforall Version 1.0.0 Copyright (C) 2020 University of Waterloo
3//
4// The contents of this file are covered under the licence agreement in the
5// file "LICENCE" distributed with Cforall.
6//
7// kernel/startup.cfa --
8//
9// Author           : Thierry Delisle
10// Created On       : Thu Jul 30 15:12:54 2020
11// Last Modified By :
12// Last Modified On :
13// Update Count     :
14//
15
16#define __cforall_thread__
17
18// #define __CFA_DEBUG_PRINT_RUNTIME_CORE__
19
20// C Includes
21#include <errno.h>                                                                              // errno
22#include <signal.h>
23#include <string.h>                                                                             // strerror
24#include <unistd.h>
25#include <limits.h>                                                                             // PTHREAD_STACK_MIN
26extern "C" {
27        #include <sys/eventfd.h>                                                        // eventfd
28        #include <sys/mman.h>                                                           // mprotect
29        #include <sys/resource.h>                                                       // getrlimit
30}
31
32// CFA Includes
33#include "kernel/private.hfa"
34#include "iofwd.hfa"
35#include "startup.hfa"                                                                  // STARTUP_PRIORITY_XXX
36#include "limits.hfa"
37#include "math.hfa"
38#include "bits/random.hfa"                                                              // prng
39
40#define CFA_PROCESSOR_USE_MMAP 0
41
42//-----------------------------------------------------------------------------
43// Some assembly required
44#if defined( __i386 )
45        #define CtxGet( ctx ) __asm__ volatile ( \
46                "movl %%esp,%0\n" \
47                "movl %%ebp,%1\n" \
48                : "=rm" (ctx.SP), \
49                  "=rm" (ctx.FP) \
50        )
51#elif defined( __x86_64 )
52        #define CtxGet( ctx ) __asm__ volatile ( \
53                "movq %%rsp,%0\n" \
54                "movq %%rbp,%1\n" \
55                : "=rm" (ctx.SP), \
56                  "=rm" (ctx.FP) \
57        )
58#elif defined( __aarch64__ )
59        #define CtxGet( ctx ) __asm__ volatile ( \
60                "mov %0, sp\n" \
61                "mov %1, fp\n" \
62                : "=rm" (ctx.SP), \
63                  "=rm" (ctx.FP) \
64        )
65#else
66        #error unknown hardware architecture
67#endif
68
69//-----------------------------------------------------------------------------
70// Start and stop routine for the kernel, declared first to make sure they run first
71static void __kernel_startup (void) __attribute__(( constructor( STARTUP_PRIORITY_KERNEL ) ));
72static void __kernel_shutdown(void) __attribute__(( destructor ( STARTUP_PRIORITY_KERNEL ) ));
73
74//-----------------------------------------------------------------------------
75// Static Forward Declarations
76struct current_stack_info_t;
77
78static void * __invoke_processor(void * arg);
79static void __kernel_first_resume( processor * this );
80static void __kernel_last_resume ( processor * this );
81static void init(processor & this, const char name[], cluster & _cltr, thread$ * initT);
82static void deinit(processor & this);
83static void doregister( struct cluster & cltr );
84static void unregister( struct cluster & cltr );
85static void register_tls( processor * this );
86static void unregister_tls( processor * this );
87static void ?{}( coroutine$ & this, current_stack_info_t * info);
88static void ?{}( thread$ & this, current_stack_info_t * info);
89static void ?{}(processorCtx_t & this) {}
90static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info);
91
92#if defined(__CFA_WITH_VERIFY__)
93        static bool verify_fwd_bck_rng(void);
94#endif
95
96//-----------------------------------------------------------------------------
97// Forward Declarations for other modules
98extern void __kernel_alarm_startup(void);
99extern void __kernel_alarm_shutdown(void);
100extern void __cfa_io_start( processor * );
101extern void __cfa_io_stop ( processor * );
102
103//-----------------------------------------------------------------------------
104// Other Forward Declarations
105extern void __wake_proc(processor *);
106extern int cfa_main_returned;                                                   // from interpose.cfa
107size_t __global_random_prime = 4_294_967_291u;
108bool __global_random_mask = false;
109
110//-----------------------------------------------------------------------------
111// Kernel storage
112KERNEL_STORAGE(cluster,              mainCluster);
113KERNEL_STORAGE(processor,            mainProcessor);
114KERNEL_STORAGE(thread$,              mainThread);
115KERNEL_STORAGE(__stack_t,            mainThreadCtx);
116#if !defined(__CFA_NO_STATISTICS__)
117KERNEL_STORAGE(__stats_t, mainProcStats);
118#endif
119
120cluster              * mainCluster libcfa_public;
121processor            * mainProcessor;
122thread$              * mainThread;
123
124extern "C" {
125        struct { __dllist_t(cluster) list; __spinlock_t lock; } __cfa_dbg_global_clusters;
126}
127
128extern size_t __page_size;
129extern int __map_prot;
130
131//-----------------------------------------------------------------------------
132// Global state
133__thread struct KernelThreadData __cfaabi_tls __attribute__ ((tls_model ( "initial-exec" ))) @= {
134        .this_thread : NULL,                                                            // cannot use 0p
135        .this_processor : NULL,
136        .sched_lock : false,
137        .preemption_state : { .disable_count : 1, .enabled : false, .in_progress : false },
138        // random_state uninitialized
139        .ready_rng : { .fwd_seed : 0, .bck_seed : 0 },
140        .this_stats : NULL,
141        #ifdef __CFA_WITH_VERIFY__
142                .in_sched_lock : false,
143                .sched_id : 0,
144        #endif
145};
146
147__scheduler_RWLock_t __scheduler_lock @= { 0 };
148
149//-----------------------------------------------------------------------------
150// Struct to steal stack
151struct current_stack_info_t {
152        __stack_t * storage;  // pointer to stack object
153        void * base;          // base of stack
154        void * limit;         // stack grows towards stack limit
155        void * context;       // address of cfa_context_t
156};
157
158static void ?{}( current_stack_info_t & this ) {
159        __stack_context_t ctx;
160        CtxGet( ctx );
161        this.base = ctx.FP;
162
163        rlimit r;
164        getrlimit( RLIMIT_STACK, &r);
165        size_t size = r.rlim_cur;
166
167        this.limit = (void *)(((intptr_t)this.base) - size);
168        this.context = &storage_mainThreadCtx;
169}
170
171
172extern void heapManagerCtor();
173extern void heapManagerDtor();
174
175//=============================================================================================
176// Kernel Setup logic
177//=============================================================================================
178//-----------------------------------------------------------------------------
179// Kernel boot procedures
180static void __kernel_startup(void) {
181        /* paranoid */ verify( ! __preemption_enabled() );
182        __cfadbg_print_safe(runtime_core, "Kernel : Starting\n");
183
184        __cfa_dbg_global_clusters.list{ __get };
185        __cfa_dbg_global_clusters.lock{};
186
187        /* paranoid */ verify( verify_fwd_bck_rng() );
188
189        // Initialize the global scheduler lock
190        // __scheduler_lock = (__scheduler_RWLock_t*)&storage___scheduler_lock;
191        (__scheduler_lock){};
192
193        // Initialize the main cluster
194        mainCluster = (cluster *)&storage_mainCluster;
195        (*mainCluster){"Main Cluster", 0};
196
197        __cfadbg_print_safe(runtime_core, "Kernel : Main cluster ready\n");
198
199        // Construct the processor context of the main processor
200        void ?{}(processorCtx_t & this, processor * proc) {
201                (this.self){ "Processor" };
202                this.self.starter = 0p;
203                this.proc = proc;
204        }
205
206        void ?{}(processor & this) with( this ) {
207                ( this.terminated ){};
208                ( this.runner ){};
209                init( this, "Main Processor", *mainCluster, 0p );
210                kernel_thread = __cfaabi_pthread_self();
211
212                runner{ &this };
213                __cfadbg_print_safe(runtime_core, "Kernel : constructed main processor context %p\n", &runner);
214        }
215
216        // Initialize the main processor and the main processor ctx
217        // (the coroutine that contains the processing control flow)
218        mainProcessor = (processor *)&storage_mainProcessor;
219        (*mainProcessor){};
220
221        __cfa_io_start( mainProcessor );
222        register_tls( mainProcessor );
223
224        // Start by initializing the main thread
225        // SKULLDUGGERY: the mainThread steals the process main thread
226        // which will then be scheduled by the mainProcessor normally
227        mainThread = (thread$ *)&storage_mainThread;
228        current_stack_info_t info;
229        info.storage = (__stack_t*)&storage_mainThreadCtx;
230        (*mainThread){ &info };
231
232        __cfadbg_print_safe(runtime_core, "Kernel : Main thread ready\n");
233
234        //initialize the global state variables
235        __cfaabi_tls.this_processor = mainProcessor;
236        __cfaabi_tls.this_thread    = mainThread;
237
238        #if !defined( __CFA_NO_STATISTICS__ )
239                __cfaabi_tls.this_stats = (__stats_t *)& storage_mainProcStats;
240                __init_stats( __cfaabi_tls.this_stats );
241        #endif
242        mainProcessor->local_data = &__cfaabi_tls;
243
244        // Enable preemption
245        __kernel_alarm_startup();
246
247        // Add the main thread to the ready queue
248        // once resume is called on mainProcessor->runner the mainThread needs to be scheduled like any normal thread
249        schedule_thread$(mainThread, UNPARK_LOCAL);
250
251        // SKULLDUGGERY: Force a context switch to the main processor to set the main thread's context to the current UNIX
252        // context. Hence, the main thread does not begin through __cfactx_invoke_thread, like all other threads. The trick here is that
253        // mainThread is on the ready queue when this call is made.
254        __kernel_first_resume( __cfaabi_tls.this_processor );
255
256
257        // THE SYSTEM IS NOW COMPLETELY RUNNING
258
259        __cfadbg_print_safe(runtime_core, "Kernel : Started\n--------------------------------------------------\n\n");
260
261        /* paranoid */ verify( ! __preemption_enabled() );
262        enable_interrupts();
263        /* paranoid */ verify( __preemption_enabled() );
264
265}
266
267extern "C"{
268        void pthread_delete_kernel_threads_();
269}
270
271
272static void __kernel_shutdown(void) {
273        if(!cfa_main_returned) return;
274
275        //delete kernel threads for pthread_concurrency
276        pthread_delete_kernel_threads_();
277
278        /* paranoid */ verify( __preemption_enabled() );
279        disable_interrupts();
280        /* paranoid */ verify( ! __preemption_enabled() );
281
282        __cfadbg_print_safe(runtime_core, "\n--------------------------------------------------\nKernel : Shutting down\n");
283
284        // SKULLDUGGERY: Notify the mainProcessor it needs to terminates.
285        // When its coroutine terminates, it return control to the mainThread
286        // which is currently here
287        /* paranoid */ verify( !__atomic_load_n(&mainProcessor->do_terminate, __ATOMIC_ACQUIRE) );
288        __atomic_store_n(&mainProcessor->do_terminate, true, __ATOMIC_RELEASE);
289        __wake_proc( mainProcessor );
290        __kernel_last_resume( __cfaabi_tls.this_processor );
291        mainThread->self_cor.state = Halted;
292
293        // THE SYSTEM IS NOW COMPLETELY STOPPED
294
295        // Disable preemption
296        __kernel_alarm_shutdown();
297
298        #if !defined( __CFA_NO_STATISTICS__ )
299                __stats_t * st = (__stats_t *)& storage_mainProcStats;
300                __tally_stats(mainCluster->stats, st);
301                if( 0 != mainProcessor->print_stats ) {
302                        __print_stats( st, mainProcessor->print_stats, "Processor ", mainProcessor->name, (void*)mainProcessor );
303                }
304                #if defined(CFA_STATS_ARRAY)
305                        __flush_stat( st, "Processor", mainProcessor );
306                #endif
307        #endif
308
309        mainProcessor->local_data = 0p;
310
311        unregister_tls( mainProcessor );
312        __cfa_io_stop( mainProcessor );
313
314        // Destroy the main processor and its context in reverse order of construction
315        // These were manually constructed so we need manually destroy them
316        void ^?{}(processor & this) with( this ){
317                deinit( this );
318
319                /* paranoid */ verify( this.do_terminate == true );
320                __cfadbg_print_safe(runtime_core, "Kernel : destroyed main processor context %p\n", &runner);
321        }
322
323        ^(*mainProcessor){};
324
325        // Final step, destroy the main thread since it is no longer needed
326
327        // Since we provided a stack to this taxk it will not destroy anything
328        /* paranoid */ verify(mainThread->self_cor.stack.storage == (__stack_t*)(((uintptr_t)&storage_mainThreadCtx)| 0x1));
329        ^(*mainThread){};
330
331        ^(*mainCluster){};
332
333        ^(__scheduler_lock){};
334
335        ^(__cfa_dbg_global_clusters.list){};
336        ^(__cfa_dbg_global_clusters.lock){};
337
338        __cfadbg_print_safe(runtime_core, "Kernel : Shutdown complete\n");
339}
340
341//=============================================================================================
342// Kernel Initial Scheduling logic
343//=============================================================================================
344
345// Context invoker for processors
346// This is the entry point for processors (kernel threads) *except* for the main processor
347// It effectively constructs a coroutine by stealing the pthread stack
348static void * __invoke_processor(void * arg) {
349        #if !defined( __CFA_NO_STATISTICS__ )
350                __stats_t local_stats;
351                __init_stats( &local_stats );
352                __cfaabi_tls.this_stats = &local_stats;
353        #endif
354
355        processor * proc = (processor *) arg;
356        __cfaabi_tls.this_processor = proc;
357        __cfaabi_tls.this_thread    = 0p;
358        __cfaabi_tls.preemption_state.[enabled, disable_count] = [false, 1];
359        proc->local_data = &__cfaabi_tls;
360
361        heapManagerCtor();                                                                      // initialize heap
362
363        __cfa_io_start( proc );
364        register_tls( proc );
365
366        // SKULLDUGGERY: We want to create a context for the processor coroutine
367        // which is needed for the 2-step context switch. However, there is no reason
368        // to waste the perfectly valid stack create by pthread.
369        current_stack_info_t info;
370        __stack_t ctx;
371        info.storage = &ctx;
372        (proc->runner){ proc, &info };
373
374        __cfadbg_print_safe(runtime_core, "Coroutine : created stack %p\n", get_coroutine(proc->runner)->stack.storage);
375
376        //Set global state
377        __cfaabi_tls.this_thread = 0p;
378
379        //We now have a proper context from which to schedule threads
380        __cfadbg_print_safe(runtime_core, "Kernel : core %p created (%p, %p)\n", proc, &proc->runner, &ctx);
381
382        // SKULLDUGGERY: Since the coroutine doesn't have its own stack, we can't
383        // resume it to start it like it normally would, it will just context switch
384        // back to here. Instead directly call the main since we already are on the
385        // appropriate stack.
386        get_coroutine(proc->runner)->state = Active;
387        main( proc->runner );
388        get_coroutine(proc->runner)->state = Halted;
389
390        // Main routine of the core returned, the core is now fully terminated
391        __cfadbg_print_safe(runtime_core, "Kernel : core %p main ended (%p)\n", proc, &proc->runner);
392
393        #if !defined(__CFA_NO_STATISTICS__)
394                __tally_stats(proc->cltr->stats, &local_stats);
395                if( 0 != proc->print_stats ) {
396                        __print_stats( &local_stats, proc->print_stats, "Processor ", proc->name, (void*)proc );
397                }
398                #if defined(CFA_STATS_ARRAY)
399                        __flush_stat( &local_stats, "Processor", proc );
400                #endif
401        #endif
402
403        proc->local_data = 0p;
404
405        unregister_tls( proc );
406        __cfa_io_stop( proc );
407
408        heapManagerDtor();                                                                      // de-initialize heap
409
410        return 0p;
411}
412
413static void __kernel_first_resume( processor * this ) {
414        thread$ * src = mainThread;
415        coroutine$ * dst = get_coroutine(this->runner);
416
417        /* paranoid */ verify( ! __preemption_enabled() );
418
419        __cfaabi_tls.this_thread->curr_cor = dst;
420        __stack_prepare( &dst->stack, DEFAULT_STACK_SIZE );
421        __cfactx_start(main, dst, this->runner, __cfactx_invoke_coroutine);
422
423        /* paranoid */ verify( ! __preemption_enabled() );
424
425        dst->last = &src->self_cor;
426        dst->starter = dst->starter ? dst->starter : &src->self_cor;
427
428        // make sure the current state is still correct
429        /* paranoid */ verify(src->state == Ready);
430        src->corctx_flag = true;
431
432        // context switch to specified coroutine
433        verify( dst->context.SP );
434        __cfactx_switch( &src->context, &dst->context );
435        // when __cfactx_switch returns we are back in the src coroutine
436
437        mainThread->curr_cor = &mainThread->self_cor;
438
439        // make sure the current state has been update
440        /* paranoid */ verify(src->state == Active);
441
442        /* paranoid */ verify( ! __preemption_enabled() );
443}
444
445// KERNEL_ONLY
446static void __kernel_last_resume( processor * this ) {
447        coroutine$ * src = &mainThread->self_cor;
448        coroutine$ * dst = get_coroutine(this->runner);
449
450        /* paranoid */ verify( ! __preemption_enabled() );
451        /* paranoid */ verify( dst->starter == src );
452        /* paranoid */ verify( dst->context.SP );
453
454        // SKULLDUGGERY in debug the processors check that the
455        // stack is still within the limit of the stack limits after running a thread.
456        // that check doesn't make sense if we context switch to the processor using the
457        // coroutine semantics. Since this is a special case, use the current context
458        // info to populate these fields.
459        __cfaabi_dbg_debug_do(
460                __stack_context_t ctx;
461                CtxGet( ctx );
462                mainThread->context.SP = ctx.SP;
463                mainThread->context.FP = ctx.FP;
464        )
465
466        // context switch to the processor
467        __cfactx_switch( &src->context, &dst->context );
468}
469
470
471//=============================================================================================
472// Kernel Object Constructors logic
473//=============================================================================================
474//-----------------------------------------------------------------------------
475// Main thread construction
476static void ?{}( coroutine$ & this, current_stack_info_t * info) with( this ) {
477        stack.storage = info->storage;
478        with(*stack.storage) {
479                limit     = info->limit;
480                base      = info->base;
481        }
482        __attribute__((may_alias)) intptr_t * istorage = (intptr_t*) &stack.storage;
483        *istorage |= 0x1;
484        name = "Main Thread";
485        state = Start;
486        starter = 0p;
487        last = 0p;
488        cancellation = 0p;
489    ehm_state.ehm_buffer{};
490    ehm_state.buffer_lock{};
491    ehm_state.ehm_enabled = false;
492}
493
494static void ?{}( thread$ & this, current_stack_info_t * info) with( this ) {
495        ticket = TICKET_RUNNING;
496        state = Start;
497        self_cor{ info };
498        curr_cor = &self_cor;
499        curr_cluster = mainCluster;
500        self_mon.owner = &this;
501        self_mon.recursion = 1;
502        self_mon_p = &self_mon;
503        rdy_link.next = 0p;
504        rdy_link.ts   = MAX;
505        user_link.next = 0p;
506        user_link.prev = 0p;
507        cltr_link.next = 0p;
508        cltr_link.prev = 0p;
509        preferred = ready_queue_new_preferred();
510        last_proc = 0p;
511        PRNG_SET_SEED( random_state,  __global_random_mask ? __global_random_prime : __global_random_prime ^ rdtscl() );
512        #if defined( __CFA_WITH_VERIFY__ )
513                executing = 0p;
514                canary = 0x0D15EA5E0D15EA5Ep;
515        #endif
516
517        doregister(curr_cluster, this);
518
519        monitors{ &self_mon_p, 1, (fptr_t)0 };
520}
521
522//-----------------------------------------------------------------------------
523// Processor
524// Construct the processor context of non-main processors
525static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info) {
526        (this.self){ info };
527        this.proc = proc;
528}
529
530static void init(processor & this, const char name[], cluster & _cltr, thread$ * initT) with( this ) {
531        this.name = name;
532        this.cltr = &_cltr;
533    __atomic_add_fetch( &_cltr.procs.constructed, 1u, __ATOMIC_RELAXED );
534        this.rdq.its = 0;
535        this.rdq.itr = 0;
536        this.rdq.id  = 0;
537        this.rdq.target = MAX;
538        this.rdq.last = MAX;
539        this.rdq.cpu = 0;
540        // this.rdq.cutoff = 0ull;
541        do_terminate = false;
542        preemption_alarm = 0p;
543        pending_preemption = false;
544
545        this.io.ctx = 0p;
546        this.io.pending = false;
547        this.io.dirty   = false;
548
549        this.init.thrd = initT;
550
551        this.local_data = 0p;
552
553        idle_wctx.evfd = eventfd(0, 0);
554        if (idle_wctx.evfd < 0) {
555                abort("KERNEL ERROR: PROCESSOR EVENTFD - %s\n", strerror(errno));
556        }
557
558        idle_wctx.sem = 0;
559        idle_wctx.wake__time = 0;
560
561        // I'm assuming these two are reserved for standard input and output
562        // so I'm using them as sentinels with idle_wctx.
563        /* paranoid */ verify( idle_wctx.evfd != 0 );
564        /* paranoid */ verify( idle_wctx.evfd != 1 );
565
566        #if !defined(__CFA_NO_STATISTICS__)
567                print_stats = 0;
568                print_halts = false;
569        #endif
570
571        __cfadbg_print_safe(runtime_core, "Kernel : core %p created\n", &this);
572}
573
574// Not a ctor, it just preps the destruction but should not destroy members
575static void deinit(processor & this) {
576        close(this.idle_wctx.evfd);
577}
578
579void ?{}(processor & this, const char name[], cluster & _cltr, thread$ * initT) libcfa_public {
580        ( this.terminated ){};
581        ( this.runner ){};
582
583        disable_interrupts();
584                init( this, name, _cltr, initT );
585        enable_interrupts();
586
587        __cfadbg_print_safe(runtime_core, "Kernel : Starting core %p\n", &this);
588
589        this.stack = __create_pthread( &this.kernel_thread, __invoke_processor, (void *)&this );
590}
591
592void ?{}(processor & this, const char name[], cluster & _cltr) libcfa_public {
593        (this){name, _cltr, 0p};
594}
595
596extern size_t __page_size;
597void ^?{}(processor & this) libcfa_public with( this ) {
598        /* paranoid */ verify( !__atomic_load_n(&do_terminate, __ATOMIC_ACQUIRE) );
599        __cfadbg_print_safe(runtime_core, "Kernel : core %p signaling termination\n", &this);
600
601    __atomic_sub_fetch( &this.cltr->procs.constructed, 1u, __ATOMIC_RELAXED );
602
603        __atomic_store_n(&do_terminate, true, __ATOMIC_RELAXED);
604        __disable_interrupts_checked();
605                __wake_proc( &this );
606        __enable_interrupts_checked();
607
608        wait( terminated );
609        /* paranoid */ verify( active_processor() != &this);
610
611        __destroy_pthread( kernel_thread, this.stack, 0p );
612
613        disable_interrupts();
614                deinit( this );
615        enable_interrupts();
616}
617
618//-----------------------------------------------------------------------------
619// Cluster
620static void ?{}(__cluster_proc_list & this) {
621        this.fdw   = 0p;
622        this.idle  = 0;
623    this.constructed = 0;
624        this.total = 0;
625}
626
627void ?{}(cluster & this, const char name[], Duration preemption_rate, unsigned num_io, const io_context_params & io_params) libcfa_public with( this ) {
628        this.name = name;
629        this.preemption_rate = preemption_rate;
630        this.sched.readyQ.data = 0p;
631        this.sched.readyQ.tscs = 0p;
632        this.sched.readyQ.count = 0;
633        this.sched.io.tscs = 0p;
634        this.sched.io.data = 0p;
635        this.sched.caches = 0p;
636
637        #if !defined(__CFA_NO_STATISTICS__)
638                print_stats = 0;
639                stats = alloc();
640                __init_stats( stats );
641        #endif
642
643        threads{};
644
645        io.arbiter = create();
646        io.params = io_params;
647
648        managed.procs = 0p;
649        managed.cnt = 0;
650
651        doregister(this);
652
653        // Lock the RWlock so no-one pushes/pops while we are changing the queue
654        disable_interrupts();
655        uint_fast32_t last_size = ready_mutate_lock();
656
657                // Adjust the ready queue size
658                ready_queue_grow( &this );
659
660        // Unlock the RWlock
661        ready_mutate_unlock( last_size );
662        enable_interrupts( false ); // Don't poll, could be in main cluster
663}
664
665void ^?{}(cluster & this) libcfa_public {
666        set_concurrency( this, 0 );
667
668        destroy(this.io.arbiter);
669
670        // Lock the RWlock so no-one pushes/pops while we are changing the queue
671        disable_interrupts();
672        uint_fast32_t last_size = ready_mutate_lock();
673
674                // Adjust the ready queue size
675                ready_queue_shrink( &this );
676
677        // Unlock the RWlock
678        ready_mutate_unlock( last_size );
679
680        ready_queue_close( &this );
681        /* paranoid */ verify( this.sched.readyQ.data == 0p );
682        /* paranoid */ verify( this.sched.readyQ.tscs == 0p );
683        /* paranoid */ verify( this.sched.readyQ.count == 0 );
684        /* paranoid */ verify( this.sched.io.tscs == 0p );
685        /* paranoid */ verify( this.sched.caches == 0p );
686
687        enable_interrupts( false ); // Don't poll, could be in main cluster
688
689
690        #if !defined(__CFA_NO_STATISTICS__)
691                if( 0 != this.print_stats ) {
692                        __print_stats( this.stats, this.print_stats, "Cluster", this.name, (void*)&this );
693                }
694                #if defined(CFA_STATS_ARRAY)
695                        __flush_stat( this.stats, "Cluster", &this );
696                #endif
697                free( this.stats );
698        #endif
699
700        unregister(this);
701}
702
703//=============================================================================================
704// Miscellaneous Initialization
705//=============================================================================================
706//-----------------------------------------------------------------------------
707// Global Queues
708static void doregister( cluster     & cltr ) {
709        lock      ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
710        push_front( __cfa_dbg_global_clusters.list, cltr );
711        unlock    ( __cfa_dbg_global_clusters.lock );
712}
713
714static void unregister( cluster     & cltr ) {
715        lock  ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
716        remove( __cfa_dbg_global_clusters.list, cltr );
717        unlock( __cfa_dbg_global_clusters.lock );
718}
719
720void doregister( cluster * cltr, thread$ & thrd ) {
721        lock      (cltr->thread_list_lock __cfaabi_dbg_ctx2);
722        cltr->nthreads += 1;
723        insert_first(cltr->threads, thrd);
724        unlock    (cltr->thread_list_lock);
725}
726
727void unregister( cluster * cltr, thread$ & thrd ) {
728        lock  (cltr->thread_list_lock __cfaabi_dbg_ctx2);
729        {
730                tytagref( dlink(thread$), dlink(thread$) ) ?`inner( thread$ & this ) = void;
731                with( DLINK_VIA( thread$, struct __thread_user_link ) )
732                        remove( thrd );
733                cltr->nthreads -= 1;
734        }
735        unlock(cltr->thread_list_lock);
736}
737
738static void register_tls( processor * this ) {
739        // Register and Lock the RWlock so no-one pushes/pops while we are changing the queue
740        uint_fast32_t last_size;
741        [this->unique_id, last_size] = ready_mutate_register();
742
743                this->rdq.cpu = __kernel_getcpu();
744
745                this->cltr->procs.total += 1u;
746                insert_last(this->cltr->procs.actives, *this);
747
748                // Adjust the ready queue size
749                ready_queue_grow( this->cltr );
750
751        // Unlock the RWlock
752        ready_mutate_unlock( last_size );
753}
754
755
756static void unregister_tls( processor * this ) {
757        // Lock the RWlock so no-one pushes/pops while we are changing the queue
758        uint_fast32_t last_size = ready_mutate_lock();
759                this->cltr->procs.total -= 1u;
760                remove(*this);
761
762                // clear the cluster so nothing gets pushed to local queues
763                cluster * cltr = this->cltr;
764                this->cltr = 0p;
765
766                // Adjust the ready queue size
767                ready_queue_shrink( cltr );
768
769        // Unlock the RWlock and unregister: we don't need the read_lock any more
770        ready_mutate_unregister( this->unique_id, last_size );
771}
772
773static void check( int ret, const char func[] ) {
774        if ( ret ) {                                                                            // pthread routines return errno values
775                abort( "%s : internal error, error(%d) %s.", func, ret, strerror( ret ) );
776        } // if
777} // Abort
778
779void * __create_pthread( pthread_t * pthread, void * (*start)(void *), void * arg ) {
780        pthread_attr_t attr;
781
782        check( __cfaabi_pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
783
784        size_t stacksize = max( PTHREAD_STACK_MIN, DEFAULT_STACK_SIZE );
785
786        void * stack;
787        #if CFA_PROCESSOR_USE_MMAP
788                stacksize = ceiling( stacksize, __page_size ) + __page_size;
789                stack = mmap(0p, stacksize, __map_prot, MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
790                if(stack == ((void*)-1)) {
791                        abort( "pthread stack creation : internal error, mmap failure, error(%d) %s.", errno, strerror( errno ) );
792                }
793                if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
794                        abort( "pthread stack creation : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
795                } // if
796        #else
797                __cfaabi_dbg_debug_do(
798                        stack = memalign( __page_size, stacksize + __page_size );
799                        // pthread has no mechanism to create the guard page in user supplied stack.
800                        if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
801                                abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
802                        } // if
803                );
804                __cfaabi_dbg_no_debug_do(
805                        stack = malloc( stacksize );
806                );
807        #endif
808
809        check( __cfaabi_pthread_attr_setstack( &attr, stack, stacksize ), "pthread_attr_setstack" );
810        check( __cfaabi_pthread_create( pthread, &attr, start, arg ), "pthread_create" );
811        return stack;
812}
813
814void __destroy_pthread( pthread_t pthread, void * stack, void ** retval ) {
815        int err = __cfaabi_pthread_join( pthread, retval );
816        if( err != 0 ) abort("KERNEL ERROR: joining pthread %p caused error %s\n", (void*)pthread, strerror(err));
817
818        #if CFA_PROCESSOR_USE_MMAP
819                pthread_attr_t attr;
820
821                check( __cfaabi_pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
822
823                size_t stacksize;
824                // default stack size, normally defined by shell limit
825                check( __cfaabi_pthread_attr_getstacksize( &attr, &stacksize ), "pthread_attr_getstacksize" );
826                assert( stacksize >= PTHREAD_STACK_MIN );
827                stacksize += __page_size;
828
829                if(munmap(stack, stacksize) == -1) {
830                        abort( "pthread stack destruction : internal error, munmap failure, error(%d) %s.", errno, strerror( errno ) );
831                }
832        #else
833                __cfaabi_dbg_debug_do(
834                        // pthread has no mechanism to create the guard page in user supplied stack.
835                        if ( mprotect( stack, __page_size, __map_prot ) == -1 ) {
836                                abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
837                        } // if
838                );
839                free( stack );
840        #endif
841}
842
843unsigned set_concurrency( cluster & this, unsigned new ) libcfa_public {
844        unsigned old = this.managed.cnt;
845
846        __cfadbg_print_safe(runtime_core, "Kernel : resizing cluster from %u to %u\n", old, (unsigned)new);
847
848        // Delete all the old unneeded procs
849        if(old > new) for(i; (unsigned)new ~ old) {
850                __cfadbg_print_safe(runtime_core, "Kernel : destroying %u\n", i);
851                delete( this.managed.procs[i] );
852        }
853
854        // Allocate new array (uses realloc and memcpies the data)
855        this.managed.procs = alloc( new, this.managed.procs`realloc );
856        this.managed.cnt = new;
857
858        // Create the desired new procs
859        if(old < new) for(i; old ~ new) {
860                __cfadbg_print_safe(runtime_core, "Kernel : constructing %u\n", i);
861                (*(this.managed.procs[i] = alloc())){ this };
862        }
863
864        // return the old count
865        return old;
866}
867
868#if defined(__CFA_WITH_VERIFY__)
869static bool verify_fwd_bck_rng(void) {
870        __cfaabi_tls.ready_rng.fwd_seed = 25214903917_l64u * (rdtscl() ^ (uintptr_t)&verify_fwd_bck_rng);
871
872        unsigned values[10];
873        for(i; 10) {
874                values[i] = __tls_rand_fwd();
875        }
876
877        __tls_rand_advance_bck();
878
879        for ( i; 9 -~= 0 ) {
880                if(values[i] != __tls_rand_bck()) {
881                        return false;
882                }
883        }
884
885        return true;
886}
887#endif
Note: See TracBrowser for help on using the repository browser.