source: libcfa/src/concurrency/kernel/startup.cfa @ dd46fd3

ADTast-experimental
Last change on this file since dd46fd3 was dd46fd3, checked in by Peter A. Buhr <pabuhr@…>, 2 years ago

generalization of PRNG

  • Property mode set to 100644
File size: 27.6 KB
Line 
1//
2// Cforall Version 1.0.0 Copyright (C) 2020 University of Waterloo
3//
4// The contents of this file are covered under the licence agreement in the
5// file "LICENCE" distributed with Cforall.
6//
7// kernel/startup.cfa --
8//
9// Author           : Thierry Delisle
10// Created On       : Thu Jul 30 15:12:54 2020
11// Last Modified By :
12// Last Modified On :
13// Update Count     :
14//
15
16#define __cforall_thread__
17#define _GNU_SOURCE
18
19// #define __CFA_DEBUG_PRINT_RUNTIME_CORE__
20
21// C Includes
22#include <errno.h>                                      // errno
23#include <signal.h>
24#include <string.h>                                     // strerror
25#include <unistd.h>                                     // sysconf
26
27extern "C" {
28        #include <limits.h>                             // PTHREAD_STACK_MIN
29        #include <unistd.h>                             // syscall
30        #include <sys/eventfd.h>                        // eventfd
31        #include <sys/mman.h>                           // mprotect
32        #include <sys/resource.h>                       // getrlimit
33}
34
35// CFA Includes
36#include "kernel/private.hfa"
37#include "iofwd.hfa"
38#include "startup.hfa"                                  // STARTUP_PRIORITY_XXX
39#include "limits.hfa"
40#include "math.hfa"
41#include "bits/random.hfa"                                                              // prng
42
43#define CFA_PROCESSOR_USE_MMAP 0
44
45//-----------------------------------------------------------------------------
46// Some assembly required
47#if defined( __i386 )
48        #define CtxGet( ctx ) __asm__ volatile ( \
49                "movl %%esp,%0\n" \
50                "movl %%ebp,%1\n" \
51                : "=rm" (ctx.SP), \
52                  "=rm" (ctx.FP) \
53        )
54#elif defined( __x86_64 )
55        #define CtxGet( ctx ) __asm__ volatile ( \
56                "movq %%rsp,%0\n" \
57                "movq %%rbp,%1\n" \
58                : "=rm" (ctx.SP), \
59                  "=rm" (ctx.FP) \
60        )
61#elif defined( __aarch64__ )
62        #define CtxGet( ctx ) __asm__ volatile ( \
63                "mov %0, sp\n" \
64                "mov %1, fp\n" \
65                : "=rm" (ctx.SP), \
66                  "=rm" (ctx.FP) \
67        )
68#else
69        #error unknown hardware architecture
70#endif
71
72//-----------------------------------------------------------------------------
73// Start and stop routine for the kernel, declared first to make sure they run first
74static void __kernel_startup (void) __attribute__(( constructor( STARTUP_PRIORITY_KERNEL ) ));
75static void __kernel_shutdown(void) __attribute__(( destructor ( STARTUP_PRIORITY_KERNEL ) ));
76
77//-----------------------------------------------------------------------------
78// Static Forward Declarations
79struct current_stack_info_t;
80
81static void * __invoke_processor(void * arg);
82static void __kernel_first_resume( processor * this );
83static void __kernel_last_resume ( processor * this );
84static void init(processor & this, const char name[], cluster & _cltr, thread$ * initT);
85static void deinit(processor & this);
86static void doregister( struct cluster & cltr );
87static void unregister( struct cluster & cltr );
88static void register_tls( processor * this );
89static void unregister_tls( processor * this );
90static void ?{}( coroutine$ & this, current_stack_info_t * info);
91static void ?{}( thread$ & this, current_stack_info_t * info);
92static void ?{}(processorCtx_t & this) {}
93static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info);
94
95#if defined(__CFA_WITH_VERIFY__)
96        static bool verify_fwd_bck_rng(void);
97#endif
98
99//-----------------------------------------------------------------------------
100// Forward Declarations for other modules
101extern void __kernel_alarm_startup(void);
102extern void __kernel_alarm_shutdown(void);
103extern void __cfa_io_start( processor * );
104extern void __cfa_io_stop ( processor * );
105
106//-----------------------------------------------------------------------------
107// Other Forward Declarations
108extern void __wake_proc(processor *);
109extern int cfa_main_returned;                                                   // from interpose.cfa
110size_t __global_random_prime = 4_294_967_291u;
111bool __global_random_mask = false;
112
113//-----------------------------------------------------------------------------
114// Kernel storage
115KERNEL_STORAGE(cluster,              mainCluster);
116KERNEL_STORAGE(processor,            mainProcessor);
117KERNEL_STORAGE(thread$,              mainThread);
118KERNEL_STORAGE(__stack_t,            mainThreadCtx);
119#if !defined(__CFA_NO_STATISTICS__)
120KERNEL_STORAGE(__stats_t, mainProcStats);
121#endif
122
123cluster              * mainCluster libcfa_public;
124processor            * mainProcessor;
125thread$              * mainThread;
126
127extern "C" {
128        struct { __dllist_t(cluster) list; __spinlock_t lock; } __cfa_dbg_global_clusters;
129}
130
131extern size_t __page_size;
132extern int __map_prot;
133
134//-----------------------------------------------------------------------------
135// Global state
136__thread struct KernelThreadData __cfaabi_tls __attribute__ ((tls_model ( "initial-exec" ))) @= {
137        .this_thread : NULL,                                                            // cannot use 0p
138        .this_processor : NULL,
139        .sched_lock : false,
140        .preemption_state : { .disable_count : 1, .enabled : false, .in_progress : false },
141        // random_state uninitialized
142        .ready_rng : { .fwd_seed : 0, .bck_seed : 0 },
143        .this_stats : NULL,
144        #ifdef __CFA_WITH_VERIFY__
145                .in_sched_lock : false,
146                .sched_id : 0,
147        #endif
148};
149
150__scheduler_RWLock_t __scheduler_lock @= { 0 };
151
152#if   defined(CFA_HAVE_LINUX_LIBRSEQ)
153        // No data needed
154#elif defined(CFA_HAVE_LINUX_RSEQ_H)
155        extern "Cforall" {
156                __attribute__((aligned(64))) __thread volatile struct rseq __cfaabi_rseq @= {
157                        .cpu_id : RSEQ_CPU_ID_UNINITIALIZED,
158                };
159        }
160#else
161        // No data needed
162#endif
163
164//-----------------------------------------------------------------------------
165// Struct to steal stack
166struct current_stack_info_t {
167        __stack_t * storage;  // pointer to stack object
168        void * base;          // base of stack
169        void * limit;         // stack grows towards stack limit
170        void * context;       // address of cfa_context_t
171};
172
173static void ?{}( current_stack_info_t & this ) {
174        __stack_context_t ctx;
175        CtxGet( ctx );
176        this.base = ctx.FP;
177
178        rlimit r;
179        getrlimit( RLIMIT_STACK, &r);
180        size_t size = r.rlim_cur;
181
182        this.limit = (void *)(((intptr_t)this.base) - size);
183        this.context = &storage_mainThreadCtx;
184}
185
186
187extern void heapManagerCtor();
188extern void heapManagerDtor();
189
190//=============================================================================================
191// Kernel Setup logic
192//=============================================================================================
193//-----------------------------------------------------------------------------
194// Kernel boot procedures
195static void __kernel_startup(void) {
196        /* paranoid */ verify( ! __preemption_enabled() );
197        __cfadbg_print_safe(runtime_core, "Kernel : Starting\n");
198
199        __cfa_dbg_global_clusters.list{ __get };
200        __cfa_dbg_global_clusters.lock{};
201
202        /* paranoid */ verify( verify_fwd_bck_rng() );
203
204        // Initialize the global scheduler lock
205        // __scheduler_lock = (__scheduler_RWLock_t*)&storage___scheduler_lock;
206        (__scheduler_lock){};
207
208        // Initialize the main cluster
209        mainCluster = (cluster *)&storage_mainCluster;
210        (*mainCluster){"Main Cluster", 0};
211
212        __cfadbg_print_safe(runtime_core, "Kernel : Main cluster ready\n");
213
214        // Construct the processor context of the main processor
215        void ?{}(processorCtx_t & this, processor * proc) {
216                (this.self){ "Processor" };
217                this.self.starter = 0p;
218                this.proc = proc;
219        }
220
221        void ?{}(processor & this) with( this ) {
222                ( this.terminated ){};
223                ( this.runner ){};
224                init( this, "Main Processor", *mainCluster, 0p );
225                kernel_thread = __cfaabi_pthread_self();
226
227                runner{ &this };
228                __cfadbg_print_safe(runtime_core, "Kernel : constructed main processor context %p\n", &runner);
229        }
230
231        // Initialize the main processor and the main processor ctx
232        // (the coroutine that contains the processing control flow)
233        mainProcessor = (processor *)&storage_mainProcessor;
234        (*mainProcessor){};
235
236        __cfa_io_start( mainProcessor );
237        register_tls( mainProcessor );
238
239        // Start by initializing the main thread
240        // SKULLDUGGERY: the mainThread steals the process main thread
241        // which will then be scheduled by the mainProcessor normally
242        mainThread = (thread$ *)&storage_mainThread;
243        current_stack_info_t info;
244        info.storage = (__stack_t*)&storage_mainThreadCtx;
245        (*mainThread){ &info };
246
247        __cfadbg_print_safe(runtime_core, "Kernel : Main thread ready\n");
248
249        //initialize the global state variables
250        __cfaabi_tls.this_processor = mainProcessor;
251        __cfaabi_tls.this_thread    = mainThread;
252
253        #if !defined( __CFA_NO_STATISTICS__ )
254                __cfaabi_tls.this_stats = (__stats_t *)& storage_mainProcStats;
255                __init_stats( __cfaabi_tls.this_stats );
256        #endif
257        mainProcessor->local_data = &__cfaabi_tls;
258
259        // Enable preemption
260        __kernel_alarm_startup();
261
262        // Add the main thread to the ready queue
263        // once resume is called on mainProcessor->runner the mainThread needs to be scheduled like any normal thread
264        schedule_thread$(mainThread, UNPARK_LOCAL);
265
266        // SKULLDUGGERY: Force a context switch to the main processor to set the main thread's context to the current UNIX
267        // context. Hence, the main thread does not begin through __cfactx_invoke_thread, like all other threads. The trick here is that
268        // mainThread is on the ready queue when this call is made.
269        __kernel_first_resume( __cfaabi_tls.this_processor );
270
271
272        // THE SYSTEM IS NOW COMPLETELY RUNNING
273
274        __cfadbg_print_safe(runtime_core, "Kernel : Started\n--------------------------------------------------\n\n");
275
276        /* paranoid */ verify( ! __preemption_enabled() );
277        enable_interrupts();
278        /* paranoid */ verify( __preemption_enabled() );
279
280}
281
282extern "C"{
283        void pthread_delete_kernel_threads_();
284}
285
286
287static void __kernel_shutdown(void) {
288        if(!cfa_main_returned) return;
289
290        //delete kernel threads for pthread_concurrency
291        pthread_delete_kernel_threads_();
292
293        /* paranoid */ verify( __preemption_enabled() );
294        disable_interrupts();
295        /* paranoid */ verify( ! __preemption_enabled() );
296
297        __cfadbg_print_safe(runtime_core, "\n--------------------------------------------------\nKernel : Shutting down\n");
298
299        // SKULLDUGGERY: Notify the mainProcessor it needs to terminates.
300        // When its coroutine terminates, it return control to the mainThread
301        // which is currently here
302        /* paranoid */ verify( !__atomic_load_n(&mainProcessor->do_terminate, __ATOMIC_ACQUIRE) );
303        __atomic_store_n(&mainProcessor->do_terminate, true, __ATOMIC_RELEASE);
304        __wake_proc( mainProcessor );
305        __kernel_last_resume( __cfaabi_tls.this_processor );
306        mainThread->self_cor.state = Halted;
307
308        // THE SYSTEM IS NOW COMPLETELY STOPPED
309
310        // Disable preemption
311        __kernel_alarm_shutdown();
312
313        #if !defined( __CFA_NO_STATISTICS__ )
314                __stats_t * st = (__stats_t *)& storage_mainProcStats;
315                __tally_stats(mainCluster->stats, st);
316                if( 0 != mainProcessor->print_stats ) {
317                        __print_stats( st, mainProcessor->print_stats, "Processor ", mainProcessor->name, (void*)mainProcessor );
318                }
319                #if defined(CFA_STATS_ARRAY)
320                        __flush_stat( st, "Processor", mainProcessor );
321                #endif
322        #endif
323
324        mainProcessor->local_data = 0p;
325
326        unregister_tls( mainProcessor );
327        __cfa_io_stop( mainProcessor );
328
329        // Destroy the main processor and its context in reverse order of construction
330        // These were manually constructed so we need manually destroy them
331        void ^?{}(processor & this) with( this ){
332                deinit( this );
333
334                /* paranoid */ verify( this.do_terminate == true );
335                __cfadbg_print_safe(runtime_core, "Kernel : destroyed main processor context %p\n", &runner);
336        }
337
338        ^(*mainProcessor){};
339
340        // Final step, destroy the main thread since it is no longer needed
341
342        // Since we provided a stack to this taxk it will not destroy anything
343        /* paranoid */ verify(mainThread->self_cor.stack.storage == (__stack_t*)(((uintptr_t)&storage_mainThreadCtx)| 0x1));
344        ^(*mainThread){};
345
346        ^(*mainCluster){};
347
348        ^(__scheduler_lock){};
349
350        ^(__cfa_dbg_global_clusters.list){};
351        ^(__cfa_dbg_global_clusters.lock){};
352
353        __cfadbg_print_safe(runtime_core, "Kernel : Shutdown complete\n");
354}
355
356//=============================================================================================
357// Kernel Initial Scheduling logic
358//=============================================================================================
359
360// Context invoker for processors
361// This is the entry point for processors (kernel threads) *except* for the main processor
362// It effectively constructs a coroutine by stealing the pthread stack
363static void * __invoke_processor(void * arg) {
364        #if !defined( __CFA_NO_STATISTICS__ )
365                __stats_t local_stats;
366                __init_stats( &local_stats );
367                __cfaabi_tls.this_stats = &local_stats;
368        #endif
369
370        processor * proc = (processor *) arg;
371        __cfaabi_tls.this_processor = proc;
372        __cfaabi_tls.this_thread    = 0p;
373        __cfaabi_tls.preemption_state.[enabled, disable_count] = [false, 1];
374        proc->local_data = &__cfaabi_tls;
375
376        heapManagerCtor();                                                                      // initialize heap
377
378        __cfa_io_start( proc );
379        register_tls( proc );
380
381        // SKULLDUGGERY: We want to create a context for the processor coroutine
382        // which is needed for the 2-step context switch. However, there is no reason
383        // to waste the perfectly valid stack create by pthread.
384        current_stack_info_t info;
385        __stack_t ctx;
386        info.storage = &ctx;
387        (proc->runner){ proc, &info };
388
389        __cfadbg_print_safe(runtime_core, "Coroutine : created stack %p\n", get_coroutine(proc->runner)->stack.storage);
390
391        //Set global state
392        __cfaabi_tls.this_thread = 0p;
393
394        //We now have a proper context from which to schedule threads
395        __cfadbg_print_safe(runtime_core, "Kernel : core %p created (%p, %p)\n", proc, &proc->runner, &ctx);
396
397        // SKULLDUGGERY: Since the coroutine doesn't have its own stack, we can't
398        // resume it to start it like it normally would, it will just context switch
399        // back to here. Instead directly call the main since we already are on the
400        // appropriate stack.
401        get_coroutine(proc->runner)->state = Active;
402        main( proc->runner );
403        get_coroutine(proc->runner)->state = Halted;
404
405        // Main routine of the core returned, the core is now fully terminated
406        __cfadbg_print_safe(runtime_core, "Kernel : core %p main ended (%p)\n", proc, &proc->runner);
407
408        #if !defined(__CFA_NO_STATISTICS__)
409                __tally_stats(proc->cltr->stats, &local_stats);
410                if( 0 != proc->print_stats ) {
411                        __print_stats( &local_stats, proc->print_stats, "Processor ", proc->name, (void*)proc );
412                }
413                #if defined(CFA_STATS_ARRAY)
414                        __flush_stat( &local_stats, "Processor", proc );
415                #endif
416        #endif
417
418        proc->local_data = 0p;
419
420        unregister_tls( proc );
421        __cfa_io_stop( proc );
422
423        heapManagerDtor();                                                                      // de-initialize heap
424
425        return 0p;
426}
427
428static void __kernel_first_resume( processor * this ) {
429        thread$ * src = mainThread;
430        coroutine$ * dst = get_coroutine(this->runner);
431
432        /* paranoid */ verify( ! __preemption_enabled() );
433
434        __cfaabi_tls.this_thread->curr_cor = dst;
435        __stack_prepare( &dst->stack, DEFAULT_STACK_SIZE );
436        __cfactx_start(main, dst, this->runner, __cfactx_invoke_coroutine);
437
438        /* paranoid */ verify( ! __preemption_enabled() );
439
440        dst->last = &src->self_cor;
441        dst->starter = dst->starter ? dst->starter : &src->self_cor;
442
443        // make sure the current state is still correct
444        /* paranoid */ verify(src->state == Ready);
445        src->corctx_flag = true;
446
447        // context switch to specified coroutine
448        verify( dst->context.SP );
449        __cfactx_switch( &src->context, &dst->context );
450        // when __cfactx_switch returns we are back in the src coroutine
451
452        mainThread->curr_cor = &mainThread->self_cor;
453
454        // make sure the current state has been update
455        /* paranoid */ verify(src->state == Active);
456
457        /* paranoid */ verify( ! __preemption_enabled() );
458}
459
460// KERNEL_ONLY
461static void __kernel_last_resume( processor * this ) {
462        coroutine$ * src = &mainThread->self_cor;
463        coroutine$ * dst = get_coroutine(this->runner);
464
465        /* paranoid */ verify( ! __preemption_enabled() );
466        /* paranoid */ verify( dst->starter == src );
467        /* paranoid */ verify( dst->context.SP );
468
469        // SKULLDUGGERY in debug the processors check that the
470        // stack is still within the limit of the stack limits after running a thread.
471        // that check doesn't make sense if we context switch to the processor using the
472        // coroutine semantics. Since this is a special case, use the current context
473        // info to populate these fields.
474        __cfaabi_dbg_debug_do(
475                __stack_context_t ctx;
476                CtxGet( ctx );
477                mainThread->context.SP = ctx.SP;
478                mainThread->context.FP = ctx.FP;
479        )
480
481        // context switch to the processor
482        __cfactx_switch( &src->context, &dst->context );
483}
484
485
486//=============================================================================================
487// Kernel Object Constructors logic
488//=============================================================================================
489//-----------------------------------------------------------------------------
490// Main thread construction
491static void ?{}( coroutine$ & this, current_stack_info_t * info) with( this ) {
492        stack.storage = info->storage;
493        with(*stack.storage) {
494                limit     = info->limit;
495                base      = info->base;
496        }
497        __attribute__((may_alias)) intptr_t * istorage = (intptr_t*) &stack.storage;
498        *istorage |= 0x1;
499        name = "Main Thread";
500        state = Start;
501        starter = 0p;
502        last = 0p;
503        cancellation = 0p;
504}
505
506static void ?{}( thread$ & this, current_stack_info_t * info) with( this ) {
507        ticket = TICKET_RUNNING;
508        state = Start;
509        self_cor{ info };
510        curr_cor = &self_cor;
511        curr_cluster = mainCluster;
512        self_mon.owner = &this;
513        self_mon.recursion = 1;
514        self_mon_p = &self_mon;
515        rdy_link.next = 0p;
516        rdy_link.ts   = MAX;
517        user_link.next = 0p;
518        user_link.prev = 0p;
519        cltr_link.next = 0p;
520        cltr_link.prev = 0p;
521        preferred = ready_queue_new_preferred();
522        last_proc = 0p;
523        PRNG_SET_SEED( random_state,  __global_random_mask ? __global_random_prime : __global_random_prime ^ rdtscl() );
524        #if defined( __CFA_WITH_VERIFY__ )
525                executing = 0p;
526                canary = 0x0D15EA5E0D15EA5Ep;
527        #endif
528
529        doregister(curr_cluster, this);
530
531        monitors{ &self_mon_p, 1, (fptr_t)0 };
532}
533
534//-----------------------------------------------------------------------------
535// Processor
536// Construct the processor context of non-main processors
537static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info) {
538        (this.self){ info };
539        this.proc = proc;
540}
541
542static void init(processor & this, const char name[], cluster & _cltr, thread$ * initT) with( this ) {
543        this.name = name;
544        this.cltr = &_cltr;
545        this.rdq.its = 0;
546        this.rdq.itr = 0;
547        this.rdq.id  = 0;
548        this.rdq.target = MAX;
549        this.rdq.last = MAX;
550        this.rdq.cpu = 0;
551        // this.rdq.cutoff = 0ull;
552        do_terminate = false;
553        preemption_alarm = 0p;
554        pending_preemption = false;
555
556        this.io.ctx = 0p;
557        this.io.pending = false;
558        this.io.dirty   = false;
559
560        this.init.thrd = initT;
561
562        this.local_data = 0p;
563
564        idle_wctx.evfd = eventfd(0, 0);
565        if (idle_wctx.evfd < 0) {
566                abort("KERNEL ERROR: PROCESSOR EVENTFD - %s\n", strerror(errno));
567        }
568
569        idle_wctx.sem = 0;
570        idle_wctx.wake__time = 0;
571
572        // I'm assuming these two are reserved for standard input and output
573        // so I'm using them as sentinels with idle_wctx.
574        /* paranoid */ verify( idle_wctx.evfd != 0 );
575        /* paranoid */ verify( idle_wctx.evfd != 1 );
576
577        #if !defined(__CFA_NO_STATISTICS__)
578                print_stats = 0;
579                print_halts = false;
580        #endif
581
582        __cfadbg_print_safe(runtime_core, "Kernel : core %p created\n", &this);
583}
584
585// Not a ctor, it just preps the destruction but should not destroy members
586static void deinit(processor & this) {
587        close(this.idle_wctx.evfd);
588}
589
590void ?{}(processor & this, const char name[], cluster & _cltr, thread$ * initT) libcfa_public {
591        ( this.terminated ){};
592        ( this.runner ){};
593
594        disable_interrupts();
595                init( this, name, _cltr, initT );
596        enable_interrupts();
597
598        __cfadbg_print_safe(runtime_core, "Kernel : Starting core %p\n", &this);
599
600        this.stack = __create_pthread( &this.kernel_thread, __invoke_processor, (void *)&this );
601}
602
603void ?{}(processor & this, const char name[], cluster & _cltr) libcfa_public {
604        (this){name, _cltr, 0p};
605}
606
607extern size_t __page_size;
608void ^?{}(processor & this) libcfa_public with( this ) {
609        /* paranoid */ verify( !__atomic_load_n(&do_terminate, __ATOMIC_ACQUIRE) );
610        __cfadbg_print_safe(runtime_core, "Kernel : core %p signaling termination\n", &this);
611
612        __atomic_store_n(&do_terminate, true, __ATOMIC_RELAXED);
613        __disable_interrupts_checked();
614                __wake_proc( &this );
615        __enable_interrupts_checked();
616
617        wait( terminated );
618        /* paranoid */ verify( active_processor() != &this);
619
620        __destroy_pthread( kernel_thread, this.stack, 0p );
621
622        disable_interrupts();
623                deinit( this );
624        enable_interrupts();
625}
626
627//-----------------------------------------------------------------------------
628// Cluster
629static void ?{}(__cluster_proc_list & this) {
630        this.fdw   = 0p;
631        this.idle  = 0;
632        this.total = 0;
633}
634
635void ?{}(cluster & this, const char name[], Duration preemption_rate, unsigned num_io, const io_context_params & io_params) libcfa_public with( this ) {
636        this.name = name;
637        this.preemption_rate = preemption_rate;
638        this.sched.readyQ.data = 0p;
639        this.sched.readyQ.tscs = 0p;
640        this.sched.readyQ.count = 0;
641        this.sched.io.tscs = 0p;
642        this.sched.io.data = 0p;
643        this.sched.caches = 0p;
644
645        #if !defined(__CFA_NO_STATISTICS__)
646                print_stats = 0;
647                stats = alloc();
648                __init_stats( stats );
649        #endif
650
651        threads{};
652
653        io.arbiter = create();
654        io.params = io_params;
655
656        managed.procs = 0p;
657        managed.cnt = 0;
658
659        doregister(this);
660
661        // Lock the RWlock so no-one pushes/pops while we are changing the queue
662        disable_interrupts();
663        uint_fast32_t last_size = ready_mutate_lock();
664
665                // Adjust the ready queue size
666                ready_queue_grow( &this );
667
668        // Unlock the RWlock
669        ready_mutate_unlock( last_size );
670        enable_interrupts( false ); // Don't poll, could be in main cluster
671}
672
673void ^?{}(cluster & this) libcfa_public {
674        set_concurrency( this, 0 );
675
676        destroy(this.io.arbiter);
677
678        // Lock the RWlock so no-one pushes/pops while we are changing the queue
679        disable_interrupts();
680        uint_fast32_t last_size = ready_mutate_lock();
681
682                // Adjust the ready queue size
683                ready_queue_shrink( &this );
684
685        // Unlock the RWlock
686        ready_mutate_unlock( last_size );
687
688        ready_queue_close( &this );
689        /* paranoid */ verify( this.sched.readyQ.data == 0p );
690        /* paranoid */ verify( this.sched.readyQ.tscs == 0p );
691        /* paranoid */ verify( this.sched.readyQ.count == 0 );
692        /* paranoid */ verify( this.sched.io.tscs == 0p );
693        /* paranoid */ verify( this.sched.caches == 0p );
694
695        enable_interrupts( false ); // Don't poll, could be in main cluster
696
697
698        #if !defined(__CFA_NO_STATISTICS__)
699                if( 0 != this.print_stats ) {
700                        __print_stats( this.stats, this.print_stats, "Cluster", this.name, (void*)&this );
701                }
702                #if defined(CFA_STATS_ARRAY)
703                        __flush_stat( this.stats, "Cluster", &this );
704                #endif
705                free( this.stats );
706        #endif
707
708        unregister(this);
709}
710
711//=============================================================================================
712// Miscellaneous Initialization
713//=============================================================================================
714//-----------------------------------------------------------------------------
715// Global Queues
716static void doregister( cluster     & cltr ) {
717        lock      ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
718        push_front( __cfa_dbg_global_clusters.list, cltr );
719        unlock    ( __cfa_dbg_global_clusters.lock );
720}
721
722static void unregister( cluster     & cltr ) {
723        lock  ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
724        remove( __cfa_dbg_global_clusters.list, cltr );
725        unlock( __cfa_dbg_global_clusters.lock );
726}
727
728void doregister( cluster * cltr, thread$ & thrd ) {
729        lock      (cltr->thread_list_lock __cfaabi_dbg_ctx2);
730        cltr->nthreads += 1;
731        insert_first(cltr->threads, thrd);
732        unlock    (cltr->thread_list_lock);
733}
734
735void unregister( cluster * cltr, thread$ & thrd ) {
736        lock  (cltr->thread_list_lock __cfaabi_dbg_ctx2);
737        {
738                tytagref( dlink(thread$), dlink(thread$) ) ?`inner( thread$ & this ) = void;
739                with( DLINK_VIA( thread$, struct __thread_user_link ) )
740                        remove( thrd );
741                cltr->nthreads -= 1;
742        }
743        unlock(cltr->thread_list_lock);
744}
745
746static void register_tls( processor * this ) {
747        // Register and Lock the RWlock so no-one pushes/pops while we are changing the queue
748        uint_fast32_t last_size;
749        [this->unique_id, last_size] = ready_mutate_register();
750
751                this->rdq.cpu = __kernel_getcpu();
752
753                this->cltr->procs.total += 1u;
754                insert_last(this->cltr->procs.actives, *this);
755
756                // Adjust the ready queue size
757                ready_queue_grow( this->cltr );
758
759        // Unlock the RWlock
760        ready_mutate_unlock( last_size );
761}
762
763
764static void unregister_tls( processor * this ) {
765        // Lock the RWlock so no-one pushes/pops while we are changing the queue
766        uint_fast32_t last_size = ready_mutate_lock();
767                this->cltr->procs.total -= 1u;
768                remove(*this);
769
770                // clear the cluster so nothing gets pushed to local queues
771                cluster * cltr = this->cltr;
772                this->cltr = 0p;
773
774                // Adjust the ready queue size
775                ready_queue_shrink( cltr );
776
777        // Unlock the RWlock and unregister: we don't need the read_lock any more
778        ready_mutate_unregister( this->unique_id, last_size );
779}
780
781static void check( int ret, const char func[] ) {
782        if ( ret ) {                                                                            // pthread routines return errno values
783                abort( "%s : internal error, error(%d) %s.", func, ret, strerror( ret ) );
784        } // if
785} // Abort
786
787void * __create_pthread( pthread_t * pthread, void * (*start)(void *), void * arg ) {
788        pthread_attr_t attr;
789
790        check( __cfaabi_pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
791
792        size_t stacksize = max( PTHREAD_STACK_MIN, DEFAULT_STACK_SIZE );
793
794        void * stack;
795        #if CFA_PROCESSOR_USE_MMAP
796                stacksize = ceiling( stacksize, __page_size ) + __page_size;
797                stack = mmap(0p, stacksize, __map_prot, MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
798                if(stack == ((void*)-1)) {
799                        abort( "pthread stack creation : internal error, mmap failure, error(%d) %s.", errno, strerror( errno ) );
800                }
801                if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
802                        abort( "pthread stack creation : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
803                } // if
804        #else
805                __cfaabi_dbg_debug_do(
806                        stack = memalign( __page_size, stacksize + __page_size );
807                        // pthread has no mechanism to create the guard page in user supplied stack.
808                        if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
809                                abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
810                        } // if
811                );
812                __cfaabi_dbg_no_debug_do(
813                        stack = malloc( stacksize );
814                );
815        #endif
816
817        check( __cfaabi_pthread_attr_setstack( &attr, stack, stacksize ), "pthread_attr_setstack" );
818        check( __cfaabi_pthread_create( pthread, &attr, start, arg ), "pthread_create" );
819        return stack;
820}
821
822void __destroy_pthread( pthread_t pthread, void * stack, void ** retval ) {
823        int err = __cfaabi_pthread_join( pthread, retval );
824        if( err != 0 ) abort("KERNEL ERROR: joining pthread %p caused error %s\n", (void*)pthread, strerror(err));
825
826        #if CFA_PROCESSOR_USE_MMAP
827                pthread_attr_t attr;
828
829                check( __cfaabi_pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
830
831                size_t stacksize;
832                // default stack size, normally defined by shell limit
833                check( __cfaabi_pthread_attr_getstacksize( &attr, &stacksize ), "pthread_attr_getstacksize" );
834                assert( stacksize >= PTHREAD_STACK_MIN );
835                stacksize += __page_size;
836
837                if(munmap(stack, stacksize) == -1) {
838                        abort( "pthread stack destruction : internal error, munmap failure, error(%d) %s.", errno, strerror( errno ) );
839                }
840        #else
841                __cfaabi_dbg_debug_do(
842                        // pthread has no mechanism to create the guard page in user supplied stack.
843                        if ( mprotect( stack, __page_size, __map_prot ) == -1 ) {
844                                abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
845                        } // if
846                );
847                free( stack );
848        #endif
849}
850
851unsigned set_concurrency( cluster & this, unsigned new ) libcfa_public {
852        unsigned old = this.managed.cnt;
853
854        __cfadbg_print_safe(runtime_core, "Kernel : resizing cluster from %u to %u\n", old, (unsigned)new);
855
856        // Delete all the old unneeded procs
857        if(old > new) for(i; (unsigned)new ~ old) {
858                __cfadbg_print_safe(runtime_core, "Kernel : destroying %u\n", i);
859                delete( this.managed.procs[i] );
860        }
861
862        // Allocate new array (uses realloc and memcpies the data)
863        this.managed.procs = alloc( new, this.managed.procs`realloc );
864        this.managed.cnt = new;
865
866        // Create the desired new procs
867        if(old < new) for(i; old ~ new) {
868                __cfadbg_print_safe(runtime_core, "Kernel : constructing %u\n", i);
869                (*(this.managed.procs[i] = alloc())){ this };
870        }
871
872        // return the old count
873        return old;
874}
875
876#if defined(__CFA_WITH_VERIFY__)
877static bool verify_fwd_bck_rng(void) {
878        __cfaabi_tls.ready_rng.fwd_seed = 25214903917_l64u * (rdtscl() ^ (uintptr_t)&verify_fwd_bck_rng);
879
880        unsigned values[10];
881        for(i; 10) {
882                values[i] = __tls_rand_fwd();
883        }
884
885        __tls_rand_advance_bck();
886
887        for ( i; 9 -~= 0 ) {
888                if(values[i] != __tls_rand_bck()) {
889                        return false;
890                }
891        }
892
893        return true;
894}
895#endif
Note: See TracBrowser for help on using the repository browser.