source: libcfa/src/concurrency/kernel/startup.cfa @ a25bcf8

ADTast-experimental
Last change on this file since a25bcf8 was 116a2ea, checked in by Peter A. Buhr <pabuhr@…>, 2 years ago

new heap and associated tests updated

  • Property mode set to 100644
File size: 26.6 KB
Line 
1//
2// Cforall Version 1.0.0 Copyright (C) 2020 University of Waterloo
3//
4// The contents of this file are covered under the licence agreement in the
5// file "LICENCE" distributed with Cforall.
6//
7// kernel/startup.cfa --
8//
9// Author           : Thierry Delisle
10// Created On       : Thu Jul 30 15:12:54 2020
11// Last Modified By :
12// Last Modified On :
13// Update Count     :
14//
15
16#define __cforall_thread__
17#define _GNU_SOURCE
18
19// C Includes
20#include <errno.h>                                      // errno
21#include <signal.h>
22#include <string.h>                                     // strerror
23#include <unistd.h>                                     // sysconf
24
25extern "C" {
26        #include <limits.h>                             // PTHREAD_STACK_MIN
27        #include <unistd.h>                             // syscall
28        #include <sys/eventfd.h>                        // eventfd
29        #include <sys/mman.h>                           // mprotect
30        #include <sys/resource.h>                       // getrlimit
31}
32
33// CFA Includes
34#include "kernel/private.hfa"
35#include "iofwd.hfa"
36#include "startup.hfa"                                  // STARTUP_PRIORITY_XXX
37#include "limits.hfa"
38#include "math.hfa"
39
40#define CFA_PROCESSOR_USE_MMAP 0
41
42//-----------------------------------------------------------------------------
43// Some assembly required
44#if defined( __i386 )
45        #define CtxGet( ctx ) __asm__ volatile ( \
46                "movl %%esp,%0\n" \
47                "movl %%ebp,%1\n" \
48                : "=rm" (ctx.SP), \
49                  "=rm" (ctx.FP) \
50        )
51#elif defined( __x86_64 )
52        #define CtxGet( ctx ) __asm__ volatile ( \
53                "movq %%rsp,%0\n" \
54                "movq %%rbp,%1\n" \
55                : "=rm" (ctx.SP), \
56                  "=rm" (ctx.FP) \
57        )
58#elif defined( __aarch64__ )
59        #define CtxGet( ctx ) __asm__ volatile ( \
60                "mov %0, sp\n" \
61                "mov %1, fp\n" \
62                : "=rm" (ctx.SP), \
63                  "=rm" (ctx.FP) \
64        )
65#else
66        #error unknown hardware architecture
67#endif
68
69//-----------------------------------------------------------------------------
70// Start and stop routine for the kernel, declared first to make sure they run first
71static void __kernel_startup (void) __attribute__(( constructor( STARTUP_PRIORITY_KERNEL ) ));
72static void __kernel_shutdown(void) __attribute__(( destructor ( STARTUP_PRIORITY_KERNEL ) ));
73
74//-----------------------------------------------------------------------------
75// Static Forward Declarations
76struct current_stack_info_t;
77
78static void * __invoke_processor(void * arg);
79static void __kernel_first_resume( processor * this );
80static void __kernel_last_resume ( processor * this );
81static void init(processor & this, const char name[], cluster & _cltr, thread$ * initT);
82static void deinit(processor & this);
83static void doregister( struct cluster & cltr );
84static void unregister( struct cluster & cltr );
85static void register_tls( processor * this );
86static void unregister_tls( processor * this );
87static void ?{}( coroutine$ & this, current_stack_info_t * info);
88static void ?{}( thread$ & this, current_stack_info_t * info);
89static void ?{}(processorCtx_t & this) {}
90static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info);
91
92#if defined(__CFA_WITH_VERIFY__)
93        static bool verify_fwd_bck_rng(void);
94#endif
95
96//-----------------------------------------------------------------------------
97// Forward Declarations for other modules
98extern void __kernel_alarm_startup(void);
99extern void __kernel_alarm_shutdown(void);
100extern void __cfa_io_start( processor * );
101extern void __cfa_io_stop ( processor * );
102
103//-----------------------------------------------------------------------------
104// Other Forward Declarations
105extern void __wake_proc(processor *);
106extern int cfa_main_returned;                                                   // from interpose.cfa
107uint32_t __global_random_prime = 4_294_967_291u, __global_random_mask = false;
108
109//-----------------------------------------------------------------------------
110// Kernel storage
111KERNEL_STORAGE(cluster,              mainCluster);
112KERNEL_STORAGE(processor,            mainProcessor);
113KERNEL_STORAGE(thread$,              mainThread);
114KERNEL_STORAGE(__stack_t,            mainThreadCtx);
115// KERNEL_STORAGE(__scheduler_RWLock_t, __scheduler_lock);
116KERNEL_STORAGE(eventfd_t,            mainIdleEventFd);
117KERNEL_STORAGE(io_future_t,          mainIdleFuture);
118#if !defined(__CFA_NO_STATISTICS__)
119KERNEL_STORAGE(__stats_t, mainProcStats);
120#endif
121
122cluster              * mainCluster libcfa_public;
123processor            * mainProcessor;
124thread$              * mainThread;
125
126extern "C" {
127        struct { __dllist_t(cluster) list; __spinlock_t lock; } __cfa_dbg_global_clusters;
128}
129
130extern size_t __page_size;
131extern int __map_prot;
132
133//-----------------------------------------------------------------------------
134// Global state
135__thread struct KernelThreadData __cfaabi_tls __attribute__ ((tls_model ( "initial-exec" ))) @= {
136        NULL,                                                                                           // cannot use 0p
137        NULL,
138        false,
139        { 1, false, false },
140        0,
141        { 0, 0 },
142        NULL,
143        #ifdef __CFA_WITH_VERIFY__
144                false,
145                0,
146        #endif
147};
148
149__scheduler_RWLock_t __scheduler_lock @= { 0 };
150
151#if   defined(CFA_HAVE_LINUX_LIBRSEQ)
152        // No data needed
153#elif defined(CFA_HAVE_LINUX_RSEQ_H)
154        extern "Cforall" {
155                __attribute__((aligned(64))) __thread volatile struct rseq __cfaabi_rseq @= {
156                        .cpu_id : RSEQ_CPU_ID_UNINITIALIZED,
157                };
158        }
159#else
160        // No data needed
161#endif
162
163//-----------------------------------------------------------------------------
164// Struct to steal stack
165struct current_stack_info_t {
166        __stack_t * storage;  // pointer to stack object
167        void * base;          // base of stack
168        void * limit;         // stack grows towards stack limit
169        void * context;       // address of cfa_context_t
170};
171
172static void ?{}( current_stack_info_t & this ) {
173        __stack_context_t ctx;
174        CtxGet( ctx );
175        this.base = ctx.FP;
176
177        rlimit r;
178        getrlimit( RLIMIT_STACK, &r);
179        size_t size = r.rlim_cur;
180
181        this.limit = (void *)(((intptr_t)this.base) - size);
182        this.context = &storage_mainThreadCtx;
183}
184
185
186extern void heapManagerCtor();
187extern void heapManagerDtor();
188
189//=============================================================================================
190// Kernel Setup logic
191//=============================================================================================
192//-----------------------------------------------------------------------------
193// Kernel boot procedures
194static void __kernel_startup(void) {
195        /* paranoid */ verify( ! __preemption_enabled() );
196        __cfadbg_print_safe(runtime_core, "Kernel : Starting\n");
197
198        __cfa_dbg_global_clusters.list{ __get };
199        __cfa_dbg_global_clusters.lock{};
200
201        /* paranoid */ verify( verify_fwd_bck_rng() );
202
203        // Initialize the global scheduler lock
204        // __scheduler_lock = (__scheduler_RWLock_t*)&storage___scheduler_lock;
205        (__scheduler_lock){};
206
207        // Initialize the main cluster
208        mainCluster = (cluster *)&storage_mainCluster;
209        (*mainCluster){"Main Cluster", 0};
210
211        __cfadbg_print_safe(runtime_core, "Kernel : Main cluster ready\n");
212
213        // Construct the processor context of the main processor
214        void ?{}(processorCtx_t & this, processor * proc) {
215                (this.self){ "Processor" };
216                this.self.starter = 0p;
217                this.proc = proc;
218        }
219
220        void ?{}(processor & this) with( this ) {
221                ( this.terminated ){};
222                ( this.runner ){};
223                init( this, "Main Processor", *mainCluster, 0p );
224                kernel_thread = pthread_self();
225
226                runner{ &this };
227                __cfadbg_print_safe(runtime_core, "Kernel : constructed main processor context %p\n", &runner);
228        }
229
230        // Initialize the main processor and the main processor ctx
231        // (the coroutine that contains the processing control flow)
232        mainProcessor = (processor *)&storage_mainProcessor;
233        (*mainProcessor){};
234
235        mainProcessor->idle_wctx.rdbuf = &storage_mainIdleEventFd;
236        mainProcessor->idle_wctx.ftr   = (io_future_t*)&storage_mainIdleFuture;
237        /* paranoid */ verify( sizeof(storage_mainIdleEventFd) == sizeof(eventfd_t) );
238
239        __cfa_io_start( mainProcessor );
240        register_tls( mainProcessor );
241
242        // Start by initializing the main thread
243        // SKULLDUGGERY: the mainThread steals the process main thread
244        // which will then be scheduled by the mainProcessor normally
245        mainThread = (thread$ *)&storage_mainThread;
246        current_stack_info_t info;
247        info.storage = (__stack_t*)&storage_mainThreadCtx;
248        (*mainThread){ &info };
249
250        __cfadbg_print_safe(runtime_core, "Kernel : Main thread ready\n");
251
252        //initialize the global state variables
253        __cfaabi_tls.this_processor = mainProcessor;
254        __cfaabi_tls.this_thread    = mainThread;
255
256        #if !defined( __CFA_NO_STATISTICS__ )
257                __cfaabi_tls.this_stats = (__stats_t *)& storage_mainProcStats;
258                __init_stats( __cfaabi_tls.this_stats );
259        #endif
260        mainProcessor->local_data = &__cfaabi_tls;
261
262        // Enable preemption
263        __kernel_alarm_startup();
264
265        // Add the main thread to the ready queue
266        // once resume is called on mainProcessor->runner the mainThread needs to be scheduled like any normal thread
267        schedule_thread$(mainThread, UNPARK_LOCAL);
268
269        // SKULLDUGGERY: Force a context switch to the main processor to set the main thread's context to the current UNIX
270        // context. Hence, the main thread does not begin through __cfactx_invoke_thread, like all other threads. The trick here is that
271        // mainThread is on the ready queue when this call is made.
272        __kernel_first_resume( __cfaabi_tls.this_processor );
273
274
275        // THE SYSTEM IS NOW COMPLETELY RUNNING
276
277        __cfadbg_print_safe(runtime_core, "Kernel : Started\n--------------------------------------------------\n\n");
278
279        /* paranoid */ verify( ! __preemption_enabled() );
280        enable_interrupts();
281        /* paranoid */ verify( __preemption_enabled() );
282
283}
284
285static void __kernel_shutdown(void) {
286        if(!cfa_main_returned) return;
287        /* paranoid */ verify( __preemption_enabled() );
288        disable_interrupts();
289        /* paranoid */ verify( ! __preemption_enabled() );
290
291        __cfadbg_print_safe(runtime_core, "\n--------------------------------------------------\nKernel : Shutting down\n");
292
293        // SKULLDUGGERY: Notify the mainProcessor it needs to terminates.
294        // When its coroutine terminates, it return control to the mainThread
295        // which is currently here
296        /* paranoid */ verify( !__atomic_load_n(&mainProcessor->do_terminate, __ATOMIC_ACQUIRE) );
297        __atomic_store_n(&mainProcessor->do_terminate, true, __ATOMIC_RELEASE);
298        __wake_proc( mainProcessor );
299        __kernel_last_resume( __cfaabi_tls.this_processor );
300        mainThread->self_cor.state = Halted;
301
302        // THE SYSTEM IS NOW COMPLETELY STOPPED
303
304        // Disable preemption
305        __kernel_alarm_shutdown();
306
307        #if !defined( __CFA_NO_STATISTICS__ )
308                __stats_t * st = (__stats_t *)& storage_mainProcStats;
309                __tally_stats(mainCluster->stats, st);
310                if( 0 != mainProcessor->print_stats ) {
311                        __print_stats( st, mainProcessor->print_stats, "Processor ", mainProcessor->name, (void*)mainProcessor );
312                }
313                #if defined(CFA_STATS_ARRAY)
314                        __flush_stat( st, "Processor", mainProcessor );
315                #endif
316        #endif
317
318        mainProcessor->local_data = 0p;
319
320        unregister_tls( mainProcessor );
321        __cfa_io_stop( mainProcessor );
322
323        // Destroy the main processor and its context in reverse order of construction
324        // These were manually constructed so we need manually destroy them
325        void ^?{}(processor & this) with( this ){
326                deinit( this );
327
328                /* paranoid */ verify( this.do_terminate == true );
329                __cfaabi_dbg_print_safe("Kernel : destroyed main processor context %p\n", &runner);
330        }
331
332        ^(*mainProcessor){};
333
334        // Final step, destroy the main thread since it is no longer needed
335
336        // Since we provided a stack to this taxk it will not destroy anything
337        /* paranoid */ verify(mainThread->self_cor.stack.storage == (__stack_t*)(((uintptr_t)&storage_mainThreadCtx)| 0x1));
338        ^(*mainThread){};
339
340        ^(*mainCluster){};
341
342        ^(__scheduler_lock){};
343
344        ^(__cfa_dbg_global_clusters.list){};
345        ^(__cfa_dbg_global_clusters.lock){};
346
347        __cfadbg_print_safe(runtime_core, "Kernel : Shutdown complete\n");
348}
349
350//=============================================================================================
351// Kernel Initial Scheduling logic
352//=============================================================================================
353
354// Context invoker for processors
355// This is the entry point for processors (kernel threads) *except* for the main processor
356// It effectively constructs a coroutine by stealing the pthread stack
357static void * __invoke_processor(void * arg) {
358        #if !defined( __CFA_NO_STATISTICS__ )
359                __stats_t local_stats;
360                __init_stats( &local_stats );
361                __cfaabi_tls.this_stats = &local_stats;
362        #endif
363
364        processor * proc = (processor *) arg;
365        __cfaabi_tls.this_processor = proc;
366        __cfaabi_tls.this_thread    = 0p;
367        __cfaabi_tls.preemption_state.[enabled, disable_count] = [false, 1];
368        proc->local_data = &__cfaabi_tls;
369
370        heapManagerCtor();                                                                      // initialize heap
371
372        __cfa_io_start( proc );
373        register_tls( proc );
374
375        // used for idle sleep when io_uring is present
376        io_future_t future;
377        eventfd_t idle_buf;
378        proc->idle_wctx.ftr = &future;
379        proc->idle_wctx.rdbuf = &idle_buf;
380
381
382        // SKULLDUGGERY: We want to create a context for the processor coroutine
383        // which is needed for the 2-step context switch. However, there is no reason
384        // to waste the perfectly valid stack create by pthread.
385        current_stack_info_t info;
386        __stack_t ctx;
387        info.storage = &ctx;
388        (proc->runner){ proc, &info };
389
390        __cfaabi_dbg_print_safe("Coroutine : created stack %p\n", get_coroutine(proc->runner)->stack.storage);
391
392        //Set global state
393        __cfaabi_tls.this_thread = 0p;
394
395        //We now have a proper context from which to schedule threads
396        __cfadbg_print_safe(runtime_core, "Kernel : core %p created (%p, %p)\n", proc, &proc->runner, &ctx);
397
398        // SKULLDUGGERY: Since the coroutine doesn't have its own stack, we can't
399        // resume it to start it like it normally would, it will just context switch
400        // back to here. Instead directly call the main since we already are on the
401        // appropriate stack.
402        get_coroutine(proc->runner)->state = Active;
403        main( proc->runner );
404        get_coroutine(proc->runner)->state = Halted;
405
406        // Main routine of the core returned, the core is now fully terminated
407        __cfadbg_print_safe(runtime_core, "Kernel : core %p main ended (%p)\n", proc, &proc->runner);
408
409        #if !defined(__CFA_NO_STATISTICS__)
410                __tally_stats(proc->cltr->stats, &local_stats);
411                if( 0 != proc->print_stats ) {
412                        __print_stats( &local_stats, proc->print_stats, "Processor ", proc->name, (void*)proc );
413                }
414                #if defined(CFA_STATS_ARRAY)
415                        __flush_stat( &local_stats, "Processor", proc );
416                #endif
417        #endif
418
419        proc->local_data = 0p;
420
421        unregister_tls( proc );
422        __cfa_io_stop( proc );
423
424        heapManagerDtor();                                                                      // de-initialize heap
425
426        return 0p;
427}
428
429static void __kernel_first_resume( processor * this ) {
430        thread$ * src = mainThread;
431        coroutine$ * dst = get_coroutine(this->runner);
432
433        /* paranoid */ verify( ! __preemption_enabled() );
434
435        __cfaabi_tls.this_thread->curr_cor = dst;
436        __stack_prepare( &dst->stack, DEFAULT_STACK_SIZE );
437        __cfactx_start(main, dst, this->runner, __cfactx_invoke_coroutine);
438
439        /* paranoid */ verify( ! __preemption_enabled() );
440
441        dst->last = &src->self_cor;
442        dst->starter = dst->starter ? dst->starter : &src->self_cor;
443
444        // make sure the current state is still correct
445        /* paranoid */ verify(src->state == Ready);
446        src->corctx_flag = true;
447
448        // context switch to specified coroutine
449        verify( dst->context.SP );
450        __cfactx_switch( &src->context, &dst->context );
451        // when __cfactx_switch returns we are back in the src coroutine
452
453        mainThread->curr_cor = &mainThread->self_cor;
454
455        // make sure the current state has been update
456        /* paranoid */ verify(src->state == Active);
457
458        /* paranoid */ verify( ! __preemption_enabled() );
459}
460
461// KERNEL_ONLY
462static void __kernel_last_resume( processor * this ) {
463        coroutine$ * src = &mainThread->self_cor;
464        coroutine$ * dst = get_coroutine(this->runner);
465
466        /* paranoid */ verify( ! __preemption_enabled() );
467        /* paranoid */ verify( dst->starter == src );
468        /* paranoid */ verify( dst->context.SP );
469
470        // SKULLDUGGERY in debug the processors check that the
471        // stack is still within the limit of the stack limits after running a thread.
472        // that check doesn't make sense if we context switch to the processor using the
473        // coroutine semantics. Since this is a special case, use the current context
474        // info to populate these fields.
475        __cfaabi_dbg_debug_do(
476                __stack_context_t ctx;
477                CtxGet( ctx );
478                mainThread->context.SP = ctx.SP;
479                mainThread->context.FP = ctx.FP;
480        )
481
482        // context switch to the processor
483        __cfactx_switch( &src->context, &dst->context );
484}
485
486
487//=============================================================================================
488// Kernel Object Constructors logic
489//=============================================================================================
490//-----------------------------------------------------------------------------
491// Main thread construction
492static void ?{}( coroutine$ & this, current_stack_info_t * info) with( this ) {
493        stack.storage = info->storage;
494        with(*stack.storage) {
495                limit     = info->limit;
496                base      = info->base;
497        }
498        __attribute__((may_alias)) intptr_t * istorage = (intptr_t*) &stack.storage;
499        *istorage |= 0x1;
500        name = "Main Thread";
501        state = Start;
502        starter = 0p;
503        last = 0p;
504        cancellation = 0p;
505}
506
507static void ?{}( thread$ & this, current_stack_info_t * info) with( this ) {
508        ticket = TICKET_RUNNING;
509        state = Start;
510        self_cor{ info };
511        curr_cor = &self_cor;
512        curr_cluster = mainCluster;
513        self_mon.owner = &this;
514        self_mon.recursion = 1;
515        self_mon_p = &self_mon;
516        link.next = 0p;
517        link.ts   = MAX;
518        preferred = ready_queue_new_preferred();
519        last_proc = 0p;
520        random_state = __global_random_mask ? __global_random_prime : __global_random_prime ^ rdtscl();
521        #if defined( __CFA_WITH_VERIFY__ )
522                canary = 0x0D15EA5E0D15EA5Ep;
523        #endif
524
525        node.next = 0p;
526        node.prev = 0p;
527        doregister(curr_cluster, this);
528
529        monitors{ &self_mon_p, 1, (fptr_t)0 };
530}
531
532//-----------------------------------------------------------------------------
533// Processor
534// Construct the processor context of non-main processors
535static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info) {
536        (this.self){ info };
537        this.proc = proc;
538}
539
540static void init(processor & this, const char name[], cluster & _cltr, thread$ * initT) with( this ) {
541        this.name = name;
542        this.cltr = &_cltr;
543        this.rdq.its = 0;
544        this.rdq.itr = 0;
545        this.rdq.id  = 0;
546        this.rdq.target = MAX;
547        this.rdq.last = MAX;
548        this.rdq.cpu = 0;
549        // this.rdq.cutoff = 0ull;
550        do_terminate = false;
551        preemption_alarm = 0p;
552        pending_preemption = false;
553
554        this.io.ctx = 0p;
555        this.io.pending = false;
556        this.io.dirty   = false;
557
558        this.init.thrd = initT;
559
560        this.local_data = 0p;
561
562        idle_wctx.evfd = eventfd(0, 0);
563        if (idle_wctx.evfd < 0) {
564                abort("KERNEL ERROR: PROCESSOR EVENTFD - %s\n", strerror(errno));
565        }
566
567        idle_wctx.sem = 0;
568        idle_wctx.wake__time = 0;
569
570        // I'm assuming these two are reserved for standard input and output
571        // so I'm using them as sentinels with idle_wctx.
572        /* paranoid */ verify( idle_wctx.evfd != 0 );
573        /* paranoid */ verify( idle_wctx.evfd != 1 );
574
575        #if !defined(__CFA_NO_STATISTICS__)
576                print_stats = 0;
577                print_halts = false;
578        #endif
579
580        __cfadbg_print_safe(runtime_core, "Kernel : core %p created\n", &this);
581}
582
583// Not a ctor, it just preps the destruction but should not destroy members
584static void deinit(processor & this) {
585        close(this.idle_wctx.evfd);
586}
587
588void ?{}(processor & this, const char name[], cluster & _cltr, thread$ * initT) libcfa_public {
589        ( this.terminated ){};
590        ( this.runner ){};
591
592        disable_interrupts();
593                init( this, name, _cltr, initT );
594        enable_interrupts();
595
596        __cfadbg_print_safe(runtime_core, "Kernel : Starting core %p\n", &this);
597
598        this.stack = __create_pthread( &this.kernel_thread, __invoke_processor, (void *)&this );
599}
600
601void ?{}(processor & this, const char name[], cluster & _cltr) libcfa_public {
602        (this){name, _cltr, 0p};
603}
604
605extern size_t __page_size;
606void ^?{}(processor & this) libcfa_public with( this ) {
607        /* paranoid */ verify( !__atomic_load_n(&do_terminate, __ATOMIC_ACQUIRE) );
608        __cfadbg_print_safe(runtime_core, "Kernel : core %p signaling termination\n", &this);
609
610        __atomic_store_n(&do_terminate, true, __ATOMIC_RELAXED);
611        __disable_interrupts_checked();
612                __wake_proc( &this );
613        __enable_interrupts_checked();
614
615        wait( terminated );
616        /* paranoid */ verify( active_processor() != &this);
617
618        __destroy_pthread( kernel_thread, this.stack, 0p );
619
620        disable_interrupts();
621                deinit( this );
622        enable_interrupts();
623}
624
625//-----------------------------------------------------------------------------
626// Cluster
627static void ?{}(__cluster_proc_list & this) {
628        this.fdw   = 0p;
629        this.idle  = 0;
630        this.total = 0;
631}
632
633void ?{}(cluster & this, const char name[], Duration preemption_rate, unsigned num_io, const io_context_params & io_params) libcfa_public with( this ) {
634        this.name = name;
635        this.preemption_rate = preemption_rate;
636        this.sched.readyQ.data = 0p;
637        this.sched.readyQ.tscs = 0p;
638        this.sched.readyQ.count = 0;
639        this.sched.io.tscs = 0p;
640        this.sched.io.data = 0p;
641        this.sched.caches = 0p;
642
643        #if !defined(__CFA_NO_STATISTICS__)
644                print_stats = 0;
645                stats = alloc();
646                __init_stats( stats );
647        #endif
648
649        threads{ __get };
650
651        io.arbiter = create();
652        io.params = io_params;
653
654        doregister(this);
655
656        // Lock the RWlock so no-one pushes/pops while we are changing the queue
657        disable_interrupts();
658        uint_fast32_t last_size = ready_mutate_lock();
659
660                // Adjust the ready queue size
661                ready_queue_grow( &this );
662
663        // Unlock the RWlock
664        ready_mutate_unlock( last_size );
665        enable_interrupts( false ); // Don't poll, could be in main cluster
666}
667
668void ^?{}(cluster & this) libcfa_public {
669        destroy(this.io.arbiter);
670
671        // Lock the RWlock so no-one pushes/pops while we are changing the queue
672        disable_interrupts();
673        uint_fast32_t last_size = ready_mutate_lock();
674
675                // Adjust the ready queue size
676                ready_queue_shrink( &this );
677
678        // Unlock the RWlock
679        ready_mutate_unlock( last_size );
680
681        ready_queue_close( &this );
682        /* paranoid */ verify( this.sched.readyQ.data == 0p );
683        /* paranoid */ verify( this.sched.readyQ.tscs == 0p );
684        /* paranoid */ verify( this.sched.readyQ.count == 0 );
685        /* paranoid */ verify( this.sched.io.tscs == 0p );
686        /* paranoid */ verify( this.sched.caches == 0p );
687
688        enable_interrupts( false ); // Don't poll, could be in main cluster
689
690
691        #if !defined(__CFA_NO_STATISTICS__)
692                if( 0 != this.print_stats ) {
693                        __print_stats( this.stats, this.print_stats, "Cluster", this.name, (void*)&this );
694                }
695                #if defined(CFA_STATS_ARRAY)
696                        __flush_stat( this.stats, "Cluster", &this );
697                #endif
698                free( this.stats );
699        #endif
700
701        unregister(this);
702}
703
704//=============================================================================================
705// Miscellaneous Initialization
706//=============================================================================================
707//-----------------------------------------------------------------------------
708// Global Queues
709static void doregister( cluster     & cltr ) {
710        lock      ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
711        push_front( __cfa_dbg_global_clusters.list, cltr );
712        unlock    ( __cfa_dbg_global_clusters.lock );
713}
714
715static void unregister( cluster     & cltr ) {
716        lock  ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
717        remove( __cfa_dbg_global_clusters.list, cltr );
718        unlock( __cfa_dbg_global_clusters.lock );
719}
720
721void doregister( cluster * cltr, thread$ & thrd ) {
722        lock      (cltr->thread_list_lock __cfaabi_dbg_ctx2);
723        cltr->nthreads += 1;
724        push_front(cltr->threads, thrd);
725        unlock    (cltr->thread_list_lock);
726}
727
728void unregister( cluster * cltr, thread$ & thrd ) {
729        lock  (cltr->thread_list_lock __cfaabi_dbg_ctx2);
730        remove(cltr->threads, thrd );
731        cltr->nthreads -= 1;
732        unlock(cltr->thread_list_lock);
733}
734
735static void register_tls( processor * this ) {
736        // Register and Lock the RWlock so no-one pushes/pops while we are changing the queue
737        uint_fast32_t last_size;
738        [this->unique_id, last_size] = ready_mutate_register();
739
740                this->rdq.cpu = __kernel_getcpu();
741
742                this->cltr->procs.total += 1u;
743                insert_last(this->cltr->procs.actives, *this);
744
745                // Adjust the ready queue size
746                ready_queue_grow( this->cltr );
747
748        // Unlock the RWlock
749        ready_mutate_unlock( last_size );
750}
751
752
753static void unregister_tls( processor * this ) {
754        // Lock the RWlock so no-one pushes/pops while we are changing the queue
755        uint_fast32_t last_size = ready_mutate_lock();
756                this->cltr->procs.total -= 1u;
757                remove(*this);
758
759                // clear the cluster so nothing gets pushed to local queues
760                cluster * cltr = this->cltr;
761                this->cltr = 0p;
762
763                // Adjust the ready queue size
764                ready_queue_shrink( cltr );
765
766        // Unlock the RWlock and unregister: we don't need the read_lock any more
767        ready_mutate_unregister( this->unique_id, last_size );
768}
769
770static void check( int ret, const char func[] ) {
771        if ( ret ) {                                                                            // pthread routines return errno values
772                abort( "%s : internal error, error(%d) %s.", func, ret, strerror( ret ) );
773        } // if
774} // Abort
775
776void * __create_pthread( pthread_t * pthread, void * (*start)(void *), void * arg ) {
777        pthread_attr_t attr;
778
779        check( pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
780
781        size_t stacksize = max( PTHREAD_STACK_MIN, DEFAULT_STACK_SIZE );
782
783        void * stack;
784        #if CFA_PROCESSOR_USE_MMAP
785                stacksize = ceiling( stacksize, __page_size ) + __page_size;
786                stack = mmap(0p, stacksize, __map_prot, MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
787                if(stack == ((void*)-1)) {
788                        abort( "pthread stack creation : internal error, mmap failure, error(%d) %s.", errno, strerror( errno ) );
789                }
790                if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
791                        abort( "pthread stack creation : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
792                } // if
793        #else
794                __cfaabi_dbg_debug_do(
795                        stack = memalign( __page_size, stacksize + __page_size );
796                        // pthread has no mechanism to create the guard page in user supplied stack.
797                        if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
798                                abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
799                        } // if
800                );
801                __cfaabi_dbg_no_debug_do(
802                        stack = malloc( stacksize );
803                );
804        #endif
805
806        check( pthread_attr_setstack( &attr, stack, stacksize ), "pthread_attr_setstack" );
807        check( pthread_create( pthread, &attr, start, arg ), "pthread_create" );
808        return stack;
809}
810
811void __destroy_pthread( pthread_t pthread, void * stack, void ** retval ) {
812        int err = pthread_join( pthread, retval );
813        if( err != 0 ) abort("KERNEL ERROR: joining pthread %p caused error %s\n", (void*)pthread, strerror(err));
814
815        #if CFA_PROCESSOR_USE_MMAP
816                pthread_attr_t attr;
817
818                check( pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
819
820                size_t stacksize;
821                // default stack size, normally defined by shell limit
822                check( pthread_attr_getstacksize( &attr, &stacksize ), "pthread_attr_getstacksize" );
823                assert( stacksize >= PTHREAD_STACK_MIN );
824                stacksize += __page_size;
825
826                if(munmap(stack, stacksize) == -1) {
827                        abort( "pthread stack destruction : internal error, munmap failure, error(%d) %s.", errno, strerror( errno ) );
828                }
829        #else
830                __cfaabi_dbg_debug_do(
831                        // pthread has no mechanism to create the guard page in user supplied stack.
832                        if ( mprotect( stack, __page_size, __map_prot ) == -1 ) {
833                                abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
834                        } // if
835                );
836                free( stack );
837        #endif
838}
839
840#if defined(__CFA_WITH_VERIFY__)
841static bool verify_fwd_bck_rng(void) {
842        __cfaabi_tls.ready_rng.fwd_seed = 25214903917_l64u * (rdtscl() ^ (uintptr_t)&verify_fwd_bck_rng);
843
844        unsigned values[10];
845        for(i; 10) {
846                values[i] = __tls_rand_fwd();
847        }
848
849        __tls_rand_advance_bck();
850
851        for ( i; 9 -~= 0 ) {
852                if(values[i] != __tls_rand_bck()) {
853                        return false;
854                }
855        }
856
857        return true;
858}
859#endif
Note: See TracBrowser for help on using the repository browser.