source: libcfa/src/concurrency/kernel/startup.cfa @ 43784ac

jacob/cs343-translationnew-ast-unique-expr
Last change on this file since 43784ac was 43784ac, checked in by Thierry Delisle <tdelisle@…>, 4 months ago

Changed libcfathread to consistently define _GNU_SOURCE

  • Property mode set to 100644
File size: 24.2 KB
RevLine 
[e660761]1//
2// Cforall Version 1.0.0 Copyright (C) 2020 University of Waterloo
3//
4// The contents of this file are covered under the licence agreement in the
5// file "LICENCE" distributed with Cforall.
6//
7// kernel/startup.cfa --
8//
9// Author           : Thierry Delisle
10// Created On       : Thu Jul 30 15:12:54 2020
11// Last Modified By :
12// Last Modified On :
13// Update Count     :
14//
15
16#define __cforall_thread__
[43784ac]17#define _GNU_SOURCE
[e660761]18
19// C Includes
20#include <errno.h>              // errno
21#include <string.h>             // strerror
22#include <unistd.h>             // sysconf
23extern "C" {
24      #include <limits.h>       // PTHREAD_STACK_MIN
[dddb3dd0]25        #include <sys/eventfd.h>  // eventfd
[e660761]26      #include <sys/mman.h>     // mprotect
27      #include <sys/resource.h> // getrlimit
28}
29
30// CFA Includes
31#include "kernel_private.hfa"
32#include "startup.hfa"          // STARTUP_PRIORITY_XXX
[bfcf6b9]33#include "math.hfa"
[e660761]34
[97229d6]35#define CFA_PROCESSOR_USE_MMAP 0
36
[e660761]37//-----------------------------------------------------------------------------
38// Some assembly required
39#if defined( __i386 )
[88cafe7]40        #define CtxGet( ctx ) __asm__ volatile ( \
41                "movl %%esp,%0\n" \
42                "movl %%ebp,%1\n" \
43                : "=rm" (ctx.SP), \
44                  "=rm" (ctx.FP) \
45        )
[e660761]46#elif defined( __x86_64 )
[88cafe7]47        #define CtxGet( ctx ) __asm__ volatile ( \
48                "movq %%rsp,%0\n" \
49                "movq %%rbp,%1\n" \
50                : "=rm" (ctx.SP), \
51                  "=rm" (ctx.FP) \
52        )
53#elif defined( __aarch64__ )
54        #define CtxGet( ctx ) __asm__ volatile ( \
55                "mov %0, sp\n" \
56                "mov %1, fp\n" \
57                : "=rm" (ctx.SP), \
58                  "=rm" (ctx.FP) \
59        )
[e660761]60#else
61        #error unknown hardware architecture
62#endif
63
64//-----------------------------------------------------------------------------
65// Start and stop routine for the kernel, declared first to make sure they run first
66static void __kernel_startup (void) __attribute__(( constructor( STARTUP_PRIORITY_KERNEL ) ));
67static void __kernel_shutdown(void) __attribute__(( destructor ( STARTUP_PRIORITY_KERNEL ) ));
68
69//-----------------------------------------------------------------------------
70// Static Forward Declarations
71struct current_stack_info_t;
72
73static void * __invoke_processor(void * arg);
74static void __kernel_first_resume( processor * this );
75static void __kernel_last_resume ( processor * this );
[a5e7233]76static void init(processor & this, const char name[], cluster & _cltr, $thread * initT);
[e660761]77static void deinit(processor & this);
78static void doregister( struct cluster & cltr );
79static void unregister( struct cluster & cltr );
[c993b15]80static void register_tls( processor * this );
81static void unregister_tls( processor * this );
[e660761]82static void ?{}( $coroutine & this, current_stack_info_t * info);
83static void ?{}( $thread & this, current_stack_info_t * info);
84static void ?{}(processorCtx_t & this) {}
85static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info);
86
[f2384c9a]87#if defined(__CFA_WITH_VERIFY__)
88        static bool verify_fwd_bck_rng(void);
89#endif
90
[e660761]91//-----------------------------------------------------------------------------
92// Forward Declarations for other modules
93extern void __kernel_alarm_startup(void);
94extern void __kernel_alarm_shutdown(void);
95
96//-----------------------------------------------------------------------------
97// Other Forward Declarations
[1eb239e4]98extern void __wake_proc(processor *);
[e660761]99
100//-----------------------------------------------------------------------------
101// Kernel storage
102KERNEL_STORAGE(cluster,              mainCluster);
103KERNEL_STORAGE(processor,            mainProcessor);
104KERNEL_STORAGE($thread,              mainThread);
105KERNEL_STORAGE(__stack_t,            mainThreadCtx);
106KERNEL_STORAGE(__scheduler_RWLock_t, __scheduler_lock);
107#if !defined(__CFA_NO_STATISTICS__)
108KERNEL_STORAGE(__stats_t, mainProcStats);
109#endif
110
111cluster              * mainCluster;
112processor            * mainProcessor;
113$thread              * mainThread;
114__scheduler_RWLock_t * __scheduler_lock;
115
116extern "C" {
117        struct { __dllist_t(cluster) list; __spinlock_t lock; } __cfa_dbg_global_clusters;
118}
119
[dd92fe9]120extern size_t __page_size;
[28c35e2]121extern int __map_prot;
[e660761]122
123//-----------------------------------------------------------------------------
124// Global state
[8fc652e0]125thread_local struct KernelThreadData __cfaabi_tls __attribute__ ((tls_model ( "initial-exec" ))) @= {
[e660761]126        NULL,                                                                                           // cannot use 0p
127        NULL,
[c993b15]128        false,
[e660761]129        { 1, false, false },
[c993b15]130        0,
131        { 0, 0 },
132        NULL,
133        #ifdef __CFA_WITH_VERIFY__
134                false,
135                0,
136        #endif
[e660761]137};
138
139//-----------------------------------------------------------------------------
140// Struct to steal stack
141struct current_stack_info_t {
142        __stack_t * storage;  // pointer to stack object
143        void * base;          // base of stack
144        void * limit;         // stack grows towards stack limit
145        void * context;       // address of cfa_context_t
146};
147
148void ?{}( current_stack_info_t & this ) {
149        __stack_context_t ctx;
150        CtxGet( ctx );
151        this.base = ctx.FP;
152
153        rlimit r;
154        getrlimit( RLIMIT_STACK, &r);
155        size_t size = r.rlim_cur;
156
157        this.limit = (void *)(((intptr_t)this.base) - size);
158        this.context = &storage_mainThreadCtx;
159}
160
161
162
163//=============================================================================================
164// Kernel Setup logic
165//=============================================================================================
166//-----------------------------------------------------------------------------
167// Kernel boot procedures
168static void __kernel_startup(void) {
[8fc652e0]169        /* paranoid */ verify( ! __preemption_enabled() );
[e660761]170        __cfadbg_print_safe(runtime_core, "Kernel : Starting\n");
171
172        __cfa_dbg_global_clusters.list{ __get };
173        __cfa_dbg_global_clusters.lock{};
174
[f2384c9a]175        /* paranoid */ verify( verify_fwd_bck_rng() );
176
[e660761]177        // Initialize the global scheduler lock
178        __scheduler_lock = (__scheduler_RWLock_t*)&storage___scheduler_lock;
179        (*__scheduler_lock){};
180
181        // Initialize the main cluster
182        mainCluster = (cluster *)&storage_mainCluster;
183        (*mainCluster){"Main Cluster", 0};
184
185        __cfadbg_print_safe(runtime_core, "Kernel : Main cluster ready\n");
186
187        // Start by initializing the main thread
188        // SKULLDUGGERY: the mainThread steals the process main thread
189        // which will then be scheduled by the mainProcessor normally
190        mainThread = ($thread *)&storage_mainThread;
191        current_stack_info_t info;
192        info.storage = (__stack_t*)&storage_mainThreadCtx;
193        (*mainThread){ &info };
194
195        __cfadbg_print_safe(runtime_core, "Kernel : Main thread ready\n");
196
197
198
199        // Construct the processor context of the main processor
200        void ?{}(processorCtx_t & this, processor * proc) {
201                (this.__cor){ "Processor" };
202                this.__cor.starter = 0p;
203                this.proc = proc;
204        }
205
206        void ?{}(processor & this) with( this ) {
[454f478]207                ( this.terminated ){};
[e660761]208                ( this.runner ){};
[a5e7233]209                init( this, "Main Processor", *mainCluster, 0p );
[e660761]210                kernel_thread = pthread_self();
211
212                runner{ &this };
213                __cfadbg_print_safe(runtime_core, "Kernel : constructed main processor context %p\n", &runner);
214        }
215
216        // Initialize the main processor and the main processor ctx
217        // (the coroutine that contains the processing control flow)
218        mainProcessor = (processor *)&storage_mainProcessor;
219        (*mainProcessor){};
220
[c993b15]221        register_tls( mainProcessor );
222
[e660761]223        //initialize the global state variables
[8fc652e0]224        __cfaabi_tls.this_processor = mainProcessor;
225        __cfaabi_tls.this_thread    = mainThread;
[e660761]226
227        #if !defined( __CFA_NO_STATISTICS__ )
[8fc652e0]228                __cfaabi_tls.this_stats = (__stats_t *)& storage_mainProcStats;
229                __init_stats( __cfaabi_tls.this_stats );
[e660761]230        #endif
[a67c5b6]231        mainProcessor->local_data = &__cfaabi_tls;
[e660761]232
233        // Enable preemption
234        __kernel_alarm_startup();
235
236        // Add the main thread to the ready queue
237        // once resume is called on mainProcessor->runner the mainThread needs to be scheduled like any normal thread
[254ad1b]238        schedule_thread$(mainThread);
[e660761]239
240        // SKULLDUGGERY: Force a context switch to the main processor to set the main thread's context to the current UNIX
241        // context. Hence, the main thread does not begin through __cfactx_invoke_thread, like all other threads. The trick here is that
242        // mainThread is on the ready queue when this call is made.
[8fc652e0]243        __kernel_first_resume( __cfaabi_tls.this_processor );
[e660761]244
245
246        // THE SYSTEM IS NOW COMPLETELY RUNNING
247
248        __cfadbg_print_safe(runtime_core, "Kernel : Started\n--------------------------------------------------\n\n");
249
[8fc652e0]250        /* paranoid */ verify( ! __preemption_enabled() );
[a3821fa]251        enable_interrupts();
[8fc652e0]252        /* paranoid */ verify( __preemption_enabled() );
253
[e660761]254}
255
256static void __kernel_shutdown(void) {
[8fc652e0]257        /* paranoid */ verify( __preemption_enabled() );
[e660761]258        disable_interrupts();
[8fc652e0]259        /* paranoid */ verify( ! __preemption_enabled() );
[e660761]260
261        __cfadbg_print_safe(runtime_core, "\n--------------------------------------------------\nKernel : Shutting down\n");
262
263        // SKULLDUGGERY: Notify the mainProcessor it needs to terminates.
264        // When its coroutine terminates, it return control to the mainThread
265        // which is currently here
266        __atomic_store_n(&mainProcessor->do_terminate, true, __ATOMIC_RELEASE);
[8fc652e0]267        __kernel_last_resume( __cfaabi_tls.this_processor );
[e660761]268        mainThread->self_cor.state = Halted;
269
270        // THE SYSTEM IS NOW COMPLETELY STOPPED
271
272        // Disable preemption
273        __kernel_alarm_shutdown();
274
[a5a01fa]275        #if !defined( __CFA_NO_STATISTICS__ )
276                __stats_t * st = (__stats_t *)& storage_mainProcStats;
277                __tally_stats(mainCluster->stats, st);
278                if( 0 != mainProcessor->print_stats ) {
279                        __print_stats( st, mainProcessor->print_stats, "Processor ", mainProcessor->name, (void*)mainProcessor );
280                }
[73f4d08]281                #if defined(CFA_STATS_ARRAY)
282                        __flush_stat( st, "Processor", mainProcessor );
283                #endif
[a5a01fa]284        #endif
285
[a67c5b6]286        mainProcessor->local_data = 0p;
287
[c993b15]288        unregister_tls( mainProcessor );
289
[e660761]290        // Destroy the main processor and its context in reverse order of construction
291        // These were manually constructed so we need manually destroy them
292        void ^?{}(processor & this) with( this ){
293                deinit( this );
294
295                /* paranoid */ verify( this.do_terminate == true );
296                __cfaabi_dbg_print_safe("Kernel : destroyed main processor context %p\n", &runner);
297        }
298
299        ^(*mainProcessor){};
300
301        // Final step, destroy the main thread since it is no longer needed
302
303        // Since we provided a stack to this taxk it will not destroy anything
304        /* paranoid */ verify(mainThread->self_cor.stack.storage == (__stack_t*)(((uintptr_t)&storage_mainThreadCtx)| 0x1));
305        ^(*mainThread){};
306
307        ^(*mainCluster){};
308
309        ^(*__scheduler_lock){};
310
311        ^(__cfa_dbg_global_clusters.list){};
312        ^(__cfa_dbg_global_clusters.lock){};
313
314        __cfadbg_print_safe(runtime_core, "Kernel : Shutdown complete\n");
315}
316
317//=============================================================================================
318// Kernel Initial Scheduling logic
319//=============================================================================================
320
321// Context invoker for processors
322// This is the entry point for processors (kernel threads) *except* for the main processor
323// It effectively constructs a coroutine by stealing the pthread stack
324static void * __invoke_processor(void * arg) {
325        #if !defined( __CFA_NO_STATISTICS__ )
326                __stats_t local_stats;
327                __init_stats( &local_stats );
[8fc652e0]328                __cfaabi_tls.this_stats = &local_stats;
[e660761]329        #endif
330
331        processor * proc = (processor *) arg;
[8fc652e0]332        __cfaabi_tls.this_processor = proc;
333        __cfaabi_tls.this_thread    = 0p;
334        __cfaabi_tls.preemption_state.[enabled, disable_count] = [false, 1];
[a67c5b6]335        proc->local_data = &__cfaabi_tls;
[c993b15]336
337        register_tls( proc );
338
[43784ac]339        #if defined(CFA_HAVE_LINUX_LIBRSEQ)
340                rseq_register_current_thread();
341        #endif
342
[e660761]343        // SKULLDUGGERY: We want to create a context for the processor coroutine
344        // which is needed for the 2-step context switch. However, there is no reason
345        // to waste the perfectly valid stack create by pthread.
346        current_stack_info_t info;
347        __stack_t ctx;
348        info.storage = &ctx;
349        (proc->runner){ proc, &info };
350
351        __cfaabi_dbg_print_safe("Coroutine : created stack %p\n", get_coroutine(proc->runner)->stack.storage);
352
353        //Set global state
[8fc652e0]354        __cfaabi_tls.this_thread = 0p;
[e660761]355
356        //We now have a proper context from which to schedule threads
357        __cfadbg_print_safe(runtime_core, "Kernel : core %p created (%p, %p)\n", proc, &proc->runner, &ctx);
358
359        // SKULLDUGGERY: Since the coroutine doesn't have its own stack, we can't
360        // resume it to start it like it normally would, it will just context switch
361        // back to here. Instead directly call the main since we already are on the
362        // appropriate stack.
363        get_coroutine(proc->runner)->state = Active;
364        main( proc->runner );
365        get_coroutine(proc->runner)->state = Halted;
366
367        // Main routine of the core returned, the core is now fully terminated
368        __cfadbg_print_safe(runtime_core, "Kernel : core %p main ended (%p)\n", proc, &proc->runner);
369
370        #if !defined(__CFA_NO_STATISTICS__)
371                __tally_stats(proc->cltr->stats, &local_stats);
372                if( 0 != proc->print_stats ) {
[1b033b8]373                        __print_stats( &local_stats, proc->print_stats, "Processor ", proc->name, (void*)proc );
[e660761]374                }
[73f4d08]375                #if defined(CFA_STATS_ARRAY)
376                        __flush_stat( &local_stats, "Processor", proc );
377                #endif
[e660761]378        #endif
379
[a67c5b6]380        proc->local_data = 0p;
381
[c993b15]382        unregister_tls( proc );
383
[e660761]384        return 0p;
385}
386
387static void __kernel_first_resume( processor * this ) {
388        $thread * src = mainThread;
389        $coroutine * dst = get_coroutine(this->runner);
390
[8fc652e0]391        /* paranoid */ verify( ! __preemption_enabled() );
[e660761]392
[8fc652e0]393        __cfaabi_tls.this_thread->curr_cor = dst;
[e660761]394        __stack_prepare( &dst->stack, 65000 );
395        __cfactx_start(main, dst, this->runner, __cfactx_invoke_coroutine);
396
[8fc652e0]397        /* paranoid */ verify( ! __preemption_enabled() );
[e660761]398
399        dst->last = &src->self_cor;
400        dst->starter = dst->starter ? dst->starter : &src->self_cor;
401
402        // make sure the current state is still correct
403        /* paranoid */ verify(src->state == Ready);
[ab5baab]404        src->corctx_flag = true;
[e660761]405
406        // context switch to specified coroutine
407        verify( dst->context.SP );
408        __cfactx_switch( &src->context, &dst->context );
409        // when __cfactx_switch returns we are back in the src coroutine
410
411        mainThread->curr_cor = &mainThread->self_cor;
412
413        // make sure the current state has been update
414        /* paranoid */ verify(src->state == Active);
415
[8fc652e0]416        /* paranoid */ verify( ! __preemption_enabled() );
[e660761]417}
418
419// KERNEL_ONLY
420static void __kernel_last_resume( processor * this ) {
421        $coroutine * src = &mainThread->self_cor;
422        $coroutine * dst = get_coroutine(this->runner);
423
[8fc652e0]424        /* paranoid */ verify( ! __preemption_enabled() );
425        /* paranoid */ verify( dst->starter == src );
426        /* paranoid */ verify( dst->context.SP );
[e660761]427
428        // SKULLDUGGERY in debug the processors check that the
429        // stack is still within the limit of the stack limits after running a thread.
430        // that check doesn't make sense if we context switch to the processor using the
431        // coroutine semantics. Since this is a special case, use the current context
432        // info to populate these fields.
433        __cfaabi_dbg_debug_do(
434                __stack_context_t ctx;
435                CtxGet( ctx );
436                mainThread->context.SP = ctx.SP;
437                mainThread->context.FP = ctx.FP;
438        )
439
440        // context switch to the processor
441        __cfactx_switch( &src->context, &dst->context );
442}
443
444
445//=============================================================================================
446// Kernel Object Constructors logic
447//=============================================================================================
448//-----------------------------------------------------------------------------
449// Main thread construction
450static void ?{}( $coroutine & this, current_stack_info_t * info) with( this ) {
451        stack.storage = info->storage;
452        with(*stack.storage) {
453                limit     = info->limit;
454                base      = info->base;
455        }
456        __attribute__((may_alias)) intptr_t * istorage = (intptr_t*) &stack.storage;
457        *istorage |= 0x1;
458        name = "Main Thread";
459        state = Start;
460        starter = 0p;
461        last = 0p;
462        cancellation = 0p;
463}
464
465static void ?{}( $thread & this, current_stack_info_t * info) with( this ) {
[6a77224]466        ticket = TICKET_RUNNING;
[e660761]467        state = Start;
468        self_cor{ info };
469        curr_cor = &self_cor;
470        curr_cluster = mainCluster;
471        self_mon.owner = &this;
472        self_mon.recursion = 1;
473        self_mon_p = &self_mon;
474        link.next = 0p;
[2b96031]475        link.ts   = 0;
[d3ba775]476        preferred = -1u;
[89eff25]477        last_proc = 0p;
[b4b63e8]478        #if defined( __CFA_WITH_VERIFY__ )
[ac12f1f]479                canary = 0x0D15EA5E0D15EA5Ep;
[b4b63e8]480        #endif
[e660761]481
482        node.next = 0p;
483        node.prev = 0p;
484        doregister(curr_cluster, this);
485
486        monitors{ &self_mon_p, 1, (fptr_t)0 };
487}
488
489//-----------------------------------------------------------------------------
490// Processor
491// Construct the processor context of non-main processors
492static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info) {
493        (this.__cor){ info };
494        this.proc = proc;
495}
496
[a5e7233]497static void init(processor & this, const char name[], cluster & _cltr, $thread * initT) with( this ) {
[e660761]498        this.name = name;
499        this.cltr = &_cltr;
[431cd4f]500        this.rdq.its = 0;
501        this.rdq.itr = 0;
502        this.rdq.id  = -1u;
503        this.rdq.target = -1u;
[3e1a705]504        this.rdq.cutoff = 0ull;
[e660761]505        do_terminate = false;
506        preemption_alarm = 0p;
507        pending_preemption = false;
508
[78da4ab]509        this.io.ctx = 0p;
[dddb3dd0]510        this.io.pending = false;
511        this.io.dirty   = false;
512
[a5e7233]513        this.init.thrd = initT;
[a1538cd]514
[a67c5b6]515        this.local_data = 0p;
516
[dddb3dd0]517        this.idle = eventfd(0, 0);
518        if (idle < 0) {
519                abort("KERNEL ERROR: PROCESSOR EVENTFD - %s\n", strerror(errno));
520        }
[78da4ab]521
[e660761]522        #if !defined(__CFA_NO_STATISTICS__)
523                print_stats = 0;
524                print_halts = false;
525        #endif
526
527        __cfadbg_print_safe(runtime_core, "Kernel : core %p created\n", &this);
528}
529
530// Not a ctor, it just preps the destruction but should not destroy members
531static void deinit(processor & this) {
[dddb3dd0]532        close(this.idle);
[e660761]533}
534
[a5e7233]535void ?{}(processor & this, const char name[], cluster & _cltr, $thread * initT) {
[454f478]536        ( this.terminated ){};
[e660761]537        ( this.runner ){};
[62502cc4]538
539        disable_interrupts();
[a5e7233]540                init( this, name, _cltr, initT );
[a3821fa]541        enable_interrupts();
[e660761]542
543        __cfadbg_print_safe(runtime_core, "Kernel : Starting core %p\n", &this);
544
545        this.stack = __create_pthread( &this.kernel_thread, __invoke_processor, (void *)&this );
[a5e7233]546}
[e660761]547
[a5e7233]548void ?{}(processor & this, const char name[], cluster & _cltr) {
549        (this){name, _cltr, 0p};
[e660761]550}
551
[bfcf6b9]552extern size_t __page_size;
[e660761]553void ^?{}(processor & this) with( this ){
554        if( ! __atomic_load_n(&do_terminate, __ATOMIC_ACQUIRE) ) {
555                __cfadbg_print_safe(runtime_core, "Kernel : core %p signaling termination\n", &this);
556
557                __atomic_store_n(&do_terminate, true, __ATOMIC_RELAXED);
558                __wake_proc( &this );
559
[454f478]560                wait( terminated );
[8fc652e0]561                /* paranoid */ verify( active_processor() != &this);
[e660761]562        }
563
[bfcf6b9]564        __destroy_pthread( kernel_thread, this.stack, 0p );
[e660761]565
[62502cc4]566        disable_interrupts();
567                deinit( this );
[a3821fa]568        enable_interrupts();
[e660761]569}
570
571//-----------------------------------------------------------------------------
572// Cluster
[6a9b12b]573static void ?{}(__cluster_proc_list & this) {
[1eb239e4]574        this.lock  = 0;
575        this.idle  = 0;
576        this.total = 0;
577}
578
[e660761]579void ?{}(cluster & this, const char name[], Duration preemption_rate, unsigned num_io, const io_context_params & io_params) with( this ) {
580        this.name = name;
581        this.preemption_rate = preemption_rate;
582        ready_queue{};
583
584        #if !defined(__CFA_NO_STATISTICS__)
585                print_stats = 0;
586                stats = alloc();
587                __init_stats( stats );
588        #endif
589
590        threads{ __get };
591
[78da4ab]592        io.arbiter = create();
593        io.params = io_params;
594
[e660761]595        doregister(this);
596
597        // Lock the RWlock so no-one pushes/pops while we are changing the queue
[772411a]598        disable_interrupts();
[e660761]599        uint_fast32_t last_size = ready_mutate_lock();
600
601                // Adjust the ready queue size
[a017ee7]602                ready_queue_grow( &this );
[e660761]603
604        // Unlock the RWlock
605        ready_mutate_unlock( last_size );
[a3821fa]606        enable_interrupts( false ); // Don't poll, could be in main cluster
[e660761]607}
608
609void ^?{}(cluster & this) {
[78da4ab]610        destroy(this.io.arbiter);
[e660761]611
612        // Lock the RWlock so no-one pushes/pops while we are changing the queue
[772411a]613        disable_interrupts();
[e660761]614        uint_fast32_t last_size = ready_mutate_lock();
615
616                // Adjust the ready queue size
[a017ee7]617                ready_queue_shrink( &this );
[e660761]618
619        // Unlock the RWlock
620        ready_mutate_unlock( last_size );
[a3821fa]621        enable_interrupts( false ); // Don't poll, could be in main cluster
[e660761]622
623        #if !defined(__CFA_NO_STATISTICS__)
624                if( 0 != this.print_stats ) {
[1b033b8]625                        __print_stats( this.stats, this.print_stats, "Cluster", this.name, (void*)&this );
[e660761]626                }
[73f4d08]627                #if defined(CFA_STATS_ARRAY)
628                        __flush_stat( this.stats, "Cluster", &this );
629                #endif
[e660761]630                free( this.stats );
631        #endif
632
633        unregister(this);
634}
635
636//=============================================================================================
637// Miscellaneous Initialization
638//=============================================================================================
639//-----------------------------------------------------------------------------
640// Global Queues
641static void doregister( cluster     & cltr ) {
642        lock      ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
643        push_front( __cfa_dbg_global_clusters.list, cltr );
644        unlock    ( __cfa_dbg_global_clusters.lock );
645}
646
647static void unregister( cluster     & cltr ) {
648        lock  ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
649        remove( __cfa_dbg_global_clusters.list, cltr );
650        unlock( __cfa_dbg_global_clusters.lock );
651}
652
653void doregister( cluster * cltr, $thread & thrd ) {
654        lock      (cltr->thread_list_lock __cfaabi_dbg_ctx2);
655        cltr->nthreads += 1;
656        push_front(cltr->threads, thrd);
657        unlock    (cltr->thread_list_lock);
658}
659
660void unregister( cluster * cltr, $thread & thrd ) {
661        lock  (cltr->thread_list_lock __cfaabi_dbg_ctx2);
662        remove(cltr->threads, thrd );
663        cltr->nthreads -= 1;
664        unlock(cltr->thread_list_lock);
665}
666
[c993b15]667static void register_tls( processor * this ) {
668        // Register and Lock the RWlock so no-one pushes/pops while we are changing the queue
669        uint_fast32_t last_size;
670        [this->unique_id, last_size] = ready_mutate_register();
671
672                this->cltr->procs.total += 1u;
673                insert_last(this->cltr->procs.actives, *this);
674
675                // Adjust the ready queue size
676                ready_queue_grow( this->cltr );
677
678        // Unlock the RWlock
679        ready_mutate_unlock( last_size );
680}
681
682
683static void unregister_tls( processor * this ) {
684        // Lock the RWlock so no-one pushes/pops while we are changing the queue
685        uint_fast32_t last_size = ready_mutate_lock();
686                this->cltr->procs.total -= 1u;
687                remove(*this);
688
689                // clear the cluster so nothing gets pushed to local queues
690                cluster * cltr = this->cltr;
691                this->cltr = 0p;
692
693                // Adjust the ready queue size
694                ready_queue_shrink( cltr );
695
696        // Unlock the RWlock and unregister: we don't need the read_lock any more
697        ready_mutate_unregister( this->unique_id, last_size );
698}
699
[e660761]700static void check( int ret, const char func[] ) {
701        if ( ret ) {                                                                            // pthread routines return errno values
702                abort( "%s : internal error, error(%d) %s.", func, ret, strerror( ret ) );
703        } // if
704} // Abort
705
706void * __create_pthread( pthread_t * pthread, void * (*start)(void *), void * arg ) {
707        pthread_attr_t attr;
708
709        check( pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
710
711        size_t stacksize;
712        // default stack size, normally defined by shell limit
713        check( pthread_attr_getstacksize( &attr, &stacksize ), "pthread_attr_getstacksize" );
714        assert( stacksize >= PTHREAD_STACK_MIN );
715
716        void * stack;
[97229d6]717        #if CFA_PROCESSOR_USE_MMAP
718                stacksize = ceiling( stacksize, __page_size ) + __page_size;
[dd92fe9]719                stack = mmap(0p, stacksize, __map_prot, MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
[97229d6]720                if(stack == ((void*)-1)) {
721                        abort( "pthread stack creation : internal error, mmap failure, error(%d) %s.", errno, strerror( errno ) );
722                }
723                if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
724                        abort( "pthread stack creation : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
725                } // if
726        #else
727                __cfaabi_dbg_debug_do(
728                        stack = memalign( __page_size, stacksize + __page_size );
729                        // pthread has no mechanism to create the guard page in user supplied stack.
730                        if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
731                                abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
732                        } // if
733                );
734                __cfaabi_dbg_no_debug_do(
735                        stack = malloc( stacksize );
736                );
737        #endif
738
[e660761]739
740        check( pthread_attr_setstack( &attr, stack, stacksize ), "pthread_attr_setstack" );
741
742        check( pthread_create( pthread, &attr, start, arg ), "pthread_create" );
743        return stack;
[88cafe7]744}
[f2384c9a]745
[bfcf6b9]746void __destroy_pthread( pthread_t pthread, void * stack, void ** retval ) {
747        int err = pthread_join( pthread, retval );
748        if( err != 0 ) abort("KERNEL ERROR: joining pthread %p caused error %s\n", (void*)pthread, strerror(err));
749
[97229d6]750        #if CFA_PROCESSOR_USE_MMAP
751                pthread_attr_t attr;
[bfcf6b9]752
[97229d6]753                check( pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
[bfcf6b9]754
[97229d6]755                size_t stacksize;
756                // default stack size, normally defined by shell limit
757                check( pthread_attr_getstacksize( &attr, &stacksize ), "pthread_attr_getstacksize" );
758                assert( stacksize >= PTHREAD_STACK_MIN );
759                stacksize += __page_size;
[bfcf6b9]760
[97229d6]761                if(munmap(stack, stacksize) == -1) {
762                        abort( "pthread stack destruction : internal error, munmap failure, error(%d) %s.", errno, strerror( errno ) );
763                }
764        #else
[72a3aff]765                __cfaabi_dbg_debug_do(
766                        // pthread has no mechanism to create the guard page in user supplied stack.
[28c35e2]767                        if ( mprotect( stack, __page_size, __map_prot ) == -1 ) {
[72a3aff]768                                abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
769                        } // if
770                );
[97229d6]771                free( stack );
772        #endif
[bfcf6b9]773}
774
[f2384c9a]775#if defined(__CFA_WITH_VERIFY__)
776static bool verify_fwd_bck_rng(void) {
[8fc652e0]777        __cfaabi_tls.ready_rng.fwd_seed = 25214903917_l64u * (rdtscl() ^ (uintptr_t)&verify_fwd_bck_rng);
[f2384c9a]778
779        unsigned values[10];
780        for(i; 10) {
781                values[i] = __tls_rand_fwd();
782        }
783
784        __tls_rand_advance_bck();
785
786        for ( i; 9 -~= 0 ) {
787                if(values[i] != __tls_rand_bck()) {
788                        return false;
789                }
790        }
791
792        return true;
793}
[e67a82d]794#endif
Note: See TracBrowser for help on using the repository browser.