source: libcfa/src/concurrency/kernel/startup.cfa @ a3769cc

ADTast-experimentalenumforall-pointer-decaypthread-emulationqualifiedEnum
Last change on this file since a3769cc was d874f59, checked in by Thierry Delisle <tdelisle@…>, 3 years ago

Fixed crash from get_cpu

  • Property mode set to 100644
File size: 24.6 KB
Line 
1//
2// Cforall Version 1.0.0 Copyright (C) 2020 University of Waterloo
3//
4// The contents of this file are covered under the licence agreement in the
5// file "LICENCE" distributed with Cforall.
6//
7// kernel/startup.cfa --
8//
9// Author           : Thierry Delisle
10// Created On       : Thu Jul 30 15:12:54 2020
11// Last Modified By :
12// Last Modified On :
13// Update Count     :
14//
15
16#define __cforall_thread__
17#define _GNU_SOURCE
18
19// C Includes
20#include <errno.h>              // errno
21#include <signal.h>
22#include <string.h>             // strerror
23#include <unistd.h>             // sysconf
24
25extern "C" {
26      #include <limits.h>       // PTHREAD_STACK_MIN
27        #include <unistd.h>       // syscall
28        #include <sys/eventfd.h>  // eventfd
29      #include <sys/mman.h>     // mprotect
30      #include <sys/resource.h> // getrlimit
31}
32
33// CFA Includes
34#include "kernel_private.hfa"
35#include "startup.hfa"          // STARTUP_PRIORITY_XXX
36#include "math.hfa"
37
38#define CFA_PROCESSOR_USE_MMAP 0
39
40//-----------------------------------------------------------------------------
41// Some assembly required
42#if defined( __i386 )
43        #define CtxGet( ctx ) __asm__ volatile ( \
44                "movl %%esp,%0\n" \
45                "movl %%ebp,%1\n" \
46                : "=rm" (ctx.SP), \
47                  "=rm" (ctx.FP) \
48        )
49#elif defined( __x86_64 )
50        #define CtxGet( ctx ) __asm__ volatile ( \
51                "movq %%rsp,%0\n" \
52                "movq %%rbp,%1\n" \
53                : "=rm" (ctx.SP), \
54                  "=rm" (ctx.FP) \
55        )
56#elif defined( __aarch64__ )
57        #define CtxGet( ctx ) __asm__ volatile ( \
58                "mov %0, sp\n" \
59                "mov %1, fp\n" \
60                : "=rm" (ctx.SP), \
61                  "=rm" (ctx.FP) \
62        )
63#else
64        #error unknown hardware architecture
65#endif
66
67//-----------------------------------------------------------------------------
68// Start and stop routine for the kernel, declared first to make sure they run first
69static void __kernel_startup (void) __attribute__(( constructor( STARTUP_PRIORITY_KERNEL ) ));
70static void __kernel_shutdown(void) __attribute__(( destructor ( STARTUP_PRIORITY_KERNEL ) ));
71
72//-----------------------------------------------------------------------------
73// Static Forward Declarations
74struct current_stack_info_t;
75
76static void * __invoke_processor(void * arg);
77static void __kernel_first_resume( processor * this );
78static void __kernel_last_resume ( processor * this );
79static void init(processor & this, const char name[], cluster & _cltr, thread$ * initT);
80static void deinit(processor & this);
81static void doregister( struct cluster & cltr );
82static void unregister( struct cluster & cltr );
83static void register_tls( processor * this );
84static void unregister_tls( processor * this );
85static void ?{}( coroutine$ & this, current_stack_info_t * info);
86static void ?{}( thread$ & this, current_stack_info_t * info);
87static void ?{}(processorCtx_t & this) {}
88static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info);
89
90#if defined(__CFA_WITH_VERIFY__)
91        static bool verify_fwd_bck_rng(void);
92#endif
93
94//-----------------------------------------------------------------------------
95// Forward Declarations for other modules
96extern void __kernel_alarm_startup(void);
97extern void __kernel_alarm_shutdown(void);
98
99//-----------------------------------------------------------------------------
100// Other Forward Declarations
101extern void __wake_proc(processor *);
102
103//-----------------------------------------------------------------------------
104// Kernel storage
105KERNEL_STORAGE(cluster,              mainCluster);
106KERNEL_STORAGE(processor,            mainProcessor);
107KERNEL_STORAGE(thread$,              mainThread);
108KERNEL_STORAGE(__stack_t,            mainThreadCtx);
109KERNEL_STORAGE(__scheduler_RWLock_t, __scheduler_lock);
110#if !defined(__CFA_NO_STATISTICS__)
111KERNEL_STORAGE(__stats_t, mainProcStats);
112#endif
113
114cluster              * mainCluster;
115processor            * mainProcessor;
116thread$              * mainThread;
117__scheduler_RWLock_t * __scheduler_lock;
118
119extern "C" {
120        struct { __dllist_t(cluster) list; __spinlock_t lock; } __cfa_dbg_global_clusters;
121}
122
123extern size_t __page_size;
124extern int __map_prot;
125
126//-----------------------------------------------------------------------------
127// Global state
128thread_local struct KernelThreadData __cfaabi_tls __attribute__ ((tls_model ( "initial-exec" ))) @= {
129        NULL,                                                                                           // cannot use 0p
130        NULL,
131        false,
132        { 1, false, false },
133        0,
134        { 0, 0 },
135        NULL,
136        #ifdef __CFA_WITH_VERIFY__
137                false,
138                0,
139        #endif
140};
141
142#if   defined(CFA_HAVE_LINUX_LIBRSEQ)
143        // No data needed
144#elif defined(CFA_HAVE_LINUX_RSEQ_H)
145        extern "Cforall" {
146                __attribute__((aligned(128))) thread_local volatile struct rseq __cfaabi_rseq @= {
147                        .cpu_id : RSEQ_CPU_ID_UNINITIALIZED,
148                };
149        }
150#else
151        // No data needed
152#endif
153
154//-----------------------------------------------------------------------------
155// Struct to steal stack
156struct current_stack_info_t {
157        __stack_t * storage;  // pointer to stack object
158        void * base;          // base of stack
159        void * limit;         // stack grows towards stack limit
160        void * context;       // address of cfa_context_t
161};
162
163void ?{}( current_stack_info_t & this ) {
164        __stack_context_t ctx;
165        CtxGet( ctx );
166        this.base = ctx.FP;
167
168        rlimit r;
169        getrlimit( RLIMIT_STACK, &r);
170        size_t size = r.rlim_cur;
171
172        this.limit = (void *)(((intptr_t)this.base) - size);
173        this.context = &storage_mainThreadCtx;
174}
175
176
177
178//=============================================================================================
179// Kernel Setup logic
180//=============================================================================================
181//-----------------------------------------------------------------------------
182// Kernel boot procedures
183static void __kernel_startup(void) {
184        /* paranoid */ verify( ! __preemption_enabled() );
185        __cfadbg_print_safe(runtime_core, "Kernel : Starting\n");
186
187        __cfa_dbg_global_clusters.list{ __get };
188        __cfa_dbg_global_clusters.lock{};
189
190        /* paranoid */ verify( verify_fwd_bck_rng() );
191
192        // Initialize the global scheduler lock
193        __scheduler_lock = (__scheduler_RWLock_t*)&storage___scheduler_lock;
194        (*__scheduler_lock){};
195
196        // Initialize the main cluster
197        mainCluster = (cluster *)&storage_mainCluster;
198        (*mainCluster){"Main Cluster", 0};
199
200        __cfadbg_print_safe(runtime_core, "Kernel : Main cluster ready\n");
201
202        // Start by initializing the main thread
203        // SKULLDUGGERY: the mainThread steals the process main thread
204        // which will then be scheduled by the mainProcessor normally
205        mainThread = (thread$ *)&storage_mainThread;
206        current_stack_info_t info;
207        info.storage = (__stack_t*)&storage_mainThreadCtx;
208        (*mainThread){ &info };
209
210        __cfadbg_print_safe(runtime_core, "Kernel : Main thread ready\n");
211
212
213
214        // Construct the processor context of the main processor
215        void ?{}(processorCtx_t & this, processor * proc) {
216                (this.__cor){ "Processor" };
217                this.__cor.starter = 0p;
218                this.proc = proc;
219        }
220
221        void ?{}(processor & this) with( this ) {
222                ( this.terminated ){};
223                ( this.runner ){};
224                init( this, "Main Processor", *mainCluster, 0p );
225                kernel_thread = pthread_self();
226
227                runner{ &this };
228                __cfadbg_print_safe(runtime_core, "Kernel : constructed main processor context %p\n", &runner);
229        }
230
231        // Initialize the main processor and the main processor ctx
232        // (the coroutine that contains the processing control flow)
233        mainProcessor = (processor *)&storage_mainProcessor;
234        (*mainProcessor){};
235
236        register_tls( mainProcessor );
237        mainThread->last_cpu = __kernel_getcpu();
238
239        //initialize the global state variables
240        __cfaabi_tls.this_processor = mainProcessor;
241        __cfaabi_tls.this_thread    = mainThread;
242
243        #if !defined( __CFA_NO_STATISTICS__ )
244                __cfaabi_tls.this_stats = (__stats_t *)& storage_mainProcStats;
245                __init_stats( __cfaabi_tls.this_stats );
246        #endif
247        mainProcessor->local_data = &__cfaabi_tls;
248
249        // Enable preemption
250        __kernel_alarm_startup();
251
252        // Add the main thread to the ready queue
253        // once resume is called on mainProcessor->runner the mainThread needs to be scheduled like any normal thread
254        schedule_thread$(mainThread);
255
256        // SKULLDUGGERY: Force a context switch to the main processor to set the main thread's context to the current UNIX
257        // context. Hence, the main thread does not begin through __cfactx_invoke_thread, like all other threads. The trick here is that
258        // mainThread is on the ready queue when this call is made.
259        __kernel_first_resume( __cfaabi_tls.this_processor );
260
261
262        // THE SYSTEM IS NOW COMPLETELY RUNNING
263
264        __cfadbg_print_safe(runtime_core, "Kernel : Started\n--------------------------------------------------\n\n");
265
266        /* paranoid */ verify( ! __preemption_enabled() );
267        enable_interrupts();
268        /* paranoid */ verify( __preemption_enabled() );
269
270}
271
272static void __kernel_shutdown(void) {
273        /* paranoid */ verify( __preemption_enabled() );
274        disable_interrupts();
275        /* paranoid */ verify( ! __preemption_enabled() );
276
277        __cfadbg_print_safe(runtime_core, "\n--------------------------------------------------\nKernel : Shutting down\n");
278
279        // SKULLDUGGERY: Notify the mainProcessor it needs to terminates.
280        // When its coroutine terminates, it return control to the mainThread
281        // which is currently here
282        __atomic_store_n(&mainProcessor->do_terminate, true, __ATOMIC_RELEASE);
283        __kernel_last_resume( __cfaabi_tls.this_processor );
284        mainThread->self_cor.state = Halted;
285
286        // THE SYSTEM IS NOW COMPLETELY STOPPED
287
288        // Disable preemption
289        __kernel_alarm_shutdown();
290
291        #if !defined( __CFA_NO_STATISTICS__ )
292                __stats_t * st = (__stats_t *)& storage_mainProcStats;
293                __tally_stats(mainCluster->stats, st);
294                if( 0 != mainProcessor->print_stats ) {
295                        __print_stats( st, mainProcessor->print_stats, "Processor ", mainProcessor->name, (void*)mainProcessor );
296                }
297                #if defined(CFA_STATS_ARRAY)
298                        __flush_stat( st, "Processor", mainProcessor );
299                #endif
300        #endif
301
302        mainProcessor->local_data = 0p;
303
304        unregister_tls( mainProcessor );
305
306        // Destroy the main processor and its context in reverse order of construction
307        // These were manually constructed so we need manually destroy them
308        void ^?{}(processor & this) with( this ){
309                deinit( this );
310
311                /* paranoid */ verify( this.do_terminate == true );
312                __cfaabi_dbg_print_safe("Kernel : destroyed main processor context %p\n", &runner);
313        }
314
315        ^(*mainProcessor){};
316
317        // Final step, destroy the main thread since it is no longer needed
318
319        // Since we provided a stack to this taxk it will not destroy anything
320        /* paranoid */ verify(mainThread->self_cor.stack.storage == (__stack_t*)(((uintptr_t)&storage_mainThreadCtx)| 0x1));
321        ^(*mainThread){};
322
323        ^(*mainCluster){};
324
325        ^(*__scheduler_lock){};
326
327        ^(__cfa_dbg_global_clusters.list){};
328        ^(__cfa_dbg_global_clusters.lock){};
329
330        __cfadbg_print_safe(runtime_core, "Kernel : Shutdown complete\n");
331}
332
333//=============================================================================================
334// Kernel Initial Scheduling logic
335//=============================================================================================
336
337// Context invoker for processors
338// This is the entry point for processors (kernel threads) *except* for the main processor
339// It effectively constructs a coroutine by stealing the pthread stack
340static void * __invoke_processor(void * arg) {
341        #if !defined( __CFA_NO_STATISTICS__ )
342                __stats_t local_stats;
343                __init_stats( &local_stats );
344                __cfaabi_tls.this_stats = &local_stats;
345        #endif
346
347        processor * proc = (processor *) arg;
348        __cfaabi_tls.this_processor = proc;
349        __cfaabi_tls.this_thread    = 0p;
350        __cfaabi_tls.preemption_state.[enabled, disable_count] = [false, 1];
351        proc->local_data = &__cfaabi_tls;
352
353        register_tls( proc );
354
355        // SKULLDUGGERY: We want to create a context for the processor coroutine
356        // which is needed for the 2-step context switch. However, there is no reason
357        // to waste the perfectly valid stack create by pthread.
358        current_stack_info_t info;
359        __stack_t ctx;
360        info.storage = &ctx;
361        (proc->runner){ proc, &info };
362
363        __cfaabi_dbg_print_safe("Coroutine : created stack %p\n", get_coroutine(proc->runner)->stack.storage);
364
365        //Set global state
366        __cfaabi_tls.this_thread = 0p;
367
368        //We now have a proper context from which to schedule threads
369        __cfadbg_print_safe(runtime_core, "Kernel : core %p created (%p, %p)\n", proc, &proc->runner, &ctx);
370
371        // SKULLDUGGERY: Since the coroutine doesn't have its own stack, we can't
372        // resume it to start it like it normally would, it will just context switch
373        // back to here. Instead directly call the main since we already are on the
374        // appropriate stack.
375        get_coroutine(proc->runner)->state = Active;
376        main( proc->runner );
377        get_coroutine(proc->runner)->state = Halted;
378
379        // Main routine of the core returned, the core is now fully terminated
380        __cfadbg_print_safe(runtime_core, "Kernel : core %p main ended (%p)\n", proc, &proc->runner);
381
382        #if !defined(__CFA_NO_STATISTICS__)
383                __tally_stats(proc->cltr->stats, &local_stats);
384                if( 0 != proc->print_stats ) {
385                        __print_stats( &local_stats, proc->print_stats, "Processor ", proc->name, (void*)proc );
386                }
387                #if defined(CFA_STATS_ARRAY)
388                        __flush_stat( &local_stats, "Processor", proc );
389                #endif
390        #endif
391
392        proc->local_data = 0p;
393
394        unregister_tls( proc );
395
396        return 0p;
397}
398
399static void __kernel_first_resume( processor * this ) {
400        thread$ * src = mainThread;
401        coroutine$ * dst = get_coroutine(this->runner);
402
403        /* paranoid */ verify( ! __preemption_enabled() );
404
405        __cfaabi_tls.this_thread->curr_cor = dst;
406        __stack_prepare( &dst->stack, 65000 );
407        __cfactx_start(main, dst, this->runner, __cfactx_invoke_coroutine);
408
409        /* paranoid */ verify( ! __preemption_enabled() );
410
411        dst->last = &src->self_cor;
412        dst->starter = dst->starter ? dst->starter : &src->self_cor;
413
414        // make sure the current state is still correct
415        /* paranoid */ verify(src->state == Ready);
416        src->corctx_flag = true;
417
418        // context switch to specified coroutine
419        verify( dst->context.SP );
420        __cfactx_switch( &src->context, &dst->context );
421        // when __cfactx_switch returns we are back in the src coroutine
422
423        mainThread->curr_cor = &mainThread->self_cor;
424
425        // make sure the current state has been update
426        /* paranoid */ verify(src->state == Active);
427
428        /* paranoid */ verify( ! __preemption_enabled() );
429}
430
431// KERNEL_ONLY
432static void __kernel_last_resume( processor * this ) {
433        coroutine$ * src = &mainThread->self_cor;
434        coroutine$ * dst = get_coroutine(this->runner);
435
436        /* paranoid */ verify( ! __preemption_enabled() );
437        /* paranoid */ verify( dst->starter == src );
438        /* paranoid */ verify( dst->context.SP );
439
440        // SKULLDUGGERY in debug the processors check that the
441        // stack is still within the limit of the stack limits after running a thread.
442        // that check doesn't make sense if we context switch to the processor using the
443        // coroutine semantics. Since this is a special case, use the current context
444        // info to populate these fields.
445        __cfaabi_dbg_debug_do(
446                __stack_context_t ctx;
447                CtxGet( ctx );
448                mainThread->context.SP = ctx.SP;
449                mainThread->context.FP = ctx.FP;
450        )
451
452        // context switch to the processor
453        __cfactx_switch( &src->context, &dst->context );
454}
455
456
457//=============================================================================================
458// Kernel Object Constructors logic
459//=============================================================================================
460//-----------------------------------------------------------------------------
461// Main thread construction
462static void ?{}( coroutine$ & this, current_stack_info_t * info) with( this ) {
463        stack.storage = info->storage;
464        with(*stack.storage) {
465                limit     = info->limit;
466                base      = info->base;
467        }
468        __attribute__((may_alias)) intptr_t * istorage = (intptr_t*) &stack.storage;
469        *istorage |= 0x1;
470        name = "Main Thread";
471        state = Start;
472        starter = 0p;
473        last = 0p;
474        cancellation = 0p;
475}
476
477static void ?{}( thread$ & this, current_stack_info_t * info) with( this ) {
478        ticket = TICKET_RUNNING;
479        state = Start;
480        self_cor{ info };
481        curr_cor = &self_cor;
482        curr_cluster = mainCluster;
483        self_mon.owner = &this;
484        self_mon.recursion = 1;
485        self_mon_p = &self_mon;
486        link.next = 0p;
487        link.ts   = -1llu;
488        preferred = -1u;
489        last_proc = 0p;
490        #if defined( __CFA_WITH_VERIFY__ )
491                canary = 0x0D15EA5E0D15EA5Ep;
492        #endif
493
494        node.next = 0p;
495        node.prev = 0p;
496        doregister(curr_cluster, this);
497
498        monitors{ &self_mon_p, 1, (fptr_t)0 };
499}
500
501//-----------------------------------------------------------------------------
502// Processor
503// Construct the processor context of non-main processors
504static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info) {
505        (this.__cor){ info };
506        this.proc = proc;
507}
508
509static void init(processor & this, const char name[], cluster & _cltr, thread$ * initT) with( this ) {
510        this.name = name;
511        this.cltr = &_cltr;
512        this.rdq.its = 0;
513        this.rdq.itr = 0;
514        this.rdq.id  = -1u;
515        this.rdq.target = -1u;
516        this.rdq.last = -1u;
517        this.rdq.cutoff = 0ull;
518        do_terminate = false;
519        preemption_alarm = 0p;
520        pending_preemption = false;
521
522        this.io.ctx = 0p;
523        this.io.pending = false;
524        this.io.dirty   = false;
525
526        this.init.thrd = initT;
527
528        this.local_data = 0p;
529
530        this.idle = eventfd(0, 0);
531        if (idle < 0) {
532                abort("KERNEL ERROR: PROCESSOR EVENTFD - %s\n", strerror(errno));
533        }
534
535        #if !defined(__CFA_NO_STATISTICS__)
536                print_stats = 0;
537                print_halts = false;
538        #endif
539
540        __cfadbg_print_safe(runtime_core, "Kernel : core %p created\n", &this);
541}
542
543// Not a ctor, it just preps the destruction but should not destroy members
544static void deinit(processor & this) {
545        close(this.idle);
546}
547
548void ?{}(processor & this, const char name[], cluster & _cltr, thread$ * initT) {
549        ( this.terminated ){};
550        ( this.runner ){};
551
552        disable_interrupts();
553                init( this, name, _cltr, initT );
554        enable_interrupts();
555
556        __cfadbg_print_safe(runtime_core, "Kernel : Starting core %p\n", &this);
557
558        this.stack = __create_pthread( &this.kernel_thread, __invoke_processor, (void *)&this );
559}
560
561void ?{}(processor & this, const char name[], cluster & _cltr) {
562        (this){name, _cltr, 0p};
563}
564
565extern size_t __page_size;
566void ^?{}(processor & this) with( this ){
567        if( ! __atomic_load_n(&do_terminate, __ATOMIC_ACQUIRE) ) {
568                __cfadbg_print_safe(runtime_core, "Kernel : core %p signaling termination\n", &this);
569
570                __atomic_store_n(&do_terminate, true, __ATOMIC_RELAXED);
571                __wake_proc( &this );
572
573                wait( terminated );
574                /* paranoid */ verify( active_processor() != &this);
575        }
576
577        __destroy_pthread( kernel_thread, this.stack, 0p );
578
579        disable_interrupts();
580                deinit( this );
581        enable_interrupts();
582}
583
584//-----------------------------------------------------------------------------
585// Cluster
586static void ?{}(__cluster_proc_list & this) {
587        this.lock  = 0;
588        this.idle  = 0;
589        this.total = 0;
590}
591
592void ?{}(cluster & this, const char name[], Duration preemption_rate, unsigned num_io, const io_context_params & io_params) with( this ) {
593        this.name = name;
594        this.preemption_rate = preemption_rate;
595        ready_queue{};
596
597        #if !defined(__CFA_NO_STATISTICS__)
598                print_stats = 0;
599                stats = alloc();
600                __init_stats( stats );
601        #endif
602
603        threads{ __get };
604
605        io.arbiter = create();
606        io.params = io_params;
607
608        doregister(this);
609
610        // Lock the RWlock so no-one pushes/pops while we are changing the queue
611        disable_interrupts();
612        uint_fast32_t last_size = ready_mutate_lock();
613
614                // Adjust the ready queue size
615                ready_queue_grow( &this );
616
617        // Unlock the RWlock
618        ready_mutate_unlock( last_size );
619        enable_interrupts( false ); // Don't poll, could be in main cluster
620}
621
622void ^?{}(cluster & this) {
623        destroy(this.io.arbiter);
624
625        // Lock the RWlock so no-one pushes/pops while we are changing the queue
626        disable_interrupts();
627        uint_fast32_t last_size = ready_mutate_lock();
628
629                // Adjust the ready queue size
630                ready_queue_shrink( &this );
631
632        // Unlock the RWlock
633        ready_mutate_unlock( last_size );
634        enable_interrupts( false ); // Don't poll, could be in main cluster
635
636        #if !defined(__CFA_NO_STATISTICS__)
637                if( 0 != this.print_stats ) {
638                        __print_stats( this.stats, this.print_stats, "Cluster", this.name, (void*)&this );
639                }
640                #if defined(CFA_STATS_ARRAY)
641                        __flush_stat( this.stats, "Cluster", &this );
642                #endif
643                free( this.stats );
644        #endif
645
646        unregister(this);
647}
648
649//=============================================================================================
650// Miscellaneous Initialization
651//=============================================================================================
652//-----------------------------------------------------------------------------
653// Global Queues
654static void doregister( cluster     & cltr ) {
655        lock      ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
656        push_front( __cfa_dbg_global_clusters.list, cltr );
657        unlock    ( __cfa_dbg_global_clusters.lock );
658}
659
660static void unregister( cluster     & cltr ) {
661        lock  ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
662        remove( __cfa_dbg_global_clusters.list, cltr );
663        unlock( __cfa_dbg_global_clusters.lock );
664}
665
666void doregister( cluster * cltr, thread$ & thrd ) {
667        lock      (cltr->thread_list_lock __cfaabi_dbg_ctx2);
668        cltr->nthreads += 1;
669        push_front(cltr->threads, thrd);
670        unlock    (cltr->thread_list_lock);
671}
672
673void unregister( cluster * cltr, thread$ & thrd ) {
674        lock  (cltr->thread_list_lock __cfaabi_dbg_ctx2);
675        remove(cltr->threads, thrd );
676        cltr->nthreads -= 1;
677        unlock(cltr->thread_list_lock);
678}
679
680static void register_tls( processor * this ) {
681        // Register and Lock the RWlock so no-one pushes/pops while we are changing the queue
682        uint_fast32_t last_size;
683        [this->unique_id, last_size] = ready_mutate_register();
684
685                this->cltr->procs.total += 1u;
686                insert_last(this->cltr->procs.actives, *this);
687
688                // Adjust the ready queue size
689                ready_queue_grow( this->cltr );
690
691        // Unlock the RWlock
692        ready_mutate_unlock( last_size );
693}
694
695
696static void unregister_tls( processor * this ) {
697        // Lock the RWlock so no-one pushes/pops while we are changing the queue
698        uint_fast32_t last_size = ready_mutate_lock();
699                this->cltr->procs.total -= 1u;
700                remove(*this);
701
702                // clear the cluster so nothing gets pushed to local queues
703                cluster * cltr = this->cltr;
704                this->cltr = 0p;
705
706                // Adjust the ready queue size
707                ready_queue_shrink( cltr );
708
709        // Unlock the RWlock and unregister: we don't need the read_lock any more
710        ready_mutate_unregister( this->unique_id, last_size );
711}
712
713static void check( int ret, const char func[] ) {
714        if ( ret ) {                                                                            // pthread routines return errno values
715                abort( "%s : internal error, error(%d) %s.", func, ret, strerror( ret ) );
716        } // if
717} // Abort
718
719void * __create_pthread( pthread_t * pthread, void * (*start)(void *), void * arg ) {
720        pthread_attr_t attr;
721
722        check( pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
723
724        size_t stacksize;
725        // default stack size, normally defined by shell limit
726        check( pthread_attr_getstacksize( &attr, &stacksize ), "pthread_attr_getstacksize" );
727        assert( stacksize >= PTHREAD_STACK_MIN );
728
729        void * stack;
730        #if CFA_PROCESSOR_USE_MMAP
731                stacksize = ceiling( stacksize, __page_size ) + __page_size;
732                stack = mmap(0p, stacksize, __map_prot, MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
733                if(stack == ((void*)-1)) {
734                        abort( "pthread stack creation : internal error, mmap failure, error(%d) %s.", errno, strerror( errno ) );
735                }
736                if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
737                        abort( "pthread stack creation : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
738                } // if
739        #else
740                __cfaabi_dbg_debug_do(
741                        stack = memalign( __page_size, stacksize + __page_size );
742                        // pthread has no mechanism to create the guard page in user supplied stack.
743                        if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
744                                abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
745                        } // if
746                );
747                __cfaabi_dbg_no_debug_do(
748                        stack = malloc( stacksize );
749                );
750        #endif
751
752
753        check( pthread_attr_setstack( &attr, stack, stacksize ), "pthread_attr_setstack" );
754
755        check( pthread_create( pthread, &attr, start, arg ), "pthread_create" );
756        return stack;
757}
758
759void __destroy_pthread( pthread_t pthread, void * stack, void ** retval ) {
760        int err = pthread_join( pthread, retval );
761        if( err != 0 ) abort("KERNEL ERROR: joining pthread %p caused error %s\n", (void*)pthread, strerror(err));
762
763        #if CFA_PROCESSOR_USE_MMAP
764                pthread_attr_t attr;
765
766                check( pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
767
768                size_t stacksize;
769                // default stack size, normally defined by shell limit
770                check( pthread_attr_getstacksize( &attr, &stacksize ), "pthread_attr_getstacksize" );
771                assert( stacksize >= PTHREAD_STACK_MIN );
772                stacksize += __page_size;
773
774                if(munmap(stack, stacksize) == -1) {
775                        abort( "pthread stack destruction : internal error, munmap failure, error(%d) %s.", errno, strerror( errno ) );
776                }
777        #else
778                __cfaabi_dbg_debug_do(
779                        // pthread has no mechanism to create the guard page in user supplied stack.
780                        if ( mprotect( stack, __page_size, __map_prot ) == -1 ) {
781                                abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
782                        } // if
783                );
784                free( stack );
785        #endif
786}
787
788#if defined(__CFA_WITH_VERIFY__)
789static bool verify_fwd_bck_rng(void) {
790        __cfaabi_tls.ready_rng.fwd_seed = 25214903917_l64u * (rdtscl() ^ (uintptr_t)&verify_fwd_bck_rng);
791
792        unsigned values[10];
793        for(i; 10) {
794                values[i] = __tls_rand_fwd();
795        }
796
797        __tls_rand_advance_bck();
798
799        for ( i; 9 -~= 0 ) {
800                if(values[i] != __tls_rand_bck()) {
801                        return false;
802                }
803        }
804
805        return true;
806}
807#endif
Note: See TracBrowser for help on using the repository browser.