source: libcfa/src/concurrency/kernel/startup.cfa @ 3e1a705

arm-ehenumforall-pointer-decayjacob/cs343-translationnew-ast-unique-exprpthread-emulationqualifiedEnum
Last change on this file since 3e1a705 was 3e1a705, checked in by Thierry Delisle <tdelisle@…>, 19 months ago

Start ready queue cutoff at zero to avoid spurious steals.

  • Property mode set to 100644
File size: 24.1 KB
Line 
1//
2// Cforall Version 1.0.0 Copyright (C) 2020 University of Waterloo
3//
4// The contents of this file are covered under the licence agreement in the
5// file "LICENCE" distributed with Cforall.
6//
7// kernel/startup.cfa --
8//
9// Author           : Thierry Delisle
10// Created On       : Thu Jul 30 15:12:54 2020
11// Last Modified By :
12// Last Modified On :
13// Update Count     :
14//
15
16#define __cforall_thread__
17
18// C Includes
19#include <errno.h>              // errno
20#include <string.h>             // strerror
21#include <unistd.h>             // sysconf
22extern "C" {
23      #include <limits.h>       // PTHREAD_STACK_MIN
24        #include <sys/eventfd.h>  // eventfd
25      #include <sys/mman.h>     // mprotect
26      #include <sys/resource.h> // getrlimit
27}
28
29// CFA Includes
30#include "kernel_private.hfa"
31#include "startup.hfa"          // STARTUP_PRIORITY_XXX
32#include "math.hfa"
33
34#define CFA_PROCESSOR_USE_MMAP 0
35
36//-----------------------------------------------------------------------------
37// Some assembly required
38#if defined( __i386 )
39        #define CtxGet( ctx ) __asm__ volatile ( \
40                "movl %%esp,%0\n" \
41                "movl %%ebp,%1\n" \
42                : "=rm" (ctx.SP), \
43                  "=rm" (ctx.FP) \
44        )
45#elif defined( __x86_64 )
46        #define CtxGet( ctx ) __asm__ volatile ( \
47                "movq %%rsp,%0\n" \
48                "movq %%rbp,%1\n" \
49                : "=rm" (ctx.SP), \
50                  "=rm" (ctx.FP) \
51        )
52#elif defined( __aarch64__ )
53        #define CtxGet( ctx ) __asm__ volatile ( \
54                "mov %0, sp\n" \
55                "mov %1, fp\n" \
56                : "=rm" (ctx.SP), \
57                  "=rm" (ctx.FP) \
58        )
59#else
60        #error unknown hardware architecture
61#endif
62
63//-----------------------------------------------------------------------------
64// Start and stop routine for the kernel, declared first to make sure they run first
65static void __kernel_startup (void) __attribute__(( constructor( STARTUP_PRIORITY_KERNEL ) ));
66static void __kernel_shutdown(void) __attribute__(( destructor ( STARTUP_PRIORITY_KERNEL ) ));
67
68//-----------------------------------------------------------------------------
69// Static Forward Declarations
70struct current_stack_info_t;
71
72static void * __invoke_processor(void * arg);
73static void __kernel_first_resume( processor * this );
74static void __kernel_last_resume ( processor * this );
75static void init(processor & this, const char name[], cluster & _cltr, $thread * initT);
76static void deinit(processor & this);
77static void doregister( struct cluster & cltr );
78static void unregister( struct cluster & cltr );
79static void register_tls( processor * this );
80static void unregister_tls( processor * this );
81static void ?{}( $coroutine & this, current_stack_info_t * info);
82static void ?{}( $thread & this, current_stack_info_t * info);
83static void ?{}(processorCtx_t & this) {}
84static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info);
85
86#if defined(__CFA_WITH_VERIFY__)
87        static bool verify_fwd_bck_rng(void);
88#endif
89
90//-----------------------------------------------------------------------------
91// Forward Declarations for other modules
92extern void __kernel_alarm_startup(void);
93extern void __kernel_alarm_shutdown(void);
94
95//-----------------------------------------------------------------------------
96// Other Forward Declarations
97extern void __wake_proc(processor *);
98
99//-----------------------------------------------------------------------------
100// Kernel storage
101KERNEL_STORAGE(cluster,              mainCluster);
102KERNEL_STORAGE(processor,            mainProcessor);
103KERNEL_STORAGE($thread,              mainThread);
104KERNEL_STORAGE(__stack_t,            mainThreadCtx);
105KERNEL_STORAGE(__scheduler_RWLock_t, __scheduler_lock);
106#if !defined(__CFA_NO_STATISTICS__)
107KERNEL_STORAGE(__stats_t, mainProcStats);
108#endif
109
110cluster              * mainCluster;
111processor            * mainProcessor;
112$thread              * mainThread;
113__scheduler_RWLock_t * __scheduler_lock;
114
115extern "C" {
116        struct { __dllist_t(cluster) list; __spinlock_t lock; } __cfa_dbg_global_clusters;
117}
118
119extern size_t __page_size;
120extern int __map_prot;
121
122//-----------------------------------------------------------------------------
123// Global state
124thread_local struct KernelThreadData __cfaabi_tls __attribute__ ((tls_model ( "initial-exec" ))) @= {
125        NULL,                                                                                           // cannot use 0p
126        NULL,
127        false,
128        { 1, false, false },
129        0,
130        { 0, 0 },
131        NULL,
132        #ifdef __CFA_WITH_VERIFY__
133                false,
134                0,
135        #endif
136};
137
138//-----------------------------------------------------------------------------
139// Struct to steal stack
140struct current_stack_info_t {
141        __stack_t * storage;  // pointer to stack object
142        void * base;          // base of stack
143        void * limit;         // stack grows towards stack limit
144        void * context;       // address of cfa_context_t
145};
146
147void ?{}( current_stack_info_t & this ) {
148        __stack_context_t ctx;
149        CtxGet( ctx );
150        this.base = ctx.FP;
151
152        rlimit r;
153        getrlimit( RLIMIT_STACK, &r);
154        size_t size = r.rlim_cur;
155
156        this.limit = (void *)(((intptr_t)this.base) - size);
157        this.context = &storage_mainThreadCtx;
158}
159
160
161
162//=============================================================================================
163// Kernel Setup logic
164//=============================================================================================
165//-----------------------------------------------------------------------------
166// Kernel boot procedures
167static void __kernel_startup(void) {
168        /* paranoid */ verify( ! __preemption_enabled() );
169        __cfadbg_print_safe(runtime_core, "Kernel : Starting\n");
170
171        __cfa_dbg_global_clusters.list{ __get };
172        __cfa_dbg_global_clusters.lock{};
173
174        /* paranoid */ verify( verify_fwd_bck_rng() );
175
176        // Initialize the global scheduler lock
177        __scheduler_lock = (__scheduler_RWLock_t*)&storage___scheduler_lock;
178        (*__scheduler_lock){};
179
180        // Initialize the main cluster
181        mainCluster = (cluster *)&storage_mainCluster;
182        (*mainCluster){"Main Cluster", 0};
183
184        __cfadbg_print_safe(runtime_core, "Kernel : Main cluster ready\n");
185
186        // Start by initializing the main thread
187        // SKULLDUGGERY: the mainThread steals the process main thread
188        // which will then be scheduled by the mainProcessor normally
189        mainThread = ($thread *)&storage_mainThread;
190        current_stack_info_t info;
191        info.storage = (__stack_t*)&storage_mainThreadCtx;
192        (*mainThread){ &info };
193
194        __cfadbg_print_safe(runtime_core, "Kernel : Main thread ready\n");
195
196
197
198        // Construct the processor context of the main processor
199        void ?{}(processorCtx_t & this, processor * proc) {
200                (this.__cor){ "Processor" };
201                this.__cor.starter = 0p;
202                this.proc = proc;
203        }
204
205        void ?{}(processor & this) with( this ) {
206                ( this.terminated ){};
207                ( this.runner ){};
208                init( this, "Main Processor", *mainCluster, 0p );
209                kernel_thread = pthread_self();
210
211                runner{ &this };
212                __cfadbg_print_safe(runtime_core, "Kernel : constructed main processor context %p\n", &runner);
213        }
214
215        // Initialize the main processor and the main processor ctx
216        // (the coroutine that contains the processing control flow)
217        mainProcessor = (processor *)&storage_mainProcessor;
218        (*mainProcessor){};
219
220        register_tls( mainProcessor );
221
222        //initialize the global state variables
223        __cfaabi_tls.this_processor = mainProcessor;
224        __cfaabi_tls.this_thread    = mainThread;
225
226        #if !defined( __CFA_NO_STATISTICS__ )
227                __cfaabi_tls.this_stats = (__stats_t *)& storage_mainProcStats;
228                __init_stats( __cfaabi_tls.this_stats );
229        #endif
230        mainProcessor->local_data = &__cfaabi_tls;
231
232        // Enable preemption
233        __kernel_alarm_startup();
234
235        // Add the main thread to the ready queue
236        // once resume is called on mainProcessor->runner the mainThread needs to be scheduled like any normal thread
237        schedule_thread$(mainThread);
238
239        // SKULLDUGGERY: Force a context switch to the main processor to set the main thread's context to the current UNIX
240        // context. Hence, the main thread does not begin through __cfactx_invoke_thread, like all other threads. The trick here is that
241        // mainThread is on the ready queue when this call is made.
242        __kernel_first_resume( __cfaabi_tls.this_processor );
243
244
245        // THE SYSTEM IS NOW COMPLETELY RUNNING
246
247        __cfadbg_print_safe(runtime_core, "Kernel : Started\n--------------------------------------------------\n\n");
248
249        /* paranoid */ verify( ! __preemption_enabled() );
250        enable_interrupts();
251        /* paranoid */ verify( __preemption_enabled() );
252
253}
254
255static void __kernel_shutdown(void) {
256        /* paranoid */ verify( __preemption_enabled() );
257        disable_interrupts();
258        /* paranoid */ verify( ! __preemption_enabled() );
259
260        __cfadbg_print_safe(runtime_core, "\n--------------------------------------------------\nKernel : Shutting down\n");
261
262        // SKULLDUGGERY: Notify the mainProcessor it needs to terminates.
263        // When its coroutine terminates, it return control to the mainThread
264        // which is currently here
265        __atomic_store_n(&mainProcessor->do_terminate, true, __ATOMIC_RELEASE);
266        __kernel_last_resume( __cfaabi_tls.this_processor );
267        mainThread->self_cor.state = Halted;
268
269        // THE SYSTEM IS NOW COMPLETELY STOPPED
270
271        // Disable preemption
272        __kernel_alarm_shutdown();
273
274        #if !defined( __CFA_NO_STATISTICS__ )
275                __stats_t * st = (__stats_t *)& storage_mainProcStats;
276                __tally_stats(mainCluster->stats, st);
277                if( 0 != mainProcessor->print_stats ) {
278                        __print_stats( st, mainProcessor->print_stats, "Processor ", mainProcessor->name, (void*)mainProcessor );
279                }
280                #if defined(CFA_STATS_ARRAY)
281                        __flush_stat( st, "Processor", mainProcessor );
282                #endif
283        #endif
284
285        mainProcessor->local_data = 0p;
286
287        unregister_tls( mainProcessor );
288
289        // Destroy the main processor and its context in reverse order of construction
290        // These were manually constructed so we need manually destroy them
291        void ^?{}(processor & this) with( this ){
292                deinit( this );
293
294                /* paranoid */ verify( this.do_terminate == true );
295                __cfaabi_dbg_print_safe("Kernel : destroyed main processor context %p\n", &runner);
296        }
297
298        ^(*mainProcessor){};
299
300        // Final step, destroy the main thread since it is no longer needed
301
302        // Since we provided a stack to this taxk it will not destroy anything
303        /* paranoid */ verify(mainThread->self_cor.stack.storage == (__stack_t*)(((uintptr_t)&storage_mainThreadCtx)| 0x1));
304        ^(*mainThread){};
305
306        ^(*mainCluster){};
307
308        ^(*__scheduler_lock){};
309
310        ^(__cfa_dbg_global_clusters.list){};
311        ^(__cfa_dbg_global_clusters.lock){};
312
313        __cfadbg_print_safe(runtime_core, "Kernel : Shutdown complete\n");
314}
315
316//=============================================================================================
317// Kernel Initial Scheduling logic
318//=============================================================================================
319
320// Context invoker for processors
321// This is the entry point for processors (kernel threads) *except* for the main processor
322// It effectively constructs a coroutine by stealing the pthread stack
323static void * __invoke_processor(void * arg) {
324        #if !defined( __CFA_NO_STATISTICS__ )
325                __stats_t local_stats;
326                __init_stats( &local_stats );
327                __cfaabi_tls.this_stats = &local_stats;
328        #endif
329
330        processor * proc = (processor *) arg;
331        __cfaabi_tls.this_processor = proc;
332        __cfaabi_tls.this_thread    = 0p;
333        __cfaabi_tls.preemption_state.[enabled, disable_count] = [false, 1];
334        proc->local_data = &__cfaabi_tls;
335
336        register_tls( proc );
337
338        // SKULLDUGGERY: We want to create a context for the processor coroutine
339        // which is needed for the 2-step context switch. However, there is no reason
340        // to waste the perfectly valid stack create by pthread.
341        current_stack_info_t info;
342        __stack_t ctx;
343        info.storage = &ctx;
344        (proc->runner){ proc, &info };
345
346        __cfaabi_dbg_print_safe("Coroutine : created stack %p\n", get_coroutine(proc->runner)->stack.storage);
347
348        //Set global state
349        __cfaabi_tls.this_thread = 0p;
350
351        //We now have a proper context from which to schedule threads
352        __cfadbg_print_safe(runtime_core, "Kernel : core %p created (%p, %p)\n", proc, &proc->runner, &ctx);
353
354        // SKULLDUGGERY: Since the coroutine doesn't have its own stack, we can't
355        // resume it to start it like it normally would, it will just context switch
356        // back to here. Instead directly call the main since we already are on the
357        // appropriate stack.
358        get_coroutine(proc->runner)->state = Active;
359        main( proc->runner );
360        get_coroutine(proc->runner)->state = Halted;
361
362        // Main routine of the core returned, the core is now fully terminated
363        __cfadbg_print_safe(runtime_core, "Kernel : core %p main ended (%p)\n", proc, &proc->runner);
364
365        #if !defined(__CFA_NO_STATISTICS__)
366                __tally_stats(proc->cltr->stats, &local_stats);
367                if( 0 != proc->print_stats ) {
368                        __print_stats( &local_stats, proc->print_stats, "Processor ", proc->name, (void*)proc );
369                }
370                #if defined(CFA_STATS_ARRAY)
371                        __flush_stat( &local_stats, "Processor", proc );
372                #endif
373        #endif
374
375        proc->local_data = 0p;
376
377        unregister_tls( proc );
378
379        return 0p;
380}
381
382static void __kernel_first_resume( processor * this ) {
383        $thread * src = mainThread;
384        $coroutine * dst = get_coroutine(this->runner);
385
386        /* paranoid */ verify( ! __preemption_enabled() );
387
388        __cfaabi_tls.this_thread->curr_cor = dst;
389        __stack_prepare( &dst->stack, 65000 );
390        __cfactx_start(main, dst, this->runner, __cfactx_invoke_coroutine);
391
392        /* paranoid */ verify( ! __preemption_enabled() );
393
394        dst->last = &src->self_cor;
395        dst->starter = dst->starter ? dst->starter : &src->self_cor;
396
397        // make sure the current state is still correct
398        /* paranoid */ verify(src->state == Ready);
399
400        // context switch to specified coroutine
401        verify( dst->context.SP );
402        __cfactx_switch( &src->context, &dst->context );
403        // when __cfactx_switch returns we are back in the src coroutine
404
405        mainThread->curr_cor = &mainThread->self_cor;
406
407        // make sure the current state has been update
408        /* paranoid */ verify(src->state == Active);
409
410        /* paranoid */ verify( ! __preemption_enabled() );
411}
412
413// KERNEL_ONLY
414static void __kernel_last_resume( processor * this ) {
415        $coroutine * src = &mainThread->self_cor;
416        $coroutine * dst = get_coroutine(this->runner);
417
418        /* paranoid */ verify( ! __preemption_enabled() );
419        /* paranoid */ verify( dst->starter == src );
420        /* paranoid */ verify( dst->context.SP );
421
422        // SKULLDUGGERY in debug the processors check that the
423        // stack is still within the limit of the stack limits after running a thread.
424        // that check doesn't make sense if we context switch to the processor using the
425        // coroutine semantics. Since this is a special case, use the current context
426        // info to populate these fields.
427        __cfaabi_dbg_debug_do(
428                __stack_context_t ctx;
429                CtxGet( ctx );
430                mainThread->context.SP = ctx.SP;
431                mainThread->context.FP = ctx.FP;
432        )
433
434        // context switch to the processor
435        __cfactx_switch( &src->context, &dst->context );
436}
437
438
439//=============================================================================================
440// Kernel Object Constructors logic
441//=============================================================================================
442//-----------------------------------------------------------------------------
443// Main thread construction
444static void ?{}( $coroutine & this, current_stack_info_t * info) with( this ) {
445        stack.storage = info->storage;
446        with(*stack.storage) {
447                limit     = info->limit;
448                base      = info->base;
449        }
450        __attribute__((may_alias)) intptr_t * istorage = (intptr_t*) &stack.storage;
451        *istorage |= 0x1;
452        name = "Main Thread";
453        state = Start;
454        starter = 0p;
455        last = 0p;
456        cancellation = 0p;
457}
458
459static void ?{}( $thread & this, current_stack_info_t * info) with( this ) {
460        ticket = TICKET_RUNNING;
461        state = Start;
462        self_cor{ info };
463        curr_cor = &self_cor;
464        curr_cluster = mainCluster;
465        self_mon.owner = &this;
466        self_mon.recursion = 1;
467        self_mon_p = &self_mon;
468        link.next = 0p;
469        link.ts   = 0;
470        preferred = -1u;
471        last_proc = 0p;
472        #if defined( __CFA_WITH_VERIFY__ )
473                canary = 0x0D15EA5E0D15EA5Ep;
474        #endif
475
476        node.next = 0p;
477        node.prev = 0p;
478        doregister(curr_cluster, this);
479
480        monitors{ &self_mon_p, 1, (fptr_t)0 };
481}
482
483//-----------------------------------------------------------------------------
484// Processor
485// Construct the processor context of non-main processors
486static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info) {
487        (this.__cor){ info };
488        this.proc = proc;
489}
490
491static void init(processor & this, const char name[], cluster & _cltr, $thread * initT) with( this ) {
492        this.name = name;
493        this.cltr = &_cltr;
494        this.rdq.its = 0;
495        this.rdq.itr = 0;
496        this.rdq.id  = -1u;
497        this.rdq.target = -1u;
498        this.rdq.cutoff = 0ull;
499        do_terminate = false;
500        preemption_alarm = 0p;
501        pending_preemption = false;
502
503        this.io.ctx = 0p;
504        this.io.pending = false;
505        this.io.dirty   = false;
506
507        this.init.thrd = initT;
508
509        this.local_data = 0p;
510
511        this.idle = eventfd(0, 0);
512        if (idle < 0) {
513                abort("KERNEL ERROR: PROCESSOR EVENTFD - %s\n", strerror(errno));
514        }
515
516        #if !defined(__CFA_NO_STATISTICS__)
517                print_stats = 0;
518                print_halts = false;
519        #endif
520
521        __cfadbg_print_safe(runtime_core, "Kernel : core %p created\n", &this);
522}
523
524// Not a ctor, it just preps the destruction but should not destroy members
525static void deinit(processor & this) {
526        close(this.idle);
527}
528
529void ?{}(processor & this, const char name[], cluster & _cltr, $thread * initT) {
530        ( this.terminated ){};
531        ( this.runner ){};
532
533        disable_interrupts();
534                init( this, name, _cltr, initT );
535        enable_interrupts();
536
537        __cfadbg_print_safe(runtime_core, "Kernel : Starting core %p\n", &this);
538
539        this.stack = __create_pthread( &this.kernel_thread, __invoke_processor, (void *)&this );
540}
541
542void ?{}(processor & this, const char name[], cluster & _cltr) {
543        (this){name, _cltr, 0p};
544}
545
546extern size_t __page_size;
547void ^?{}(processor & this) with( this ){
548        if( ! __atomic_load_n(&do_terminate, __ATOMIC_ACQUIRE) ) {
549                __cfadbg_print_safe(runtime_core, "Kernel : core %p signaling termination\n", &this);
550
551                __atomic_store_n(&do_terminate, true, __ATOMIC_RELAXED);
552                __wake_proc( &this );
553
554                wait( terminated );
555                /* paranoid */ verify( active_processor() != &this);
556        }
557
558        __destroy_pthread( kernel_thread, this.stack, 0p );
559
560        disable_interrupts();
561                deinit( this );
562        enable_interrupts();
563}
564
565//-----------------------------------------------------------------------------
566// Cluster
567static void ?{}(__cluster_proc_list & this) {
568        this.lock  = 0;
569        this.idle  = 0;
570        this.total = 0;
571}
572
573void ?{}(cluster & this, const char name[], Duration preemption_rate, unsigned num_io, const io_context_params & io_params) with( this ) {
574        this.name = name;
575        this.preemption_rate = preemption_rate;
576        ready_queue{};
577
578        #if !defined(__CFA_NO_STATISTICS__)
579                print_stats = 0;
580                stats = alloc();
581                __init_stats( stats );
582        #endif
583
584        threads{ __get };
585
586        io.arbiter = create();
587        io.params = io_params;
588
589        doregister(this);
590
591        // Lock the RWlock so no-one pushes/pops while we are changing the queue
592        disable_interrupts();
593        uint_fast32_t last_size = ready_mutate_lock();
594
595                // Adjust the ready queue size
596                ready_queue_grow( &this );
597
598        // Unlock the RWlock
599        ready_mutate_unlock( last_size );
600        enable_interrupts( false ); // Don't poll, could be in main cluster
601}
602
603void ^?{}(cluster & this) {
604        destroy(this.io.arbiter);
605
606        // Lock the RWlock so no-one pushes/pops while we are changing the queue
607        disable_interrupts();
608        uint_fast32_t last_size = ready_mutate_lock();
609
610                // Adjust the ready queue size
611                ready_queue_shrink( &this );
612
613        // Unlock the RWlock
614        ready_mutate_unlock( last_size );
615        enable_interrupts( false ); // Don't poll, could be in main cluster
616
617        #if !defined(__CFA_NO_STATISTICS__)
618                if( 0 != this.print_stats ) {
619                        __print_stats( this.stats, this.print_stats, "Cluster", this.name, (void*)&this );
620                }
621                #if defined(CFA_STATS_ARRAY)
622                        __flush_stat( this.stats, "Cluster", &this );
623                #endif
624                free( this.stats );
625        #endif
626
627        unregister(this);
628}
629
630//=============================================================================================
631// Miscellaneous Initialization
632//=============================================================================================
633//-----------------------------------------------------------------------------
634// Global Queues
635static void doregister( cluster     & cltr ) {
636        lock      ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
637        push_front( __cfa_dbg_global_clusters.list, cltr );
638        unlock    ( __cfa_dbg_global_clusters.lock );
639}
640
641static void unregister( cluster     & cltr ) {
642        lock  ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
643        remove( __cfa_dbg_global_clusters.list, cltr );
644        unlock( __cfa_dbg_global_clusters.lock );
645}
646
647void doregister( cluster * cltr, $thread & thrd ) {
648        lock      (cltr->thread_list_lock __cfaabi_dbg_ctx2);
649        cltr->nthreads += 1;
650        push_front(cltr->threads, thrd);
651        unlock    (cltr->thread_list_lock);
652}
653
654void unregister( cluster * cltr, $thread & thrd ) {
655        lock  (cltr->thread_list_lock __cfaabi_dbg_ctx2);
656        remove(cltr->threads, thrd );
657        cltr->nthreads -= 1;
658        unlock(cltr->thread_list_lock);
659}
660
661static void register_tls( processor * this ) {
662        // Register and Lock the RWlock so no-one pushes/pops while we are changing the queue
663        uint_fast32_t last_size;
664        [this->unique_id, last_size] = ready_mutate_register();
665
666                this->cltr->procs.total += 1u;
667                insert_last(this->cltr->procs.actives, *this);
668
669                // Adjust the ready queue size
670                ready_queue_grow( this->cltr );
671
672        // Unlock the RWlock
673        ready_mutate_unlock( last_size );
674}
675
676
677static void unregister_tls( processor * this ) {
678        // Lock the RWlock so no-one pushes/pops while we are changing the queue
679        uint_fast32_t last_size = ready_mutate_lock();
680                this->cltr->procs.total -= 1u;
681                remove(*this);
682
683                // clear the cluster so nothing gets pushed to local queues
684                cluster * cltr = this->cltr;
685                this->cltr = 0p;
686
687                // Adjust the ready queue size
688                ready_queue_shrink( cltr );
689
690        // Unlock the RWlock and unregister: we don't need the read_lock any more
691        ready_mutate_unregister( this->unique_id, last_size );
692}
693
694static void check( int ret, const char func[] ) {
695        if ( ret ) {                                                                            // pthread routines return errno values
696                abort( "%s : internal error, error(%d) %s.", func, ret, strerror( ret ) );
697        } // if
698} // Abort
699
700void * __create_pthread( pthread_t * pthread, void * (*start)(void *), void * arg ) {
701        pthread_attr_t attr;
702
703        check( pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
704
705        size_t stacksize;
706        // default stack size, normally defined by shell limit
707        check( pthread_attr_getstacksize( &attr, &stacksize ), "pthread_attr_getstacksize" );
708        assert( stacksize >= PTHREAD_STACK_MIN );
709
710        void * stack;
711        #if CFA_PROCESSOR_USE_MMAP
712                stacksize = ceiling( stacksize, __page_size ) + __page_size;
713                stack = mmap(0p, stacksize, __map_prot, MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
714                if(stack == ((void*)-1)) {
715                        abort( "pthread stack creation : internal error, mmap failure, error(%d) %s.", errno, strerror( errno ) );
716                }
717                if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
718                        abort( "pthread stack creation : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
719                } // if
720        #else
721                __cfaabi_dbg_debug_do(
722                        stack = memalign( __page_size, stacksize + __page_size );
723                        // pthread has no mechanism to create the guard page in user supplied stack.
724                        if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
725                                abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
726                        } // if
727                );
728                __cfaabi_dbg_no_debug_do(
729                        stack = malloc( stacksize );
730                );
731        #endif
732
733
734        check( pthread_attr_setstack( &attr, stack, stacksize ), "pthread_attr_setstack" );
735
736        check( pthread_create( pthread, &attr, start, arg ), "pthread_create" );
737        return stack;
738}
739
740void __destroy_pthread( pthread_t pthread, void * stack, void ** retval ) {
741        int err = pthread_join( pthread, retval );
742        if( err != 0 ) abort("KERNEL ERROR: joining pthread %p caused error %s\n", (void*)pthread, strerror(err));
743
744        #if CFA_PROCESSOR_USE_MMAP
745                pthread_attr_t attr;
746
747                check( pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
748
749                size_t stacksize;
750                // default stack size, normally defined by shell limit
751                check( pthread_attr_getstacksize( &attr, &stacksize ), "pthread_attr_getstacksize" );
752                assert( stacksize >= PTHREAD_STACK_MIN );
753                stacksize += __page_size;
754
755                if(munmap(stack, stacksize) == -1) {
756                        abort( "pthread stack destruction : internal error, munmap failure, error(%d) %s.", errno, strerror( errno ) );
757                }
758        #else
759                __cfaabi_dbg_debug_do(
760                        // pthread has no mechanism to create the guard page in user supplied stack.
761                        if ( mprotect( stack, __page_size, __map_prot ) == -1 ) {
762                                abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
763                        } // if
764                );
765                free( stack );
766        #endif
767}
768
769#if defined(__CFA_WITH_VERIFY__)
770static bool verify_fwd_bck_rng(void) {
771        __cfaabi_tls.ready_rng.fwd_seed = 25214903917_l64u * (rdtscl() ^ (uintptr_t)&verify_fwd_bck_rng);
772
773        unsigned values[10];
774        for(i; 10) {
775                values[i] = __tls_rand_fwd();
776        }
777
778        __tls_rand_advance_bck();
779
780        for ( i; 9 -~= 0 ) {
781                if(values[i] != __tls_rand_bck()) {
782                        return false;
783                }
784        }
785
786        return true;
787}
788#endif
Note: See TracBrowser for help on using the repository browser.