source: libcfa/src/concurrency/kernel/startup.cfa @ bd0bdd37

ADTarm-ehast-experimentalenumforall-pointer-decayjacob/cs343-translationnew-ast-unique-exprpthread-emulationqualifiedEnum
Last change on this file since bd0bdd37 was bd0bdd37, checked in by Thierry Delisle <tdelisle@…>, 3 years ago

Fix how bias is handled in the ready queue to be more consistent with multiple clusters.
Does not handle churn of processors yet.

  • Property mode set to 100644
File size: 23.1 KB
RevLine 
[e660761]1//
2// Cforall Version 1.0.0 Copyright (C) 2020 University of Waterloo
3//
4// The contents of this file are covered under the licence agreement in the
5// file "LICENCE" distributed with Cforall.
6//
7// kernel/startup.cfa --
8//
9// Author           : Thierry Delisle
10// Created On       : Thu Jul 30 15:12:54 2020
11// Last Modified By :
12// Last Modified On :
13// Update Count     :
14//
15
16#define __cforall_thread__
17
18// C Includes
19#include <errno.h>              // errno
20#include <string.h>             // strerror
21#include <unistd.h>             // sysconf
22extern "C" {
23      #include <limits.h>       // PTHREAD_STACK_MIN
[dddb3dd0]24        #include <sys/eventfd.h>  // eventfd
[e660761]25      #include <sys/mman.h>     // mprotect
26      #include <sys/resource.h> // getrlimit
27}
28
29// CFA Includes
30#include "kernel_private.hfa"
31#include "startup.hfa"          // STARTUP_PRIORITY_XXX
[bfcf6b9]32#include "math.hfa"
[e660761]33
[97229d6]34#define CFA_PROCESSOR_USE_MMAP 0
35
[e660761]36//-----------------------------------------------------------------------------
37// Some assembly required
38#if defined( __i386 )
[88cafe7]39        #define CtxGet( ctx ) __asm__ volatile ( \
40                "movl %%esp,%0\n" \
41                "movl %%ebp,%1\n" \
42                : "=rm" (ctx.SP), \
43                  "=rm" (ctx.FP) \
44        )
[e660761]45#elif defined( __x86_64 )
[88cafe7]46        #define CtxGet( ctx ) __asm__ volatile ( \
47                "movq %%rsp,%0\n" \
48                "movq %%rbp,%1\n" \
49                : "=rm" (ctx.SP), \
50                  "=rm" (ctx.FP) \
51        )
52#elif defined( __aarch64__ )
53        #define CtxGet( ctx ) __asm__ volatile ( \
54                "mov %0, sp\n" \
55                "mov %1, fp\n" \
56                : "=rm" (ctx.SP), \
57                  "=rm" (ctx.FP) \
58        )
[e660761]59#else
60        #error unknown hardware architecture
61#endif
62
63//-----------------------------------------------------------------------------
64// Start and stop routine for the kernel, declared first to make sure they run first
65static void __kernel_startup (void) __attribute__(( constructor( STARTUP_PRIORITY_KERNEL ) ));
66static void __kernel_shutdown(void) __attribute__(( destructor ( STARTUP_PRIORITY_KERNEL ) ));
67
68//-----------------------------------------------------------------------------
69// Static Forward Declarations
70struct current_stack_info_t;
71
72static void * __invoke_processor(void * arg);
73static void __kernel_first_resume( processor * this );
74static void __kernel_last_resume ( processor * this );
[a5e7233]75static void init(processor & this, const char name[], cluster & _cltr, $thread * initT);
[e660761]76static void deinit(processor & this);
77static void doregister( struct cluster & cltr );
78static void unregister( struct cluster & cltr );
79static void ?{}( $coroutine & this, current_stack_info_t * info);
80static void ?{}( $thread & this, current_stack_info_t * info);
81static void ?{}(processorCtx_t & this) {}
82static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info);
83
[f2384c9a]84#if defined(__CFA_WITH_VERIFY__)
85        static bool verify_fwd_bck_rng(void);
86#endif
87
[e660761]88//-----------------------------------------------------------------------------
89// Forward Declarations for other modules
90extern void __kernel_alarm_startup(void);
91extern void __kernel_alarm_shutdown(void);
92
93//-----------------------------------------------------------------------------
94// Other Forward Declarations
[1eb239e4]95extern void __wake_proc(processor *);
[e660761]96
97//-----------------------------------------------------------------------------
98// Kernel storage
99KERNEL_STORAGE(cluster,              mainCluster);
100KERNEL_STORAGE(processor,            mainProcessor);
101KERNEL_STORAGE($thread,              mainThread);
102KERNEL_STORAGE(__stack_t,            mainThreadCtx);
103KERNEL_STORAGE(__scheduler_RWLock_t, __scheduler_lock);
104#if !defined(__CFA_NO_STATISTICS__)
105KERNEL_STORAGE(__stats_t, mainProcStats);
106#endif
107
108cluster              * mainCluster;
109processor            * mainProcessor;
110$thread              * mainThread;
111__scheduler_RWLock_t * __scheduler_lock;
112
113extern "C" {
114        struct { __dllist_t(cluster) list; __spinlock_t lock; } __cfa_dbg_global_clusters;
115}
116
[dd92fe9]117extern size_t __page_size;
[28c35e2]118extern int __map_prot;
[e660761]119
120//-----------------------------------------------------------------------------
121// Global state
[8fc652e0]122thread_local struct KernelThreadData __cfaabi_tls __attribute__ ((tls_model ( "initial-exec" ))) @= {
[e660761]123        NULL,                                                                                           // cannot use 0p
124        NULL,
125        NULL,
[e873838]126        NULL,
[e660761]127        { 1, false, false },
128};
129
130//-----------------------------------------------------------------------------
131// Struct to steal stack
132struct current_stack_info_t {
133        __stack_t * storage;  // pointer to stack object
134        void * base;          // base of stack
135        void * limit;         // stack grows towards stack limit
136        void * context;       // address of cfa_context_t
137};
138
139void ?{}( current_stack_info_t & this ) {
140        __stack_context_t ctx;
141        CtxGet( ctx );
142        this.base = ctx.FP;
143
144        rlimit r;
145        getrlimit( RLIMIT_STACK, &r);
146        size_t size = r.rlim_cur;
147
148        this.limit = (void *)(((intptr_t)this.base) - size);
149        this.context = &storage_mainThreadCtx;
150}
151
152
153
154//=============================================================================================
155// Kernel Setup logic
156//=============================================================================================
157//-----------------------------------------------------------------------------
158// Kernel boot procedures
159static void __kernel_startup(void) {
[8fc652e0]160        /* paranoid */ verify( ! __preemption_enabled() );
[e660761]161        __cfadbg_print_safe(runtime_core, "Kernel : Starting\n");
162
163        __cfa_dbg_global_clusters.list{ __get };
164        __cfa_dbg_global_clusters.lock{};
165
[f2384c9a]166        /* paranoid */ verify( verify_fwd_bck_rng() );
167
[e660761]168        // Initialize the global scheduler lock
169        __scheduler_lock = (__scheduler_RWLock_t*)&storage___scheduler_lock;
170        (*__scheduler_lock){};
171
172        // Initialize the main cluster
173        mainCluster = (cluster *)&storage_mainCluster;
174        (*mainCluster){"Main Cluster", 0};
175
176        __cfadbg_print_safe(runtime_core, "Kernel : Main cluster ready\n");
177
178        // Start by initializing the main thread
179        // SKULLDUGGERY: the mainThread steals the process main thread
180        // which will then be scheduled by the mainProcessor normally
181        mainThread = ($thread *)&storage_mainThread;
182        current_stack_info_t info;
183        info.storage = (__stack_t*)&storage_mainThreadCtx;
184        (*mainThread){ &info };
185
186        __cfadbg_print_safe(runtime_core, "Kernel : Main thread ready\n");
187
188
189
190        // Construct the processor context of the main processor
191        void ?{}(processorCtx_t & this, processor * proc) {
192                (this.__cor){ "Processor" };
193                this.__cor.starter = 0p;
194                this.proc = proc;
195        }
196
197        void ?{}(processor & this) with( this ) {
[454f478]198                ( this.terminated ){};
[e660761]199                ( this.runner ){};
[a5e7233]200                init( this, "Main Processor", *mainCluster, 0p );
[e660761]201                kernel_thread = pthread_self();
202
203                runner{ &this };
204                __cfadbg_print_safe(runtime_core, "Kernel : constructed main processor context %p\n", &runner);
205        }
206
207        // Initialize the main processor and the main processor ctx
208        // (the coroutine that contains the processing control flow)
209        mainProcessor = (processor *)&storage_mainProcessor;
210        (*mainProcessor){};
211
212        //initialize the global state variables
[8fc652e0]213        __cfaabi_tls.this_processor = mainProcessor;
214        __cfaabi_tls.this_proc_id   = (__processor_id_t*)mainProcessor;
215        __cfaabi_tls.this_thread    = mainThread;
[e660761]216
217        #if !defined( __CFA_NO_STATISTICS__ )
[8fc652e0]218                __cfaabi_tls.this_stats = (__stats_t *)& storage_mainProcStats;
219                __init_stats( __cfaabi_tls.this_stats );
[e660761]220        #endif
221
222        // Enable preemption
223        __kernel_alarm_startup();
224
225        // Add the main thread to the ready queue
226        // once resume is called on mainProcessor->runner the mainThread needs to be scheduled like any normal thread
[e873838]227        __schedule_thread(mainThread);
[e660761]228
229        // SKULLDUGGERY: Force a context switch to the main processor to set the main thread's context to the current UNIX
230        // context. Hence, the main thread does not begin through __cfactx_invoke_thread, like all other threads. The trick here is that
231        // mainThread is on the ready queue when this call is made.
[8fc652e0]232        __kernel_first_resume( __cfaabi_tls.this_processor );
[e660761]233
234
235        // THE SYSTEM IS NOW COMPLETELY RUNNING
236
237        __cfadbg_print_safe(runtime_core, "Kernel : Started\n--------------------------------------------------\n\n");
238
[8fc652e0]239        /* paranoid */ verify( ! __preemption_enabled() );
[e660761]240        enable_interrupts( __cfaabi_dbg_ctx );
[8fc652e0]241        /* paranoid */ verify( __preemption_enabled() );
242
[e660761]243}
244
245static void __kernel_shutdown(void) {
[8fc652e0]246        /* paranoid */ verify( __preemption_enabled() );
[e660761]247        disable_interrupts();
[8fc652e0]248        /* paranoid */ verify( ! __preemption_enabled() );
[e660761]249
250        __cfadbg_print_safe(runtime_core, "\n--------------------------------------------------\nKernel : Shutting down\n");
251
252        // SKULLDUGGERY: Notify the mainProcessor it needs to terminates.
253        // When its coroutine terminates, it return control to the mainThread
254        // which is currently here
255        __atomic_store_n(&mainProcessor->do_terminate, true, __ATOMIC_RELEASE);
[8fc652e0]256        __kernel_last_resume( __cfaabi_tls.this_processor );
[e660761]257        mainThread->self_cor.state = Halted;
258
259        // THE SYSTEM IS NOW COMPLETELY STOPPED
260
261        // Disable preemption
262        __kernel_alarm_shutdown();
263
264        // Destroy the main processor and its context in reverse order of construction
265        // These were manually constructed so we need manually destroy them
266        void ^?{}(processor & this) with( this ){
267                deinit( this );
268
269                /* paranoid */ verify( this.do_terminate == true );
270                __cfaabi_dbg_print_safe("Kernel : destroyed main processor context %p\n", &runner);
271        }
272
273        ^(*mainProcessor){};
274
275        // Final step, destroy the main thread since it is no longer needed
276
277        // Since we provided a stack to this taxk it will not destroy anything
278        /* paranoid */ verify(mainThread->self_cor.stack.storage == (__stack_t*)(((uintptr_t)&storage_mainThreadCtx)| 0x1));
279        ^(*mainThread){};
280
281        ^(*mainCluster){};
282
283        ^(*__scheduler_lock){};
284
285        ^(__cfa_dbg_global_clusters.list){};
286        ^(__cfa_dbg_global_clusters.lock){};
287
288        __cfadbg_print_safe(runtime_core, "Kernel : Shutdown complete\n");
289}
290
291//=============================================================================================
292// Kernel Initial Scheduling logic
293//=============================================================================================
294
295// Context invoker for processors
296// This is the entry point for processors (kernel threads) *except* for the main processor
297// It effectively constructs a coroutine by stealing the pthread stack
298static void * __invoke_processor(void * arg) {
299        #if !defined( __CFA_NO_STATISTICS__ )
300                __stats_t local_stats;
301                __init_stats( &local_stats );
[8fc652e0]302                __cfaabi_tls.this_stats = &local_stats;
[e660761]303        #endif
304
305        processor * proc = (processor *) arg;
[8fc652e0]306        __cfaabi_tls.this_processor = proc;
307        __cfaabi_tls.this_proc_id   = (__processor_id_t*)proc;
308        __cfaabi_tls.this_thread    = 0p;
309        __cfaabi_tls.preemption_state.[enabled, disable_count] = [false, 1];
[e660761]310        // SKULLDUGGERY: We want to create a context for the processor coroutine
311        // which is needed for the 2-step context switch. However, there is no reason
312        // to waste the perfectly valid stack create by pthread.
313        current_stack_info_t info;
314        __stack_t ctx;
315        info.storage = &ctx;
316        (proc->runner){ proc, &info };
317
318        __cfaabi_dbg_print_safe("Coroutine : created stack %p\n", get_coroutine(proc->runner)->stack.storage);
319
320        //Set global state
[8fc652e0]321        __cfaabi_tls.this_thread = 0p;
[e660761]322
323        //We now have a proper context from which to schedule threads
324        __cfadbg_print_safe(runtime_core, "Kernel : core %p created (%p, %p)\n", proc, &proc->runner, &ctx);
325
326        // SKULLDUGGERY: Since the coroutine doesn't have its own stack, we can't
327        // resume it to start it like it normally would, it will just context switch
328        // back to here. Instead directly call the main since we already are on the
329        // appropriate stack.
330        get_coroutine(proc->runner)->state = Active;
331        main( proc->runner );
332        get_coroutine(proc->runner)->state = Halted;
333
334        // Main routine of the core returned, the core is now fully terminated
335        __cfadbg_print_safe(runtime_core, "Kernel : core %p main ended (%p)\n", proc, &proc->runner);
336
337        #if !defined(__CFA_NO_STATISTICS__)
338                __tally_stats(proc->cltr->stats, &local_stats);
339                if( 0 != proc->print_stats ) {
[1b033b8]340                        __print_stats( &local_stats, proc->print_stats, "Processor ", proc->name, (void*)proc );
[e660761]341                }
342        #endif
343
344        return 0p;
345}
346
347static void __kernel_first_resume( processor * this ) {
348        $thread * src = mainThread;
349        $coroutine * dst = get_coroutine(this->runner);
350
[8fc652e0]351        /* paranoid */ verify( ! __preemption_enabled() );
[e660761]352
[8fc652e0]353        __cfaabi_tls.this_thread->curr_cor = dst;
[e660761]354        __stack_prepare( &dst->stack, 65000 );
355        __cfactx_start(main, dst, this->runner, __cfactx_invoke_coroutine);
356
[8fc652e0]357        /* paranoid */ verify( ! __preemption_enabled() );
[e660761]358
359        dst->last = &src->self_cor;
360        dst->starter = dst->starter ? dst->starter : &src->self_cor;
361
362        // make sure the current state is still correct
363        /* paranoid */ verify(src->state == Ready);
364
365        // context switch to specified coroutine
366        verify( dst->context.SP );
367        __cfactx_switch( &src->context, &dst->context );
368        // when __cfactx_switch returns we are back in the src coroutine
369
370        mainThread->curr_cor = &mainThread->self_cor;
371
372        // make sure the current state has been update
373        /* paranoid */ verify(src->state == Active);
374
[8fc652e0]375        /* paranoid */ verify( ! __preemption_enabled() );
[e660761]376}
377
378// KERNEL_ONLY
379static void __kernel_last_resume( processor * this ) {
380        $coroutine * src = &mainThread->self_cor;
381        $coroutine * dst = get_coroutine(this->runner);
382
[8fc652e0]383        /* paranoid */ verify( ! __preemption_enabled() );
384        /* paranoid */ verify( dst->starter == src );
385        /* paranoid */ verify( dst->context.SP );
[e660761]386
387        // SKULLDUGGERY in debug the processors check that the
388        // stack is still within the limit of the stack limits after running a thread.
389        // that check doesn't make sense if we context switch to the processor using the
390        // coroutine semantics. Since this is a special case, use the current context
391        // info to populate these fields.
392        __cfaabi_dbg_debug_do(
393                __stack_context_t ctx;
394                CtxGet( ctx );
395                mainThread->context.SP = ctx.SP;
396                mainThread->context.FP = ctx.FP;
397        )
398
399        // context switch to the processor
400        __cfactx_switch( &src->context, &dst->context );
401}
402
403
404//=============================================================================================
405// Kernel Object Constructors logic
406//=============================================================================================
407//-----------------------------------------------------------------------------
408// Main thread construction
409static void ?{}( $coroutine & this, current_stack_info_t * info) with( this ) {
410        stack.storage = info->storage;
411        with(*stack.storage) {
412                limit     = info->limit;
413                base      = info->base;
414        }
415        __attribute__((may_alias)) intptr_t * istorage = (intptr_t*) &stack.storage;
416        *istorage |= 0x1;
417        name = "Main Thread";
418        state = Start;
419        starter = 0p;
420        last = 0p;
421        cancellation = 0p;
422}
423
424static void ?{}( $thread & this, current_stack_info_t * info) with( this ) {
[6a77224]425        ticket = TICKET_RUNNING;
[e660761]426        state = Start;
427        self_cor{ info };
428        curr_cor = &self_cor;
429        curr_cluster = mainCluster;
430        self_mon.owner = &this;
431        self_mon.recursion = 1;
432        self_mon_p = &self_mon;
433        link.next = 0p;
434        link.prev = 0p;
[b4b63e8]435        #if defined( __CFA_WITH_VERIFY__ )
[ac12f1f]436                canary = 0x0D15EA5E0D15EA5Ep;
[b4b63e8]437        #endif
[e660761]438
439        node.next = 0p;
440        node.prev = 0p;
441        doregister(curr_cluster, this);
442
443        monitors{ &self_mon_p, 1, (fptr_t)0 };
444}
445
446//-----------------------------------------------------------------------------
447// Processor
448// Construct the processor context of non-main processors
449static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info) {
450        (this.__cor){ info };
451        this.proc = proc;
452}
453
[a5e7233]454static void init(processor & this, const char name[], cluster & _cltr, $thread * initT) with( this ) {
[e660761]455        this.name = name;
456        this.cltr = &_cltr;
[58d64a4]457        full_proc = true;
[e660761]458        do_terminate = false;
459        preemption_alarm = 0p;
460        pending_preemption = false;
461
[78da4ab]462        this.io.ctx = 0p;
[dddb3dd0]463        this.io.pending = false;
464        this.io.dirty   = false;
465
[a5e7233]466        this.init.thrd = initT;
[a1538cd]467
[dddb3dd0]468        this.idle = eventfd(0, 0);
469        if (idle < 0) {
470                abort("KERNEL ERROR: PROCESSOR EVENTFD - %s\n", strerror(errno));
471        }
[78da4ab]472
[e660761]473        #if !defined(__CFA_NO_STATISTICS__)
474                print_stats = 0;
475                print_halts = false;
476        #endif
477
[1eb239e4]478        lock( this.cltr->idles );
479                int target = this.cltr->idles.total += 1u;
480        unlock( this.cltr->idles );
[e660761]481
482        id = doregister((__processor_id_t*)&this);
483
484        // Lock the RWlock so no-one pushes/pops while we are changing the queue
485        uint_fast32_t last_size = ready_mutate_lock();
486
487                // Adjust the ready queue size
[bd0bdd37]488                this.cltr_id = ready_queue_grow( cltr, target );
[e660761]489
490        // Unlock the RWlock
491        ready_mutate_unlock( last_size );
492
493        __cfadbg_print_safe(runtime_core, "Kernel : core %p created\n", &this);
494}
495
496// Not a ctor, it just preps the destruction but should not destroy members
497static void deinit(processor & this) {
[1eb239e4]498        lock( this.cltr->idles );
499                int target = this.cltr->idles.total -= 1u;
500        unlock( this.cltr->idles );
[e660761]501
502        // Lock the RWlock so no-one pushes/pops while we are changing the queue
503        uint_fast32_t last_size = ready_mutate_lock();
504
505                // Adjust the ready queue size
506                ready_queue_shrink( this.cltr, target );
507
508        // Unlock the RWlock
509        ready_mutate_unlock( last_size );
510
511        // Finally we don't need the read_lock any more
512        unregister((__processor_id_t*)&this);
[dddb3dd0]513
514        close(this.idle);
[e660761]515}
516
[a5e7233]517void ?{}(processor & this, const char name[], cluster & _cltr, $thread * initT) {
[454f478]518        ( this.terminated ){};
[e660761]519        ( this.runner ){};
[62502cc4]520
521        disable_interrupts();
[a5e7233]522                init( this, name, _cltr, initT );
[62502cc4]523        enable_interrupts( __cfaabi_dbg_ctx );
[e660761]524
525        __cfadbg_print_safe(runtime_core, "Kernel : Starting core %p\n", &this);
526
527        this.stack = __create_pthread( &this.kernel_thread, __invoke_processor, (void *)&this );
[a5e7233]528}
[e660761]529
[a5e7233]530void ?{}(processor & this, const char name[], cluster & _cltr) {
531        (this){name, _cltr, 0p};
[e660761]532}
533
[bfcf6b9]534extern size_t __page_size;
[e660761]535void ^?{}(processor & this) with( this ){
536        if( ! __atomic_load_n(&do_terminate, __ATOMIC_ACQUIRE) ) {
537                __cfadbg_print_safe(runtime_core, "Kernel : core %p signaling termination\n", &this);
538
539                __atomic_store_n(&do_terminate, true, __ATOMIC_RELAXED);
540                __wake_proc( &this );
541
[454f478]542                wait( terminated );
[8fc652e0]543                /* paranoid */ verify( active_processor() != &this);
[e660761]544        }
545
[bfcf6b9]546        __destroy_pthread( kernel_thread, this.stack, 0p );
[e660761]547
[62502cc4]548        disable_interrupts();
549                deinit( this );
550        enable_interrupts( __cfaabi_dbg_ctx );
[e660761]551}
552
553//-----------------------------------------------------------------------------
554// Cluster
[1eb239e4]555static void ?{}(__cluster_idles & this) {
556        this.lock  = 0;
557        this.idle  = 0;
558        this.total = 0;
559        (this.list){};
560}
561
[e660761]562void ?{}(cluster & this, const char name[], Duration preemption_rate, unsigned num_io, const io_context_params & io_params) with( this ) {
563        this.name = name;
564        this.preemption_rate = preemption_rate;
565        ready_queue{};
566
567        #if !defined(__CFA_NO_STATISTICS__)
568                print_stats = 0;
569                stats = alloc();
570                __init_stats( stats );
571        #endif
572
573        threads{ __get };
574
[78da4ab]575        io.arbiter = create();
576        io.params = io_params;
577
[e660761]578        doregister(this);
579
580        // Lock the RWlock so no-one pushes/pops while we are changing the queue
[772411a]581        disable_interrupts();
[e660761]582        uint_fast32_t last_size = ready_mutate_lock();
583
584                // Adjust the ready queue size
585                ready_queue_grow( &this, 0 );
586
587        // Unlock the RWlock
588        ready_mutate_unlock( last_size );
[772411a]589        enable_interrupts_noPoll(); // Don't poll, could be in main cluster
[e660761]590}
591
592void ^?{}(cluster & this) {
[78da4ab]593        destroy(this.io.arbiter);
[e660761]594
595        // Lock the RWlock so no-one pushes/pops while we are changing the queue
[772411a]596        disable_interrupts();
[e660761]597        uint_fast32_t last_size = ready_mutate_lock();
598
599                // Adjust the ready queue size
600                ready_queue_shrink( &this, 0 );
601
602        // Unlock the RWlock
603        ready_mutate_unlock( last_size );
[772411a]604        enable_interrupts_noPoll(); // Don't poll, could be in main cluster
[e660761]605
606        #if !defined(__CFA_NO_STATISTICS__)
607                if( 0 != this.print_stats ) {
[1b033b8]608                        __print_stats( this.stats, this.print_stats, "Cluster", this.name, (void*)&this );
[e660761]609                }
610                free( this.stats );
611        #endif
612
613        unregister(this);
614}
615
616//=============================================================================================
617// Miscellaneous Initialization
618//=============================================================================================
619//-----------------------------------------------------------------------------
620// Global Queues
621static void doregister( cluster     & cltr ) {
622        lock      ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
623        push_front( __cfa_dbg_global_clusters.list, cltr );
624        unlock    ( __cfa_dbg_global_clusters.lock );
625}
626
627static void unregister( cluster     & cltr ) {
628        lock  ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
629        remove( __cfa_dbg_global_clusters.list, cltr );
630        unlock( __cfa_dbg_global_clusters.lock );
631}
632
633void doregister( cluster * cltr, $thread & thrd ) {
634        lock      (cltr->thread_list_lock __cfaabi_dbg_ctx2);
635        cltr->nthreads += 1;
636        push_front(cltr->threads, thrd);
637        unlock    (cltr->thread_list_lock);
638}
639
640void unregister( cluster * cltr, $thread & thrd ) {
641        lock  (cltr->thread_list_lock __cfaabi_dbg_ctx2);
642        remove(cltr->threads, thrd );
643        cltr->nthreads -= 1;
644        unlock(cltr->thread_list_lock);
645}
646
647static void check( int ret, const char func[] ) {
648        if ( ret ) {                                                                            // pthread routines return errno values
649                abort( "%s : internal error, error(%d) %s.", func, ret, strerror( ret ) );
650        } // if
651} // Abort
652
653void * __create_pthread( pthread_t * pthread, void * (*start)(void *), void * arg ) {
654        pthread_attr_t attr;
655
656        check( pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
657
658        size_t stacksize;
659        // default stack size, normally defined by shell limit
660        check( pthread_attr_getstacksize( &attr, &stacksize ), "pthread_attr_getstacksize" );
661        assert( stacksize >= PTHREAD_STACK_MIN );
662
663        void * stack;
[97229d6]664        #if CFA_PROCESSOR_USE_MMAP
665                stacksize = ceiling( stacksize, __page_size ) + __page_size;
[dd92fe9]666                stack = mmap(0p, stacksize, __map_prot, MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
[97229d6]667                if(stack == ((void*)-1)) {
668                        abort( "pthread stack creation : internal error, mmap failure, error(%d) %s.", errno, strerror( errno ) );
669                }
670                if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
671                        abort( "pthread stack creation : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
672                } // if
673        #else
674                __cfaabi_dbg_debug_do(
675                        stack = memalign( __page_size, stacksize + __page_size );
676                        // pthread has no mechanism to create the guard page in user supplied stack.
677                        if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
678                                abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
679                        } // if
680                );
681                __cfaabi_dbg_no_debug_do(
682                        stack = malloc( stacksize );
683                );
684        #endif
685
[e660761]686
687        check( pthread_attr_setstack( &attr, stack, stacksize ), "pthread_attr_setstack" );
688
689        check( pthread_create( pthread, &attr, start, arg ), "pthread_create" );
690        return stack;
[88cafe7]691}
[f2384c9a]692
[bfcf6b9]693void __destroy_pthread( pthread_t pthread, void * stack, void ** retval ) {
694        int err = pthread_join( pthread, retval );
695        if( err != 0 ) abort("KERNEL ERROR: joining pthread %p caused error %s\n", (void*)pthread, strerror(err));
696
[97229d6]697        #if CFA_PROCESSOR_USE_MMAP
698                pthread_attr_t attr;
[bfcf6b9]699
[97229d6]700                check( pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
[bfcf6b9]701
[97229d6]702                size_t stacksize;
703                // default stack size, normally defined by shell limit
704                check( pthread_attr_getstacksize( &attr, &stacksize ), "pthread_attr_getstacksize" );
705                assert( stacksize >= PTHREAD_STACK_MIN );
706                stacksize += __page_size;
[bfcf6b9]707
[97229d6]708                if(munmap(stack, stacksize) == -1) {
709                        abort( "pthread stack destruction : internal error, munmap failure, error(%d) %s.", errno, strerror( errno ) );
710                }
711        #else
[72a3aff]712                __cfaabi_dbg_debug_do(
713                        // pthread has no mechanism to create the guard page in user supplied stack.
[28c35e2]714                        if ( mprotect( stack, __page_size, __map_prot ) == -1 ) {
[72a3aff]715                                abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
716                        } // if
717                );
[97229d6]718                free( stack );
719        #endif
[bfcf6b9]720}
721
[f2384c9a]722#if defined(__CFA_WITH_VERIFY__)
723static bool verify_fwd_bck_rng(void) {
[8fc652e0]724        __cfaabi_tls.ready_rng.fwd_seed = 25214903917_l64u * (rdtscl() ^ (uintptr_t)&verify_fwd_bck_rng);
[f2384c9a]725
726        unsigned values[10];
727        for(i; 10) {
728                values[i] = __tls_rand_fwd();
729        }
730
731        __tls_rand_advance_bck();
732
733        for ( i; 9 -~= 0 ) {
734                if(values[i] != __tls_rand_bck()) {
735                        return false;
736                }
737        }
738
739        return true;
740}
[e67a82d]741#endif
Note: See TracBrowser for help on using the repository browser.