source: libcfa/src/concurrency/kernel/startup.cfa @ 7d0ebd0

ADTast-experimentalenumforall-pointer-decaypthread-emulationqualifiedEnum
Last change on this file since 7d0ebd0 was 7d0ebd0, checked in by Thierry Delisle <tdelisle@…>, 3 years ago

Processors should now correctly be unconditionnaly woken-up on termination

  • Property mode set to 100644
File size: 24.7 KB
Line 
1//
2// Cforall Version 1.0.0 Copyright (C) 2020 University of Waterloo
3//
4// The contents of this file are covered under the licence agreement in the
5// file "LICENCE" distributed with Cforall.
6//
7// kernel/startup.cfa --
8//
9// Author           : Thierry Delisle
10// Created On       : Thu Jul 30 15:12:54 2020
11// Last Modified By :
12// Last Modified On :
13// Update Count     :
14//
15
16#define __cforall_thread__
17#define _GNU_SOURCE
18
19// C Includes
20#include <errno.h>              // errno
21#include <signal.h>
22#include <string.h>             // strerror
23#include <unistd.h>             // sysconf
24
25extern "C" {
26      #include <limits.h>       // PTHREAD_STACK_MIN
27        #include <unistd.h>       // syscall
28        #include <sys/eventfd.h>  // eventfd
29      #include <sys/mman.h>     // mprotect
30      #include <sys/resource.h> // getrlimit
31}
32
33// CFA Includes
34#include "kernel_private.hfa"
35#include "startup.hfa"          // STARTUP_PRIORITY_XXX
36#include "math.hfa"
37
38#define CFA_PROCESSOR_USE_MMAP 0
39
40//-----------------------------------------------------------------------------
41// Some assembly required
42#if defined( __i386 )
43        #define CtxGet( ctx ) __asm__ volatile ( \
44                "movl %%esp,%0\n" \
45                "movl %%ebp,%1\n" \
46                : "=rm" (ctx.SP), \
47                  "=rm" (ctx.FP) \
48        )
49#elif defined( __x86_64 )
50        #define CtxGet( ctx ) __asm__ volatile ( \
51                "movq %%rsp,%0\n" \
52                "movq %%rbp,%1\n" \
53                : "=rm" (ctx.SP), \
54                  "=rm" (ctx.FP) \
55        )
56#elif defined( __aarch64__ )
57        #define CtxGet( ctx ) __asm__ volatile ( \
58                "mov %0, sp\n" \
59                "mov %1, fp\n" \
60                : "=rm" (ctx.SP), \
61                  "=rm" (ctx.FP) \
62        )
63#else
64        #error unknown hardware architecture
65#endif
66
67//-----------------------------------------------------------------------------
68// Start and stop routine for the kernel, declared first to make sure they run first
69static void __kernel_startup (void) __attribute__(( constructor( STARTUP_PRIORITY_KERNEL ) ));
70static void __kernel_shutdown(void) __attribute__(( destructor ( STARTUP_PRIORITY_KERNEL ) ));
71
72//-----------------------------------------------------------------------------
73// Static Forward Declarations
74struct current_stack_info_t;
75
76static void * __invoke_processor(void * arg);
77static void __kernel_first_resume( processor * this );
78static void __kernel_last_resume ( processor * this );
79static void init(processor & this, const char name[], cluster & _cltr, thread$ * initT);
80static void deinit(processor & this);
81static void doregister( struct cluster & cltr );
82static void unregister( struct cluster & cltr );
83static void register_tls( processor * this );
84static void unregister_tls( processor * this );
85static void ?{}( coroutine$ & this, current_stack_info_t * info);
86static void ?{}( thread$ & this, current_stack_info_t * info);
87static void ?{}(processorCtx_t & this) {}
88static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info);
89
90#if defined(__CFA_WITH_VERIFY__)
91        static bool verify_fwd_bck_rng(void);
92#endif
93
94//-----------------------------------------------------------------------------
95// Forward Declarations for other modules
96extern void __kernel_alarm_startup(void);
97extern void __kernel_alarm_shutdown(void);
98
99//-----------------------------------------------------------------------------
100// Other Forward Declarations
101extern void __wake_proc(processor *);
102extern int cfa_main_returned;                                                   // from interpose.cfa
103
104//-----------------------------------------------------------------------------
105// Kernel storage
106KERNEL_STORAGE(cluster,              mainCluster);
107KERNEL_STORAGE(processor,            mainProcessor);
108KERNEL_STORAGE(thread$,              mainThread);
109KERNEL_STORAGE(__stack_t,            mainThreadCtx);
110KERNEL_STORAGE(__scheduler_RWLock_t, __scheduler_lock);
111#if !defined(__CFA_NO_STATISTICS__)
112KERNEL_STORAGE(__stats_t, mainProcStats);
113#endif
114
115cluster              * mainCluster;
116processor            * mainProcessor;
117thread$              * mainThread;
118__scheduler_RWLock_t * __scheduler_lock;
119
120extern "C" {
121        struct { __dllist_t(cluster) list; __spinlock_t lock; } __cfa_dbg_global_clusters;
122}
123
124extern size_t __page_size;
125extern int __map_prot;
126
127//-----------------------------------------------------------------------------
128// Global state
129thread_local struct KernelThreadData __cfaabi_tls __attribute__ ((tls_model ( "initial-exec" ))) @= {
130        NULL,                                                                                           // cannot use 0p
131        NULL,
132        false,
133        { 1, false, false },
134        0,
135        { 0, 0 },
136        NULL,
137        #ifdef __CFA_WITH_VERIFY__
138                false,
139                0,
140        #endif
141};
142
143#if   defined(CFA_HAVE_LINUX_LIBRSEQ)
144        // No data needed
145#elif defined(CFA_HAVE_LINUX_RSEQ_H)
146        extern "Cforall" {
147                __attribute__((aligned(128))) thread_local volatile struct rseq __cfaabi_rseq @= {
148                        .cpu_id : RSEQ_CPU_ID_UNINITIALIZED,
149                };
150        }
151#else
152        // No data needed
153#endif
154
155//-----------------------------------------------------------------------------
156// Struct to steal stack
157struct current_stack_info_t {
158        __stack_t * storage;  // pointer to stack object
159        void * base;          // base of stack
160        void * limit;         // stack grows towards stack limit
161        void * context;       // address of cfa_context_t
162};
163
164void ?{}( current_stack_info_t & this ) {
165        __stack_context_t ctx;
166        CtxGet( ctx );
167        this.base = ctx.FP;
168
169        rlimit r;
170        getrlimit( RLIMIT_STACK, &r);
171        size_t size = r.rlim_cur;
172
173        this.limit = (void *)(((intptr_t)this.base) - size);
174        this.context = &storage_mainThreadCtx;
175}
176
177
178
179//=============================================================================================
180// Kernel Setup logic
181//=============================================================================================
182//-----------------------------------------------------------------------------
183// Kernel boot procedures
184static void __kernel_startup(void) {
185        /* paranoid */ verify( ! __preemption_enabled() );
186        __cfadbg_print_safe(runtime_core, "Kernel : Starting\n");
187
188        __cfa_dbg_global_clusters.list{ __get };
189        __cfa_dbg_global_clusters.lock{};
190
191        /* paranoid */ verify( verify_fwd_bck_rng() );
192
193        // Initialize the global scheduler lock
194        __scheduler_lock = (__scheduler_RWLock_t*)&storage___scheduler_lock;
195        (*__scheduler_lock){};
196
197        // Initialize the main cluster
198        mainCluster = (cluster *)&storage_mainCluster;
199        (*mainCluster){"Main Cluster", 0};
200
201        __cfadbg_print_safe(runtime_core, "Kernel : Main cluster ready\n");
202
203        // Construct the processor context of the main processor
204        void ?{}(processorCtx_t & this, processor * proc) {
205                (this.__cor){ "Processor" };
206                this.__cor.starter = 0p;
207                this.proc = proc;
208        }
209
210        void ?{}(processor & this) with( this ) {
211                ( this.terminated ){};
212                ( this.runner ){};
213                init( this, "Main Processor", *mainCluster, 0p );
214                kernel_thread = pthread_self();
215
216                runner{ &this };
217                __cfadbg_print_safe(runtime_core, "Kernel : constructed main processor context %p\n", &runner);
218        }
219
220        // Initialize the main processor and the main processor ctx
221        // (the coroutine that contains the processing control flow)
222        mainProcessor = (processor *)&storage_mainProcessor;
223        (*mainProcessor){};
224
225        register_tls( mainProcessor );
226
227        // Start by initializing the main thread
228        // SKULLDUGGERY: the mainThread steals the process main thread
229        // which will then be scheduled by the mainProcessor normally
230        mainThread = (thread$ *)&storage_mainThread;
231        current_stack_info_t info;
232        info.storage = (__stack_t*)&storage_mainThreadCtx;
233        (*mainThread){ &info };
234
235        __cfadbg_print_safe(runtime_core, "Kernel : Main thread ready\n");
236
237        //initialize the global state variables
238        __cfaabi_tls.this_processor = mainProcessor;
239        __cfaabi_tls.this_thread    = mainThread;
240
241        #if !defined( __CFA_NO_STATISTICS__ )
242                __cfaabi_tls.this_stats = (__stats_t *)& storage_mainProcStats;
243                __init_stats( __cfaabi_tls.this_stats );
244        #endif
245        mainProcessor->local_data = &__cfaabi_tls;
246
247        // Enable preemption
248        __kernel_alarm_startup();
249
250        // Add the main thread to the ready queue
251        // once resume is called on mainProcessor->runner the mainThread needs to be scheduled like any normal thread
252        schedule_thread$(mainThread, UNPARK_LOCAL);
253
254        // SKULLDUGGERY: Force a context switch to the main processor to set the main thread's context to the current UNIX
255        // context. Hence, the main thread does not begin through __cfactx_invoke_thread, like all other threads. The trick here is that
256        // mainThread is on the ready queue when this call is made.
257        __kernel_first_resume( __cfaabi_tls.this_processor );
258
259
260        // THE SYSTEM IS NOW COMPLETELY RUNNING
261
262        __cfadbg_print_safe(runtime_core, "Kernel : Started\n--------------------------------------------------\n\n");
263
264        /* paranoid */ verify( ! __preemption_enabled() );
265        enable_interrupts();
266        /* paranoid */ verify( __preemption_enabled() );
267
268}
269
270static void __kernel_shutdown(void) {
271        if(!cfa_main_returned) return;
272        /* paranoid */ verify( __preemption_enabled() );
273        disable_interrupts();
274        /* paranoid */ verify( ! __preemption_enabled() );
275
276        __cfadbg_print_safe(runtime_core, "\n--------------------------------------------------\nKernel : Shutting down\n");
277
278        // SKULLDUGGERY: Notify the mainProcessor it needs to terminates.
279        // When its coroutine terminates, it return control to the mainThread
280        // which is currently here
281        /* paranoid */ verify( !__atomic_load_n(&mainProcessor->do_terminate, __ATOMIC_ACQUIRE) );
282        __atomic_store_n(&mainProcessor->do_terminate, true, __ATOMIC_RELEASE);
283        __wake_proc( mainProcessor );
284        __kernel_last_resume( __cfaabi_tls.this_processor );
285        mainThread->self_cor.state = Halted;
286
287        // THE SYSTEM IS NOW COMPLETELY STOPPED
288
289        // Disable preemption
290        __kernel_alarm_shutdown();
291
292        #if !defined( __CFA_NO_STATISTICS__ )
293                __stats_t * st = (__stats_t *)& storage_mainProcStats;
294                __tally_stats(mainCluster->stats, st);
295                if( 0 != mainProcessor->print_stats ) {
296                        __print_stats( st, mainProcessor->print_stats, "Processor ", mainProcessor->name, (void*)mainProcessor );
297                }
298                #if defined(CFA_STATS_ARRAY)
299                        __flush_stat( st, "Processor", mainProcessor );
300                #endif
301        #endif
302
303        mainProcessor->local_data = 0p;
304
305        unregister_tls( mainProcessor );
306
307        // Destroy the main processor and its context in reverse order of construction
308        // These were manually constructed so we need manually destroy them
309        void ^?{}(processor & this) with( this ){
310                deinit( this );
311
312                /* paranoid */ verify( this.do_terminate == true );
313                __cfaabi_dbg_print_safe("Kernel : destroyed main processor context %p\n", &runner);
314        }
315
316        ^(*mainProcessor){};
317
318        // Final step, destroy the main thread since it is no longer needed
319
320        // Since we provided a stack to this taxk it will not destroy anything
321        /* paranoid */ verify(mainThread->self_cor.stack.storage == (__stack_t*)(((uintptr_t)&storage_mainThreadCtx)| 0x1));
322        ^(*mainThread){};
323
324        ^(*mainCluster){};
325
326        ^(*__scheduler_lock){};
327
328        ^(__cfa_dbg_global_clusters.list){};
329        ^(__cfa_dbg_global_clusters.lock){};
330
331        __cfadbg_print_safe(runtime_core, "Kernel : Shutdown complete\n");
332}
333
334//=============================================================================================
335// Kernel Initial Scheduling logic
336//=============================================================================================
337
338// Context invoker for processors
339// This is the entry point for processors (kernel threads) *except* for the main processor
340// It effectively constructs a coroutine by stealing the pthread stack
341static void * __invoke_processor(void * arg) {
342        #if !defined( __CFA_NO_STATISTICS__ )
343                __stats_t local_stats;
344                __init_stats( &local_stats );
345                __cfaabi_tls.this_stats = &local_stats;
346        #endif
347
348        processor * proc = (processor *) arg;
349        __cfaabi_tls.this_processor = proc;
350        __cfaabi_tls.this_thread    = 0p;
351        __cfaabi_tls.preemption_state.[enabled, disable_count] = [false, 1];
352        proc->local_data = &__cfaabi_tls;
353
354        register_tls( proc );
355
356        // SKULLDUGGERY: We want to create a context for the processor coroutine
357        // which is needed for the 2-step context switch. However, there is no reason
358        // to waste the perfectly valid stack create by pthread.
359        current_stack_info_t info;
360        __stack_t ctx;
361        info.storage = &ctx;
362        (proc->runner){ proc, &info };
363
364        __cfaabi_dbg_print_safe("Coroutine : created stack %p\n", get_coroutine(proc->runner)->stack.storage);
365
366        //Set global state
367        __cfaabi_tls.this_thread = 0p;
368
369        //We now have a proper context from which to schedule threads
370        __cfadbg_print_safe(runtime_core, "Kernel : core %p created (%p, %p)\n", proc, &proc->runner, &ctx);
371
372        // SKULLDUGGERY: Since the coroutine doesn't have its own stack, we can't
373        // resume it to start it like it normally would, it will just context switch
374        // back to here. Instead directly call the main since we already are on the
375        // appropriate stack.
376        get_coroutine(proc->runner)->state = Active;
377        main( proc->runner );
378        get_coroutine(proc->runner)->state = Halted;
379
380        // Main routine of the core returned, the core is now fully terminated
381        __cfadbg_print_safe(runtime_core, "Kernel : core %p main ended (%p)\n", proc, &proc->runner);
382
383        #if !defined(__CFA_NO_STATISTICS__)
384                __tally_stats(proc->cltr->stats, &local_stats);
385                if( 0 != proc->print_stats ) {
386                        __print_stats( &local_stats, proc->print_stats, "Processor ", proc->name, (void*)proc );
387                }
388                #if defined(CFA_STATS_ARRAY)
389                        __flush_stat( &local_stats, "Processor", proc );
390                #endif
391        #endif
392
393        proc->local_data = 0p;
394
395        unregister_tls( proc );
396
397        return 0p;
398}
399
400static void __kernel_first_resume( processor * this ) {
401        thread$ * src = mainThread;
402        coroutine$ * dst = get_coroutine(this->runner);
403
404        /* paranoid */ verify( ! __preemption_enabled() );
405
406        __cfaabi_tls.this_thread->curr_cor = dst;
407        __stack_prepare( &dst->stack, DEFAULT_STACK_SIZE );
408        __cfactx_start(main, dst, this->runner, __cfactx_invoke_coroutine);
409
410        /* paranoid */ verify( ! __preemption_enabled() );
411
412        dst->last = &src->self_cor;
413        dst->starter = dst->starter ? dst->starter : &src->self_cor;
414
415        // make sure the current state is still correct
416        /* paranoid */ verify(src->state == Ready);
417        src->corctx_flag = true;
418
419        // context switch to specified coroutine
420        verify( dst->context.SP );
421        __cfactx_switch( &src->context, &dst->context );
422        // when __cfactx_switch returns we are back in the src coroutine
423
424        mainThread->curr_cor = &mainThread->self_cor;
425
426        // make sure the current state has been update
427        /* paranoid */ verify(src->state == Active);
428
429        /* paranoid */ verify( ! __preemption_enabled() );
430}
431
432// KERNEL_ONLY
433static void __kernel_last_resume( processor * this ) {
434        coroutine$ * src = &mainThread->self_cor;
435        coroutine$ * dst = get_coroutine(this->runner);
436
437        /* paranoid */ verify( ! __preemption_enabled() );
438        /* paranoid */ verify( dst->starter == src );
439        /* paranoid */ verify( dst->context.SP );
440
441        // SKULLDUGGERY in debug the processors check that the
442        // stack is still within the limit of the stack limits after running a thread.
443        // that check doesn't make sense if we context switch to the processor using the
444        // coroutine semantics. Since this is a special case, use the current context
445        // info to populate these fields.
446        __cfaabi_dbg_debug_do(
447                __stack_context_t ctx;
448                CtxGet( ctx );
449                mainThread->context.SP = ctx.SP;
450                mainThread->context.FP = ctx.FP;
451        )
452
453        // context switch to the processor
454        __cfactx_switch( &src->context, &dst->context );
455}
456
457
458//=============================================================================================
459// Kernel Object Constructors logic
460//=============================================================================================
461//-----------------------------------------------------------------------------
462// Main thread construction
463static void ?{}( coroutine$ & this, current_stack_info_t * info) with( this ) {
464        stack.storage = info->storage;
465        with(*stack.storage) {
466                limit     = info->limit;
467                base      = info->base;
468        }
469        __attribute__((may_alias)) intptr_t * istorage = (intptr_t*) &stack.storage;
470        *istorage |= 0x1;
471        name = "Main Thread";
472        state = Start;
473        starter = 0p;
474        last = 0p;
475        cancellation = 0p;
476}
477
478static void ?{}( thread$ & this, current_stack_info_t * info) with( this ) {
479        ticket = TICKET_RUNNING;
480        state = Start;
481        self_cor{ info };
482        curr_cor = &self_cor;
483        curr_cluster = mainCluster;
484        self_mon.owner = &this;
485        self_mon.recursion = 1;
486        self_mon_p = &self_mon;
487        link.next = 0p;
488        link.ts   = -1llu;
489        preferred = ready_queue_new_preferred();
490        last_proc = 0p;
491        #if defined( __CFA_WITH_VERIFY__ )
492                canary = 0x0D15EA5E0D15EA5Ep;
493        #endif
494
495        node.next = 0p;
496        node.prev = 0p;
497        doregister(curr_cluster, this);
498
499        monitors{ &self_mon_p, 1, (fptr_t)0 };
500}
501
502//-----------------------------------------------------------------------------
503// Processor
504// Construct the processor context of non-main processors
505static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info) {
506        (this.__cor){ info };
507        this.proc = proc;
508}
509
510static void init(processor & this, const char name[], cluster & _cltr, thread$ * initT) with( this ) {
511        this.name = name;
512        this.cltr = &_cltr;
513        this.rdq.its = 0;
514        this.rdq.itr = 0;
515        this.rdq.id  = -1u;
516        this.rdq.target = -1u;
517        this.rdq.last = -1u;
518        this.rdq.cutoff = 0ull;
519        do_terminate = false;
520        preemption_alarm = 0p;
521        pending_preemption = false;
522
523        this.io.ctx = 0p;
524        this.io.pending = false;
525        this.io.dirty   = false;
526
527        this.init.thrd = initT;
528
529        this.local_data = 0p;
530
531        this.idle_fd = eventfd(0, 0);
532        if (idle_fd < 0) {
533                abort("KERNEL ERROR: PROCESSOR EVENTFD - %s\n", strerror(errno));
534        }
535
536        #if !defined(__CFA_NO_STATISTICS__)
537                print_stats = 0;
538                print_halts = false;
539        #endif
540
541        __cfadbg_print_safe(runtime_core, "Kernel : core %p created\n", &this);
542}
543
544// Not a ctor, it just preps the destruction but should not destroy members
545static void deinit(processor & this) {
546        close(this.idle_fd);
547}
548
549void ?{}(processor & this, const char name[], cluster & _cltr, thread$ * initT) {
550        ( this.terminated ){};
551        ( this.runner ){};
552
553        disable_interrupts();
554                init( this, name, _cltr, initT );
555        enable_interrupts();
556
557        __cfadbg_print_safe(runtime_core, "Kernel : Starting core %p\n", &this);
558
559        this.stack = __create_pthread( &this.kernel_thread, __invoke_processor, (void *)&this );
560}
561
562void ?{}(processor & this, const char name[], cluster & _cltr) {
563        (this){name, _cltr, 0p};
564}
565
566extern size_t __page_size;
567void ^?{}(processor & this) with( this ){
568        /* paranoid */ verify( !__atomic_load_n(&do_terminate, __ATOMIC_ACQUIRE) );
569        __cfadbg_print_safe(runtime_core, "Kernel : core %p signaling termination\n", &this);
570
571        __atomic_store_n(&do_terminate, true, __ATOMIC_RELAXED);
572        __disable_interrupts_checked();
573                __wake_proc( &this );
574        __enable_interrupts_checked();
575
576        wait( terminated );
577        /* paranoid */ verify( active_processor() != &this);
578
579        __destroy_pthread( kernel_thread, this.stack, 0p );
580
581        disable_interrupts();
582                deinit( this );
583        enable_interrupts();
584}
585
586//-----------------------------------------------------------------------------
587// Cluster
588static void ?{}(__cluster_proc_list & this) {
589        this.fd    = 0;
590        this.idle  = 0;
591        this.total = 0;
592}
593
594void ?{}(cluster & this, const char name[], Duration preemption_rate, unsigned num_io, const io_context_params & io_params) with( this ) {
595        this.name = name;
596        this.preemption_rate = preemption_rate;
597        ready_queue{};
598
599        #if !defined(__CFA_NO_STATISTICS__)
600                print_stats = 0;
601                stats = alloc();
602                __init_stats( stats );
603        #endif
604
605        threads{ __get };
606
607        io.arbiter = create();
608        io.params = io_params;
609
610        doregister(this);
611
612        // Lock the RWlock so no-one pushes/pops while we are changing the queue
613        disable_interrupts();
614        uint_fast32_t last_size = ready_mutate_lock();
615
616                // Adjust the ready queue size
617                ready_queue_grow( &this );
618
619        // Unlock the RWlock
620        ready_mutate_unlock( last_size );
621        enable_interrupts( false ); // Don't poll, could be in main cluster
622}
623
624void ^?{}(cluster & this) {
625        destroy(this.io.arbiter);
626
627        // Lock the RWlock so no-one pushes/pops while we are changing the queue
628        disable_interrupts();
629        uint_fast32_t last_size = ready_mutate_lock();
630
631                // Adjust the ready queue size
632                ready_queue_shrink( &this );
633
634        // Unlock the RWlock
635        ready_mutate_unlock( last_size );
636        enable_interrupts( false ); // Don't poll, could be in main cluster
637
638        #if !defined(__CFA_NO_STATISTICS__)
639                if( 0 != this.print_stats ) {
640                        __print_stats( this.stats, this.print_stats, "Cluster", this.name, (void*)&this );
641                }
642                #if defined(CFA_STATS_ARRAY)
643                        __flush_stat( this.stats, "Cluster", &this );
644                #endif
645                free( this.stats );
646        #endif
647
648        unregister(this);
649}
650
651//=============================================================================================
652// Miscellaneous Initialization
653//=============================================================================================
654//-----------------------------------------------------------------------------
655// Global Queues
656static void doregister( cluster     & cltr ) {
657        lock      ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
658        push_front( __cfa_dbg_global_clusters.list, cltr );
659        unlock    ( __cfa_dbg_global_clusters.lock );
660}
661
662static void unregister( cluster     & cltr ) {
663        lock  ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
664        remove( __cfa_dbg_global_clusters.list, cltr );
665        unlock( __cfa_dbg_global_clusters.lock );
666}
667
668void doregister( cluster * cltr, thread$ & thrd ) {
669        lock      (cltr->thread_list_lock __cfaabi_dbg_ctx2);
670        cltr->nthreads += 1;
671        push_front(cltr->threads, thrd);
672        unlock    (cltr->thread_list_lock);
673}
674
675void unregister( cluster * cltr, thread$ & thrd ) {
676        lock  (cltr->thread_list_lock __cfaabi_dbg_ctx2);
677        remove(cltr->threads, thrd );
678        cltr->nthreads -= 1;
679        unlock(cltr->thread_list_lock);
680}
681
682static void register_tls( processor * this ) {
683        // Register and Lock the RWlock so no-one pushes/pops while we are changing the queue
684        uint_fast32_t last_size;
685        [this->unique_id, last_size] = ready_mutate_register();
686
687                this->cltr->procs.total += 1u;
688                insert_last(this->cltr->procs.actives, *this);
689
690                // Adjust the ready queue size
691                ready_queue_grow( this->cltr );
692
693        // Unlock the RWlock
694        ready_mutate_unlock( last_size );
695}
696
697
698static void unregister_tls( processor * this ) {
699        // Lock the RWlock so no-one pushes/pops while we are changing the queue
700        uint_fast32_t last_size = ready_mutate_lock();
701                this->cltr->procs.total -= 1u;
702                remove(*this);
703
704                // clear the cluster so nothing gets pushed to local queues
705                cluster * cltr = this->cltr;
706                this->cltr = 0p;
707
708                // Adjust the ready queue size
709                ready_queue_shrink( cltr );
710
711        // Unlock the RWlock and unregister: we don't need the read_lock any more
712        ready_mutate_unregister( this->unique_id, last_size );
713}
714
715static void check( int ret, const char func[] ) {
716        if ( ret ) {                                                                            // pthread routines return errno values
717                abort( "%s : internal error, error(%d) %s.", func, ret, strerror( ret ) );
718        } // if
719} // Abort
720
721void * __create_pthread( pthread_t * pthread, void * (*start)(void *), void * arg ) {
722        pthread_attr_t attr;
723
724        check( pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
725
726        size_t stacksize = DEFAULT_STACK_SIZE;
727
728        void * stack;
729        #if CFA_PROCESSOR_USE_MMAP
730                stacksize = ceiling( stacksize, __page_size ) + __page_size;
731                stack = mmap(0p, stacksize, __map_prot, MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
732                if(stack == ((void*)-1)) {
733                        abort( "pthread stack creation : internal error, mmap failure, error(%d) %s.", errno, strerror( errno ) );
734                }
735                if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
736                        abort( "pthread stack creation : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
737                } // if
738        #else
739                __cfaabi_dbg_debug_do(
740                        stack = memalign( __page_size, stacksize + __page_size );
741                        // pthread has no mechanism to create the guard page in user supplied stack.
742                        if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
743                                abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
744                        } // if
745                );
746                __cfaabi_dbg_no_debug_do(
747                        stack = malloc( stacksize );
748                );
749        #endif
750
751        check( pthread_attr_setstack( &attr, stack, stacksize ), "pthread_attr_setstack" );
752        check( pthread_create( pthread, &attr, start, arg ), "pthread_create" );
753        return stack;
754}
755
756void __destroy_pthread( pthread_t pthread, void * stack, void ** retval ) {
757        int err = pthread_join( pthread, retval );
758        if( err != 0 ) abort("KERNEL ERROR: joining pthread %p caused error %s\n", (void*)pthread, strerror(err));
759
760        #if CFA_PROCESSOR_USE_MMAP
761                pthread_attr_t attr;
762
763                check( pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
764
765                size_t stacksize;
766                // default stack size, normally defined by shell limit
767                check( pthread_attr_getstacksize( &attr, &stacksize ), "pthread_attr_getstacksize" );
768                assert( stacksize >= PTHREAD_STACK_MIN );
769                stacksize += __page_size;
770
771                if(munmap(stack, stacksize) == -1) {
772                        abort( "pthread stack destruction : internal error, munmap failure, error(%d) %s.", errno, strerror( errno ) );
773                }
774        #else
775                __cfaabi_dbg_debug_do(
776                        // pthread has no mechanism to create the guard page in user supplied stack.
777                        if ( mprotect( stack, __page_size, __map_prot ) == -1 ) {
778                                abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
779                        } // if
780                );
781                free( stack );
782        #endif
783}
784
785#if defined(__CFA_WITH_VERIFY__)
786static bool verify_fwd_bck_rng(void) {
787        __cfaabi_tls.ready_rng.fwd_seed = 25214903917_l64u * (rdtscl() ^ (uintptr_t)&verify_fwd_bck_rng);
788
789        unsigned values[10];
790        for(i; 10) {
791                values[i] = __tls_rand_fwd();
792        }
793
794        __tls_rand_advance_bck();
795
796        for ( i; 9 -~= 0 ) {
797                if(values[i] != __tls_rand_bck()) {
798                        return false;
799                }
800        }
801
802        return true;
803}
804#endif
Note: See TracBrowser for help on using the repository browser.