source: libcfa/src/concurrency/kernel/startup.cfa @ 145dcd5

ADTast-experimentalenumforall-pointer-decaypthread-emulationqualifiedEnum
Last change on this file since 145dcd5 was 145dcd5, checked in by Thierry Delisle <tdelisle@…>, 2 years ago

Now using MAX instead of -1u since it's more appropriate.

  • Property mode set to 100644
File size: 24.8 KB
Line 
1//
2// Cforall Version 1.0.0 Copyright (C) 2020 University of Waterloo
3//
4// The contents of this file are covered under the licence agreement in the
5// file "LICENCE" distributed with Cforall.
6//
7// kernel/startup.cfa --
8//
9// Author           : Thierry Delisle
10// Created On       : Thu Jul 30 15:12:54 2020
11// Last Modified By :
12// Last Modified On :
13// Update Count     :
14//
15
16#define __cforall_thread__
17#define _GNU_SOURCE
18
19// C Includes
20#include <errno.h>              // errno
21#include <signal.h>
22#include <string.h>             // strerror
23#include <unistd.h>             // sysconf
24
25extern "C" {
26      #include <limits.h>       // PTHREAD_STACK_MIN
27        #include <unistd.h>       // syscall
28        #include <sys/eventfd.h>  // eventfd
29      #include <sys/mman.h>     // mprotect
30      #include <sys/resource.h> // getrlimit
31}
32
33// CFA Includes
34#include "kernel_private.hfa"
35#include "startup.hfa"          // STARTUP_PRIORITY_XXX
36#include "limits.hfa"
37#include "math.hfa"
38
39#define CFA_PROCESSOR_USE_MMAP 0
40
41//-----------------------------------------------------------------------------
42// Some assembly required
43#if defined( __i386 )
44        #define CtxGet( ctx ) __asm__ volatile ( \
45                "movl %%esp,%0\n" \
46                "movl %%ebp,%1\n" \
47                : "=rm" (ctx.SP), \
48                  "=rm" (ctx.FP) \
49        )
50#elif defined( __x86_64 )
51        #define CtxGet( ctx ) __asm__ volatile ( \
52                "movq %%rsp,%0\n" \
53                "movq %%rbp,%1\n" \
54                : "=rm" (ctx.SP), \
55                  "=rm" (ctx.FP) \
56        )
57#elif defined( __aarch64__ )
58        #define CtxGet( ctx ) __asm__ volatile ( \
59                "mov %0, sp\n" \
60                "mov %1, fp\n" \
61                : "=rm" (ctx.SP), \
62                  "=rm" (ctx.FP) \
63        )
64#else
65        #error unknown hardware architecture
66#endif
67
68//-----------------------------------------------------------------------------
69// Start and stop routine for the kernel, declared first to make sure they run first
70static void __kernel_startup (void) __attribute__(( constructor( STARTUP_PRIORITY_KERNEL ) ));
71static void __kernel_shutdown(void) __attribute__(( destructor ( STARTUP_PRIORITY_KERNEL ) ));
72
73//-----------------------------------------------------------------------------
74// Static Forward Declarations
75struct current_stack_info_t;
76
77static void * __invoke_processor(void * arg);
78static void __kernel_first_resume( processor * this );
79static void __kernel_last_resume ( processor * this );
80static void init(processor & this, const char name[], cluster & _cltr, thread$ * initT);
81static void deinit(processor & this);
82static void doregister( struct cluster & cltr );
83static void unregister( struct cluster & cltr );
84static void register_tls( processor * this );
85static void unregister_tls( processor * this );
86static void ?{}( coroutine$ & this, current_stack_info_t * info);
87static void ?{}( thread$ & this, current_stack_info_t * info);
88static void ?{}(processorCtx_t & this) {}
89static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info);
90
91#if defined(__CFA_WITH_VERIFY__)
92        static bool verify_fwd_bck_rng(void);
93#endif
94
95//-----------------------------------------------------------------------------
96// Forward Declarations for other modules
97extern void __kernel_alarm_startup(void);
98extern void __kernel_alarm_shutdown(void);
99
100//-----------------------------------------------------------------------------
101// Other Forward Declarations
102extern void __wake_proc(processor *);
103extern int cfa_main_returned;                                                   // from interpose.cfa
104
105//-----------------------------------------------------------------------------
106// Kernel storage
107KERNEL_STORAGE(cluster,              mainCluster);
108KERNEL_STORAGE(processor,            mainProcessor);
109KERNEL_STORAGE(thread$,              mainThread);
110KERNEL_STORAGE(__stack_t,            mainThreadCtx);
111KERNEL_STORAGE(__scheduler_RWLock_t, __scheduler_lock);
112#if !defined(__CFA_NO_STATISTICS__)
113KERNEL_STORAGE(__stats_t, mainProcStats);
114#endif
115
116cluster              * mainCluster;
117processor            * mainProcessor;
118thread$              * mainThread;
119__scheduler_RWLock_t * __scheduler_lock;
120
121extern "C" {
122        struct { __dllist_t(cluster) list; __spinlock_t lock; } __cfa_dbg_global_clusters;
123}
124
125extern size_t __page_size;
126extern int __map_prot;
127
128//-----------------------------------------------------------------------------
129// Global state
130thread_local struct KernelThreadData __cfaabi_tls __attribute__ ((tls_model ( "initial-exec" ))) @= {
131        NULL,                                                                                           // cannot use 0p
132        NULL,
133        false,
134        { 1, false, false },
135        0,
136        { 0, 0 },
137        NULL,
138        #ifdef __CFA_WITH_VERIFY__
139                false,
140                0,
141        #endif
142};
143
144#if   defined(CFA_HAVE_LINUX_LIBRSEQ)
145        // No data needed
146#elif defined(CFA_HAVE_LINUX_RSEQ_H)
147        extern "Cforall" {
148                __attribute__((aligned(128))) thread_local volatile struct rseq __cfaabi_rseq @= {
149                        .cpu_id : RSEQ_CPU_ID_UNINITIALIZED,
150                };
151        }
152#else
153        // No data needed
154#endif
155
156//-----------------------------------------------------------------------------
157// Struct to steal stack
158struct current_stack_info_t {
159        __stack_t * storage;  // pointer to stack object
160        void * base;          // base of stack
161        void * limit;         // stack grows towards stack limit
162        void * context;       // address of cfa_context_t
163};
164
165void ?{}( current_stack_info_t & this ) {
166        __stack_context_t ctx;
167        CtxGet( ctx );
168        this.base = ctx.FP;
169
170        rlimit r;
171        getrlimit( RLIMIT_STACK, &r);
172        size_t size = r.rlim_cur;
173
174        this.limit = (void *)(((intptr_t)this.base) - size);
175        this.context = &storage_mainThreadCtx;
176}
177
178
179
180//=============================================================================================
181// Kernel Setup logic
182//=============================================================================================
183//-----------------------------------------------------------------------------
184// Kernel boot procedures
185static void __kernel_startup(void) {
186        /* paranoid */ verify( ! __preemption_enabled() );
187        __cfadbg_print_safe(runtime_core, "Kernel : Starting\n");
188
189        __cfa_dbg_global_clusters.list{ __get };
190        __cfa_dbg_global_clusters.lock{};
191
192        /* paranoid */ verify( verify_fwd_bck_rng() );
193
194        // Initialize the global scheduler lock
195        __scheduler_lock = (__scheduler_RWLock_t*)&storage___scheduler_lock;
196        (*__scheduler_lock){};
197
198        // Initialize the main cluster
199        mainCluster = (cluster *)&storage_mainCluster;
200        (*mainCluster){"Main Cluster", 0};
201
202        __cfadbg_print_safe(runtime_core, "Kernel : Main cluster ready\n");
203
204        // Construct the processor context of the main processor
205        void ?{}(processorCtx_t & this, processor * proc) {
206                (this.__cor){ "Processor" };
207                this.__cor.starter = 0p;
208                this.proc = proc;
209        }
210
211        void ?{}(processor & this) with( this ) {
212                ( this.terminated ){};
213                ( this.runner ){};
214                init( this, "Main Processor", *mainCluster, 0p );
215                kernel_thread = pthread_self();
216
217                runner{ &this };
218                __cfadbg_print_safe(runtime_core, "Kernel : constructed main processor context %p\n", &runner);
219        }
220
221        // Initialize the main processor and the main processor ctx
222        // (the coroutine that contains the processing control flow)
223        mainProcessor = (processor *)&storage_mainProcessor;
224        (*mainProcessor){};
225
226        register_tls( mainProcessor );
227
228        // Start by initializing the main thread
229        // SKULLDUGGERY: the mainThread steals the process main thread
230        // which will then be scheduled by the mainProcessor normally
231        mainThread = (thread$ *)&storage_mainThread;
232        current_stack_info_t info;
233        info.storage = (__stack_t*)&storage_mainThreadCtx;
234        (*mainThread){ &info };
235
236        __cfadbg_print_safe(runtime_core, "Kernel : Main thread ready\n");
237
238        //initialize the global state variables
239        __cfaabi_tls.this_processor = mainProcessor;
240        __cfaabi_tls.this_thread    = mainThread;
241
242        #if !defined( __CFA_NO_STATISTICS__ )
243                __cfaabi_tls.this_stats = (__stats_t *)& storage_mainProcStats;
244                __init_stats( __cfaabi_tls.this_stats );
245        #endif
246        mainProcessor->local_data = &__cfaabi_tls;
247
248        // Enable preemption
249        __kernel_alarm_startup();
250
251        // Add the main thread to the ready queue
252        // once resume is called on mainProcessor->runner the mainThread needs to be scheduled like any normal thread
253        schedule_thread$(mainThread, UNPARK_LOCAL);
254
255        // SKULLDUGGERY: Force a context switch to the main processor to set the main thread's context to the current UNIX
256        // context. Hence, the main thread does not begin through __cfactx_invoke_thread, like all other threads. The trick here is that
257        // mainThread is on the ready queue when this call is made.
258        __kernel_first_resume( __cfaabi_tls.this_processor );
259
260
261        // THE SYSTEM IS NOW COMPLETELY RUNNING
262
263        __cfadbg_print_safe(runtime_core, "Kernel : Started\n--------------------------------------------------\n\n");
264
265        /* paranoid */ verify( ! __preemption_enabled() );
266        enable_interrupts();
267        /* paranoid */ verify( __preemption_enabled() );
268
269}
270
271static void __kernel_shutdown(void) {
272        if(!cfa_main_returned) return;
273        /* paranoid */ verify( __preemption_enabled() );
274        disable_interrupts();
275        /* paranoid */ verify( ! __preemption_enabled() );
276
277        __cfadbg_print_safe(runtime_core, "\n--------------------------------------------------\nKernel : Shutting down\n");
278
279        // SKULLDUGGERY: Notify the mainProcessor it needs to terminates.
280        // When its coroutine terminates, it return control to the mainThread
281        // which is currently here
282        /* paranoid */ verify( !__atomic_load_n(&mainProcessor->do_terminate, __ATOMIC_ACQUIRE) );
283        __atomic_store_n(&mainProcessor->do_terminate, true, __ATOMIC_RELEASE);
284        __wake_proc( mainProcessor );
285        __kernel_last_resume( __cfaabi_tls.this_processor );
286        mainThread->self_cor.state = Halted;
287
288        // THE SYSTEM IS NOW COMPLETELY STOPPED
289
290        // Disable preemption
291        __kernel_alarm_shutdown();
292
293        #if !defined( __CFA_NO_STATISTICS__ )
294                __stats_t * st = (__stats_t *)& storage_mainProcStats;
295                __tally_stats(mainCluster->stats, st);
296                if( 0 != mainProcessor->print_stats ) {
297                        __print_stats( st, mainProcessor->print_stats, "Processor ", mainProcessor->name, (void*)mainProcessor );
298                }
299                #if defined(CFA_STATS_ARRAY)
300                        __flush_stat( st, "Processor", mainProcessor );
301                #endif
302        #endif
303
304        mainProcessor->local_data = 0p;
305
306        unregister_tls( mainProcessor );
307
308        // Destroy the main processor and its context in reverse order of construction
309        // These were manually constructed so we need manually destroy them
310        void ^?{}(processor & this) with( this ){
311                deinit( this );
312
313                /* paranoid */ verify( this.do_terminate == true );
314                __cfaabi_dbg_print_safe("Kernel : destroyed main processor context %p\n", &runner);
315        }
316
317        ^(*mainProcessor){};
318
319        // Final step, destroy the main thread since it is no longer needed
320
321        // Since we provided a stack to this taxk it will not destroy anything
322        /* paranoid */ verify(mainThread->self_cor.stack.storage == (__stack_t*)(((uintptr_t)&storage_mainThreadCtx)| 0x1));
323        ^(*mainThread){};
324
325        ^(*mainCluster){};
326
327        ^(*__scheduler_lock){};
328
329        ^(__cfa_dbg_global_clusters.list){};
330        ^(__cfa_dbg_global_clusters.lock){};
331
332        __cfadbg_print_safe(runtime_core, "Kernel : Shutdown complete\n");
333}
334
335//=============================================================================================
336// Kernel Initial Scheduling logic
337//=============================================================================================
338
339// Context invoker for processors
340// This is the entry point for processors (kernel threads) *except* for the main processor
341// It effectively constructs a coroutine by stealing the pthread stack
342static void * __invoke_processor(void * arg) {
343        #if !defined( __CFA_NO_STATISTICS__ )
344                __stats_t local_stats;
345                __init_stats( &local_stats );
346                __cfaabi_tls.this_stats = &local_stats;
347        #endif
348
349        processor * proc = (processor *) arg;
350        __cfaabi_tls.this_processor = proc;
351        __cfaabi_tls.this_thread    = 0p;
352        __cfaabi_tls.preemption_state.[enabled, disable_count] = [false, 1];
353        proc->local_data = &__cfaabi_tls;
354
355        register_tls( proc );
356
357        // SKULLDUGGERY: We want to create a context for the processor coroutine
358        // which is needed for the 2-step context switch. However, there is no reason
359        // to waste the perfectly valid stack create by pthread.
360        current_stack_info_t info;
361        __stack_t ctx;
362        info.storage = &ctx;
363        (proc->runner){ proc, &info };
364
365        __cfaabi_dbg_print_safe("Coroutine : created stack %p\n", get_coroutine(proc->runner)->stack.storage);
366
367        //Set global state
368        __cfaabi_tls.this_thread = 0p;
369
370        //We now have a proper context from which to schedule threads
371        __cfadbg_print_safe(runtime_core, "Kernel : core %p created (%p, %p)\n", proc, &proc->runner, &ctx);
372
373        // SKULLDUGGERY: Since the coroutine doesn't have its own stack, we can't
374        // resume it to start it like it normally would, it will just context switch
375        // back to here. Instead directly call the main since we already are on the
376        // appropriate stack.
377        get_coroutine(proc->runner)->state = Active;
378        main( proc->runner );
379        get_coroutine(proc->runner)->state = Halted;
380
381        // Main routine of the core returned, the core is now fully terminated
382        __cfadbg_print_safe(runtime_core, "Kernel : core %p main ended (%p)\n", proc, &proc->runner);
383
384        #if !defined(__CFA_NO_STATISTICS__)
385                __tally_stats(proc->cltr->stats, &local_stats);
386                if( 0 != proc->print_stats ) {
387                        __print_stats( &local_stats, proc->print_stats, "Processor ", proc->name, (void*)proc );
388                }
389                #if defined(CFA_STATS_ARRAY)
390                        __flush_stat( &local_stats, "Processor", proc );
391                #endif
392        #endif
393
394        proc->local_data = 0p;
395
396        unregister_tls( proc );
397
398        return 0p;
399}
400
401static void __kernel_first_resume( processor * this ) {
402        thread$ * src = mainThread;
403        coroutine$ * dst = get_coroutine(this->runner);
404
405        /* paranoid */ verify( ! __preemption_enabled() );
406
407        __cfaabi_tls.this_thread->curr_cor = dst;
408        __stack_prepare( &dst->stack, DEFAULT_STACK_SIZE );
409        __cfactx_start(main, dst, this->runner, __cfactx_invoke_coroutine);
410
411        /* paranoid */ verify( ! __preemption_enabled() );
412
413        dst->last = &src->self_cor;
414        dst->starter = dst->starter ? dst->starter : &src->self_cor;
415
416        // make sure the current state is still correct
417        /* paranoid */ verify(src->state == Ready);
418        src->corctx_flag = true;
419
420        // context switch to specified coroutine
421        verify( dst->context.SP );
422        __cfactx_switch( &src->context, &dst->context );
423        // when __cfactx_switch returns we are back in the src coroutine
424
425        mainThread->curr_cor = &mainThread->self_cor;
426
427        // make sure the current state has been update
428        /* paranoid */ verify(src->state == Active);
429
430        /* paranoid */ verify( ! __preemption_enabled() );
431}
432
433// KERNEL_ONLY
434static void __kernel_last_resume( processor * this ) {
435        coroutine$ * src = &mainThread->self_cor;
436        coroutine$ * dst = get_coroutine(this->runner);
437
438        /* paranoid */ verify( ! __preemption_enabled() );
439        /* paranoid */ verify( dst->starter == src );
440        /* paranoid */ verify( dst->context.SP );
441
442        // SKULLDUGGERY in debug the processors check that the
443        // stack is still within the limit of the stack limits after running a thread.
444        // that check doesn't make sense if we context switch to the processor using the
445        // coroutine semantics. Since this is a special case, use the current context
446        // info to populate these fields.
447        __cfaabi_dbg_debug_do(
448                __stack_context_t ctx;
449                CtxGet( ctx );
450                mainThread->context.SP = ctx.SP;
451                mainThread->context.FP = ctx.FP;
452        )
453
454        // context switch to the processor
455        __cfactx_switch( &src->context, &dst->context );
456}
457
458
459//=============================================================================================
460// Kernel Object Constructors logic
461//=============================================================================================
462//-----------------------------------------------------------------------------
463// Main thread construction
464static void ?{}( coroutine$ & this, current_stack_info_t * info) with( this ) {
465        stack.storage = info->storage;
466        with(*stack.storage) {
467                limit     = info->limit;
468                base      = info->base;
469        }
470        __attribute__((may_alias)) intptr_t * istorage = (intptr_t*) &stack.storage;
471        *istorage |= 0x1;
472        name = "Main Thread";
473        state = Start;
474        starter = 0p;
475        last = 0p;
476        cancellation = 0p;
477}
478
479static void ?{}( thread$ & this, current_stack_info_t * info) with( this ) {
480        ticket = TICKET_RUNNING;
481        state = Start;
482        self_cor{ info };
483        curr_cor = &self_cor;
484        curr_cluster = mainCluster;
485        self_mon.owner = &this;
486        self_mon.recursion = 1;
487        self_mon_p = &self_mon;
488        link.next = 0p;
489        link.ts   = -1llu;
490        preferred = ready_queue_new_preferred();
491        last_proc = 0p;
492        #if defined( __CFA_WITH_VERIFY__ )
493                canary = 0x0D15EA5E0D15EA5Ep;
494        #endif
495
496        node.next = 0p;
497        node.prev = 0p;
498        doregister(curr_cluster, this);
499
500        monitors{ &self_mon_p, 1, (fptr_t)0 };
501}
502
503//-----------------------------------------------------------------------------
504// Processor
505// Construct the processor context of non-main processors
506static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info) {
507        (this.__cor){ info };
508        this.proc = proc;
509}
510
511static void init(processor & this, const char name[], cluster & _cltr, thread$ * initT) with( this ) {
512        this.name = name;
513        this.cltr = &_cltr;
514        this.rdq.its = 0;
515        this.rdq.itr = 0;
516        this.rdq.id  = MAX;
517        this.rdq.target = MAX;
518        this.rdq.last = MAX;
519        this.rdq.cpu = 0;
520        // this.rdq.cutoff = 0ull;
521        do_terminate = false;
522        preemption_alarm = 0p;
523        pending_preemption = false;
524
525        this.io.ctx = 0p;
526        this.io.pending = false;
527        this.io.dirty   = false;
528
529        this.init.thrd = initT;
530
531        this.local_data = 0p;
532
533        this.idle_fd = eventfd(0, 0);
534        if (idle_fd < 0) {
535                abort("KERNEL ERROR: PROCESSOR EVENTFD - %s\n", strerror(errno));
536        }
537
538        #if !defined(__CFA_NO_STATISTICS__)
539                print_stats = 0;
540                print_halts = false;
541        #endif
542
543        __cfadbg_print_safe(runtime_core, "Kernel : core %p created\n", &this);
544}
545
546// Not a ctor, it just preps the destruction but should not destroy members
547static void deinit(processor & this) {
548        close(this.idle_fd);
549}
550
551void ?{}(processor & this, const char name[], cluster & _cltr, thread$ * initT) {
552        ( this.terminated ){};
553        ( this.runner ){};
554
555        disable_interrupts();
556                init( this, name, _cltr, initT );
557        enable_interrupts();
558
559        __cfadbg_print_safe(runtime_core, "Kernel : Starting core %p\n", &this);
560
561        this.stack = __create_pthread( &this.kernel_thread, __invoke_processor, (void *)&this );
562}
563
564void ?{}(processor & this, const char name[], cluster & _cltr) {
565        (this){name, _cltr, 0p};
566}
567
568extern size_t __page_size;
569void ^?{}(processor & this) with( this ){
570        /* paranoid */ verify( !__atomic_load_n(&do_terminate, __ATOMIC_ACQUIRE) );
571        __cfadbg_print_safe(runtime_core, "Kernel : core %p signaling termination\n", &this);
572
573        __atomic_store_n(&do_terminate, true, __ATOMIC_RELAXED);
574        __disable_interrupts_checked();
575                __wake_proc( &this );
576        __enable_interrupts_checked();
577
578        wait( terminated );
579        /* paranoid */ verify( active_processor() != &this);
580
581        __destroy_pthread( kernel_thread, this.stack, 0p );
582
583        disable_interrupts();
584                deinit( this );
585        enable_interrupts();
586}
587
588//-----------------------------------------------------------------------------
589// Cluster
590static void ?{}(__cluster_proc_list & this) {
591        this.fd    = 0;
592        this.idle  = 0;
593        this.total = 0;
594}
595
596void ?{}(cluster & this, const char name[], Duration preemption_rate, unsigned num_io, const io_context_params & io_params) with( this ) {
597        this.name = name;
598        this.preemption_rate = preemption_rate;
599        ready_queue{};
600
601        #if !defined(__CFA_NO_STATISTICS__)
602                print_stats = 0;
603                stats = alloc();
604                __init_stats( stats );
605        #endif
606
607        threads{ __get };
608
609        io.arbiter = create();
610        io.params = io_params;
611
612        doregister(this);
613
614        // Lock the RWlock so no-one pushes/pops while we are changing the queue
615        disable_interrupts();
616        uint_fast32_t last_size = ready_mutate_lock();
617
618                // Adjust the ready queue size
619                ready_queue_grow( &this );
620
621        // Unlock the RWlock
622        ready_mutate_unlock( last_size );
623        enable_interrupts( false ); // Don't poll, could be in main cluster
624}
625
626void ^?{}(cluster & this) {
627        destroy(this.io.arbiter);
628
629        // Lock the RWlock so no-one pushes/pops while we are changing the queue
630        disable_interrupts();
631        uint_fast32_t last_size = ready_mutate_lock();
632
633                // Adjust the ready queue size
634                ready_queue_shrink( &this );
635
636        // Unlock the RWlock
637        ready_mutate_unlock( last_size );
638        enable_interrupts( false ); // Don't poll, could be in main cluster
639
640        #if !defined(__CFA_NO_STATISTICS__)
641                if( 0 != this.print_stats ) {
642                        __print_stats( this.stats, this.print_stats, "Cluster", this.name, (void*)&this );
643                }
644                #if defined(CFA_STATS_ARRAY)
645                        __flush_stat( this.stats, "Cluster", &this );
646                #endif
647                free( this.stats );
648        #endif
649
650        unregister(this);
651}
652
653//=============================================================================================
654// Miscellaneous Initialization
655//=============================================================================================
656//-----------------------------------------------------------------------------
657// Global Queues
658static void doregister( cluster     & cltr ) {
659        lock      ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
660        push_front( __cfa_dbg_global_clusters.list, cltr );
661        unlock    ( __cfa_dbg_global_clusters.lock );
662}
663
664static void unregister( cluster     & cltr ) {
665        lock  ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
666        remove( __cfa_dbg_global_clusters.list, cltr );
667        unlock( __cfa_dbg_global_clusters.lock );
668}
669
670void doregister( cluster * cltr, thread$ & thrd ) {
671        lock      (cltr->thread_list_lock __cfaabi_dbg_ctx2);
672        cltr->nthreads += 1;
673        push_front(cltr->threads, thrd);
674        unlock    (cltr->thread_list_lock);
675}
676
677void unregister( cluster * cltr, thread$ & thrd ) {
678        lock  (cltr->thread_list_lock __cfaabi_dbg_ctx2);
679        remove(cltr->threads, thrd );
680        cltr->nthreads -= 1;
681        unlock(cltr->thread_list_lock);
682}
683
684static void register_tls( processor * this ) {
685        // Register and Lock the RWlock so no-one pushes/pops while we are changing the queue
686        uint_fast32_t last_size;
687        [this->unique_id, last_size] = ready_mutate_register();
688
689                this->rdq.cpu = __kernel_getcpu();
690
691                this->cltr->procs.total += 1u;
692                insert_last(this->cltr->procs.actives, *this);
693
694                // Adjust the ready queue size
695                ready_queue_grow( this->cltr );
696
697        // Unlock the RWlock
698        ready_mutate_unlock( last_size );
699}
700
701
702static void unregister_tls( processor * this ) {
703        // Lock the RWlock so no-one pushes/pops while we are changing the queue
704        uint_fast32_t last_size = ready_mutate_lock();
705                this->cltr->procs.total -= 1u;
706                remove(*this);
707
708                // clear the cluster so nothing gets pushed to local queues
709                cluster * cltr = this->cltr;
710                this->cltr = 0p;
711
712                // Adjust the ready queue size
713                ready_queue_shrink( cltr );
714
715        // Unlock the RWlock and unregister: we don't need the read_lock any more
716        ready_mutate_unregister( this->unique_id, last_size );
717}
718
719static void check( int ret, const char func[] ) {
720        if ( ret ) {                                                                            // pthread routines return errno values
721                abort( "%s : internal error, error(%d) %s.", func, ret, strerror( ret ) );
722        } // if
723} // Abort
724
725void * __create_pthread( pthread_t * pthread, void * (*start)(void *), void * arg ) {
726        pthread_attr_t attr;
727
728        check( pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
729
730        size_t stacksize = DEFAULT_STACK_SIZE;
731
732        void * stack;
733        #if CFA_PROCESSOR_USE_MMAP
734                stacksize = ceiling( stacksize, __page_size ) + __page_size;
735                stack = mmap(0p, stacksize, __map_prot, MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
736                if(stack == ((void*)-1)) {
737                        abort( "pthread stack creation : internal error, mmap failure, error(%d) %s.", errno, strerror( errno ) );
738                }
739                if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
740                        abort( "pthread stack creation : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
741                } // if
742        #else
743                __cfaabi_dbg_debug_do(
744                        stack = memalign( __page_size, stacksize + __page_size );
745                        // pthread has no mechanism to create the guard page in user supplied stack.
746                        if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
747                                abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
748                        } // if
749                );
750                __cfaabi_dbg_no_debug_do(
751                        stack = malloc( stacksize );
752                );
753        #endif
754
755        check( pthread_attr_setstack( &attr, stack, stacksize ), "pthread_attr_setstack" );
756        check( pthread_create( pthread, &attr, start, arg ), "pthread_create" );
757        return stack;
758}
759
760void __destroy_pthread( pthread_t pthread, void * stack, void ** retval ) {
761        int err = pthread_join( pthread, retval );
762        if( err != 0 ) abort("KERNEL ERROR: joining pthread %p caused error %s\n", (void*)pthread, strerror(err));
763
764        #if CFA_PROCESSOR_USE_MMAP
765                pthread_attr_t attr;
766
767                check( pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
768
769                size_t stacksize;
770                // default stack size, normally defined by shell limit
771                check( pthread_attr_getstacksize( &attr, &stacksize ), "pthread_attr_getstacksize" );
772                assert( stacksize >= PTHREAD_STACK_MIN );
773                stacksize += __page_size;
774
775                if(munmap(stack, stacksize) == -1) {
776                        abort( "pthread stack destruction : internal error, munmap failure, error(%d) %s.", errno, strerror( errno ) );
777                }
778        #else
779                __cfaabi_dbg_debug_do(
780                        // pthread has no mechanism to create the guard page in user supplied stack.
781                        if ( mprotect( stack, __page_size, __map_prot ) == -1 ) {
782                                abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
783                        } // if
784                );
785                free( stack );
786        #endif
787}
788
789#if defined(__CFA_WITH_VERIFY__)
790static bool verify_fwd_bck_rng(void) {
791        __cfaabi_tls.ready_rng.fwd_seed = 25214903917_l64u * (rdtscl() ^ (uintptr_t)&verify_fwd_bck_rng);
792
793        unsigned values[10];
794        for(i; 10) {
795                values[i] = __tls_rand_fwd();
796        }
797
798        __tls_rand_advance_bck();
799
800        for ( i; 9 -~= 0 ) {
801                if(values[i] != __tls_rand_bck()) {
802                        return false;
803                }
804        }
805
806        return true;
807}
808#endif
Note: See TracBrowser for help on using the repository browser.