source: libcfa/src/concurrency/kernel/startup.cfa @ fde879b3

ADTast-experimentalenumforall-pointer-decayjacob/cs343-translationnew-ast-unique-exprpthread-emulationqualifiedEnum
Last change on this file since fde879b3 was fde879b3, checked in by Thierry Delisle <tdelisle@…>, 3 years ago

Fixed typo in startup.cfa without rseq support.
  • Property mode set to 100644
File size: 27.5 KB
Line 
1//
2// Cforall Version 1.0.0 Copyright (C) 2020 University of Waterloo
3//
4// The contents of this file are covered under the licence agreement in the
5// file "LICENCE" distributed with Cforall.
6//
7// kernel/startup.cfa --
8//
9// Author           : Thierry Delisle
10// Created On       : Thu Jul 30 15:12:54 2020
11// Last Modified By :
12// Last Modified On :
13// Update Count     :
14//
15
16#define __cforall_thread__
17#define _GNU_SOURCE
18
19// C Includes
20#include <errno.h>              // errno
21#include <signal.h>
22#include <string.h>             // strerror
23#include <unistd.h>             // sysconf
24
25extern "C" {
26      #include <limits.h>       // PTHREAD_STACK_MIN
27        #include <unistd.h>       // syscall
28        #include <sys/eventfd.h>  // eventfd
29      #include <sys/mman.h>     // mprotect
30      #include <sys/resource.h> // getrlimit
31        #include <sys/syscall.h>  // __NR_xxx
32}
33
34// CFA Includes
35#include "kernel_private.hfa"
36#include "startup.hfa"          // STARTUP_PRIORITY_XXX
37#include "math.hfa"
38
39#define CFA_PROCESSOR_USE_MMAP 0
40
41//-----------------------------------------------------------------------------
42// Some assembly required
43#if defined( __i386 )
44        #define CtxGet( ctx ) __asm__ volatile ( \
45                "movl %%esp,%0\n" \
46                "movl %%ebp,%1\n" \
47                : "=rm" (ctx.SP), \
48                  "=rm" (ctx.FP) \
49        )
50#elif defined( __x86_64 )
51        #define CtxGet( ctx ) __asm__ volatile ( \
52                "movq %%rsp,%0\n" \
53                "movq %%rbp,%1\n" \
54                : "=rm" (ctx.SP), \
55                  "=rm" (ctx.FP) \
56        )
57#elif defined( __aarch64__ )
58        #define CtxGet( ctx ) __asm__ volatile ( \
59                "mov %0, sp\n" \
60                "mov %1, fp\n" \
61                : "=rm" (ctx.SP), \
62                  "=rm" (ctx.FP) \
63        )
64#else
65        #error unknown hardware architecture
66#endif
67
68//-----------------------------------------------------------------------------
69// Start and stop routine for the kernel, declared first to make sure they run first
70static void __kernel_startup (void) __attribute__(( constructor( STARTUP_PRIORITY_KERNEL ) ));
71static void __kernel_shutdown(void) __attribute__(( destructor ( STARTUP_PRIORITY_KERNEL ) ));
72
73//-----------------------------------------------------------------------------
74// Static Forward Declarations
75struct current_stack_info_t;
76
77static void * __invoke_processor(void * arg);
78static void __kernel_first_resume( processor * this );
79static void __kernel_last_resume ( processor * this );
80static void init(processor & this, const char name[], cluster & _cltr, $thread * initT);
81static void deinit(processor & this);
82static void doregister( struct cluster & cltr );
83static void unregister( struct cluster & cltr );
84static void register_tls( processor * this );
85static void unregister_tls( processor * this );
86static void ?{}( $coroutine & this, current_stack_info_t * info);
87static void ?{}( $thread & this, current_stack_info_t * info);
88static void ?{}(processorCtx_t & this) {}
89static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info);
90
91#if defined(__CFA_WITH_VERIFY__)
92        static bool verify_fwd_bck_rng(void);
93#endif
94
95//-----------------------------------------------------------------------------
96// Forward Declarations for other modules
97extern void __kernel_alarm_startup(void);
98extern void __kernel_alarm_shutdown(void);
99
100//-----------------------------------------------------------------------------
101// Other Forward Declarations
102extern void __wake_proc(processor *);
103
104#if   defined(CFA_HAVE_LINUX_LIBRSEQ)
105        // No forward declaration needed
106        #define __kernel_rseq_register rseq_register_current_thread
107        #define __kernel_rseq_unregister rseq_unregister_current_thread
108#elif defined(CFA_HAVE_LINUX_RSEQ_H)
109        void __kernel_raw_rseq_register  (void);
110        void __kernel_raw_rseq_unregister(void);
111
112        #define __kernel_rseq_register __kernel_raw_rseq_register
113        #define __kernel_rseq_unregister __kernel_raw_rseq_unregister
114#else
115        // No forward declaration needed
116        // No initialization needed
117        static inline void noop(void) {}
118
119        #define __kernel_rseq_register noop
120        #define __kernel_rseq_unregister noop
121#endif
122
123//-----------------------------------------------------------------------------
124// Kernel storage
125KERNEL_STORAGE(cluster,              mainCluster);
126KERNEL_STORAGE(processor,            mainProcessor);
127KERNEL_STORAGE($thread,              mainThread);
128KERNEL_STORAGE(__stack_t,            mainThreadCtx);
129KERNEL_STORAGE(__scheduler_RWLock_t, __scheduler_lock);
130#if !defined(__CFA_NO_STATISTICS__)
131KERNEL_STORAGE(__stats_t, mainProcStats);
132#endif
133
134cluster              * mainCluster;
135processor            * mainProcessor;
136$thread              * mainThread;
137__scheduler_RWLock_t * __scheduler_lock;
138
139extern "C" {
140        struct { __dllist_t(cluster) list; __spinlock_t lock; } __cfa_dbg_global_clusters;
141}
142
143extern size_t __page_size;
144extern int __map_prot;
145
146//-----------------------------------------------------------------------------
147// Global state
148thread_local struct KernelThreadData __cfaabi_tls __attribute__ ((tls_model ( "initial-exec" ))) @= {
149        NULL,                                                                                           // cannot use 0p
150        NULL,
151        false,
152        { 1, false, false },
153        0,
154        { 0, 0 },
155        NULL,
156        #ifdef __CFA_WITH_VERIFY__
157                false,
158                0,
159        #endif
160};
161
162#if   defined(CFA_HAVE_LINUX_LIBRSEQ)
163        // No data needed
164#elif defined(CFA_HAVE_LINUX_RSEQ_H)
165        extern "Cforall" {
166                __attribute__((aligned(128))) thread_local volatile struct rseq __cfaabi_rseq @= {
167                        .cpu_id : RSEQ_CPU_ID_UNINITIALIZED,
168                };
169        }
170#else
171        // No data needed
172#endif
173
174//-----------------------------------------------------------------------------
175// Struct to steal stack
176struct current_stack_info_t {
177        __stack_t * storage;  // pointer to stack object
178        void * base;          // base of stack
179        void * limit;         // stack grows towards stack limit
180        void * context;       // address of cfa_context_t
181};
182
183void ?{}( current_stack_info_t & this ) {
184        __stack_context_t ctx;
185        CtxGet( ctx );
186        this.base = ctx.FP;
187
188        rlimit r;
189        getrlimit( RLIMIT_STACK, &r);
190        size_t size = r.rlim_cur;
191
192        this.limit = (void *)(((intptr_t)this.base) - size);
193        this.context = &storage_mainThreadCtx;
194}
195
196
197
198//=============================================================================================
199// Kernel Setup logic
200//=============================================================================================
201//-----------------------------------------------------------------------------
202// Kernel boot procedures
203static void __kernel_startup(void) {
204        /* paranoid */ verify( ! __preemption_enabled() );
205        __cfadbg_print_safe(runtime_core, "Kernel : Starting\n");
206
207        __cfa_dbg_global_clusters.list{ __get };
208        __cfa_dbg_global_clusters.lock{};
209
210        /* paranoid */ verify( verify_fwd_bck_rng() );
211
212        // Initialize the global scheduler lock
213        __scheduler_lock = (__scheduler_RWLock_t*)&storage___scheduler_lock;
214        (*__scheduler_lock){};
215
216        // Initialize the main cluster
217        mainCluster = (cluster *)&storage_mainCluster;
218        (*mainCluster){"Main Cluster", 0};
219
220        __cfadbg_print_safe(runtime_core, "Kernel : Main cluster ready\n");
221
222        // Start by initializing the main thread
223        // SKULLDUGGERY: the mainThread steals the process main thread
224        // which will then be scheduled by the mainProcessor normally
225        mainThread = ($thread *)&storage_mainThread;
226        current_stack_info_t info;
227        info.storage = (__stack_t*)&storage_mainThreadCtx;
228        (*mainThread){ &info };
229
230        __cfadbg_print_safe(runtime_core, "Kernel : Main thread ready\n");
231
232
233
234        // Construct the processor context of the main processor
235        void ?{}(processorCtx_t & this, processor * proc) {
236                (this.__cor){ "Processor" };
237                this.__cor.starter = 0p;
238                this.proc = proc;
239        }
240
241        void ?{}(processor & this) with( this ) {
242                ( this.terminated ){};
243                ( this.runner ){};
244                init( this, "Main Processor", *mainCluster, 0p );
245                kernel_thread = pthread_self();
246
247                runner{ &this };
248                __cfadbg_print_safe(runtime_core, "Kernel : constructed main processor context %p\n", &runner);
249        }
250
251        // Initialize the main processor and the main processor ctx
252        // (the coroutine that contains the processing control flow)
253        mainProcessor = (processor *)&storage_mainProcessor;
254        (*mainProcessor){};
255
256        register_tls( mainProcessor );
257
258        __kernel_rseq_register();
259
260        //initialize the global state variables
261        __cfaabi_tls.this_processor = mainProcessor;
262        __cfaabi_tls.this_thread    = mainThread;
263
264        #if !defined( __CFA_NO_STATISTICS__ )
265                __cfaabi_tls.this_stats = (__stats_t *)& storage_mainProcStats;
266                __init_stats( __cfaabi_tls.this_stats );
267        #endif
268        mainProcessor->local_data = &__cfaabi_tls;
269
270        // Enable preemption
271        __kernel_alarm_startup();
272
273        // Add the main thread to the ready queue
274        // once resume is called on mainProcessor->runner the mainThread needs to be scheduled like any normal thread
275        schedule_thread$(mainThread);
276
277        // SKULLDUGGERY: Force a context switch to the main processor to set the main thread's context to the current UNIX
278        // context. Hence, the main thread does not begin through __cfactx_invoke_thread, like all other threads. The trick here is that
279        // mainThread is on the ready queue when this call is made.
280        __kernel_first_resume( __cfaabi_tls.this_processor );
281
282
283        // THE SYSTEM IS NOW COMPLETELY RUNNING
284
285        __cfadbg_print_safe(runtime_core, "Kernel : Started\n--------------------------------------------------\n\n");
286
287        /* paranoid */ verify( ! __preemption_enabled() );
288        enable_interrupts();
289        /* paranoid */ verify( __preemption_enabled() );
290
291}
292
293static void __kernel_shutdown(void) {
294        /* paranoid */ verify( __preemption_enabled() );
295        disable_interrupts();
296        /* paranoid */ verify( ! __preemption_enabled() );
297
298        __cfadbg_print_safe(runtime_core, "\n--------------------------------------------------\nKernel : Shutting down\n");
299
300        // SKULLDUGGERY: Notify the mainProcessor it needs to terminates.
301        // When its coroutine terminates, it return control to the mainThread
302        // which is currently here
303        __atomic_store_n(&mainProcessor->do_terminate, true, __ATOMIC_RELEASE);
304        __kernel_last_resume( __cfaabi_tls.this_processor );
305        mainThread->self_cor.state = Halted;
306
307        // THE SYSTEM IS NOW COMPLETELY STOPPED
308
309        // Disable preemption
310        __kernel_alarm_shutdown();
311
312        #if !defined( __CFA_NO_STATISTICS__ )
313                __stats_t * st = (__stats_t *)& storage_mainProcStats;
314                __tally_stats(mainCluster->stats, st);
315                if( 0 != mainProcessor->print_stats ) {
316                        __print_stats( st, mainProcessor->print_stats, "Processor ", mainProcessor->name, (void*)mainProcessor );
317                }
318                #if defined(CFA_STATS_ARRAY)
319                        __flush_stat( st, "Processor", mainProcessor );
320                #endif
321        #endif
322
323        mainProcessor->local_data = 0p;
324
325        __kernel_rseq_unregister();
326
327        unregister_tls( mainProcessor );
328
329        // Destroy the main processor and its context in reverse order of construction
330        // These were manually constructed so we need manually destroy them
331        void ^?{}(processor & this) with( this ){
332                deinit( this );
333
334                /* paranoid */ verify( this.do_terminate == true );
335                __cfaabi_dbg_print_safe("Kernel : destroyed main processor context %p\n", &runner);
336        }
337
338        ^(*mainProcessor){};
339
340        // Final step, destroy the main thread since it is no longer needed
341
342        // Since we provided a stack to this taxk it will not destroy anything
343        /* paranoid */ verify(mainThread->self_cor.stack.storage == (__stack_t*)(((uintptr_t)&storage_mainThreadCtx)| 0x1));
344        ^(*mainThread){};
345
346        ^(*mainCluster){};
347
348        ^(*__scheduler_lock){};
349
350        ^(__cfa_dbg_global_clusters.list){};
351        ^(__cfa_dbg_global_clusters.lock){};
352
353        __cfadbg_print_safe(runtime_core, "Kernel : Shutdown complete\n");
354}
355
356//=============================================================================================
357// Kernel Initial Scheduling logic
358//=============================================================================================
359
360// Context invoker for processors
361// This is the entry point for processors (kernel threads) *except* for the main processor
362// It effectively constructs a coroutine by stealing the pthread stack
363static void * __invoke_processor(void * arg) {
364        #if !defined( __CFA_NO_STATISTICS__ )
365                __stats_t local_stats;
366                __init_stats( &local_stats );
367                __cfaabi_tls.this_stats = &local_stats;
368        #endif
369
370        processor * proc = (processor *) arg;
371        __cfaabi_tls.this_processor = proc;
372        __cfaabi_tls.this_thread    = 0p;
373        __cfaabi_tls.preemption_state.[enabled, disable_count] = [false, 1];
374        proc->local_data = &__cfaabi_tls;
375
376        register_tls( proc );
377
378        __kernel_rseq_register();
379
380        // SKULLDUGGERY: We want to create a context for the processor coroutine
381        // which is needed for the 2-step context switch. However, there is no reason
382        // to waste the perfectly valid stack create by pthread.
383        current_stack_info_t info;
384        __stack_t ctx;
385        info.storage = &ctx;
386        (proc->runner){ proc, &info };
387
388        __cfaabi_dbg_print_safe("Coroutine : created stack %p\n", get_coroutine(proc->runner)->stack.storage);
389
390        //Set global state
391        __cfaabi_tls.this_thread = 0p;
392
393        //We now have a proper context from which to schedule threads
394        __cfadbg_print_safe(runtime_core, "Kernel : core %p created (%p, %p)\n", proc, &proc->runner, &ctx);
395
396        // SKULLDUGGERY: Since the coroutine doesn't have its own stack, we can't
397        // resume it to start it like it normally would, it will just context switch
398        // back to here. Instead directly call the main since we already are on the
399        // appropriate stack.
400        get_coroutine(proc->runner)->state = Active;
401        main( proc->runner );
402        get_coroutine(proc->runner)->state = Halted;
403
404        // Main routine of the core returned, the core is now fully terminated
405        __cfadbg_print_safe(runtime_core, "Kernel : core %p main ended (%p)\n", proc, &proc->runner);
406
407        #if !defined(__CFA_NO_STATISTICS__)
408                __tally_stats(proc->cltr->stats, &local_stats);
409                if( 0 != proc->print_stats ) {
410                        __print_stats( &local_stats, proc->print_stats, "Processor ", proc->name, (void*)proc );
411                }
412                #if defined(CFA_STATS_ARRAY)
413                        __flush_stat( &local_stats, "Processor", proc );
414                #endif
415        #endif
416
417        proc->local_data = 0p;
418
419        __kernel_rseq_unregister();
420
421        unregister_tls( proc );
422
423        return 0p;
424}
425
426static void __kernel_first_resume( processor * this ) {
427        $thread * src = mainThread;
428        $coroutine * dst = get_coroutine(this->runner);
429
430        /* paranoid */ verify( ! __preemption_enabled() );
431
432        __cfaabi_tls.this_thread->curr_cor = dst;
433        __stack_prepare( &dst->stack, 65000 );
434        __cfactx_start(main, dst, this->runner, __cfactx_invoke_coroutine);
435
436        /* paranoid */ verify( ! __preemption_enabled() );
437
438        dst->last = &src->self_cor;
439        dst->starter = dst->starter ? dst->starter : &src->self_cor;
440
441        // make sure the current state is still correct
442        /* paranoid */ verify(src->state == Ready);
443        src->corctx_flag = true;
444
445        // context switch to specified coroutine
446        verify( dst->context.SP );
447        __cfactx_switch( &src->context, &dst->context );
448        // when __cfactx_switch returns we are back in the src coroutine
449
450        mainThread->curr_cor = &mainThread->self_cor;
451
452        // make sure the current state has been update
453        /* paranoid */ verify(src->state == Active);
454
455        /* paranoid */ verify( ! __preemption_enabled() );
456}
457
458// KERNEL_ONLY
459static void __kernel_last_resume( processor * this ) {
460        $coroutine * src = &mainThread->self_cor;
461        $coroutine * dst = get_coroutine(this->runner);
462
463        /* paranoid */ verify( ! __preemption_enabled() );
464        /* paranoid */ verify( dst->starter == src );
465        /* paranoid */ verify( dst->context.SP );
466
467        // SKULLDUGGERY in debug the processors check that the
468        // stack is still within the limit of the stack limits after running a thread.
469        // that check doesn't make sense if we context switch to the processor using the
470        // coroutine semantics. Since this is a special case, use the current context
471        // info to populate these fields.
472        __cfaabi_dbg_debug_do(
473                __stack_context_t ctx;
474                CtxGet( ctx );
475                mainThread->context.SP = ctx.SP;
476                mainThread->context.FP = ctx.FP;
477        )
478
479        // context switch to the processor
480        __cfactx_switch( &src->context, &dst->context );
481}
482
483
484//=============================================================================================
485// Kernel Object Constructors logic
486//=============================================================================================
487//-----------------------------------------------------------------------------
488// Main thread construction
489static void ?{}( $coroutine & this, current_stack_info_t * info) with( this ) {
490        stack.storage = info->storage;
491        with(*stack.storage) {
492                limit     = info->limit;
493                base      = info->base;
494        }
495        __attribute__((may_alias)) intptr_t * istorage = (intptr_t*) &stack.storage;
496        *istorage |= 0x1;
497        name = "Main Thread";
498        state = Start;
499        starter = 0p;
500        last = 0p;
501        cancellation = 0p;
502}
503
504static void ?{}( $thread & this, current_stack_info_t * info) with( this ) {
505        ticket = TICKET_RUNNING;
506        state = Start;
507        self_cor{ info };
508        curr_cor = &self_cor;
509        curr_cluster = mainCluster;
510        self_mon.owner = &this;
511        self_mon.recursion = 1;
512        self_mon_p = &self_mon;
513        link.next = 0p;
514        link.ts   = 0;
515        preferred = -1u;
516        last_proc = 0p;
517        #if defined( __CFA_WITH_VERIFY__ )
518                canary = 0x0D15EA5E0D15EA5Ep;
519        #endif
520
521        node.next = 0p;
522        node.prev = 0p;
523        doregister(curr_cluster, this);
524
525        monitors{ &self_mon_p, 1, (fptr_t)0 };
526}
527
528//-----------------------------------------------------------------------------
529// Processor
530// Construct the processor context of non-main processors
531static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info) {
532        (this.__cor){ info };
533        this.proc = proc;
534}
535
536static void init(processor & this, const char name[], cluster & _cltr, $thread * initT) with( this ) {
537        this.name = name;
538        this.cltr = &_cltr;
539        this.rdq.its = 0;
540        this.rdq.itr = 0;
541        this.rdq.id  = -1u;
542        this.rdq.target = -1u;
543        this.rdq.cutoff = 0ull;
544        do_terminate = false;
545        preemption_alarm = 0p;
546        pending_preemption = false;
547
548        this.io.ctx = 0p;
549        this.io.pending = false;
550        this.io.dirty   = false;
551
552        this.init.thrd = initT;
553
554        this.local_data = 0p;
555
556        this.idle = eventfd(0, 0);
557        if (idle < 0) {
558                abort("KERNEL ERROR: PROCESSOR EVENTFD - %s\n", strerror(errno));
559        }
560
561        #if !defined(__CFA_NO_STATISTICS__)
562                print_stats = 0;
563                print_halts = false;
564        #endif
565
566        __cfadbg_print_safe(runtime_core, "Kernel : core %p created\n", &this);
567}
568
569// Not a ctor, it just preps the destruction but should not destroy members
570static void deinit(processor & this) {
571        close(this.idle);
572}
573
574void ?{}(processor & this, const char name[], cluster & _cltr, $thread * initT) {
575        ( this.terminated ){};
576        ( this.runner ){};
577
578        disable_interrupts();
579                init( this, name, _cltr, initT );
580        enable_interrupts();
581
582        __cfadbg_print_safe(runtime_core, "Kernel : Starting core %p\n", &this);
583
584        this.stack = __create_pthread( &this.kernel_thread, __invoke_processor, (void *)&this );
585}
586
587void ?{}(processor & this, const char name[], cluster & _cltr) {
588        (this){name, _cltr, 0p};
589}
590
591extern size_t __page_size;
592void ^?{}(processor & this) with( this ){
593        if( ! __atomic_load_n(&do_terminate, __ATOMIC_ACQUIRE) ) {
594                __cfadbg_print_safe(runtime_core, "Kernel : core %p signaling termination\n", &this);
595
596                __atomic_store_n(&do_terminate, true, __ATOMIC_RELAXED);
597                __wake_proc( &this );
598
599                wait( terminated );
600                /* paranoid */ verify( active_processor() != &this);
601        }
602
603        __destroy_pthread( kernel_thread, this.stack, 0p );
604
605        disable_interrupts();
606                deinit( this );
607        enable_interrupts();
608}
609
610//-----------------------------------------------------------------------------
611// Cluster
612static void ?{}(__cluster_proc_list & this) {
613        this.lock  = 0;
614        this.idle  = 0;
615        this.total = 0;
616}
617
618void ?{}(cluster & this, const char name[], Duration preemption_rate, unsigned num_io, const io_context_params & io_params) with( this ) {
619        this.name = name;
620        this.preemption_rate = preemption_rate;
621        ready_queue{};
622
623        #if !defined(__CFA_NO_STATISTICS__)
624                print_stats = 0;
625                stats = alloc();
626                __init_stats( stats );
627        #endif
628
629        threads{ __get };
630
631        io.arbiter = create();
632        io.params = io_params;
633
634        doregister(this);
635
636        // Lock the RWlock so no-one pushes/pops while we are changing the queue
637        disable_interrupts();
638        uint_fast32_t last_size = ready_mutate_lock();
639
640                // Adjust the ready queue size
641                ready_queue_grow( &this );
642
643        // Unlock the RWlock
644        ready_mutate_unlock( last_size );
645        enable_interrupts( false ); // Don't poll, could be in main cluster
646}
647
648void ^?{}(cluster & this) {
649        destroy(this.io.arbiter);
650
651        // Lock the RWlock so no-one pushes/pops while we are changing the queue
652        disable_interrupts();
653        uint_fast32_t last_size = ready_mutate_lock();
654
655                // Adjust the ready queue size
656                ready_queue_shrink( &this );
657
658        // Unlock the RWlock
659        ready_mutate_unlock( last_size );
660        enable_interrupts( false ); // Don't poll, could be in main cluster
661
662        #if !defined(__CFA_NO_STATISTICS__)
663                if( 0 != this.print_stats ) {
664                        __print_stats( this.stats, this.print_stats, "Cluster", this.name, (void*)&this );
665                }
666                #if defined(CFA_STATS_ARRAY)
667                        __flush_stat( this.stats, "Cluster", &this );
668                #endif
669                free( this.stats );
670        #endif
671
672        unregister(this);
673}
674
675//=============================================================================================
676// Miscellaneous Initialization
677//=============================================================================================
678//-----------------------------------------------------------------------------
679// Global Queues
680static void doregister( cluster     & cltr ) {
681        lock      ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
682        push_front( __cfa_dbg_global_clusters.list, cltr );
683        unlock    ( __cfa_dbg_global_clusters.lock );
684}
685
686static void unregister( cluster     & cltr ) {
687        lock  ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
688        remove( __cfa_dbg_global_clusters.list, cltr );
689        unlock( __cfa_dbg_global_clusters.lock );
690}
691
692void doregister( cluster * cltr, $thread & thrd ) {
693        lock      (cltr->thread_list_lock __cfaabi_dbg_ctx2);
694        cltr->nthreads += 1;
695        push_front(cltr->threads, thrd);
696        unlock    (cltr->thread_list_lock);
697}
698
699void unregister( cluster * cltr, $thread & thrd ) {
700        lock  (cltr->thread_list_lock __cfaabi_dbg_ctx2);
701        remove(cltr->threads, thrd );
702        cltr->nthreads -= 1;
703        unlock(cltr->thread_list_lock);
704}
705
706static void register_tls( processor * this ) {
707        // Register and Lock the RWlock so no-one pushes/pops while we are changing the queue
708        uint_fast32_t last_size;
709        [this->unique_id, last_size] = ready_mutate_register();
710
711                this->cltr->procs.total += 1u;
712                insert_last(this->cltr->procs.actives, *this);
713
714                // Adjust the ready queue size
715                ready_queue_grow( this->cltr );
716
717        // Unlock the RWlock
718        ready_mutate_unlock( last_size );
719}
720
721
722static void unregister_tls( processor * this ) {
723        // Lock the RWlock so no-one pushes/pops while we are changing the queue
724        uint_fast32_t last_size = ready_mutate_lock();
725                this->cltr->procs.total -= 1u;
726                remove(*this);
727
728                // clear the cluster so nothing gets pushed to local queues
729                cluster * cltr = this->cltr;
730                this->cltr = 0p;
731
732                // Adjust the ready queue size
733                ready_queue_shrink( cltr );
734
735        // Unlock the RWlock and unregister: we don't need the read_lock any more
736        ready_mutate_unregister( this->unique_id, last_size );
737}
738
739static void check( int ret, const char func[] ) {
740        if ( ret ) {                                                                            // pthread routines return errno values
741                abort( "%s : internal error, error(%d) %s.", func, ret, strerror( ret ) );
742        } // if
743} // Abort
744
745void * __create_pthread( pthread_t * pthread, void * (*start)(void *), void * arg ) {
746        pthread_attr_t attr;
747
748        check( pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
749
750        size_t stacksize;
751        // default stack size, normally defined by shell limit
752        check( pthread_attr_getstacksize( &attr, &stacksize ), "pthread_attr_getstacksize" );
753        assert( stacksize >= PTHREAD_STACK_MIN );
754
755        void * stack;
756        #if CFA_PROCESSOR_USE_MMAP
757                stacksize = ceiling( stacksize, __page_size ) + __page_size;
758                stack = mmap(0p, stacksize, __map_prot, MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
759                if(stack == ((void*)-1)) {
760                        abort( "pthread stack creation : internal error, mmap failure, error(%d) %s.", errno, strerror( errno ) );
761                }
762                if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
763                        abort( "pthread stack creation : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
764                } // if
765        #else
766                __cfaabi_dbg_debug_do(
767                        stack = memalign( __page_size, stacksize + __page_size );
768                        // pthread has no mechanism to create the guard page in user supplied stack.
769                        if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
770                                abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
771                        } // if
772                );
773                __cfaabi_dbg_no_debug_do(
774                        stack = malloc( stacksize );
775                );
776        #endif
777
778
779        check( pthread_attr_setstack( &attr, stack, stacksize ), "pthread_attr_setstack" );
780
781        check( pthread_create( pthread, &attr, start, arg ), "pthread_create" );
782        return stack;
783}
784
785void __destroy_pthread( pthread_t pthread, void * stack, void ** retval ) {
786        int err = pthread_join( pthread, retval );
787        if( err != 0 ) abort("KERNEL ERROR: joining pthread %p caused error %s\n", (void*)pthread, strerror(err));
788
789        #if CFA_PROCESSOR_USE_MMAP
790                pthread_attr_t attr;
791
792                check( pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
793
794                size_t stacksize;
795                // default stack size, normally defined by shell limit
796                check( pthread_attr_getstacksize( &attr, &stacksize ), "pthread_attr_getstacksize" );
797                assert( stacksize >= PTHREAD_STACK_MIN );
798                stacksize += __page_size;
799
800                if(munmap(stack, stacksize) == -1) {
801                        abort( "pthread stack destruction : internal error, munmap failure, error(%d) %s.", errno, strerror( errno ) );
802                }
803        #else
804                __cfaabi_dbg_debug_do(
805                        // pthread has no mechanism to create the guard page in user supplied stack.
806                        if ( mprotect( stack, __page_size, __map_prot ) == -1 ) {
807                                abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
808                        } // if
809                );
810                free( stack );
811        #endif
812}
813
814#if defined(__CFA_WITH_VERIFY__)
815static bool verify_fwd_bck_rng(void) {
816        __cfaabi_tls.ready_rng.fwd_seed = 25214903917_l64u * (rdtscl() ^ (uintptr_t)&verify_fwd_bck_rng);
817
818        unsigned values[10];
819        for(i; 10) {
820                values[i] = __tls_rand_fwd();
821        }
822
823        __tls_rand_advance_bck();
824
825        for ( i; 9 -~= 0 ) {
826                if(values[i] != __tls_rand_bck()) {
827                        return false;
828                }
829        }
830
831        return true;
832}
833#endif
834
835#if   defined(CFA_HAVE_LINUX_LIBRSEQ)
836        // No definition needed
837#elif defined(CFA_HAVE_LINUX_RSEQ_H)
838
839        #if defined( __x86_64 ) || defined( __i386 )
840                #define RSEQ_SIG        0x53053053
841        #elif defined( __ARM_ARCH )
842                #ifdef __ARMEB__
843                #define RSEQ_SIG    0xf3def5e7      /* udf    #24035    ; 0x5de3 (ARMv6+) */
844                #else
845                #define RSEQ_SIG    0xe7f5def3      /* udf    #24035    ; 0x5de3 */
846                #endif
847        #endif
848
849        extern void __disable_interrupts_hard();
850        extern void __enable_interrupts_hard();
851
852        void __kernel_raw_rseq_register  (void) {
853                /* paranoid */ verify( __cfaabi_rseq.cpu_id == RSEQ_CPU_ID_UNINITIALIZED );
854
855                // int ret = syscall(__NR_rseq, &__cfaabi_rseq, sizeof(struct rseq), 0, (sigset_t *)0p, _NSIG / 8);
856                int ret = syscall(__NR_rseq, &__cfaabi_rseq, sizeof(struct rseq), 0, RSEQ_SIG);
857                if(ret != 0) {
858                        int e = errno;
859                        switch(e) {
860                        case EINVAL: abort("KERNEL ERROR: rseq register invalid argument");
861                        case ENOSYS: abort("KERNEL ERROR: rseq register no supported");
862                        case EFAULT: abort("KERNEL ERROR: rseq register with invalid argument");
863                        case EBUSY : abort("KERNEL ERROR: rseq register already registered");
864                        case EPERM : abort("KERNEL ERROR: rseq register sig  argument  on unregistration does not match the signature received on registration");
865                        default: abort("KERNEL ERROR: rseq register unexpected return %d", e);
866                        }
867                }
868        }
869
870        void __kernel_raw_rseq_unregister(void) {
871                /* paranoid */ verify( __cfaabi_rseq.cpu_id >= 0 );
872
873                // int ret = syscall(__NR_rseq, &__cfaabi_rseq, sizeof(struct rseq), RSEQ_FLAG_UNREGISTER, (sigset_t *)0p, _NSIG / 8);
874                int ret = syscall(__NR_rseq, &__cfaabi_rseq, sizeof(struct rseq), RSEQ_FLAG_UNREGISTER, RSEQ_SIG);
875                if(ret != 0) {
876                        int e = errno;
877                        switch(e) {
878                        case EINVAL: abort("KERNEL ERROR: rseq unregister invalid argument");
879                        case ENOSYS: abort("KERNEL ERROR: rseq unregister no supported");
880                        case EFAULT: abort("KERNEL ERROR: rseq unregister with invalid argument");
881                        case EBUSY : abort("KERNEL ERROR: rseq unregister already registered");
882                        case EPERM : abort("KERNEL ERROR: rseq unregister sig  argument  on unregistration does not match the signature received on registration");
883                        default: abort("KERNEL ERROR: rseq unregisteunexpected return %d", e);
884                        }
885                }
886        }
887#else
888        // No definition needed
889#endif
Note: See TracBrowser for help on using the repository browser.