source: libcfa/src/concurrency/kernel/fwd.hfa @ cd6a6ff

//
// Cforall Version 1.0.0 Copyright (C) 2020 University of Waterloo
//
// The contents of this file are covered under the licence agreement in the
// file "LICENCE" distributed with Cforall.
//
// kernel/fwd.hfa --
//
// Author           : Thierry Delisle
// Created On       : Thu Jul 30 16:46:41 2020
// Last Modified By :
// Last Modified On :
// Update Count     :
//

#pragma once

#include "bits/defs.hfa"
#include "bits/debug.hfa"

#ifdef __cforall
#include "bits/random.hfa"
#endif

struct $thread;
struct processor;
struct cluster;

enum __Preemption_Reason { __NO_PREEMPTION, __ALARM_PREEMPTION, __POLL_PREEMPTION, __MANUAL_PREEMPTION };

#define KERNEL_STORAGE(T,X) __attribute((aligned(__alignof__(T)))) static char storage_##X[sizeof(T)]

#ifdef __cforall
extern "C" {
        extern "Cforall" {
                extern __attribute__((aligned(128))) thread_local struct KernelThreadData {
                        struct $thread          * volatile this_thread;
                        struct processor        * volatile this_processor;
                        struct __processor_id_t * volatile this_proc_id;
                        struct __stats_t        * volatile this_stats;

                        struct {
                                volatile unsigned short disable_count;
                                volatile bool enabled;
                                volatile bool in_progress;
                        } preemption_state;

                        #if defined(__SIZEOF_INT128__)
                                __uint128_t rand_seed;
                        #else
                                uint64_t rand_seed;
                        #endif
                        struct {
                                uint64_t fwd_seed;
                                uint64_t bck_seed;
                        } ready_rng;
                } __cfaabi_tls __attribute__ ((tls_model ( "initial-exec" )));

                extern bool __preemption_enabled();

                static inline KernelThreadData & kernelTLS( void ) {
                        /* paranoid */ verify( ! __preemption_enabled() );
                        return __cfaabi_tls;
                }

                extern uintptr_t __cfatls_get( unsigned long int member );
                #define publicTLS_get( member ) ((typeof(__cfaabi_tls.member))__cfatls_get( __builtin_offsetof(KernelThreadData, member) ))

                static inline uint64_t __tls_rand() {
                        #if defined(__SIZEOF_INT128__)
                                return __lehmer64( kernelTLS().rand_seed );
                        #else
                                return __xorshift64( kernelTLS().rand_seed );
                        #endif
                }

                #define M  (1_l64u << 48_l64u)
                #define A  (25214903917_l64u)
                #define AI (18446708753438544741_l64u)
                #define C  (11_l64u)
                #define D  (16_l64u)

                static inline unsigned __tls_rand_fwd() {

                        kernelTLS().ready_rng.fwd_seed = (A * kernelTLS().ready_rng.fwd_seed + C) & (M - 1);
                        return kernelTLS().ready_rng.fwd_seed >> D;
                }

                static inline unsigned __tls_rand_bck() {
                        unsigned int r = kernelTLS().ready_rng.bck_seed >> D;
                        kernelTLS().ready_rng.bck_seed = AI * (kernelTLS().ready_rng.bck_seed - C) & (M - 1);
                        return r;
                }

                #undef M
                #undef A
                #undef AI
                #undef C
                #undef D

                static inline void __tls_rand_advance_bck(void) {
                        kernelTLS().ready_rng.bck_seed = kernelTLS().ready_rng.fwd_seed;
                }
        }



        extern void disable_interrupts();
        extern void enable_interrupts_noPoll();
        extern void enable_interrupts( __cfaabi_dbg_ctx_param );

        extern "Cforall" {
                extern void park( void );
                extern void unpark( struct $thread * this );
                static inline struct $thread * active_thread () {
                        struct $thread * t = publicTLS_get( this_thread );
                        /* paranoid */ verify( t );
                        return t;
                }

                extern bool force_yield( enum __Preemption_Reason );

                static inline void yield() {
                        force_yield(__MANUAL_PREEMPTION);
                }

                // Yield: yield N times
                static inline void yield( unsigned times ) {
                        for( times ) {
                                yield();
                        }
                }

                //-----------------------------------------------------------------------
                // Statics call at the end of each thread to register statistics
                #if !defined(__CFA_NO_STATISTICS__)
                        static inline struct __stats_t * __tls_stats() {
                                /* paranoid */ verify( ! __preemption_enabled() );
                                /* paranoid */ verify( kernelTLS().this_stats );
                                return kernelTLS().this_stats;
                        }

                        #define __STATS__(in_kernel, ...) { \
                                if( !(in_kernel) ) disable_interrupts(); \
                                with( *__tls_stats() ) { \
                                        __VA_ARGS__ \
                                } \
                                if( !(in_kernel) ) enable_interrupts( __cfaabi_dbg_ctx ); \
                        }
                #else
                        #define __STATS__(in_kernel, ...)
                #endif
        }
}
#endif