| 1 | //
|
|---|
| 2 | // Cforall Version 1.0.0 Copyright (C) 2020 University of Waterloo
|
|---|
| 3 | //
|
|---|
| 4 | // The contents of this file are covered under the licence agreement in the
|
|---|
| 5 | // file "LICENCE" distributed with Cforall.
|
|---|
| 6 | //
|
|---|
| 7 | // kernel/fwd.hfa --
|
|---|
| 8 | //
|
|---|
| 9 | // Author : Thierry Delisle
|
|---|
| 10 | // Created On : Thu Jul 30 16:46:41 2020
|
|---|
| 11 | // Last Modified By :
|
|---|
| 12 | // Last Modified On :
|
|---|
| 13 | // Update Count :
|
|---|
| 14 | //
|
|---|
| 15 |
|
|---|
| 16 | #pragma once
|
|---|
| 17 |
|
|---|
| 18 | #include "bits/defs.hfa"
|
|---|
| 19 | #include "bits/debug.hfa"
|
|---|
| 20 |
|
|---|
| 21 | #ifdef __cforall
|
|---|
| 22 | #include "bits/random.hfa"
|
|---|
| 23 | #endif
|
|---|
| 24 |
|
|---|
| 25 | struct $thread;
|
|---|
| 26 | struct processor;
|
|---|
| 27 | struct cluster;
|
|---|
| 28 |
|
|---|
| 29 | enum __Preemption_Reason { __NO_PREEMPTION, __ALARM_PREEMPTION, __POLL_PREEMPTION, __MANUAL_PREEMPTION };
|
|---|
| 30 |
|
|---|
| 31 | #define KERNEL_STORAGE(T,X) __attribute((aligned(__alignof__(T)))) static char storage_##X[sizeof(T)]
|
|---|
| 32 |
|
|---|
| 33 | #ifdef __cforall
|
|---|
| 34 | extern "C" {
|
|---|
| 35 | extern "Cforall" {
|
|---|
| 36 | extern __attribute__((aligned(128))) thread_local struct KernelThreadData {
|
|---|
| 37 | struct $thread * volatile this_thread;
|
|---|
| 38 | struct processor * volatile this_processor;
|
|---|
| 39 | struct __stats_t * volatile this_stats;
|
|---|
| 40 |
|
|---|
| 41 | struct {
|
|---|
| 42 | volatile unsigned short disable_count;
|
|---|
| 43 | volatile bool enabled;
|
|---|
| 44 | volatile bool in_progress;
|
|---|
| 45 | } preemption_state;
|
|---|
| 46 |
|
|---|
| 47 | #if defined(__SIZEOF_INT128__)
|
|---|
| 48 | __uint128_t rand_seed;
|
|---|
| 49 | #else
|
|---|
| 50 | uint64_t rand_seed;
|
|---|
| 51 | #endif
|
|---|
| 52 | } kernelTLS __attribute__ ((tls_model ( "initial-exec" )));
|
|---|
| 53 |
|
|---|
| 54 | static inline uint64_t __tls_rand() {
|
|---|
| 55 | #if defined(__SIZEOF_INT128__)
|
|---|
| 56 | return __lehmer64( kernelTLS.rand_seed );
|
|---|
| 57 | #else
|
|---|
| 58 | return __xorshift64( kernelTLS.rand_seed );
|
|---|
| 59 | #endif
|
|---|
| 60 | }
|
|---|
| 61 | }
|
|---|
| 62 |
|
|---|
| 63 | #ifdef __ARM_ARCH
|
|---|
| 64 | // function prototypes are only really used by these macros on ARM
|
|---|
| 65 | void disable_global_interrupts();
|
|---|
| 66 | void enable_global_interrupts();
|
|---|
| 67 |
|
|---|
| 68 | #define TL_GET( member ) ( { __typeof__( kernelTLS.member ) target; \
|
|---|
| 69 | disable_global_interrupts(); \
|
|---|
| 70 | target = kernelTLS.member; \
|
|---|
| 71 | enable_global_interrupts(); \
|
|---|
| 72 | target; } )
|
|---|
| 73 | #define TL_SET( member, value ) disable_global_interrupts(); \
|
|---|
| 74 | kernelTLS.member = value; \
|
|---|
| 75 | enable_global_interrupts();
|
|---|
| 76 | #else
|
|---|
| 77 | #define TL_GET( member ) kernelTLS.member
|
|---|
| 78 | #define TL_SET( member, value ) kernelTLS.member = value;
|
|---|
| 79 | #endif
|
|---|
| 80 |
|
|---|
| 81 | extern void disable_interrupts();
|
|---|
| 82 | extern void enable_interrupts_noPoll();
|
|---|
| 83 | extern void enable_interrupts( __cfaabi_dbg_ctx_param );
|
|---|
| 84 |
|
|---|
| 85 | extern "Cforall" {
|
|---|
| 86 | extern void park( __cfaabi_dbg_ctx_param );
|
|---|
| 87 | extern void unpark( struct $thread * this __cfaabi_dbg_ctx_param2 );
|
|---|
| 88 | static inline struct $thread * active_thread () { return TL_GET( this_thread ); }
|
|---|
| 89 |
|
|---|
| 90 | extern bool force_yield( enum __Preemption_Reason );
|
|---|
| 91 |
|
|---|
| 92 | static inline void yield() {
|
|---|
| 93 | force_yield(__MANUAL_PREEMPTION);
|
|---|
| 94 | }
|
|---|
| 95 |
|
|---|
| 96 | // Yield: yield N times
|
|---|
| 97 | static inline void yield( unsigned times ) {
|
|---|
| 98 | for( times ) {
|
|---|
| 99 | yield();
|
|---|
| 100 | }
|
|---|
| 101 | }
|
|---|
| 102 |
|
|---|
| 103 | //-----------------------------------------------------------------------
|
|---|
| 104 | // Statics call at the end of each thread to register statistics
|
|---|
| 105 | #if !defined(__CFA_NO_STATISTICS__)
|
|---|
| 106 | static inline struct __stats_t * __tls_stats() {
|
|---|
| 107 | /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
|
|---|
| 108 | /* paranoid */ verify( kernelTLS.this_stats );
|
|---|
| 109 | return kernelTLS.this_stats;
|
|---|
| 110 | }
|
|---|
| 111 |
|
|---|
| 112 | #define __STATS__(in_kernel, ...) { \
|
|---|
| 113 | if( !(in_kernel) ) disable_interrupts(); \
|
|---|
| 114 | with( *__tls_stats() ) { \
|
|---|
| 115 | __VA_ARGS__ \
|
|---|
| 116 | } \
|
|---|
| 117 | if( !(in_kernel) ) enable_interrupts( __cfaabi_dbg_ctx ); \
|
|---|
| 118 | }
|
|---|
| 119 | #else
|
|---|
| 120 | #define __STATS__(in_kernel, ...)
|
|---|
| 121 | #endif
|
|---|
| 122 | }
|
|---|
| 123 | }
|
|---|
| 124 | #endif
|
|---|
