1 | // |
---|
2 | // Cforall Version 1.0.0 Copyright (C) 2020 University of Waterloo |
---|
3 | // |
---|
4 | // The contents of this file are covered under the licence agreement in the |
---|
5 | // file "LICENCE" distributed with Cforall. |
---|
6 | // |
---|
7 | // kernel/fwd.hfa -- |
---|
8 | // |
---|
9 | // Author : Thierry Delisle |
---|
10 | // Created On : Thu Jul 30 16:46:41 2020 |
---|
11 | // Last Modified By : |
---|
12 | // Last Modified On : |
---|
13 | // Update Count : |
---|
14 | // |
---|
15 | |
---|
16 | #pragma once |
---|
17 | |
---|
18 | #include "bits/defs.hfa" |
---|
19 | #include "bits/debug.hfa" |
---|
20 | |
---|
21 | #ifdef __cforall |
---|
22 | #include "bits/random.hfa" |
---|
23 | #endif |
---|
24 | |
---|
25 | struct $thread; |
---|
26 | struct processor; |
---|
27 | struct cluster; |
---|
28 | |
---|
29 | enum __Preemption_Reason { __NO_PREEMPTION, __ALARM_PREEMPTION, __POLL_PREEMPTION, __MANUAL_PREEMPTION }; |
---|
30 | |
---|
31 | #define KERNEL_STORAGE(T,X) __attribute((aligned(__alignof__(T)))) static char storage_##X[sizeof(T)] |
---|
32 | |
---|
33 | #ifdef __cforall |
---|
34 | extern "C" { |
---|
35 | extern "Cforall" { |
---|
36 | extern __attribute__((aligned(128))) thread_local struct KernelThreadData { |
---|
37 | struct $thread * volatile this_thread; |
---|
38 | struct processor * volatile this_processor; |
---|
39 | struct __processor_id_t * volatile this_proc_id; |
---|
40 | struct __stats_t * volatile this_stats; |
---|
41 | |
---|
42 | struct { |
---|
43 | volatile unsigned short disable_count; |
---|
44 | volatile bool enabled; |
---|
45 | volatile bool in_progress; |
---|
46 | } preemption_state; |
---|
47 | |
---|
48 | #if defined(__SIZEOF_INT128__) |
---|
49 | __uint128_t rand_seed; |
---|
50 | #else |
---|
51 | uint64_t rand_seed; |
---|
52 | #endif |
---|
53 | struct { |
---|
54 | uint64_t fwd_seed; |
---|
55 | uint64_t bck_seed; |
---|
56 | } ready_rng; |
---|
57 | } __cfaabi_tls __attribute__ ((tls_model ( "initial-exec" ))); |
---|
58 | |
---|
59 | extern bool __preemption_enabled(); |
---|
60 | |
---|
61 | static inline KernelThreadData & kernelTLS( void ) { |
---|
62 | /* paranoid */ verify( ! __preemption_enabled() ); |
---|
63 | return __cfaabi_tls; |
---|
64 | } |
---|
65 | |
---|
66 | extern uintptr_t __cfatls_get( unsigned long int member ); |
---|
67 | #define publicTLS_get( member ) ((typeof(__cfaabi_tls.member))__cfatls_get( __builtin_offsetof(KernelThreadData, member) )) |
---|
68 | |
---|
69 | static inline uint64_t __tls_rand() { |
---|
70 | #if defined(__SIZEOF_INT128__) |
---|
71 | return __lehmer64( kernelTLS().rand_seed ); |
---|
72 | #else |
---|
73 | return __xorshift64( kernelTLS().rand_seed ); |
---|
74 | #endif |
---|
75 | } |
---|
76 | |
---|
77 | #define M (1_l64u << 48_l64u) |
---|
78 | #define A (25214903917_l64u) |
---|
79 | #define AI (18446708753438544741_l64u) |
---|
80 | #define C (11_l64u) |
---|
81 | #define D (16_l64u) |
---|
82 | |
---|
83 | static inline unsigned __tls_rand_fwd() { |
---|
84 | |
---|
85 | kernelTLS().ready_rng.fwd_seed = (A * kernelTLS().ready_rng.fwd_seed + C) & (M - 1); |
---|
86 | return kernelTLS().ready_rng.fwd_seed >> D; |
---|
87 | } |
---|
88 | |
---|
89 | static inline unsigned __tls_rand_bck() { |
---|
90 | unsigned int r = kernelTLS().ready_rng.bck_seed >> D; |
---|
91 | kernelTLS().ready_rng.bck_seed = AI * (kernelTLS().ready_rng.bck_seed - C) & (M - 1); |
---|
92 | return r; |
---|
93 | } |
---|
94 | |
---|
95 | #undef M |
---|
96 | #undef A |
---|
97 | #undef AI |
---|
98 | #undef C |
---|
99 | #undef D |
---|
100 | |
---|
101 | static inline void __tls_rand_advance_bck(void) { |
---|
102 | kernelTLS().ready_rng.bck_seed = kernelTLS().ready_rng.fwd_seed; |
---|
103 | } |
---|
104 | } |
---|
105 | |
---|
106 | |
---|
107 | |
---|
108 | extern void disable_interrupts(); |
---|
109 | extern void enable_interrupts_noPoll(); |
---|
110 | extern void enable_interrupts( __cfaabi_dbg_ctx_param ); |
---|
111 | |
---|
112 | extern "Cforall" { |
---|
113 | extern void park( void ); |
---|
114 | extern void unpark( struct $thread * this ); |
---|
115 | static inline struct $thread * active_thread () { |
---|
116 | struct $thread * t = publicTLS_get( this_thread ); |
---|
117 | /* paranoid */ verify( t ); |
---|
118 | return t; |
---|
119 | } |
---|
120 | |
---|
121 | extern bool force_yield( enum __Preemption_Reason ); |
---|
122 | |
---|
123 | static inline void yield() { |
---|
124 | force_yield(__MANUAL_PREEMPTION); |
---|
125 | } |
---|
126 | |
---|
127 | // Yield: yield N times |
---|
128 | static inline void yield( unsigned times ) { |
---|
129 | for( times ) { |
---|
130 | yield(); |
---|
131 | } |
---|
132 | } |
---|
133 | |
---|
134 | //----------------------------------------------------------------------- |
---|
135 | // Statics call at the end of each thread to register statistics |
---|
136 | #if !defined(__CFA_NO_STATISTICS__) |
---|
137 | static inline struct __stats_t * __tls_stats() { |
---|
138 | /* paranoid */ verify( ! __preemption_enabled() ); |
---|
139 | /* paranoid */ verify( kernelTLS().this_stats ); |
---|
140 | return kernelTLS().this_stats; |
---|
141 | } |
---|
142 | |
---|
143 | #define __STATS__(in_kernel, ...) { \ |
---|
144 | if( !(in_kernel) ) disable_interrupts(); \ |
---|
145 | with( *__tls_stats() ) { \ |
---|
146 | __VA_ARGS__ \ |
---|
147 | } \ |
---|
148 | if( !(in_kernel) ) enable_interrupts( __cfaabi_dbg_ctx ); \ |
---|
149 | } |
---|
150 | #else |
---|
151 | #define __STATS__(in_kernel, ...) |
---|
152 | #endif |
---|
153 | } |
---|
154 | } |
---|
155 | #endif |
---|