1 | //
|
---|
2 | // Cforall Version 1.0.0 Copyright (C) 2016 University of Waterloo
|
---|
3 | //
|
---|
4 | // The contents of this file are covered under the licence agreement in the
|
---|
5 | // file "LICENCE" distributed with Cforall.
|
---|
6 | //
|
---|
7 | // kernel.c --
|
---|
8 | //
|
---|
9 | // Author : Thierry Delisle
|
---|
10 | // Created On : Tue Jan 17 12:27:26 2017
|
---|
11 | // Last Modified By : Peter A. Buhr
|
---|
12 | // Last Modified On : Mon Aug 31 07:08:20 2020
|
---|
13 | // Update Count : 71
|
---|
14 | //
|
---|
15 |
|
---|
16 | #define __cforall_thread__
|
---|
17 | // #define __CFA_DEBUG_PRINT_RUNTIME_CORE__
|
---|
18 |
|
---|
19 | //C Includes
|
---|
20 | #include <errno.h>
|
---|
21 | #include <stdio.h>
|
---|
22 | #include <signal.h>
|
---|
23 | #include <unistd.h>
|
---|
24 |
|
---|
25 | //CFA Includes
|
---|
26 | #include "kernel_private.hfa"
|
---|
27 | #include "preemption.hfa"
|
---|
28 |
|
---|
29 | //Private includes
|
---|
30 | #define __CFA_INVOKE_PRIVATE__
|
---|
31 | #include "invoke.h"
|
---|
32 |
|
---|
33 |
|
---|
34 | //-----------------------------------------------------------------------------
|
---|
35 | // Some assembly required
|
---|
36 | #if defined( __i386 )
|
---|
37 | // mxcr : SSE Status and Control bits (control bits are preserved across function calls)
|
---|
38 | // fcw : X87 FPU control word (preserved across function calls)
|
---|
39 | #define __x87_store \
|
---|
40 | uint32_t __mxcr; \
|
---|
41 | uint16_t __fcw; \
|
---|
42 | __asm__ volatile ( \
|
---|
43 | "stmxcsr %0\n" \
|
---|
44 | "fnstcw %1\n" \
|
---|
45 | : "=m" (__mxcr),\
|
---|
46 | "=m" (__fcw) \
|
---|
47 | )
|
---|
48 |
|
---|
49 | #define __x87_load \
|
---|
50 | __asm__ volatile ( \
|
---|
51 | "fldcw %1\n" \
|
---|
52 | "ldmxcsr %0\n" \
|
---|
53 | ::"m" (__mxcr),\
|
---|
54 | "m" (__fcw) \
|
---|
55 | )
|
---|
56 |
|
---|
57 | #elif defined( __x86_64 )
|
---|
58 | #define __x87_store \
|
---|
59 | uint32_t __mxcr; \
|
---|
60 | uint16_t __fcw; \
|
---|
61 | __asm__ volatile ( \
|
---|
62 | "stmxcsr %0\n" \
|
---|
63 | "fnstcw %1\n" \
|
---|
64 | : "=m" (__mxcr),\
|
---|
65 | "=m" (__fcw) \
|
---|
66 | )
|
---|
67 |
|
---|
68 | #define __x87_load \
|
---|
69 | __asm__ volatile ( \
|
---|
70 | "fldcw %1\n" \
|
---|
71 | "ldmxcsr %0\n" \
|
---|
72 | :: "m" (__mxcr),\
|
---|
73 | "m" (__fcw) \
|
---|
74 | )
|
---|
75 |
|
---|
76 | #elif defined( __arm__ )
|
---|
77 | #define __x87_store
|
---|
78 | #define __x87_load
|
---|
79 |
|
---|
80 | #elif defined( __aarch64__ )
|
---|
81 | #define __x87_store \
|
---|
82 | uint32_t __fpcntl[2]; \
|
---|
83 | __asm__ volatile ( \
|
---|
84 | "mrs x9, FPCR\n" \
|
---|
85 | "mrs x10, FPSR\n" \
|
---|
86 | "stp x9, x10, %0\n" \
|
---|
87 | : "=m" (__fpcntl) : : "x9", "x10" \
|
---|
88 | )
|
---|
89 |
|
---|
90 | #define __x87_load \
|
---|
91 | __asm__ volatile ( \
|
---|
92 | "ldp x9, x10, %0\n" \
|
---|
93 | "msr FPSR, x10\n" \
|
---|
94 | "msr FPCR, x9\n" \
|
---|
95 | : "=m" (__fpcntl) : : "x9", "x10" \
|
---|
96 | )
|
---|
97 |
|
---|
98 | #else
|
---|
99 | #error unsupported hardware architecture
|
---|
100 | #endif
|
---|
101 |
|
---|
102 | extern $thread * mainThread;
|
---|
103 | extern processor * mainProcessor;
|
---|
104 |
|
---|
105 | //-----------------------------------------------------------------------------
|
---|
106 | // Kernel Scheduling logic
|
---|
107 | static $thread * __next_thread(cluster * this);
|
---|
108 | static $thread * __next_thread_slow(cluster * this);
|
---|
109 | static void __run_thread(processor * this, $thread * dst);
|
---|
110 | static void __wake_one(struct __processor_id_t * id, cluster * cltr);
|
---|
111 |
|
---|
112 | static void push (__cluster_idles & idles, processor & proc);
|
---|
113 | static void remove(__cluster_idles & idles, processor & proc);
|
---|
114 | static [unsigned idle, unsigned total, * processor] query( & __cluster_idles idles );
|
---|
115 |
|
---|
116 |
|
---|
117 | //=============================================================================================
|
---|
118 | // Kernel Scheduling logic
|
---|
119 | //=============================================================================================
|
---|
120 | //Main of the processor contexts
|
---|
121 | void main(processorCtx_t & runner) {
|
---|
122 | // Because of a bug, we couldn't initialized the seed on construction
|
---|
123 | // Do it here
|
---|
124 | kernelTLS.rand_seed ^= rdtscl();
|
---|
125 | kernelTLS.ready_rng.fwd_seed = 25214903917_l64u * (rdtscl() ^ (uintptr_t)&runner);
|
---|
126 | __tls_rand_advance_bck();
|
---|
127 |
|
---|
128 | processor * this = runner.proc;
|
---|
129 | verify(this);
|
---|
130 |
|
---|
131 | __cfadbg_print_safe(runtime_core, "Kernel : core %p starting\n", this);
|
---|
132 | #if !defined(__CFA_NO_STATISTICS__)
|
---|
133 | if( this->print_halts ) {
|
---|
134 | __cfaabi_bits_print_safe( STDOUT_FILENO, "Processor : %d - %s (%p)\n", this->id, this->name, (void*)this);
|
---|
135 | }
|
---|
136 | #endif
|
---|
137 |
|
---|
138 | {
|
---|
139 | // Setup preemption data
|
---|
140 | preemption_scope scope = { this };
|
---|
141 |
|
---|
142 | __cfadbg_print_safe(runtime_core, "Kernel : core %p started\n", this);
|
---|
143 |
|
---|
144 | $thread * readyThread = 0p;
|
---|
145 | MAIN_LOOP:
|
---|
146 | for() {
|
---|
147 | // Try to get the next thread
|
---|
148 | readyThread = __next_thread( this->cltr );
|
---|
149 |
|
---|
150 | if( !readyThread ) {
|
---|
151 | readyThread = __next_thread_slow( this->cltr );
|
---|
152 | }
|
---|
153 |
|
---|
154 | HALT:
|
---|
155 | if( !readyThread ) {
|
---|
156 | // Don't block if we are done
|
---|
157 | if( __atomic_load_n(&this->do_terminate, __ATOMIC_SEQ_CST) ) break MAIN_LOOP;
|
---|
158 |
|
---|
159 | #if !defined(__CFA_NO_STATISTICS__)
|
---|
160 | __tls_stats()->ready.sleep.halts++;
|
---|
161 | #endif
|
---|
162 |
|
---|
163 | // Push self to idle stack
|
---|
164 | push(this->cltr->idles, * this);
|
---|
165 |
|
---|
166 | // Confirm the ready-queue is empty
|
---|
167 | readyThread = __next_thread_slow( this->cltr );
|
---|
168 | if( readyThread ) {
|
---|
169 | // A thread was found, cancel the halt
|
---|
170 | remove(this->cltr->idles, * this);
|
---|
171 |
|
---|
172 | #if !defined(__CFA_NO_STATISTICS__)
|
---|
173 | __tls_stats()->ready.sleep.cancels++;
|
---|
174 | #endif
|
---|
175 |
|
---|
176 | // continue the mai loop
|
---|
177 | break HALT;
|
---|
178 | }
|
---|
179 |
|
---|
180 | #if !defined(__CFA_NO_STATISTICS__)
|
---|
181 | if(this->print_halts) {
|
---|
182 | __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 0\n", this->id, rdtscl());
|
---|
183 | }
|
---|
184 | #endif
|
---|
185 |
|
---|
186 | wait( this->idle );
|
---|
187 |
|
---|
188 | #if !defined(__CFA_NO_STATISTICS__)
|
---|
189 | if(this->print_halts) {
|
---|
190 | __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 1\n", this->id, rdtscl());
|
---|
191 | }
|
---|
192 | #endif
|
---|
193 |
|
---|
194 | // We were woken up, remove self from idle
|
---|
195 | remove(this->cltr->idles, * this);
|
---|
196 |
|
---|
197 | // DON'T just proceed, start looking again
|
---|
198 | continue MAIN_LOOP;
|
---|
199 | }
|
---|
200 |
|
---|
201 | /* paranoid */ verify( readyThread );
|
---|
202 |
|
---|
203 | // We found a thread run it
|
---|
204 | __run_thread(this, readyThread);
|
---|
205 |
|
---|
206 | // Are we done?
|
---|
207 | if( __atomic_load_n(&this->do_terminate, __ATOMIC_SEQ_CST) ) break MAIN_LOOP;
|
---|
208 | }
|
---|
209 |
|
---|
210 | __cfadbg_print_safe(runtime_core, "Kernel : core %p stopping\n", this);
|
---|
211 | }
|
---|
212 |
|
---|
213 | V( this->terminated );
|
---|
214 |
|
---|
215 | if(this == mainProcessor) {
|
---|
216 | // HACK : the coroutine context switch expects this_thread to be set
|
---|
217 | // and it make sense for it to be set in all other cases except here
|
---|
218 | // fake it
|
---|
219 | kernelTLS.this_thread = mainThread;
|
---|
220 | }
|
---|
221 |
|
---|
222 | __cfadbg_print_safe(runtime_core, "Kernel : core %p terminated\n", this);
|
---|
223 | }
|
---|
224 |
|
---|
225 | static int * __volatile_errno() __attribute__((noinline));
|
---|
226 | static int * __volatile_errno() { asm(""); return &errno; }
|
---|
227 |
|
---|
228 | // KERNEL ONLY
|
---|
229 | // runThread runs a thread by context switching
|
---|
230 | // from the processor coroutine to the target thread
|
---|
231 | static void __run_thread(processor * this, $thread * thrd_dst) {
|
---|
232 | /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
|
---|
233 | /* paranoid */ verifyf( thrd_dst->state == Ready || thrd_dst->preempted != __NO_PREEMPTION, "state : %d, preempted %d\n", thrd_dst->state, thrd_dst->preempted);
|
---|
234 | /* paranoid */ verifyf( thrd_dst->link.next == 0p, "Expected null got %p", thrd_dst->link.next );
|
---|
235 | __builtin_prefetch( thrd_dst->context.SP );
|
---|
236 |
|
---|
237 | $coroutine * proc_cor = get_coroutine(this->runner);
|
---|
238 |
|
---|
239 | // set state of processor coroutine to inactive
|
---|
240 | verify(proc_cor->state == Active);
|
---|
241 | proc_cor->state = Blocked;
|
---|
242 |
|
---|
243 | // Actually run the thread
|
---|
244 | RUNNING: while(true) {
|
---|
245 | thrd_dst->preempted = __NO_PREEMPTION;
|
---|
246 | thrd_dst->state = Active;
|
---|
247 |
|
---|
248 | __cfaabi_dbg_debug_do(
|
---|
249 | thrd_dst->park_stale = true;
|
---|
250 | thrd_dst->unpark_stale = true;
|
---|
251 | )
|
---|
252 | // Update global state
|
---|
253 | kernelTLS.this_thread = thrd_dst;
|
---|
254 |
|
---|
255 | /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
|
---|
256 | /* paranoid */ verify( kernelTLS.this_thread == thrd_dst );
|
---|
257 | /* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) < ((uintptr_t)__get_stack(thrd_dst->curr_cor)->base ) || thrd_dst->curr_cor == proc_cor, "ERROR : Destination $thread %p has been corrupted.\n StackPointer too small.\n", thrd_dst ); // add escape condition if we are setting up the processor
|
---|
258 | /* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) > ((uintptr_t)__get_stack(thrd_dst->curr_cor)->limit) || thrd_dst->curr_cor == proc_cor, "ERROR : Destination $thread %p has been corrupted.\n StackPointer too large.\n", thrd_dst ); // add escape condition if we are setting up the processor
|
---|
259 |
|
---|
260 |
|
---|
261 | // set context switch to the thread that the processor is executing
|
---|
262 | verify( thrd_dst->context.SP );
|
---|
263 | __cfactx_switch( &proc_cor->context, &thrd_dst->context );
|
---|
264 | // when __cfactx_switch returns we are back in the processor coroutine
|
---|
265 |
|
---|
266 | /* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) > ((uintptr_t)__get_stack(thrd_dst->curr_cor)->limit), "ERROR : Destination $thread %p has been corrupted.\n StackPointer too large.\n", thrd_dst );
|
---|
267 | /* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) < ((uintptr_t)__get_stack(thrd_dst->curr_cor)->base ), "ERROR : Destination $thread %p has been corrupted.\n StackPointer too small.\n", thrd_dst );
|
---|
268 | /* paranoid */ verify( kernelTLS.this_thread == thrd_dst );
|
---|
269 | /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
|
---|
270 |
|
---|
271 | // Reset global state
|
---|
272 | kernelTLS.this_thread = 0p;
|
---|
273 |
|
---|
274 | // We just finished running a thread, there are a few things that could have happened.
|
---|
275 | // 1 - Regular case : the thread has blocked and now one has scheduled it yet.
|
---|
276 | // 2 - Racy case : the thread has blocked but someone has already tried to schedule it.
|
---|
277 | // 4 - Preempted
|
---|
278 | // In case 1, we may have won a race so we can't write to the state again.
|
---|
279 | // In case 2, we lost the race so we now own the thread.
|
---|
280 |
|
---|
281 | if(unlikely(thrd_dst->preempted != __NO_PREEMPTION)) {
|
---|
282 | // The thread was preempted, reschedule it and reset the flag
|
---|
283 | __schedule_thread( (__processor_id_t*)this, thrd_dst );
|
---|
284 | break RUNNING;
|
---|
285 | }
|
---|
286 |
|
---|
287 | if(unlikely(thrd_dst->state == Halted)) {
|
---|
288 | // The thread has halted, it should never be scheduled/run again
|
---|
289 | // We may need to wake someone up here since
|
---|
290 | unpark( this->destroyer __cfaabi_dbg_ctx2 );
|
---|
291 | this->destroyer = 0p;
|
---|
292 | break RUNNING;
|
---|
293 | }
|
---|
294 |
|
---|
295 | /* paranoid */ verify( thrd_dst->state == Active );
|
---|
296 | thrd_dst->state = Blocked;
|
---|
297 |
|
---|
298 | // set state of processor coroutine to active and the thread to inactive
|
---|
299 | int old_ticket = __atomic_fetch_sub(&thrd_dst->ticket, 1, __ATOMIC_SEQ_CST);
|
---|
300 | __cfaabi_dbg_debug_do( thrd_dst->park_result = old_ticket; )
|
---|
301 | switch(old_ticket) {
|
---|
302 | case 1:
|
---|
303 | // This is case 1, the regular case, nothing more is needed
|
---|
304 | break RUNNING;
|
---|
305 | case 2:
|
---|
306 | // This is case 2, the racy case, someone tried to run this thread before it finished blocking
|
---|
307 | // In this case, just run it again.
|
---|
308 | continue RUNNING;
|
---|
309 | default:
|
---|
310 | // This makes no sense, something is wrong abort
|
---|
311 | abort();
|
---|
312 | }
|
---|
313 | }
|
---|
314 |
|
---|
315 | // Just before returning to the processor, set the processor coroutine to active
|
---|
316 | proc_cor->state = Active;
|
---|
317 |
|
---|
318 | /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
|
---|
319 | }
|
---|
320 |
|
---|
321 | // KERNEL_ONLY
|
---|
322 | void returnToKernel() {
|
---|
323 | /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
|
---|
324 | $coroutine * proc_cor = get_coroutine(kernelTLS.this_processor->runner);
|
---|
325 | $thread * thrd_src = kernelTLS.this_thread;
|
---|
326 |
|
---|
327 | #if !defined(__CFA_NO_STATISTICS__)
|
---|
328 | struct processor * last_proc = kernelTLS.this_processor;
|
---|
329 | #endif
|
---|
330 |
|
---|
331 | // Run the thread on this processor
|
---|
332 | {
|
---|
333 | int local_errno = *__volatile_errno();
|
---|
334 | #if defined( __i386 ) || defined( __x86_64 )
|
---|
335 | __x87_store;
|
---|
336 | #endif
|
---|
337 | verify( proc_cor->context.SP );
|
---|
338 | __cfactx_switch( &thrd_src->context, &proc_cor->context );
|
---|
339 | #if defined( __i386 ) || defined( __x86_64 )
|
---|
340 | __x87_load;
|
---|
341 | #endif
|
---|
342 | *__volatile_errno() = local_errno;
|
---|
343 | }
|
---|
344 |
|
---|
345 | #if !defined(__CFA_NO_STATISTICS__)
|
---|
346 | if(last_proc != kernelTLS.this_processor) {
|
---|
347 | __tls_stats()->ready.threads.migration++;
|
---|
348 | }
|
---|
349 | #endif
|
---|
350 |
|
---|
351 | /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
|
---|
352 | /* paranoid */ verifyf( ((uintptr_t)thrd_src->context.SP) < ((uintptr_t)__get_stack(thrd_src->curr_cor)->base ), "ERROR : Returning $thread %p has been corrupted.\n StackPointer too small.\n", thrd_src );
|
---|
353 | /* paranoid */ verifyf( ((uintptr_t)thrd_src->context.SP) > ((uintptr_t)__get_stack(thrd_src->curr_cor)->limit), "ERROR : Returning $thread %p has been corrupted.\n StackPointer too large.\n", thrd_src );
|
---|
354 | }
|
---|
355 |
|
---|
356 | //-----------------------------------------------------------------------------
|
---|
357 | // Scheduler routines
|
---|
358 | // KERNEL ONLY
|
---|
359 | void __schedule_thread( struct __processor_id_t * id, $thread * thrd ) {
|
---|
360 | /* paranoid */ verify( thrd );
|
---|
361 | /* paranoid */ verify( thrd->state != Halted );
|
---|
362 | /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
|
---|
363 | /* paranoid */ #if defined( __CFA_WITH_VERIFY__ )
|
---|
364 | /* paranoid */ if( thrd->state == Blocked || thrd->state == Start ) assertf( thrd->preempted == __NO_PREEMPTION,
|
---|
365 | "Error inactive thread marked as preempted, state %d, preemption %d\n", thrd->state, thrd->preempted );
|
---|
366 | /* paranoid */ if( thrd->preempted != __NO_PREEMPTION ) assertf(thrd->state == Active,
|
---|
367 | "Error preempted thread marked as not currently running, state %d, preemption %d\n", thrd->state, thrd->preempted );
|
---|
368 | /* paranoid */ #endif
|
---|
369 | /* paranoid */ verifyf( thrd->link.next == 0p, "Expected null got %p", thrd->link.next );
|
---|
370 |
|
---|
371 | if (thrd->preempted == __NO_PREEMPTION) thrd->state = Ready;
|
---|
372 |
|
---|
373 | ready_schedule_lock ( id );
|
---|
374 | push( thrd->curr_cluster, thrd );
|
---|
375 | __wake_one(id, thrd->curr_cluster);
|
---|
376 | ready_schedule_unlock( id );
|
---|
377 |
|
---|
378 | /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
|
---|
379 | }
|
---|
380 |
|
---|
381 | // KERNEL ONLY
|
---|
382 | static inline $thread * __next_thread(cluster * this) with( *this ) {
|
---|
383 | /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
|
---|
384 |
|
---|
385 | ready_schedule_lock ( (__processor_id_t*)kernelTLS.this_processor );
|
---|
386 | $thread * thrd = pop( this );
|
---|
387 | ready_schedule_unlock( (__processor_id_t*)kernelTLS.this_processor );
|
---|
388 |
|
---|
389 | /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
|
---|
390 | return thrd;
|
---|
391 | }
|
---|
392 |
|
---|
393 | // KERNEL ONLY
|
---|
394 | static inline $thread * __next_thread_slow(cluster * this) with( *this ) {
|
---|
395 | /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
|
---|
396 |
|
---|
397 | ready_schedule_lock ( (__processor_id_t*)kernelTLS.this_processor );
|
---|
398 | $thread * thrd = pop_slow( this );
|
---|
399 | ready_schedule_unlock( (__processor_id_t*)kernelTLS.this_processor );
|
---|
400 |
|
---|
401 | /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
|
---|
402 | return thrd;
|
---|
403 | }
|
---|
404 |
|
---|
405 | // KERNEL ONLY unpark with out disabling interrupts
|
---|
406 | void __unpark( struct __processor_id_t * id, $thread * thrd __cfaabi_dbg_ctx_param2 ) {
|
---|
407 | // record activity
|
---|
408 | __cfaabi_dbg_record_thrd( *thrd, false, caller );
|
---|
409 |
|
---|
410 | int old_ticket = __atomic_fetch_add(&thrd->ticket, 1, __ATOMIC_SEQ_CST);
|
---|
411 | __cfaabi_dbg_debug_do( thrd->unpark_result = old_ticket; thrd->unpark_state = thrd->state; )
|
---|
412 | switch(old_ticket) {
|
---|
413 | case 1:
|
---|
414 | // Wake won the race, the thread will reschedule/rerun itself
|
---|
415 | break;
|
---|
416 | case 0:
|
---|
417 | /* paranoid */ verify( ! thrd->preempted != __NO_PREEMPTION );
|
---|
418 | /* paranoid */ verify( thrd->state == Blocked );
|
---|
419 |
|
---|
420 | // Wake lost the race,
|
---|
421 | __schedule_thread( id, thrd );
|
---|
422 | break;
|
---|
423 | default:
|
---|
424 | // This makes no sense, something is wrong abort
|
---|
425 | abort();
|
---|
426 | }
|
---|
427 | }
|
---|
428 |
|
---|
429 | void unpark( $thread * thrd __cfaabi_dbg_ctx_param2 ) {
|
---|
430 | if( !thrd ) return;
|
---|
431 |
|
---|
432 | disable_interrupts();
|
---|
433 | __unpark( (__processor_id_t*)kernelTLS.this_processor, thrd __cfaabi_dbg_ctx_fwd2 );
|
---|
434 | enable_interrupts( __cfaabi_dbg_ctx );
|
---|
435 | }
|
---|
436 |
|
---|
437 | void park( __cfaabi_dbg_ctx_param ) {
|
---|
438 | /* paranoid */ verify( kernelTLS.preemption_state.enabled );
|
---|
439 | disable_interrupts();
|
---|
440 | /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
|
---|
441 | /* paranoid */ verify( kernelTLS.this_thread->preempted == __NO_PREEMPTION );
|
---|
442 |
|
---|
443 | // record activity
|
---|
444 | __cfaabi_dbg_record_thrd( *kernelTLS.this_thread, true, caller );
|
---|
445 |
|
---|
446 | returnToKernel();
|
---|
447 |
|
---|
448 | /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
|
---|
449 | enable_interrupts( __cfaabi_dbg_ctx );
|
---|
450 | /* paranoid */ verify( kernelTLS.preemption_state.enabled );
|
---|
451 |
|
---|
452 | }
|
---|
453 |
|
---|
454 | // KERNEL ONLY
|
---|
455 | void __leave_thread() {
|
---|
456 | /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
|
---|
457 | returnToKernel();
|
---|
458 | abort();
|
---|
459 | }
|
---|
460 |
|
---|
461 | // KERNEL ONLY
|
---|
462 | bool force_yield( __Preemption_Reason reason ) {
|
---|
463 | /* paranoid */ verify( kernelTLS.preemption_state.enabled );
|
---|
464 | disable_interrupts();
|
---|
465 | /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
|
---|
466 |
|
---|
467 | $thread * thrd = kernelTLS.this_thread;
|
---|
468 | /* paranoid */ verify(thrd->state == Active);
|
---|
469 |
|
---|
470 | // SKULLDUGGERY: It is possible that we are preempting this thread just before
|
---|
471 | // it was going to park itself. If that is the case and it is already using the
|
---|
472 | // intrusive fields then we can't use them to preempt the thread
|
---|
473 | // If that is the case, abandon the preemption.
|
---|
474 | bool preempted = false;
|
---|
475 | if(thrd->link.next == 0p) {
|
---|
476 | preempted = true;
|
---|
477 | thrd->preempted = reason;
|
---|
478 | returnToKernel();
|
---|
479 | }
|
---|
480 |
|
---|
481 | /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
|
---|
482 | enable_interrupts_noPoll();
|
---|
483 | /* paranoid */ verify( kernelTLS.preemption_state.enabled );
|
---|
484 |
|
---|
485 | return preempted;
|
---|
486 | }
|
---|
487 |
|
---|
488 | //=============================================================================================
|
---|
489 | // Kernel Idle Sleep
|
---|
490 | //=============================================================================================
|
---|
491 | // Wake a thread from the front if there are any
|
---|
492 | static void __wake_one(struct __processor_id_t * id, cluster * this) {
|
---|
493 | /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
|
---|
494 | /* paranoid */ verify( ready_schedule_islocked( id ) );
|
---|
495 |
|
---|
496 | // Check if there is a sleeping processor
|
---|
497 | processor * p;
|
---|
498 | unsigned idle;
|
---|
499 | unsigned total;
|
---|
500 | [idle, total, p] = query(this->idles);
|
---|
501 |
|
---|
502 | // If no one is sleeping, we are done
|
---|
503 | if( idle == 0 ) return;
|
---|
504 |
|
---|
505 | // We found a processor, wake it up
|
---|
506 | post( p->idle );
|
---|
507 |
|
---|
508 | #if !defined(__CFA_NO_STATISTICS__)
|
---|
509 | __tls_stats()->ready.sleep.wakes++;
|
---|
510 | #endif
|
---|
511 |
|
---|
512 | /* paranoid */ verify( ready_schedule_islocked( id ) );
|
---|
513 | /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
|
---|
514 |
|
---|
515 | return;
|
---|
516 | }
|
---|
517 |
|
---|
518 | // Unconditionnaly wake a thread
|
---|
519 | void __wake_proc(processor * this) {
|
---|
520 | __cfadbg_print_safe(runtime_core, "Kernel : waking Processor %p\n", this);
|
---|
521 |
|
---|
522 | disable_interrupts();
|
---|
523 | /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
|
---|
524 | post( this->idle );
|
---|
525 | enable_interrupts( __cfaabi_dbg_ctx );
|
---|
526 | }
|
---|
527 |
|
---|
528 | static void push (__cluster_idles & this, processor & proc) {
|
---|
529 | /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
|
---|
530 | lock( this );
|
---|
531 | this.idle++;
|
---|
532 | /* paranoid */ verify( this.idle <= this.total );
|
---|
533 |
|
---|
534 | insert_first(this.list, proc);
|
---|
535 | unlock( this );
|
---|
536 | /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
|
---|
537 | }
|
---|
538 |
|
---|
539 | static void remove(__cluster_idles & this, processor & proc) {
|
---|
540 | /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
|
---|
541 | lock( this );
|
---|
542 | this.idle--;
|
---|
543 | /* paranoid */ verify( this.idle >= 0 );
|
---|
544 |
|
---|
545 | remove(proc);
|
---|
546 | unlock( this );
|
---|
547 | /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
|
---|
548 | }
|
---|
549 |
|
---|
550 | static [unsigned idle, unsigned total, * processor] query( & __cluster_idles this ) {
|
---|
551 | for() {
|
---|
552 | uint64_t l = __atomic_load_n(&this.lock, __ATOMIC_SEQ_CST);
|
---|
553 | if( 1 == (l % 2) ) { Pause(); continue; }
|
---|
554 | unsigned idle = this.idle;
|
---|
555 | unsigned total = this.total;
|
---|
556 | processor * proc = &this.list`first;
|
---|
557 | // Compiler fence is unnecessary, but gcc-8 and older incorrectly reorder code without it
|
---|
558 | asm volatile("": : :"memory");
|
---|
559 | if(l != __atomic_load_n(&this.lock, __ATOMIC_SEQ_CST)) { Pause(); continue; }
|
---|
560 | return [idle, total, proc];
|
---|
561 | }
|
---|
562 | }
|
---|
563 |
|
---|
564 | //=============================================================================================
|
---|
565 | // Unexpected Terminating logic
|
---|
566 | //=============================================================================================
|
---|
567 | static __spinlock_t kernel_abort_lock;
|
---|
568 | static bool kernel_abort_called = false;
|
---|
569 |
|
---|
570 | void * kernel_abort(void) __attribute__ ((__nothrow__)) {
|
---|
571 | // abort cannot be recursively entered by the same or different processors because all signal handlers return when
|
---|
572 | // the globalAbort flag is true.
|
---|
573 | lock( kernel_abort_lock __cfaabi_dbg_ctx2 );
|
---|
574 |
|
---|
575 | // first task to abort ?
|
---|
576 | if ( kernel_abort_called ) { // not first task to abort ?
|
---|
577 | unlock( kernel_abort_lock );
|
---|
578 |
|
---|
579 | sigset_t mask;
|
---|
580 | sigemptyset( &mask );
|
---|
581 | sigaddset( &mask, SIGALRM ); // block SIGALRM signals
|
---|
582 | sigaddset( &mask, SIGUSR1 ); // block SIGALRM signals
|
---|
583 | sigsuspend( &mask ); // block the processor to prevent further damage during abort
|
---|
584 | _exit( EXIT_FAILURE ); // if processor unblocks before it is killed, terminate it
|
---|
585 | }
|
---|
586 | else {
|
---|
587 | kernel_abort_called = true;
|
---|
588 | unlock( kernel_abort_lock );
|
---|
589 | }
|
---|
590 |
|
---|
591 | return kernelTLS.this_thread;
|
---|
592 | }
|
---|
593 |
|
---|
594 | void kernel_abort_msg( void * kernel_data, char * abort_text, int abort_text_size ) {
|
---|
595 | $thread * thrd = ( $thread * ) kernel_data;
|
---|
596 |
|
---|
597 | if(thrd) {
|
---|
598 | int len = snprintf( abort_text, abort_text_size, "Error occurred while executing thread %.256s (%p)", thrd->self_cor.name, thrd );
|
---|
599 | __cfaabi_bits_write( STDERR_FILENO, abort_text, len );
|
---|
600 |
|
---|
601 | if ( &thrd->self_cor != thrd->curr_cor ) {
|
---|
602 | len = snprintf( abort_text, abort_text_size, " in coroutine %.256s (%p).\n", thrd->curr_cor->name, thrd->curr_cor );
|
---|
603 | __cfaabi_bits_write( STDERR_FILENO, abort_text, len );
|
---|
604 | }
|
---|
605 | else {
|
---|
606 | __cfaabi_bits_write( STDERR_FILENO, ".\n", 2 );
|
---|
607 | }
|
---|
608 | }
|
---|
609 | else {
|
---|
610 | int len = snprintf( abort_text, abort_text_size, "Error occurred outside of any thread.\n" );
|
---|
611 | __cfaabi_bits_write( STDERR_FILENO, abort_text, len );
|
---|
612 | }
|
---|
613 | }
|
---|
614 |
|
---|
615 | int kernel_abort_lastframe( void ) __attribute__ ((__nothrow__)) {
|
---|
616 | return get_coroutine(kernelTLS.this_thread) == get_coroutine(mainThread) ? 4 : 2;
|
---|
617 | }
|
---|
618 |
|
---|
619 | static __spinlock_t kernel_debug_lock;
|
---|
620 |
|
---|
621 | extern "C" {
|
---|
622 | void __cfaabi_bits_acquire() {
|
---|
623 | lock( kernel_debug_lock __cfaabi_dbg_ctx2 );
|
---|
624 | }
|
---|
625 |
|
---|
626 | void __cfaabi_bits_release() {
|
---|
627 | unlock( kernel_debug_lock );
|
---|
628 | }
|
---|
629 | }
|
---|
630 |
|
---|
631 | //=============================================================================================
|
---|
632 | // Kernel Utilities
|
---|
633 | //=============================================================================================
|
---|
634 | //-----------------------------------------------------------------------------
|
---|
635 | // Locks
|
---|
636 | void ?{}( semaphore & this, int count = 1 ) {
|
---|
637 | (this.lock){};
|
---|
638 | this.count = count;
|
---|
639 | (this.waiting){};
|
---|
640 | }
|
---|
641 | void ^?{}(semaphore & this) {}
|
---|
642 |
|
---|
643 | bool P(semaphore & this) with( this ){
|
---|
644 | lock( lock __cfaabi_dbg_ctx2 );
|
---|
645 | count -= 1;
|
---|
646 | if ( count < 0 ) {
|
---|
647 | // queue current task
|
---|
648 | append( waiting, kernelTLS.this_thread );
|
---|
649 |
|
---|
650 | // atomically release spin lock and block
|
---|
651 | unlock( lock );
|
---|
652 | park( __cfaabi_dbg_ctx );
|
---|
653 | return true;
|
---|
654 | }
|
---|
655 | else {
|
---|
656 | unlock( lock );
|
---|
657 | return false;
|
---|
658 | }
|
---|
659 | }
|
---|
660 |
|
---|
661 | bool V(semaphore & this) with( this ) {
|
---|
662 | $thread * thrd = 0p;
|
---|
663 | lock( lock __cfaabi_dbg_ctx2 );
|
---|
664 | count += 1;
|
---|
665 | if ( count <= 0 ) {
|
---|
666 | // remove task at head of waiting list
|
---|
667 | thrd = pop_head( waiting );
|
---|
668 | }
|
---|
669 |
|
---|
670 | unlock( lock );
|
---|
671 |
|
---|
672 | // make new owner
|
---|
673 | unpark( thrd __cfaabi_dbg_ctx2 );
|
---|
674 |
|
---|
675 | return thrd != 0p;
|
---|
676 | }
|
---|
677 |
|
---|
678 | bool V(semaphore & this, unsigned diff) with( this ) {
|
---|
679 | $thread * thrd = 0p;
|
---|
680 | lock( lock __cfaabi_dbg_ctx2 );
|
---|
681 | int release = max(-count, (int)diff);
|
---|
682 | count += diff;
|
---|
683 | for(release) {
|
---|
684 | unpark( pop_head( waiting ) __cfaabi_dbg_ctx2 );
|
---|
685 | }
|
---|
686 |
|
---|
687 | unlock( lock );
|
---|
688 |
|
---|
689 | return thrd != 0p;
|
---|
690 | }
|
---|
691 |
|
---|
692 | //-----------------------------------------------------------------------------
|
---|
693 | // Debug
|
---|
694 | __cfaabi_dbg_debug_do(
|
---|
695 | extern "C" {
|
---|
696 | void __cfaabi_dbg_record_lock(__spinlock_t & this, const char prev_name[]) {
|
---|
697 | this.prev_name = prev_name;
|
---|
698 | this.prev_thrd = kernelTLS.this_thread;
|
---|
699 | }
|
---|
700 |
|
---|
701 | void __cfaabi_dbg_record_thrd($thread & this, bool park, const char prev_name[]) {
|
---|
702 | if(park) {
|
---|
703 | this.park_caller = prev_name;
|
---|
704 | this.park_stale = false;
|
---|
705 | }
|
---|
706 | else {
|
---|
707 | this.unpark_caller = prev_name;
|
---|
708 | this.unpark_stale = false;
|
---|
709 | }
|
---|
710 | }
|
---|
711 | }
|
---|
712 | )
|
---|
713 |
|
---|
714 | //-----------------------------------------------------------------------------
|
---|
715 | // Debug
|
---|
716 | bool threading_enabled(void) __attribute__((const)) {
|
---|
717 | return true;
|
---|
718 | }
|
---|
719 |
|
---|
720 | //-----------------------------------------------------------------------------
|
---|
721 | // Statistics
|
---|
722 | #if !defined(__CFA_NO_STATISTICS__)
|
---|
723 | void print_halts( processor & this ) {
|
---|
724 | this.print_halts = true;
|
---|
725 | }
|
---|
726 | #endif
|
---|
727 | // Local Variables: //
|
---|
728 | // mode: c //
|
---|
729 | // tab-width: 4 //
|
---|
730 | // End: //
|
---|