1 | // |
---|
2 | // Cforall Version 1.0.0 Copyright (C) 2016 University of Waterloo |
---|
3 | // |
---|
4 | // The contents of this file are covered under the licence agreement in the |
---|
5 | // file "LICENCE" distributed with Cforall. |
---|
6 | // |
---|
7 | // signal.c -- |
---|
8 | // |
---|
9 | // Author : Thierry Delisle |
---|
10 | // Created On : Mon Jun 5 14:20:42 2017 |
---|
11 | // Last Modified By : Peter A. Buhr |
---|
12 | // Last Modified On : Thu Feb 17 11:18:57 2022 |
---|
13 | // Update Count : 59 |
---|
14 | // |
---|
15 | |
---|
16 | #define __cforall_thread__ |
---|
17 | #define _GNU_SOURCE |
---|
18 | |
---|
19 | // #define __CFA_DEBUG_PRINT_PREEMPTION__ |
---|
20 | |
---|
21 | #include "preemption.hfa" |
---|
22 | |
---|
23 | #include <assert.h> |
---|
24 | |
---|
25 | #include <errno.h> |
---|
26 | #include <stdio.h> |
---|
27 | #include <string.h> |
---|
28 | #include <unistd.h> |
---|
29 | #include <limits.h> // PTHREAD_STACK_MIN |
---|
30 | |
---|
31 | #include "bits/debug.hfa" |
---|
32 | #include "bits/signal.hfa" |
---|
33 | #include "kernel_private.hfa" |
---|
34 | |
---|
35 | |
---|
36 | #if !defined(__CFA_DEFAULT_PREEMPTION__) |
---|
37 | #define __CFA_DEFAULT_PREEMPTION__ 10`ms |
---|
38 | #endif |
---|
39 | |
---|
40 | __attribute__((weak)) Duration default_preemption() { |
---|
41 | const char * preempt_rate_s = getenv("CFA_DEFAULT_PREEMPTION"); |
---|
42 | if(!preempt_rate_s) { |
---|
43 | __cfadbg_print_safe(preemption, "No CFA_DEFAULT_PREEMPTION in ENV\n"); |
---|
44 | return __CFA_DEFAULT_PREEMPTION__; |
---|
45 | } |
---|
46 | |
---|
47 | char * endptr = 0p; |
---|
48 | long int preempt_rate_l = strtol(preempt_rate_s, &endptr, 10); |
---|
49 | if(preempt_rate_l < 0 || preempt_rate_l > 65535) { |
---|
50 | __cfadbg_print_safe(preemption, "CFA_DEFAULT_PREEMPTION out of range : %ld\n", preempt_rate_l); |
---|
51 | return __CFA_DEFAULT_PREEMPTION__; |
---|
52 | } |
---|
53 | if('\0' != *endptr) { |
---|
54 | __cfadbg_print_safe(preemption, "CFA_DEFAULT_PREEMPTION not a decimal number : %s\n", preempt_rate_s); |
---|
55 | return __CFA_DEFAULT_PREEMPTION__; |
---|
56 | } |
---|
57 | |
---|
58 | return preempt_rate_l`ms; |
---|
59 | } |
---|
60 | |
---|
61 | // FwdDeclarations : timeout handlers |
---|
62 | static void preempt( processor * this ); |
---|
63 | static void timeout( thread$ * this ); |
---|
64 | |
---|
65 | // FwdDeclarations : Signal handlers |
---|
66 | static void sigHandler_ctxSwitch( __CFA_SIGPARMS__ ); |
---|
67 | static void sigHandler_alarm ( __CFA_SIGPARMS__ ); |
---|
68 | static void sigHandler_segv ( __CFA_SIGPARMS__ ); |
---|
69 | static void sigHandler_ill ( __CFA_SIGPARMS__ ); |
---|
70 | static void sigHandler_fpe ( __CFA_SIGPARMS__ ); |
---|
71 | static void sigHandler_abort ( __CFA_SIGPARMS__ ); |
---|
72 | |
---|
73 | // FwdDeclarations : alarm thread main |
---|
74 | static void * alarm_loop( __attribute__((unused)) void * args ); |
---|
75 | |
---|
76 | // Machine specific register name |
---|
77 | #if defined( __i386 ) |
---|
78 | #define CFA_REG_IP gregs[REG_EIP] |
---|
79 | #elif defined( __x86_64 ) |
---|
80 | #define CFA_REG_IP gregs[REG_RIP] |
---|
81 | #elif defined( __arm__ ) |
---|
82 | #define CFA_REG_IP arm_pc |
---|
83 | #elif defined( __aarch64__ ) |
---|
84 | #define CFA_REG_IP pc |
---|
85 | #else |
---|
86 | #error unsupported hardware architecture |
---|
87 | #endif |
---|
88 | |
---|
89 | KERNEL_STORAGE(event_kernel_t, event_kernel); // private storage for event kernel |
---|
90 | event_kernel_t * event_kernel; // kernel public handle to even kernel |
---|
91 | static pthread_t alarm_thread; // pthread handle to alarm thread |
---|
92 | static void * alarm_stack; // pthread stack for alarm thread |
---|
93 | |
---|
94 | static void ?{}(event_kernel_t & this) with( this ) { |
---|
95 | alarms{}; |
---|
96 | lock{}; |
---|
97 | } |
---|
98 | |
---|
99 | enum { |
---|
100 | PREEMPT_NORMAL = 0, |
---|
101 | PREEMPT_TERMINATE = 1, |
---|
102 | }; |
---|
103 | |
---|
104 | //============================================================================================= |
---|
105 | // Kernel Preemption logic |
---|
106 | //============================================================================================= |
---|
107 | |
---|
108 | // Get next expired node |
---|
109 | static inline alarm_node_t * get_expired( alarm_list_t * alarms, Time currtime ) { |
---|
110 | if( ! & (*alarms)`first ) return 0p; // If no alarms return null |
---|
111 | if( (*alarms)`first.timeval >= currtime ) return 0p; // If alarms head not expired return null |
---|
112 | return pop(alarms); // Otherwise just pop head |
---|
113 | } |
---|
114 | |
---|
115 | // Tick one frame of the Discrete Event Simulation for alarms |
---|
116 | static void tick_preemption(void) { |
---|
117 | alarm_node_t * node = 0p; // Used in the while loop but cannot be declared in the while condition |
---|
118 | alarm_list_t * alarms = &event_kernel->alarms; // Local copy for ease of reading |
---|
119 | Time currtime = __kernel_get_time(); // Check current time once so everything "happens at once" |
---|
120 | |
---|
121 | //Loop throught every thing expired |
---|
122 | while( node = get_expired( alarms, currtime ) ) { |
---|
123 | __cfadbg_print_buffer_decl( preemption, " KERNEL: preemption tick %lu\n", currtime.tn); |
---|
124 | Duration period = node->period; |
---|
125 | if( period == 0) { |
---|
126 | node->set = false; // Node is one-shot, just mark it as not pending |
---|
127 | } |
---|
128 | |
---|
129 | __cfadbg_print_buffer_local( preemption, " KERNEL: alarm ticking node %p.\n", node ); |
---|
130 | |
---|
131 | |
---|
132 | // Check if this is a kernel |
---|
133 | if( node->type == Kernel ) { |
---|
134 | preempt( node->proc ); |
---|
135 | } |
---|
136 | else if( node->type == User ) { |
---|
137 | __cfadbg_print_buffer_local( preemption, " KERNEL: alarm unparking %p.\n", node->thrd ); |
---|
138 | timeout( node->thrd ); |
---|
139 | } |
---|
140 | else { |
---|
141 | node->callback(*node); |
---|
142 | } |
---|
143 | |
---|
144 | // Check if this is a periodic alarm |
---|
145 | if( period > 0 ) { |
---|
146 | __cfadbg_print_buffer_local( preemption, " KERNEL: alarm period is %lu.\n", period`ns ); |
---|
147 | node->timeval = currtime + period; // Alarm is periodic, add currtime to it (used cached current time) |
---|
148 | insert( alarms, node ); // Reinsert the node for the next time it triggers |
---|
149 | } |
---|
150 | } |
---|
151 | |
---|
152 | // If there are still alarms pending, reset the timer |
---|
153 | if( & (*alarms)`first ) { |
---|
154 | Duration delta = (*alarms)`first.timeval - currtime; |
---|
155 | __kernel_set_timer( delta ); |
---|
156 | } |
---|
157 | } |
---|
158 | |
---|
159 | // Update the preemption of a processor and notify interested parties |
---|
160 | void update_preemption( processor * this, Duration duration ) { |
---|
161 | alarm_node_t * alarm = this->preemption_alarm; |
---|
162 | |
---|
163 | // Alarms need to be enabled |
---|
164 | if ( duration > 0 && ! alarm->set ) { |
---|
165 | alarm->initial = duration; |
---|
166 | alarm->period = duration; |
---|
167 | register_self( alarm ); |
---|
168 | } |
---|
169 | // Zero duration but alarm is set |
---|
170 | else if ( duration == 0 && alarm->set ) { |
---|
171 | unregister_self( alarm ); |
---|
172 | alarm->initial = 0; |
---|
173 | alarm->period = 0; |
---|
174 | } |
---|
175 | // If alarm is different from previous, change it |
---|
176 | else if ( duration > 0 && alarm->period != duration ) { |
---|
177 | unregister_self( alarm ); |
---|
178 | alarm->initial = duration; |
---|
179 | alarm->period = duration; |
---|
180 | register_self( alarm ); |
---|
181 | } |
---|
182 | } |
---|
183 | |
---|
184 | //============================================================================================= |
---|
185 | // Kernel Signal Tools |
---|
186 | //============================================================================================= |
---|
187 | // In a user-level threading system, there are handful of thread-local variables where this problem occurs on the ARM. |
---|
188 | // |
---|
189 | // For each kernel thread running user-level threads, there is a flag variable to indicate if interrupts are |
---|
190 | // enabled/disabled for that kernel thread. Therefore, this variable is made thread local. |
---|
191 | // |
---|
192 | // For example, this code fragment sets the state of the "interrupt" variable in thread-local memory. |
---|
193 | // |
---|
194 | // _Thread_local volatile int interrupts; |
---|
195 | // int main() { |
---|
196 | // interrupts = 0; // disable interrupts } |
---|
197 | // |
---|
198 | // which generates the following code on the ARM |
---|
199 | // |
---|
200 | // (gdb) disassemble main |
---|
201 | // Dump of assembler code for function main: |
---|
202 | // 0x0000000000000610 <+0>: mrs x1, tpidr_el0 |
---|
203 | // 0x0000000000000614 <+4>: mov w0, #0x0 // #0 |
---|
204 | // 0x0000000000000618 <+8>: add x1, x1, #0x0, lsl #12 |
---|
205 | // 0x000000000000061c <+12>: add x1, x1, #0x10 |
---|
206 | // 0x0000000000000620 <+16>: str wzr, [x1] |
---|
207 | // 0x0000000000000624 <+20>: ret |
---|
208 | // |
---|
209 | // The mrs moves a pointer from coprocessor register tpidr_el0 into register x1. Register w0 is set to 0. The two adds |
---|
210 | // increase the TLS pointer with the displacement (offset) 0x10, which is the location in the TSL of variable |
---|
211 | // "interrupts". Finally, 0 is stored into "interrupts" through the pointer in register x1 that points into the |
---|
212 | // TSL. Now once x1 has the pointer to the location of the TSL for kernel thread N, it can be be preempted at a |
---|
213 | // user-level and the user thread is put on the user-level ready-queue. When the preempted thread gets to the front of |
---|
214 | // the user-level ready-queue it is run on kernel thread M. It now stores 0 into "interrupts" back on kernel thread N, |
---|
215 | // turning off interrupt on the wrong kernel thread. |
---|
216 | // |
---|
217 | // On the x86, the following code is generated for the same code fragment. |
---|
218 | // |
---|
219 | // (gdb) disassemble main |
---|
220 | // Dump of assembler code for function main: |
---|
221 | // 0x0000000000400420 <+0>: movl $0x0,%fs:0xfffffffffffffffc |
---|
222 | // 0x000000000040042c <+12>: xor %eax,%eax |
---|
223 | // 0x000000000040042e <+14>: retq |
---|
224 | // |
---|
225 | // and there is base-displacement addressing used to atomically reset variable "interrupts" off of the TSL pointer in |
---|
226 | // register "fs". |
---|
227 | // |
---|
228 | // Hence, the ARM has base-displacement address for the general purpose registers, BUT not to the coprocessor |
---|
229 | // registers. As a result, generating the address for the write into variable "interrupts" is no longer atomic. |
---|
230 | // |
---|
231 | // Note this problem does NOT occur when just using multiple kernel threads because the preemption ALWAYS restarts the |
---|
232 | // thread on the same kernel thread. |
---|
233 | // |
---|
234 | // The obvious question is why does ARM use a coprocessor register to store the TSL pointer given that coprocessor |
---|
235 | // registers are second-class registers with respect to the instruction set. One possible answer is that they did not |
---|
236 | // want to dedicate one of the general registers to hold the TLS pointer and there was a free coprocessor register |
---|
237 | // available. |
---|
238 | |
---|
239 | //----------------------------------------------------------------------------- |
---|
240 | // Some assembly required |
---|
241 | #define __cfaasm_label(label, when) when: asm volatile goto(".global __cfaasm_" #label "_" #when "\n" "__cfaasm_" #label "_" #when ":":::"memory":when) |
---|
242 | |
---|
243 | //---------- |
---|
244 | // special case for preemption since used often |
---|
245 | __attribute__((optimize("no-reorder-blocks"))) bool __preemption_enabled() { |
---|
246 | // create a assembler label before |
---|
247 | // marked as clobber all to avoid movement |
---|
248 | __cfaasm_label(check, before); |
---|
249 | |
---|
250 | // access tls as normal |
---|
251 | bool enabled = __cfaabi_tls.preemption_state.enabled; |
---|
252 | |
---|
253 | // Check if there is a pending preemption |
---|
254 | processor * proc = __cfaabi_tls.this_processor; |
---|
255 | bool pending = proc ? proc->pending_preemption : false; |
---|
256 | if( enabled && pending ) proc->pending_preemption = false; |
---|
257 | |
---|
258 | // create a assembler label after |
---|
259 | // marked as clobber all to avoid movement |
---|
260 | __cfaasm_label(check, after); |
---|
261 | |
---|
262 | // If we can preempt and there is a pending one |
---|
263 | // this is a good time to yield |
---|
264 | if( enabled && pending ) { |
---|
265 | force_yield( __POLL_PREEMPTION ); |
---|
266 | } |
---|
267 | return enabled; |
---|
268 | } |
---|
269 | |
---|
270 | struct asm_region { |
---|
271 | void * before; |
---|
272 | void * after; |
---|
273 | }; |
---|
274 | |
---|
275 | static inline bool __cfaasm_in( void * ip, struct asm_region & region ) { |
---|
276 | return ip >= region.before && ip <= region.after; |
---|
277 | } |
---|
278 | |
---|
279 | |
---|
280 | //---------- |
---|
281 | // Get data from the TLS block |
---|
282 | // struct asm_region __cfaasm_get; |
---|
283 | uintptr_t __cfatls_get( unsigned long int offset ) __attribute__((__noinline__)); //no inline to avoid problems |
---|
284 | uintptr_t __cfatls_get( unsigned long int offset ) { |
---|
285 | // create a assembler label before |
---|
286 | // marked as clobber all to avoid movement |
---|
287 | __cfaasm_label(get, before); |
---|
288 | |
---|
289 | // access tls as normal (except for pointer arithmetic) |
---|
290 | uintptr_t val = *(uintptr_t*)((uintptr_t)&__cfaabi_tls + offset); |
---|
291 | |
---|
292 | // create a assembler label after |
---|
293 | // marked as clobber all to avoid movement |
---|
294 | __cfaasm_label(get, after); |
---|
295 | |
---|
296 | // This is used everywhere, to avoid cost, we DO NOT poll pending preemption |
---|
297 | return val; |
---|
298 | } |
---|
299 | |
---|
300 | extern "C" { |
---|
301 | // Disable interrupts by incrementing the counter |
---|
302 | void disable_interrupts() { |
---|
303 | // create a assembler label before |
---|
304 | // marked as clobber all to avoid movement |
---|
305 | __cfaasm_label(dsable, before); |
---|
306 | |
---|
307 | with( __cfaabi_tls.preemption_state ) { |
---|
308 | #if GCC_VERSION > 50000 |
---|
309 | static_assert(__atomic_always_lock_free(sizeof(enabled), &enabled), "Must be lock-free"); |
---|
310 | #endif |
---|
311 | |
---|
312 | // Set enabled flag to false |
---|
313 | // should be atomic to avoid preemption in the middle of the operation. |
---|
314 | // use memory order RELAXED since there is no inter-thread on this variable requirements |
---|
315 | __atomic_store_n(&enabled, false, __ATOMIC_RELAXED); |
---|
316 | |
---|
317 | // Signal the compiler that a fence is needed but only for signal handlers |
---|
318 | __atomic_signal_fence(__ATOMIC_ACQUIRE); |
---|
319 | |
---|
320 | __attribute__((unused)) unsigned short new_val = disable_count + 1; |
---|
321 | disable_count = new_val; |
---|
322 | verify( new_val < 65_000u ); // If this triggers someone is disabling interrupts without enabling them |
---|
323 | } |
---|
324 | |
---|
325 | // create a assembler label after |
---|
326 | // marked as clobber all to avoid movement |
---|
327 | __cfaasm_label(dsable, after); |
---|
328 | |
---|
329 | } |
---|
330 | |
---|
331 | // Enable interrupts by decrementing the counter |
---|
332 | // If counter reaches 0, execute any pending __cfactx_switch |
---|
333 | void enable_interrupts( bool poll ) { |
---|
334 | // Cache the processor now since interrupts can start happening after the atomic store |
---|
335 | processor * proc = __cfaabi_tls.this_processor; |
---|
336 | /* paranoid */ verify( !poll || proc ); |
---|
337 | |
---|
338 | with( __cfaabi_tls.preemption_state ){ |
---|
339 | unsigned short prev = disable_count; |
---|
340 | disable_count -= 1; |
---|
341 | |
---|
342 | // If this triggers someone is enabled already enabled interruptsverify( prev != 0u ); |
---|
343 | /* paranoid */ verify( prev != 0u ); |
---|
344 | |
---|
345 | // Check if we need to prempt the thread because an interrupt was missed |
---|
346 | if( prev == 1 ) { |
---|
347 | #if GCC_VERSION > 50000 |
---|
348 | static_assert(__atomic_always_lock_free(sizeof(enabled), &enabled), "Must be lock-free"); |
---|
349 | #endif |
---|
350 | |
---|
351 | // Set enabled flag to true |
---|
352 | // should be atomic to avoid preemption in the middle of the operation. |
---|
353 | // use memory order RELAXED since there is no inter-thread on this variable requirements |
---|
354 | __atomic_store_n(&enabled, true, __ATOMIC_RELAXED); |
---|
355 | |
---|
356 | // Signal the compiler that a fence is needed but only for signal handlers |
---|
357 | __atomic_signal_fence(__ATOMIC_RELEASE); |
---|
358 | if( poll && proc->pending_preemption ) { |
---|
359 | proc->pending_preemption = false; |
---|
360 | force_yield( __POLL_PREEMPTION ); |
---|
361 | } |
---|
362 | } |
---|
363 | } |
---|
364 | } |
---|
365 | } |
---|
366 | |
---|
367 | //----------------------------------------------------------------------------- |
---|
368 | // Kernel Signal Debug |
---|
369 | void __cfaabi_check_preemption() { |
---|
370 | bool ready = __preemption_enabled(); |
---|
371 | if(!ready) { abort("Preemption should be ready"); } |
---|
372 | |
---|
373 | sigset_t oldset; |
---|
374 | int ret; |
---|
375 | ret = pthread_sigmask(0, ( const sigset_t * ) 0p, &oldset); // workaround trac#208: cast should be unnecessary |
---|
376 | if(ret != 0) { abort("ERROR sigprocmask returned %d", ret); } |
---|
377 | |
---|
378 | ret = sigismember(&oldset, SIGUSR1); |
---|
379 | if(ret < 0) { abort("ERROR sigismember returned %d", ret); } |
---|
380 | if(ret == 1) { abort("ERROR SIGUSR1 is disabled"); } |
---|
381 | |
---|
382 | ret = sigismember(&oldset, SIGALRM); |
---|
383 | if(ret < 0) { abort("ERROR sigismember returned %d", ret); } |
---|
384 | if(ret == 0) { abort("ERROR SIGALRM is enabled"); } |
---|
385 | |
---|
386 | ret = sigismember(&oldset, SIGTERM); |
---|
387 | if(ret < 0) { abort("ERROR sigismember returned %d", ret); } |
---|
388 | if(ret == 1) { abort("ERROR SIGTERM is disabled"); } |
---|
389 | } |
---|
390 | |
---|
391 | #ifdef __CFA_WITH_VERIFY__ |
---|
392 | bool __cfaabi_dbg_in_kernel() { |
---|
393 | return !__preemption_enabled(); |
---|
394 | } |
---|
395 | #endif |
---|
396 | |
---|
397 | #undef __cfaasm_label |
---|
398 | |
---|
399 | //----------------------------------------------------------------------------- |
---|
400 | // Signal handling |
---|
401 | |
---|
402 | // sigprocmask wrapper : unblock a single signal |
---|
403 | static inline void signal_unblock( int sig ) { |
---|
404 | sigset_t mask; |
---|
405 | sigemptyset( &mask ); |
---|
406 | sigaddset( &mask, sig ); |
---|
407 | |
---|
408 | if ( pthread_sigmask( SIG_UNBLOCK, &mask, 0p ) == -1 ) { |
---|
409 | abort( "internal error, pthread_sigmask" ); |
---|
410 | } |
---|
411 | } |
---|
412 | |
---|
413 | // sigprocmask wrapper : block a single signal |
---|
414 | static inline void signal_block( int sig ) { |
---|
415 | sigset_t mask; |
---|
416 | sigemptyset( &mask ); |
---|
417 | sigaddset( &mask, sig ); |
---|
418 | |
---|
419 | if ( pthread_sigmask( SIG_BLOCK, &mask, 0p ) == -1 ) { |
---|
420 | abort( "internal error, pthread_sigmask" ); |
---|
421 | } |
---|
422 | } |
---|
423 | |
---|
424 | // kill wrapper : signal a processor |
---|
425 | static void preempt( processor * this ) { |
---|
426 | sigval_t value = { PREEMPT_NORMAL }; |
---|
427 | pthread_sigqueue( this->kernel_thread, SIGUSR1, value ); |
---|
428 | } |
---|
429 | |
---|
430 | // reserved for future use |
---|
431 | static void timeout( thread$ * this ) { |
---|
432 | unpark( this ); |
---|
433 | } |
---|
434 | |
---|
435 | void __disable_interrupts_hard() { |
---|
436 | sigset_t oldset; |
---|
437 | int ret; |
---|
438 | ret = pthread_sigmask(0, ( const sigset_t * ) 0p, &oldset); // workaround trac#208: cast should be unnecessary |
---|
439 | if(ret != 0) { abort("ERROR sigprocmask returned %d", ret); } |
---|
440 | |
---|
441 | ret = sigismember(&oldset, SIGUSR1); |
---|
442 | if(ret < 0) { abort("ERROR sigismember returned %d", ret); } |
---|
443 | if(ret == 1) { abort("ERROR SIGUSR1 is disabled"); } |
---|
444 | |
---|
445 | ret = sigismember(&oldset, SIGALRM); |
---|
446 | if(ret < 0) { abort("ERROR sigismember returned %d", ret); } |
---|
447 | if(ret == 0) { abort("ERROR SIGALRM is enabled"); } |
---|
448 | |
---|
449 | signal_block( SIGUSR1 ); |
---|
450 | } |
---|
451 | |
---|
452 | void __enable_interrupts_hard() { |
---|
453 | signal_unblock( SIGUSR1 ); |
---|
454 | |
---|
455 | sigset_t oldset; |
---|
456 | int ret; |
---|
457 | ret = pthread_sigmask(0, ( const sigset_t * ) 0p, &oldset); // workaround trac#208: cast should be unnecessary |
---|
458 | if(ret != 0) { abort("ERROR sigprocmask returned %d", ret); } |
---|
459 | |
---|
460 | ret = sigismember(&oldset, SIGUSR1); |
---|
461 | if(ret < 0) { abort("ERROR sigismember returned %d", ret); } |
---|
462 | if(ret == 1) { abort("ERROR SIGUSR1 is disabled"); } |
---|
463 | |
---|
464 | ret = sigismember(&oldset, SIGALRM); |
---|
465 | if(ret < 0) { abort("ERROR sigismember returned %d", ret); } |
---|
466 | if(ret == 0) { abort("ERROR SIGALRM is enabled"); } |
---|
467 | } |
---|
468 | |
---|
469 | //----------------------------------------------------------------------------- |
---|
470 | // Some assembly required |
---|
471 | #if defined( __i386 ) |
---|
472 | #ifdef __PIC__ |
---|
473 | #define RELOC_PRELUDE( label ) \ |
---|
474 | "calll .Lcfaasm_prelude_" #label "$pb\n\t" \ |
---|
475 | ".Lcfaasm_prelude_" #label "$pb:\n\t" \ |
---|
476 | "popl %%eax\n\t" \ |
---|
477 | ".Lcfaasm_prelude_" #label "_end:\n\t" \ |
---|
478 | "addl $_GLOBAL_OFFSET_TABLE_+(.Lcfaasm_prelude_" #label "_end-.Lcfaasm_prelude_" #label "$pb), %%eax\n\t" |
---|
479 | #define RELOC_PREFIX "" |
---|
480 | #define RELOC_SUFFIX "@GOT(%%eax)" |
---|
481 | #else |
---|
482 | #define RELOC_PREFIX "$" |
---|
483 | #define RELOC_SUFFIX "" |
---|
484 | #endif |
---|
485 | #define __cfaasm_label( label ) struct asm_region label = \ |
---|
486 | ({ \ |
---|
487 | struct asm_region region; \ |
---|
488 | asm( \ |
---|
489 | RELOC_PRELUDE( label ) \ |
---|
490 | "movl " RELOC_PREFIX "__cfaasm_" #label "_before" RELOC_SUFFIX ", %[vb]\n\t" \ |
---|
491 | "movl " RELOC_PREFIX "__cfaasm_" #label "_after" RELOC_SUFFIX ", %[va]\n\t" \ |
---|
492 | : [vb]"=r"(region.before), [va]"=r"(region.after) \ |
---|
493 | ); \ |
---|
494 | region; \ |
---|
495 | }); |
---|
496 | #elif defined( __x86_64 ) |
---|
497 | #ifdef __PIC__ |
---|
498 | #define RELOC_PREFIX "" |
---|
499 | #define RELOC_SUFFIX "@GOTPCREL(%%rip)" |
---|
500 | #else |
---|
501 | #define RELOC_PREFIX "$" |
---|
502 | #define RELOC_SUFFIX "" |
---|
503 | #endif |
---|
504 | #define __cfaasm_label( label ) struct asm_region label = \ |
---|
505 | ({ \ |
---|
506 | struct asm_region region; \ |
---|
507 | asm( \ |
---|
508 | "movq " RELOC_PREFIX "__cfaasm_" #label "_before" RELOC_SUFFIX ", %[vb]\n\t" \ |
---|
509 | "movq " RELOC_PREFIX "__cfaasm_" #label "_after" RELOC_SUFFIX ", %[va]\n\t" \ |
---|
510 | : [vb]"=r"(region.before), [va]"=r"(region.after) \ |
---|
511 | ); \ |
---|
512 | region; \ |
---|
513 | }); |
---|
514 | #elif defined( __aarch64__ ) |
---|
515 | #ifdef __PIC__ |
---|
516 | // Note that this works only for gcc |
---|
517 | #define __cfaasm_label( label ) struct asm_region label = \ |
---|
518 | ({ \ |
---|
519 | struct asm_region region; \ |
---|
520 | asm( \ |
---|
521 | "adrp %[vb], _GLOBAL_OFFSET_TABLE_" "\n\t" \ |
---|
522 | "ldr %[vb], [%[vb], #:gotpage_lo15:__cfaasm_" #label "_before]" "\n\t" \ |
---|
523 | "adrp %[va], _GLOBAL_OFFSET_TABLE_" "\n\t" \ |
---|
524 | "ldr %[va], [%[va], #:gotpage_lo15:__cfaasm_" #label "_after]" "\n\t" \ |
---|
525 | : [vb]"=r"(region.before), [va]"=r"(region.after) \ |
---|
526 | ); \ |
---|
527 | region; \ |
---|
528 | }); |
---|
529 | #else |
---|
530 | #error this is not the right thing to do |
---|
531 | /* |
---|
532 | #define __cfaasm_label( label ) struct asm_region label = \ |
---|
533 | ({ \ |
---|
534 | struct asm_region region; \ |
---|
535 | asm( \ |
---|
536 | "adrp %[vb], __cfaasm_" #label "_before" "\n\t" \ |
---|
537 | "add %[vb], %[vb], :lo12:__cfaasm_" #label "_before" "\n\t" \ |
---|
538 | "adrp %[va], :got:__cfaasm_" #label "_after" "\n\t" \ |
---|
539 | "add %[va], %[va], :lo12:__cfaasm_" #label "_after" "\n\t" \ |
---|
540 | : [vb]"=r"(region.before), [va]"=r"(region.after) \ |
---|
541 | ); \ |
---|
542 | region; \ |
---|
543 | }); |
---|
544 | */ |
---|
545 | #endif |
---|
546 | #else |
---|
547 | #error unknown hardware architecture |
---|
548 | #endif |
---|
549 | |
---|
550 | // KERNEL ONLY |
---|
551 | // Check if a __cfactx_switch signal handler shoud defer |
---|
552 | // If true : preemption is safe |
---|
553 | // If false : preemption is unsafe and marked as pending |
---|
554 | static inline bool preemption_ready( void * ip ) { |
---|
555 | // Get all the region for which it is not safe to preempt |
---|
556 | __cfaasm_label( get ); |
---|
557 | __cfaasm_label( check ); |
---|
558 | __cfaasm_label( dsable ); |
---|
559 | // __cfaasm_label( debug ); |
---|
560 | |
---|
561 | // Check if preemption is safe |
---|
562 | bool ready = true; |
---|
563 | if( __cfaasm_in( ip, get ) ) { ready = false; goto EXIT; }; |
---|
564 | if( __cfaasm_in( ip, check ) ) { ready = false; goto EXIT; }; |
---|
565 | if( __cfaasm_in( ip, dsable ) ) { ready = false; goto EXIT; }; |
---|
566 | // if( __cfaasm_in( ip, debug ) ) { ready = false; goto EXIT; }; |
---|
567 | if( !__cfaabi_tls.preemption_state.enabled) { ready = false; goto EXIT; }; |
---|
568 | if( __cfaabi_tls.preemption_state.in_progress ) { ready = false; goto EXIT; }; |
---|
569 | |
---|
570 | EXIT: |
---|
571 | // Adjust the pending flag accordingly |
---|
572 | __cfaabi_tls.this_processor->pending_preemption = !ready; |
---|
573 | return ready; |
---|
574 | } |
---|
575 | |
---|
576 | //============================================================================================= |
---|
577 | // Kernel Signal Startup/Shutdown logic |
---|
578 | //============================================================================================= |
---|
579 | |
---|
580 | // Startup routine to activate preemption |
---|
581 | // Called from kernel_startup |
---|
582 | void __kernel_alarm_startup() { |
---|
583 | __cfaabi_dbg_print_safe( "Kernel : Starting preemption\n" ); |
---|
584 | |
---|
585 | // Start with preemption disabled until ready |
---|
586 | __cfaabi_tls.preemption_state.enabled = false; |
---|
587 | __cfaabi_tls.preemption_state.disable_count = 1; |
---|
588 | |
---|
589 | // Initialize the event kernel |
---|
590 | event_kernel = (event_kernel_t *)&storage_event_kernel; |
---|
591 | (*event_kernel){}; |
---|
592 | |
---|
593 | // Setup proper signal handlers |
---|
594 | __cfaabi_sigaction( SIGUSR1, sigHandler_ctxSwitch, SA_SIGINFO ); // __cfactx_switch handler |
---|
595 | __cfaabi_sigaction( SIGALRM, sigHandler_alarm , SA_SIGINFO ); // debug handler |
---|
596 | |
---|
597 | signal_block( SIGALRM ); |
---|
598 | |
---|
599 | alarm_stack = __create_pthread( &alarm_thread, alarm_loop, 0p ); |
---|
600 | } |
---|
601 | |
---|
602 | // Shutdown routine to deactivate preemption |
---|
603 | // Called from kernel_shutdown |
---|
604 | void __kernel_alarm_shutdown() { |
---|
605 | __cfaabi_dbg_print_safe( "Kernel : Preemption stopping\n" ); |
---|
606 | |
---|
607 | // Block all signals since we are already shutting down |
---|
608 | sigset_t mask; |
---|
609 | sigfillset( &mask ); |
---|
610 | sigprocmask( SIG_BLOCK, &mask, 0p ); |
---|
611 | |
---|
612 | // Notify the alarm thread of the shutdown |
---|
613 | sigval val; |
---|
614 | val.sival_int = 0; |
---|
615 | pthread_sigqueue( alarm_thread, SIGALRM, val ); |
---|
616 | |
---|
617 | // Wait for the preemption thread to finish |
---|
618 | |
---|
619 | __destroy_pthread( alarm_thread, alarm_stack, 0p ); |
---|
620 | |
---|
621 | // Preemption is now fully stopped |
---|
622 | |
---|
623 | __cfaabi_dbg_print_safe( "Kernel : Preemption stopped\n" ); |
---|
624 | } |
---|
625 | |
---|
626 | // Prevent preemption since we are about to start terminating things |
---|
627 | void __kernel_abort_lock(void) { |
---|
628 | signal_block( SIGUSR1 ); |
---|
629 | } |
---|
630 | |
---|
631 | // Raii ctor/dtor for the preemption_scope |
---|
632 | // Used by thread to control when they want to receive preemption signals |
---|
633 | void ?{}( preemption_scope & this, processor * proc ) { |
---|
634 | (this.alarm){ proc, 0`s, 0`s }; |
---|
635 | this.proc = proc; |
---|
636 | this.proc->preemption_alarm = &this.alarm; |
---|
637 | |
---|
638 | update_preemption( this.proc, this.proc->cltr->preemption_rate ); |
---|
639 | } |
---|
640 | |
---|
641 | void ^?{}( preemption_scope & this ) { |
---|
642 | disable_interrupts(); |
---|
643 | |
---|
644 | update_preemption( this.proc, 0`s ); |
---|
645 | } |
---|
646 | |
---|
647 | //============================================================================================= |
---|
648 | // Kernel Signal Handlers |
---|
649 | //============================================================================================= |
---|
650 | __cfaabi_dbg_debug_do( static thread_local void * last_interrupt = 0; ) |
---|
651 | |
---|
652 | // Context switch signal handler |
---|
653 | // Receives SIGUSR1 signal and causes the current thread to yield |
---|
654 | static void sigHandler_ctxSwitch( __CFA_SIGPARMS__ ) { |
---|
655 | void * ip = (void *)(cxt->uc_mcontext.CFA_REG_IP); |
---|
656 | __cfaabi_dbg_debug_do( last_interrupt = ip; ) |
---|
657 | |
---|
658 | // SKULLDUGGERY: if a thread creates a processor and the immediately deletes it, |
---|
659 | // the interrupt that is supposed to force the kernel thread to preempt might arrive |
---|
660 | // before the kernel thread has even started running. When that happens, an interrupt |
---|
661 | // with a null 'this_processor' will be caught, just ignore it. |
---|
662 | if(! __cfaabi_tls.this_processor ) return; |
---|
663 | |
---|
664 | choose(sfp->si_value.sival_int) { |
---|
665 | case PREEMPT_NORMAL : ;// Normal case, nothing to do here |
---|
666 | case PREEMPT_TERMINATE: verify( __atomic_load_n( &__cfaabi_tls.this_processor->do_terminate, __ATOMIC_SEQ_CST ) ); |
---|
667 | default: |
---|
668 | abort( "internal error, signal value is %d", sfp->si_value.sival_int ); |
---|
669 | } |
---|
670 | |
---|
671 | // Check if it is safe to preempt here |
---|
672 | if( !preemption_ready( ip ) ) { |
---|
673 | #if !defined(__CFA_NO_STATISTICS__) |
---|
674 | __cfaabi_tls.this_stats->ready.threads.preempt.rllfwd++; |
---|
675 | #endif |
---|
676 | return; |
---|
677 | } |
---|
678 | |
---|
679 | __cfaabi_dbg_print_buffer_decl( " KERNEL: preempting core %p (%p @ %p).\n", __cfaabi_tls.this_processor, __cfaabi_tls.this_thread, (void *)(cxt->uc_mcontext.CFA_REG_IP) ); |
---|
680 | |
---|
681 | // Sync flag : prevent recursive calls to the signal handler |
---|
682 | __cfaabi_tls.preemption_state.in_progress = true; |
---|
683 | |
---|
684 | // Clear sighandler mask before context switching. |
---|
685 | #if GCC_VERSION > 50000 |
---|
686 | static_assert( sizeof( sigset_t ) == sizeof( cxt->uc_sigmask ), "Expected cxt->uc_sigmask to be of sigset_t" ); |
---|
687 | #endif |
---|
688 | if ( pthread_sigmask( SIG_SETMASK, (sigset_t *)&(cxt->uc_sigmask), 0p ) == -1 ) { |
---|
689 | abort( "internal error, sigprocmask" ); |
---|
690 | } |
---|
691 | |
---|
692 | // Clear the in progress flag |
---|
693 | __cfaabi_tls.preemption_state.in_progress = false; |
---|
694 | |
---|
695 | // Preemption can occur here |
---|
696 | |
---|
697 | #if !defined(__CFA_NO_STATISTICS__) |
---|
698 | __cfaabi_tls.this_stats->ready.threads.preempt.yield++; |
---|
699 | #endif |
---|
700 | |
---|
701 | force_yield( __ALARM_PREEMPTION ); // Do the actual __cfactx_switch |
---|
702 | } |
---|
703 | |
---|
704 | static void sigHandler_alarm( __CFA_SIGPARMS__ ) { |
---|
705 | abort("SIGALRM should never reach the signal handler"); |
---|
706 | } |
---|
707 | |
---|
708 | // Main of the alarm thread |
---|
709 | // Waits on SIGALRM and send SIGUSR1 to whom ever needs it |
---|
710 | static void * alarm_loop( __attribute__((unused)) void * args ) { |
---|
711 | unsigned id = register_proc_id(); |
---|
712 | |
---|
713 | // Block sigalrms to control when they arrive |
---|
714 | sigset_t mask; |
---|
715 | sigfillset(&mask); |
---|
716 | if ( pthread_sigmask( SIG_BLOCK, &mask, 0p ) == -1 ) { |
---|
717 | abort( "internal error, pthread_sigmask" ); |
---|
718 | } |
---|
719 | |
---|
720 | sigemptyset( &mask ); |
---|
721 | sigaddset( &mask, SIGALRM ); |
---|
722 | |
---|
723 | // Main loop |
---|
724 | while( true ) { |
---|
725 | // Wait for a sigalrm |
---|
726 | siginfo_t info; |
---|
727 | int sig = sigwaitinfo( &mask, &info ); |
---|
728 | |
---|
729 | __cfadbg_print_buffer_decl ( preemption, " KERNEL: sigwaitinfo returned %d, c: %d, v: %d\n", sig, info.si_code, info.si_value.sival_int ); |
---|
730 | __cfadbg_print_buffer_local( preemption, " KERNEL: SI_QUEUE %d, SI_TIMER %d, SI_KERNEL %d\n", SI_QUEUE, SI_TIMER, SI_KERNEL ); |
---|
731 | |
---|
732 | if( sig < 0 ) { |
---|
733 | //Error! |
---|
734 | int err = errno; |
---|
735 | switch( err ) { |
---|
736 | case EAGAIN : |
---|
737 | case EINTR : |
---|
738 | {__cfadbg_print_buffer_local( preemption, " KERNEL: Spurious wakeup %d.\n", err );} |
---|
739 | continue; |
---|
740 | case EINVAL : |
---|
741 | abort( "Timeout was invalid." ); |
---|
742 | default: |
---|
743 | abort( "Unhandled error %d", err); |
---|
744 | } |
---|
745 | } |
---|
746 | |
---|
747 | // If another signal arrived something went wrong |
---|
748 | assertf(sig == SIGALRM, "Kernel Internal Error, sigwait: Unexpected signal %d (%d : %d)\n", sig, info.si_code, info.si_value.sival_int); |
---|
749 | |
---|
750 | // Switch on the code (a.k.a. the sender) to |
---|
751 | switch( info.si_code ) |
---|
752 | { |
---|
753 | // Signal was not sent by the kernel but by an other thread |
---|
754 | case SI_QUEUE: |
---|
755 | // other threads may signal the alarm thread to shut it down |
---|
756 | // or to manual cause the preemption tick |
---|
757 | // use info.si_value and handle the case here |
---|
758 | switch( info.si_value.sival_int ) { |
---|
759 | case 0: |
---|
760 | goto EXIT; |
---|
761 | default: |
---|
762 | abort( "SI_QUEUE with val %d", info.si_value.sival_int); |
---|
763 | } |
---|
764 | // fallthrough |
---|
765 | // Timers can apparently be marked as sent for the kernel |
---|
766 | // In either case, tick preemption |
---|
767 | case SI_TIMER: |
---|
768 | case SI_KERNEL: |
---|
769 | // __cfaabi_dbg_print_safe( "Kernel : Preemption thread tick\n" ); |
---|
770 | lock( event_kernel->lock __cfaabi_dbg_ctx2 ); |
---|
771 | tick_preemption(); |
---|
772 | unlock( event_kernel->lock ); |
---|
773 | break; |
---|
774 | } |
---|
775 | } |
---|
776 | |
---|
777 | EXIT: |
---|
778 | __cfaabi_dbg_print_safe( "Kernel : Preemption thread stopping\n" ); |
---|
779 | unregister_proc_id(id); |
---|
780 | |
---|
781 | return 0p; |
---|
782 | } |
---|
783 | |
---|
784 | // Local Variables: // |
---|
785 | // mode: c // |
---|
786 | // tab-width: 4 // |
---|
787 | // End: // |
---|