1 | //
|
---|
2 | // Cforall Version 1.0.0 Copyright (C) 2016 University of Waterloo
|
---|
3 | //
|
---|
4 | // The contents of this file are covered under the licence agreement in the
|
---|
5 | // file "LICENCE" distributed with Cforall.
|
---|
6 | //
|
---|
7 | // kernel.c --
|
---|
8 | //
|
---|
9 | // Author : Thierry Delisle
|
---|
10 | // Created On : Tue Jan 17 12:27:26 2017
|
---|
11 | // Last Modified By : Peter A. Buhr
|
---|
12 | // Last Modified On : Fri Mar 30 18:26:11 2018
|
---|
13 | // Update Count : 23
|
---|
14 | //
|
---|
15 |
|
---|
16 | //C Includes
|
---|
17 | #include <stddef.h>
|
---|
18 | extern "C" {
|
---|
19 | #include <stdio.h>
|
---|
20 | #include <fenv.h>
|
---|
21 | #include <sys/resource.h>
|
---|
22 | #include <signal.h>
|
---|
23 | #include <unistd.h>
|
---|
24 | }
|
---|
25 |
|
---|
26 | //CFA Includes
|
---|
27 | #include "kernel_private.h"
|
---|
28 | #include "preemption.h"
|
---|
29 | #include "startup.h"
|
---|
30 |
|
---|
31 | //Private includes
|
---|
32 | #define __CFA_INVOKE_PRIVATE__
|
---|
33 | #include "invoke.h"
|
---|
34 |
|
---|
35 | //Start and stop routine for the kernel, declared first to make sure they run first
|
---|
36 | void kernel_startup(void) __attribute__(( constructor( STARTUP_PRIORITY_KERNEL ) ));
|
---|
37 | void kernel_shutdown(void) __attribute__(( destructor ( STARTUP_PRIORITY_KERNEL ) ));
|
---|
38 |
|
---|
39 | //-----------------------------------------------------------------------------
|
---|
40 | // Kernel storage
|
---|
41 | KERNEL_STORAGE(cluster, mainCluster);
|
---|
42 | KERNEL_STORAGE(processor, mainProcessor);
|
---|
43 | KERNEL_STORAGE(processorCtx_t, mainProcessorCtx);
|
---|
44 | KERNEL_STORAGE(thread_desc, mainThread);
|
---|
45 | KERNEL_STORAGE(machine_context_t, mainThreadCtx);
|
---|
46 |
|
---|
47 | cluster * mainCluster;
|
---|
48 | processor * mainProcessor;
|
---|
49 | thread_desc * mainThread;
|
---|
50 |
|
---|
51 | //-----------------------------------------------------------------------------
|
---|
52 | // Global state
|
---|
53 |
|
---|
54 | // volatile thread_local bool preemption_in_progress = 0;
|
---|
55 | // volatile thread_local bool preemption_enabled = false;
|
---|
56 | // volatile thread_local unsigned short disable_preempt_count = 1;
|
---|
57 |
|
---|
58 | thread_local struct KernelThreadData kernelThreadData = {
|
---|
59 | NULL,
|
---|
60 | NULL,
|
---|
61 | NULL,
|
---|
62 | { 1, false, false }
|
---|
63 | };
|
---|
64 |
|
---|
65 | //-----------------------------------------------------------------------------
|
---|
66 | // Main thread construction
|
---|
67 | struct current_stack_info_t {
|
---|
68 | machine_context_t ctx;
|
---|
69 | unsigned int size; // size of stack
|
---|
70 | void *base; // base of stack
|
---|
71 | void *storage; // pointer to stack
|
---|
72 | void *limit; // stack grows towards stack limit
|
---|
73 | void *context; // address of cfa_context_t
|
---|
74 | void *top; // address of top of storage
|
---|
75 | };
|
---|
76 |
|
---|
77 | void ?{}( current_stack_info_t & this ) {
|
---|
78 | CtxGet( this.ctx );
|
---|
79 | this.base = this.ctx.FP;
|
---|
80 | this.storage = this.ctx.SP;
|
---|
81 |
|
---|
82 | rlimit r;
|
---|
83 | getrlimit( RLIMIT_STACK, &r);
|
---|
84 | this.size = r.rlim_cur;
|
---|
85 |
|
---|
86 | this.limit = (void *)(((intptr_t)this.base) - this.size);
|
---|
87 | this.context = &storage_mainThreadCtx;
|
---|
88 | this.top = this.base;
|
---|
89 | }
|
---|
90 |
|
---|
91 | void ?{}( coStack_t & this, current_stack_info_t * info) with( this ) {
|
---|
92 | size = info->size;
|
---|
93 | storage = info->storage;
|
---|
94 | limit = info->limit;
|
---|
95 | base = info->base;
|
---|
96 | context = info->context;
|
---|
97 | top = info->top;
|
---|
98 | userStack = true;
|
---|
99 | }
|
---|
100 |
|
---|
101 | void ?{}( coroutine_desc & this, current_stack_info_t * info) with( this ) {
|
---|
102 | stack{ info };
|
---|
103 | name = "Main Thread";
|
---|
104 | errno_ = 0;
|
---|
105 | state = Start;
|
---|
106 | starter = NULL;
|
---|
107 | }
|
---|
108 |
|
---|
109 | void ?{}( thread_desc & this, current_stack_info_t * info) with( this ) {
|
---|
110 | self_cor{ info };
|
---|
111 | curr_cor = &self_cor;
|
---|
112 | self_mon.owner = &this;
|
---|
113 | self_mon.recursion = 1;
|
---|
114 | self_mon_p = &self_mon;
|
---|
115 | next = NULL;
|
---|
116 | __cfaabi_dbg_debug_do(
|
---|
117 | dbg_next = NULL;
|
---|
118 | dbg_prev = NULL;
|
---|
119 | __cfaabi_dbg_thread_register(&this);
|
---|
120 | )
|
---|
121 |
|
---|
122 | monitors{ &self_mon_p, 1, (fptr_t)0 };
|
---|
123 | }
|
---|
124 |
|
---|
125 | //-----------------------------------------------------------------------------
|
---|
126 | // Processor coroutine
|
---|
127 | void ?{}(processorCtx_t & this) {}
|
---|
128 |
|
---|
129 | // Construct the processor context of the main processor
|
---|
130 | void ?{}(processorCtx_t & this, processor * proc) {
|
---|
131 | (this.__cor){ "Processor" };
|
---|
132 | this.__cor.starter = NULL;
|
---|
133 | this.proc = proc;
|
---|
134 | }
|
---|
135 |
|
---|
136 | // Construct the processor context of non-main processors
|
---|
137 | void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info) {
|
---|
138 | (this.__cor){ info };
|
---|
139 | this.proc = proc;
|
---|
140 | }
|
---|
141 |
|
---|
142 | void ?{}(processor & this) {
|
---|
143 | this{ mainCluster };
|
---|
144 | }
|
---|
145 |
|
---|
146 | void ?{}(processor & this, cluster * cltr) with( this ) {
|
---|
147 | this.cltr = cltr;
|
---|
148 | terminated{ 0 };
|
---|
149 | do_terminate = false;
|
---|
150 | preemption_alarm = NULL;
|
---|
151 | pending_preemption = false;
|
---|
152 | runner.proc = &this;
|
---|
153 |
|
---|
154 | start( &this );
|
---|
155 | }
|
---|
156 |
|
---|
157 | void ?{}(processor & this, cluster * cltr, processorCtx_t & runner) with( this ) {
|
---|
158 | this.cltr = cltr;
|
---|
159 | terminated{ 0 };
|
---|
160 | do_terminate = false;
|
---|
161 | preemption_alarm = NULL;
|
---|
162 | pending_preemption = false;
|
---|
163 | kernel_thread = pthread_self();
|
---|
164 | runner.proc = &this;
|
---|
165 |
|
---|
166 | __cfaabi_dbg_print_safe("Kernel : constructing main processor context %p\n", &runner);
|
---|
167 | runner{ &this };
|
---|
168 | }
|
---|
169 |
|
---|
170 | void ^?{}(processor & this) with( this ){
|
---|
171 | if( ! do_terminate ) {
|
---|
172 | __cfaabi_dbg_print_safe("Kernel : core %p signaling termination\n", &this);
|
---|
173 | terminate(&this);
|
---|
174 | verify(this.do_terminate);
|
---|
175 | verify(TL_GET( this_processor ) != &this);
|
---|
176 | P( terminated );
|
---|
177 | verify(TL_GET( this_processor ) != &this);
|
---|
178 | pthread_join( kernel_thread, NULL );
|
---|
179 | }
|
---|
180 | }
|
---|
181 |
|
---|
182 | void ?{}(cluster & this) with( this ) {
|
---|
183 | ready_queue{};
|
---|
184 | ready_queue_lock{};
|
---|
185 |
|
---|
186 | preemption_rate = default_preemption();
|
---|
187 | }
|
---|
188 |
|
---|
189 | void ^?{}(cluster & this) {
|
---|
190 |
|
---|
191 | }
|
---|
192 |
|
---|
193 | //=============================================================================================
|
---|
194 | // Kernel Scheduling logic
|
---|
195 | //=============================================================================================
|
---|
196 | //Main of the processor contexts
|
---|
197 | void main(processorCtx_t & runner) {
|
---|
198 | processor * this = runner.proc;
|
---|
199 | verify(this);
|
---|
200 |
|
---|
201 | __cfaabi_dbg_print_safe("Kernel : core %p starting\n", this);
|
---|
202 |
|
---|
203 | {
|
---|
204 | // Setup preemption data
|
---|
205 | preemption_scope scope = { this };
|
---|
206 |
|
---|
207 | __cfaabi_dbg_print_safe("Kernel : core %p started\n", this);
|
---|
208 |
|
---|
209 | thread_desc * readyThread = NULL;
|
---|
210 | for( unsigned int spin_count = 0; ! this->do_terminate; spin_count++ )
|
---|
211 | {
|
---|
212 | readyThread = nextThread( this->cltr );
|
---|
213 |
|
---|
214 | if(readyThread)
|
---|
215 | {
|
---|
216 | verify( ! TL_GET( preemption_state ).enabled );
|
---|
217 |
|
---|
218 | runThread(this, readyThread);
|
---|
219 |
|
---|
220 | verify( ! TL_GET( preemption_state ).enabled );
|
---|
221 |
|
---|
222 | //Some actions need to be taken from the kernel
|
---|
223 | finishRunning(this);
|
---|
224 |
|
---|
225 | spin_count = 0;
|
---|
226 | }
|
---|
227 | else
|
---|
228 | {
|
---|
229 | spin(this, &spin_count);
|
---|
230 | }
|
---|
231 | }
|
---|
232 |
|
---|
233 | __cfaabi_dbg_print_safe("Kernel : core %p stopping\n", this);
|
---|
234 | }
|
---|
235 |
|
---|
236 | V( this->terminated );
|
---|
237 |
|
---|
238 | __cfaabi_dbg_print_safe("Kernel : core %p terminated\n", this);
|
---|
239 | }
|
---|
240 |
|
---|
241 | // runThread runs a thread by context switching
|
---|
242 | // from the processor coroutine to the target thread
|
---|
243 | void runThread(processor * this, thread_desc * dst) {
|
---|
244 | assert(dst->curr_cor);
|
---|
245 | coroutine_desc * proc_cor = get_coroutine(this->runner);
|
---|
246 | coroutine_desc * thrd_cor = dst->curr_cor;
|
---|
247 |
|
---|
248 | //Reset the terminating actions here
|
---|
249 | this->finish.action_code = No_Action;
|
---|
250 |
|
---|
251 | //Update global state
|
---|
252 | TL_SET( this_thread, dst );
|
---|
253 |
|
---|
254 | // Context Switch to the thread
|
---|
255 | ThreadCtxSwitch(proc_cor, thrd_cor);
|
---|
256 | // when ThreadCtxSwitch returns we are back in the processor coroutine
|
---|
257 | }
|
---|
258 |
|
---|
259 | void returnToKernel() {
|
---|
260 | coroutine_desc * proc_cor = get_coroutine(TL_GET( this_processor )->runner);
|
---|
261 | coroutine_desc * thrd_cor = TL_GET( this_thread )->curr_cor = TL_GET( this_coroutine );
|
---|
262 | ThreadCtxSwitch(thrd_cor, proc_cor);
|
---|
263 | }
|
---|
264 |
|
---|
265 | // Once a thread has finished running, some of
|
---|
266 | // its final actions must be executed from the kernel
|
---|
267 | void finishRunning(processor * this) with( this->finish ) {
|
---|
268 | if( action_code == Release ) {
|
---|
269 | verify( ! TL_GET( preemption_state ).enabled );
|
---|
270 | unlock( *lock );
|
---|
271 | }
|
---|
272 | else if( action_code == Schedule ) {
|
---|
273 | ScheduleThread( thrd );
|
---|
274 | }
|
---|
275 | else if( action_code == Release_Schedule ) {
|
---|
276 | verify( ! TL_GET( preemption_state ).enabled );
|
---|
277 | unlock( *lock );
|
---|
278 | ScheduleThread( thrd );
|
---|
279 | }
|
---|
280 | else if( action_code == Release_Multi ) {
|
---|
281 | verify( ! TL_GET( preemption_state ).enabled );
|
---|
282 | for(int i = 0; i < lock_count; i++) {
|
---|
283 | unlock( *locks[i] );
|
---|
284 | }
|
---|
285 | }
|
---|
286 | else if( action_code == Release_Multi_Schedule ) {
|
---|
287 | for(int i = 0; i < lock_count; i++) {
|
---|
288 | unlock( *locks[i] );
|
---|
289 | }
|
---|
290 | for(int i = 0; i < thrd_count; i++) {
|
---|
291 | ScheduleThread( thrds[i] );
|
---|
292 | }
|
---|
293 | }
|
---|
294 | else {
|
---|
295 | assert(action_code == No_Action);
|
---|
296 | }
|
---|
297 | }
|
---|
298 |
|
---|
299 | // Handles spinning logic
|
---|
300 | // TODO : find some strategy to put cores to sleep after some time
|
---|
301 | void spin(processor * this, unsigned int * spin_count) {
|
---|
302 | (*spin_count)++;
|
---|
303 | }
|
---|
304 |
|
---|
305 | // Context invoker for processors
|
---|
306 | // This is the entry point for processors (kernel threads)
|
---|
307 | // It effectively constructs a coroutine by stealing the pthread stack
|
---|
308 | void * CtxInvokeProcessor(void * arg) {
|
---|
309 | processor * proc = (processor *) arg;
|
---|
310 | TL_SET( this_processor, proc );
|
---|
311 | TL_SET( this_coroutine, NULL );
|
---|
312 | TL_SET( this_thread, NULL );
|
---|
313 | TL_GET( preemption_state ).enabled = false;
|
---|
314 | TL_GET( preemption_state ).disable_count = 1;
|
---|
315 | // SKULLDUGGERY: We want to create a context for the processor coroutine
|
---|
316 | // which is needed for the 2-step context switch. However, there is no reason
|
---|
317 | // to waste the perfectly valid stack create by pthread.
|
---|
318 | current_stack_info_t info;
|
---|
319 | machine_context_t ctx;
|
---|
320 | info.context = &ctx;
|
---|
321 | (proc->runner){ proc, &info };
|
---|
322 |
|
---|
323 | __cfaabi_dbg_print_safe("Coroutine : created stack %p\n", get_coroutine(proc->runner)->stack.base);
|
---|
324 |
|
---|
325 | //Set global state
|
---|
326 | TL_SET( this_coroutine, get_coroutine(proc->runner) );
|
---|
327 | TL_SET( this_thread, NULL );
|
---|
328 |
|
---|
329 | //We now have a proper context from which to schedule threads
|
---|
330 | __cfaabi_dbg_print_safe("Kernel : core %p created (%p, %p)\n", proc, &proc->runner, &ctx);
|
---|
331 |
|
---|
332 | // SKULLDUGGERY: Since the coroutine doesn't have its own stack, we can't
|
---|
333 | // resume it to start it like it normally would, it will just context switch
|
---|
334 | // back to here. Instead directly call the main since we already are on the
|
---|
335 | // appropriate stack.
|
---|
336 | get_coroutine(proc->runner)->state = Active;
|
---|
337 | main( proc->runner );
|
---|
338 | get_coroutine(proc->runner)->state = Halted;
|
---|
339 |
|
---|
340 | // Main routine of the core returned, the core is now fully terminated
|
---|
341 | __cfaabi_dbg_print_safe("Kernel : core %p main ended (%p)\n", proc, &proc->runner);
|
---|
342 |
|
---|
343 | return NULL;
|
---|
344 | }
|
---|
345 |
|
---|
346 | void start(processor * this) {
|
---|
347 | __cfaabi_dbg_print_safe("Kernel : Starting core %p\n", this);
|
---|
348 |
|
---|
349 | pthread_create( &this->kernel_thread, NULL, CtxInvokeProcessor, (void*)this );
|
---|
350 |
|
---|
351 | __cfaabi_dbg_print_safe("Kernel : core %p started\n", this);
|
---|
352 | }
|
---|
353 |
|
---|
354 | void kernel_first_resume(processor * this) {
|
---|
355 | coroutine_desc * src = TL_GET( this_coroutine );
|
---|
356 | coroutine_desc * dst = get_coroutine(this->runner);
|
---|
357 |
|
---|
358 | verify( ! TL_GET( preemption_state ).enabled );
|
---|
359 |
|
---|
360 | create_stack(&dst->stack, dst->stack.size);
|
---|
361 | CtxStart(&this->runner, CtxInvokeCoroutine);
|
---|
362 |
|
---|
363 | verify( ! TL_GET( preemption_state ).enabled );
|
---|
364 |
|
---|
365 | dst->last = src;
|
---|
366 | dst->starter = dst->starter ? dst->starter : src;
|
---|
367 |
|
---|
368 | // set state of current coroutine to inactive
|
---|
369 | src->state = src->state == Halted ? Halted : Inactive;
|
---|
370 |
|
---|
371 | // set new coroutine that task is executing
|
---|
372 | TL_SET( this_coroutine, dst );
|
---|
373 |
|
---|
374 | // SKULLDUGGERY normally interrupts are enable before leaving a coroutine ctxswitch.
|
---|
375 | // Therefore, when first creating a coroutine, interrupts are enable before calling the main.
|
---|
376 | // This is consistent with thread creation. However, when creating the main processor coroutine,
|
---|
377 | // we wan't interrupts to be disabled. Therefore, we double-disable interrupts here so they will
|
---|
378 | // stay disabled.
|
---|
379 | disable_interrupts();
|
---|
380 |
|
---|
381 | // context switch to specified coroutine
|
---|
382 | assert( src->stack.context );
|
---|
383 | CtxSwitch( src->stack.context, dst->stack.context );
|
---|
384 | // when CtxSwitch returns we are back in the src coroutine
|
---|
385 |
|
---|
386 | // set state of new coroutine to active
|
---|
387 | src->state = Active;
|
---|
388 |
|
---|
389 | verify( ! TL_GET( preemption_state ).enabled );
|
---|
390 | }
|
---|
391 |
|
---|
392 | //-----------------------------------------------------------------------------
|
---|
393 | // Scheduler routines
|
---|
394 | void ScheduleThread( thread_desc * thrd ) {
|
---|
395 | // if( ! thrd ) return;
|
---|
396 | verify( thrd );
|
---|
397 | verify( thrd->self_cor.state != Halted );
|
---|
398 |
|
---|
399 | verify( ! TL_GET( preemption_state ).enabled );
|
---|
400 |
|
---|
401 | verifyf( thrd->next == NULL, "Expected null got %p", thrd->next );
|
---|
402 |
|
---|
403 | with( *TL_GET( this_processor )->cltr ) {
|
---|
404 | lock ( ready_queue_lock __cfaabi_dbg_ctx2 );
|
---|
405 | append( ready_queue, thrd );
|
---|
406 | unlock( ready_queue_lock );
|
---|
407 | }
|
---|
408 |
|
---|
409 | verify( ! TL_GET( preemption_state ).enabled );
|
---|
410 | }
|
---|
411 |
|
---|
412 | thread_desc * nextThread(cluster * this) with( *this ) {
|
---|
413 | verify( ! TL_GET( preemption_state ).enabled );
|
---|
414 | lock( ready_queue_lock __cfaabi_dbg_ctx2 );
|
---|
415 | thread_desc * head = pop_head( ready_queue );
|
---|
416 | unlock( ready_queue_lock );
|
---|
417 | verify( ! TL_GET( preemption_state ).enabled );
|
---|
418 | return head;
|
---|
419 | }
|
---|
420 |
|
---|
421 | void BlockInternal() {
|
---|
422 | disable_interrupts();
|
---|
423 | verify( ! TL_GET( preemption_state ).enabled );
|
---|
424 | returnToKernel();
|
---|
425 | verify( ! TL_GET( preemption_state ).enabled );
|
---|
426 | enable_interrupts( __cfaabi_dbg_ctx );
|
---|
427 | }
|
---|
428 |
|
---|
429 | void BlockInternal( __spinlock_t * lock ) {
|
---|
430 | disable_interrupts();
|
---|
431 | TL_GET( this_processor )->finish.action_code = Release;
|
---|
432 | TL_GET( this_processor )->finish.lock = lock;
|
---|
433 |
|
---|
434 | verify( ! TL_GET( preemption_state ).enabled );
|
---|
435 | returnToKernel();
|
---|
436 | verify( ! TL_GET( preemption_state ).enabled );
|
---|
437 |
|
---|
438 | enable_interrupts( __cfaabi_dbg_ctx );
|
---|
439 | }
|
---|
440 |
|
---|
441 | void BlockInternal( thread_desc * thrd ) {
|
---|
442 | disable_interrupts();
|
---|
443 | TL_GET( this_processor )->finish.action_code = Schedule;
|
---|
444 | TL_GET( this_processor )->finish.thrd = thrd;
|
---|
445 |
|
---|
446 | verify( ! TL_GET( preemption_state ).enabled );
|
---|
447 | returnToKernel();
|
---|
448 | verify( ! TL_GET( preemption_state ).enabled );
|
---|
449 |
|
---|
450 | enable_interrupts( __cfaabi_dbg_ctx );
|
---|
451 | }
|
---|
452 |
|
---|
453 | void BlockInternal( __spinlock_t * lock, thread_desc * thrd ) {
|
---|
454 | assert(thrd);
|
---|
455 | disable_interrupts();
|
---|
456 | TL_GET( this_processor )->finish.action_code = Release_Schedule;
|
---|
457 | TL_GET( this_processor )->finish.lock = lock;
|
---|
458 | TL_GET( this_processor )->finish.thrd = thrd;
|
---|
459 |
|
---|
460 | verify( ! TL_GET( preemption_state ).enabled );
|
---|
461 | returnToKernel();
|
---|
462 | verify( ! TL_GET( preemption_state ).enabled );
|
---|
463 |
|
---|
464 | enable_interrupts( __cfaabi_dbg_ctx );
|
---|
465 | }
|
---|
466 |
|
---|
467 | void BlockInternal(__spinlock_t * locks [], unsigned short count) {
|
---|
468 | disable_interrupts();
|
---|
469 | TL_GET( this_processor )->finish.action_code = Release_Multi;
|
---|
470 | TL_GET( this_processor )->finish.locks = locks;
|
---|
471 | TL_GET( this_processor )->finish.lock_count = count;
|
---|
472 |
|
---|
473 | verify( ! TL_GET( preemption_state ).enabled );
|
---|
474 | returnToKernel();
|
---|
475 | verify( ! TL_GET( preemption_state ).enabled );
|
---|
476 |
|
---|
477 | enable_interrupts( __cfaabi_dbg_ctx );
|
---|
478 | }
|
---|
479 |
|
---|
480 | void BlockInternal(__spinlock_t * locks [], unsigned short lock_count, thread_desc * thrds [], unsigned short thrd_count) {
|
---|
481 | disable_interrupts();
|
---|
482 | TL_GET( this_processor )->finish.action_code = Release_Multi_Schedule;
|
---|
483 | TL_GET( this_processor )->finish.locks = locks;
|
---|
484 | TL_GET( this_processor )->finish.lock_count = lock_count;
|
---|
485 | TL_GET( this_processor )->finish.thrds = thrds;
|
---|
486 | TL_GET( this_processor )->finish.thrd_count = thrd_count;
|
---|
487 |
|
---|
488 | verify( ! TL_GET( preemption_state ).enabled );
|
---|
489 | returnToKernel();
|
---|
490 | verify( ! TL_GET( preemption_state ).enabled );
|
---|
491 |
|
---|
492 | enable_interrupts( __cfaabi_dbg_ctx );
|
---|
493 | }
|
---|
494 |
|
---|
495 | void LeaveThread(__spinlock_t * lock, thread_desc * thrd) {
|
---|
496 | verify( ! TL_GET( preemption_state ).enabled );
|
---|
497 | TL_GET( this_processor )->finish.action_code = thrd ? Release_Schedule : Release;
|
---|
498 | TL_GET( this_processor )->finish.lock = lock;
|
---|
499 | TL_GET( this_processor )->finish.thrd = thrd;
|
---|
500 |
|
---|
501 | returnToKernel();
|
---|
502 | }
|
---|
503 |
|
---|
504 | //=============================================================================================
|
---|
505 | // Kernel Setup logic
|
---|
506 | //=============================================================================================
|
---|
507 | //-----------------------------------------------------------------------------
|
---|
508 | // Kernel boot procedures
|
---|
509 | void kernel_startup(void) {
|
---|
510 | verify( ! TL_GET( preemption_state ).enabled );
|
---|
511 | __cfaabi_dbg_print_safe("Kernel : Starting\n");
|
---|
512 |
|
---|
513 | // Start by initializing the main thread
|
---|
514 | // SKULLDUGGERY: the mainThread steals the process main thread
|
---|
515 | // which will then be scheduled by the mainProcessor normally
|
---|
516 | mainThread = (thread_desc *)&storage_mainThread;
|
---|
517 | current_stack_info_t info;
|
---|
518 | (*mainThread){ &info };
|
---|
519 |
|
---|
520 | __cfaabi_dbg_print_safe("Kernel : Main thread ready\n");
|
---|
521 |
|
---|
522 | // Initialize the main cluster
|
---|
523 | mainCluster = (cluster *)&storage_mainCluster;
|
---|
524 | (*mainCluster){};
|
---|
525 |
|
---|
526 | __cfaabi_dbg_print_safe("Kernel : main cluster ready\n");
|
---|
527 |
|
---|
528 | // Initialize the main processor and the main processor ctx
|
---|
529 | // (the coroutine that contains the processing control flow)
|
---|
530 | mainProcessor = (processor *)&storage_mainProcessor;
|
---|
531 | (*mainProcessor){ mainCluster, *(processorCtx_t *)&storage_mainProcessorCtx };
|
---|
532 |
|
---|
533 | //initialize the global state variables
|
---|
534 | TL_SET( this_processor, mainProcessor );
|
---|
535 | TL_SET( this_thread, mainThread );
|
---|
536 | TL_SET( this_coroutine, &mainThread->self_cor );
|
---|
537 |
|
---|
538 | // Enable preemption
|
---|
539 | kernel_start_preemption();
|
---|
540 |
|
---|
541 | // Add the main thread to the ready queue
|
---|
542 | // once resume is called on mainProcessor->runner the mainThread needs to be scheduled like any normal thread
|
---|
543 | ScheduleThread(mainThread);
|
---|
544 |
|
---|
545 | // SKULLDUGGERY: Force a context switch to the main processor to set the main thread's context to the current UNIX
|
---|
546 | // context. Hence, the main thread does not begin through CtxInvokeThread, like all other threads. The trick here is that
|
---|
547 | // mainThread is on the ready queue when this call is made.
|
---|
548 | kernel_first_resume( TL_GET( this_processor ) );
|
---|
549 |
|
---|
550 |
|
---|
551 |
|
---|
552 | // THE SYSTEM IS NOW COMPLETELY RUNNING
|
---|
553 | __cfaabi_dbg_print_safe("Kernel : Started\n--------------------------------------------------\n\n");
|
---|
554 |
|
---|
555 | verify( ! TL_GET( preemption_state ).enabled );
|
---|
556 | enable_interrupts( __cfaabi_dbg_ctx );
|
---|
557 | verify( TL_GET( preemption_state ).enabled );
|
---|
558 | }
|
---|
559 |
|
---|
560 | void kernel_shutdown(void) {
|
---|
561 | __cfaabi_dbg_print_safe("\n--------------------------------------------------\nKernel : Shutting down\n");
|
---|
562 |
|
---|
563 | verify( TL_GET( preemption_state ).enabled );
|
---|
564 | disable_interrupts();
|
---|
565 | verify( ! TL_GET( preemption_state ).enabled );
|
---|
566 |
|
---|
567 | // SKULLDUGGERY: Notify the mainProcessor it needs to terminates.
|
---|
568 | // When its coroutine terminates, it return control to the mainThread
|
---|
569 | // which is currently here
|
---|
570 | mainProcessor->do_terminate = true;
|
---|
571 | returnToKernel();
|
---|
572 |
|
---|
573 | // THE SYSTEM IS NOW COMPLETELY STOPPED
|
---|
574 |
|
---|
575 | // Disable preemption
|
---|
576 | kernel_stop_preemption();
|
---|
577 |
|
---|
578 | // Destroy the main processor and its context in reverse order of construction
|
---|
579 | // These were manually constructed so we need manually destroy them
|
---|
580 | ^(mainProcessor->runner){};
|
---|
581 | ^(mainProcessor){};
|
---|
582 |
|
---|
583 | // Final step, destroy the main thread since it is no longer needed
|
---|
584 | // Since we provided a stack to this taxk it will not destroy anything
|
---|
585 | ^(mainThread){};
|
---|
586 |
|
---|
587 | __cfaabi_dbg_print_safe("Kernel : Shutdown complete\n");
|
---|
588 | }
|
---|
589 |
|
---|
590 | //=============================================================================================
|
---|
591 | // Unexpected Terminating logic
|
---|
592 | //=============================================================================================
|
---|
593 |
|
---|
594 |
|
---|
595 | static __spinlock_t kernel_abort_lock;
|
---|
596 | static __spinlock_t kernel_debug_lock;
|
---|
597 | static bool kernel_abort_called = false;
|
---|
598 |
|
---|
599 | void * kernel_abort (void) __attribute__ ((__nothrow__)) {
|
---|
600 | // abort cannot be recursively entered by the same or different processors because all signal handlers return when
|
---|
601 | // the globalAbort flag is true.
|
---|
602 | lock( kernel_abort_lock __cfaabi_dbg_ctx2 );
|
---|
603 |
|
---|
604 | // first task to abort ?
|
---|
605 | if ( ! kernel_abort_called ) { // not first task to abort ?
|
---|
606 | kernel_abort_called = true;
|
---|
607 | unlock( kernel_abort_lock );
|
---|
608 | }
|
---|
609 | else {
|
---|
610 | unlock( kernel_abort_lock );
|
---|
611 |
|
---|
612 | sigset_t mask;
|
---|
613 | sigemptyset( &mask );
|
---|
614 | sigaddset( &mask, SIGALRM ); // block SIGALRM signals
|
---|
615 | sigaddset( &mask, SIGUSR1 ); // block SIGUSR1 signals
|
---|
616 | sigsuspend( &mask ); // block the processor to prevent further damage during abort
|
---|
617 | _exit( EXIT_FAILURE ); // if processor unblocks before it is killed, terminate it
|
---|
618 | }
|
---|
619 |
|
---|
620 | return TL_GET( this_thread );
|
---|
621 | }
|
---|
622 |
|
---|
623 | void kernel_abort_msg( void * kernel_data, char * abort_text, int abort_text_size ) {
|
---|
624 | thread_desc * thrd = kernel_data;
|
---|
625 |
|
---|
626 | int len = snprintf( abort_text, abort_text_size, "Error occurred while executing task %.256s (%p)", thrd->self_cor.name, thrd );
|
---|
627 | __cfaabi_dbg_bits_write( abort_text, len );
|
---|
628 |
|
---|
629 | if ( thrd != TL_GET( this_coroutine ) ) {
|
---|
630 | len = snprintf( abort_text, abort_text_size, " in coroutine %.256s (%p).\n", TL_GET( this_coroutine )->name, TL_GET( this_coroutine ) );
|
---|
631 | __cfaabi_dbg_bits_write( abort_text, len );
|
---|
632 | }
|
---|
633 | else {
|
---|
634 | __cfaabi_dbg_bits_write( ".\n", 2 );
|
---|
635 | }
|
---|
636 | }
|
---|
637 |
|
---|
638 | int kernel_abort_lastframe( void ) __attribute__ ((__nothrow__)) {
|
---|
639 | return get_coroutine(TL_GET( this_thread )) == get_coroutine(mainThread) ? 4 : 2;
|
---|
640 | }
|
---|
641 |
|
---|
642 | extern "C" {
|
---|
643 | void __cfaabi_dbg_bits_acquire() {
|
---|
644 | lock( kernel_debug_lock __cfaabi_dbg_ctx2 );
|
---|
645 | }
|
---|
646 |
|
---|
647 | void __cfaabi_dbg_bits_release() {
|
---|
648 | unlock( kernel_debug_lock );
|
---|
649 | }
|
---|
650 | }
|
---|
651 |
|
---|
652 | //=============================================================================================
|
---|
653 | // Kernel Utilities
|
---|
654 | //=============================================================================================
|
---|
655 | //-----------------------------------------------------------------------------
|
---|
656 | // Locks
|
---|
657 | void ?{}( semaphore & this, int count = 1 ) {
|
---|
658 | (this.lock){};
|
---|
659 | this.count = count;
|
---|
660 | (this.waiting){};
|
---|
661 | }
|
---|
662 | void ^?{}(semaphore & this) {}
|
---|
663 |
|
---|
664 | void P(semaphore & this) with( this ){
|
---|
665 | lock( lock __cfaabi_dbg_ctx2 );
|
---|
666 | count -= 1;
|
---|
667 | if ( count < 0 ) {
|
---|
668 | // queue current task
|
---|
669 | append( waiting, (thread_desc *)TL_GET( this_thread ) );
|
---|
670 |
|
---|
671 | // atomically release spin lock and block
|
---|
672 | BlockInternal( &lock );
|
---|
673 | }
|
---|
674 | else {
|
---|
675 | unlock( lock );
|
---|
676 | }
|
---|
677 | }
|
---|
678 |
|
---|
679 | void V(semaphore & this) with( this ) {
|
---|
680 | thread_desc * thrd = NULL;
|
---|
681 | lock( lock __cfaabi_dbg_ctx2 );
|
---|
682 | count += 1;
|
---|
683 | if ( count <= 0 ) {
|
---|
684 | // remove task at head of waiting list
|
---|
685 | thrd = pop_head( waiting );
|
---|
686 | }
|
---|
687 |
|
---|
688 | unlock( lock );
|
---|
689 |
|
---|
690 | // make new owner
|
---|
691 | WakeThread( thrd );
|
---|
692 | }
|
---|
693 |
|
---|
694 | //-----------------------------------------------------------------------------
|
---|
695 | // Debug
|
---|
696 | __cfaabi_dbg_debug_do(
|
---|
697 | struct {
|
---|
698 | thread_desc * tail;
|
---|
699 | } __cfaabi_dbg_thread_list = { NULL };
|
---|
700 |
|
---|
701 | void __cfaabi_dbg_thread_register( thread_desc * thrd ) {
|
---|
702 | if( !__cfaabi_dbg_thread_list.tail ) {
|
---|
703 | __cfaabi_dbg_thread_list.tail = thrd;
|
---|
704 | return;
|
---|
705 | }
|
---|
706 | __cfaabi_dbg_thread_list.tail->dbg_next = thrd;
|
---|
707 | thrd->dbg_prev = __cfaabi_dbg_thread_list.tail;
|
---|
708 | __cfaabi_dbg_thread_list.tail = thrd;
|
---|
709 | }
|
---|
710 |
|
---|
711 | void __cfaabi_dbg_thread_unregister( thread_desc * thrd ) {
|
---|
712 | thread_desc * prev = thrd->dbg_prev;
|
---|
713 | thread_desc * next = thrd->dbg_next;
|
---|
714 |
|
---|
715 | if( next ) { next->dbg_prev = prev; }
|
---|
716 | else {
|
---|
717 | assert( __cfaabi_dbg_thread_list.tail == thrd );
|
---|
718 | __cfaabi_dbg_thread_list.tail = prev;
|
---|
719 | }
|
---|
720 |
|
---|
721 | if( prev ) { prev->dbg_next = next; }
|
---|
722 |
|
---|
723 | thrd->dbg_prev = NULL;
|
---|
724 | thrd->dbg_next = NULL;
|
---|
725 | }
|
---|
726 | )
|
---|
727 | // Local Variables: //
|
---|
728 | // mode: c //
|
---|
729 | // tab-width: 4 //
|
---|
730 | // End: //
|
---|