Context Navigation

kernel.cfa@ 6a490b2

Visit:

ADT arm-eh ast-experimental enum forall-pointer-decay jacob/cs343-translation new-ast new-ast-unique-expr pthread-emulation qualifiedEnum

Last change on this file since 6a490b2 was 6a490b2, checked in by Thierry Delisle <tdelisle@…>, 6 years ago
Merge branch 'master' into relaxed_ready
Property mode set to `100644`
File size: 35.6 KB

Line
1	//
2	// Cforall Version 1.0.0 Copyright (C) 2016 University of Waterloo
3	//
4	// The contents of this file are covered under the licence agreement in the
5	// file "LICENCE" distributed with Cforall.
6	//
7	// kernel.c --
8	//
9	// Author : Thierry Delisle
10	// Created On : Tue Jan 17 12:27:26 2017
11	// Last Modified By : Peter A. Buhr
12	// Last Modified On : Tue Feb 4 13:03:15 2020
13	// Update Count : 58
14	//
15
16	#define __cforall_thread__
17	// #define __CFA_DEBUG_PRINT_RUNTIME_CORE__
18
19	//C Includes
20	#include <stddef.h>
21	#include <errno.h>
22	#include <string.h>
23	extern "C" {
24	#include <stdio.h>
25	#include <fenv.h>
26	#include <sys/resource.h>
27	#include <signal.h>
28	#include <unistd.h>
29	#include <limits.h> // PTHREAD_STACK_MIN
30	#include <sys/mman.h> // mprotect
31	}
32
33	//CFA Includes
34	#include "time.hfa"
35	#include "kernel_private.hfa"
36	#include "preemption.hfa"
37	#include "startup.hfa"
38
39	//Private includes
40	#define __CFA_INVOKE_PRIVATE__
41	#include "invoke.h"
42
43
44	//-----------------------------------------------------------------------------
45	// Some assembly required
46	#if defined( __i386 )
47	#define CtxGet( ctx ) \
48	__asm__ volatile ( \
49	"movl %%esp,%0\n"\
50	"movl %%ebp,%1\n"\
51	: "=rm" (ctx.SP),\
52	"=rm" (ctx.FP) \
53	)
54
55	// mxcr : SSE Status and Control bits (control bits are preserved across function calls)
56	// fcw : X87 FPU control word (preserved across function calls)
57	#define __x87_store \
58	uint32_t __mxcr; \
59	uint16_t __fcw; \
60	__asm__ volatile ( \
61	"stmxcsr %0\n" \
62	"fnstcw %1\n" \
63	: "=m" (__mxcr),\
64	"=m" (__fcw) \
65	)
66
67	#define __x87_load \
68	__asm__ volatile ( \
69	"fldcw %1\n" \
70	"ldmxcsr %0\n" \
71	::"m" (__mxcr),\
72	"m" (__fcw) \
73	)
74
75	#elif defined( __x86_64 )
76	#define CtxGet( ctx ) \
77	__asm__ volatile ( \
78	"movq %%rsp,%0\n"\
79	"movq %%rbp,%1\n"\
80	: "=rm" (ctx.SP),\
81	"=rm" (ctx.FP) \
82	)
83
84	#define __x87_store \
85	uint32_t __mxcr; \
86	uint16_t __fcw; \
87	__asm__ volatile ( \
88	"stmxcsr %0\n" \
89	"fnstcw %1\n" \
90	: "=m" (__mxcr),\
91	"=m" (__fcw) \
92	)
93
94	#define __x87_load \
95	__asm__ volatile ( \
96	"fldcw %1\n" \
97	"ldmxcsr %0\n" \
98	:: "m" (__mxcr),\
99	"m" (__fcw) \
100	)
101
102
103	#elif defined( __ARM_ARCH )
104	#define CtxGet( ctx ) __asm__ ( \
105	"mov %0,%%sp\n" \
106	"mov %1,%%r11\n" \
107	: "=rm" (ctx.SP), "=rm" (ctx.FP) )
108	#else
109	#error unknown hardware architecture
110	#endif
111
112	//-----------------------------------------------------------------------------
113	//Start and stop routine for the kernel, declared first to make sure they run first
114	static void __kernel_startup (void) __attribute__(( constructor( STARTUP_PRIORITY_KERNEL ) ));
115	static void __kernel_shutdown(void) __attribute__(( destructor ( STARTUP_PRIORITY_KERNEL ) ));
116
117	//-----------------------------------------------------------------------------
118	// Kernel Scheduling logic
119	static $thread * __next_thread(cluster * this);
120	static void __run_thread(processor * this, $thread * dst);
121	static $thread * __halt(processor * this);
122	static bool __wake_one(cluster * cltr, bool was_empty);
123	static bool __wake_proc(processor *);
124
125	//-----------------------------------------------------------------------------
126	// Kernel storage
127	KERNEL_STORAGE(cluster, mainCluster);
128	KERNEL_STORAGE(processor, mainProcessor);
129	KERNEL_STORAGE($thread, mainThread);
130	KERNEL_STORAGE(__stack_t, mainThreadCtx);
131
132	cluster * mainCluster;
133	processor * mainProcessor;
134	$thread * mainThread;
135
136	extern "C" {
137	struct { __dllist_t(cluster) list; __spinlock_t lock; } __cfa_dbg_global_clusters;
138	}
139
140	size_t __page_size = 0;
141
142	//-----------------------------------------------------------------------------
143	// Global state
144	thread_local struct KernelThreadData kernelTLS __attribute__ ((tls_model ( "initial-exec" ))) = {
145	NULL, // cannot use 0p
146	NULL,
147	{ 1, false, false },
148	6u //this should be seeded better but due to a bug calling rdtsc doesn't work
149	};
150
151	//-----------------------------------------------------------------------------
152	// Struct to steal stack
153	struct current_stack_info_t {
154	__stack_t * storage; // pointer to stack object
155	void * base; // base of stack
156	void * limit; // stack grows towards stack limit
157	void * context; // address of cfa_context_t
158	};
159
160	void ?{}( current_stack_info_t & this ) {
161	__stack_context_t ctx;
162	CtxGet( ctx );
163	this.base = ctx.FP;
164
165	rlimit r;
166	getrlimit( RLIMIT_STACK, &r);
167	size_t size = r.rlim_cur;
168
169	this.limit = (void *)(((intptr_t)this.base) - size);
170	this.context = &storage_mainThreadCtx;
171	}
172
173	//-----------------------------------------------------------------------------
174	// Main thread construction
175
176	void ?{}( $coroutine & this, current_stack_info_t * info) with( this ) {
177	stack.storage = info->storage;
178	with(*stack.storage) {
179	limit = info->limit;
180	base = info->base;
181	}
182	__attribute__((may_alias)) intptr_t * istorage = (intptr_t*) &stack.storage;
183	*istorage \|= 0x1;
184	name = "Main Thread";
185	state = Start;
186	starter = 0p;
187	last = 0p;
188	cancellation = 0p;
189	}
190
191	void ?{}( $thread & this, current_stack_info_t * info) with( this ) {
192	state = Start;
193	self_cor{ info };
194	curr_cor = &self_cor;
195	curr_cluster = mainCluster;
196	self_mon.owner = &this;
197	self_mon.recursion = 1;
198	self_mon_p = &self_mon;
199	link.next = 0p;
200	link.prev = 0p;
201
202	node.next = 0p;
203	node.prev = 0p;
204	doregister(curr_cluster, this);
205
206	monitors{ &self_mon_p, 1, (fptr_t)0 };
207	}
208
209	//-----------------------------------------------------------------------------
210	// Processor coroutine
211	void ?{}(processorCtx_t & this) {
212
213	}
214
215	// Construct the processor context of non-main processors
216	static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info) {
217	(this.__cor){ info };
218	this.proc = proc;
219	}
220
221	static void * __invoke_processor(void * arg);
222
223	void ?{}(processor & this, const char name[], cluster & cltr) with( this ) {
224	this.name = name;
225	this.cltr = &cltr;
226	id = -1u;
227	terminated{ 0 };
228	destroyer = 0p;
229	do_terminate = false;
230	preemption_alarm = 0p;
231	pending_preemption = false;
232	runner.proc = &this;
233
234	idle{};
235
236	__cfadbg_print_safe(runtime_core, "Kernel : Starting core %p\n", &this);
237
238	this.stack = __create_pthread( &this.kernel_thread, __invoke_processor, (void *)&this );
239
240	__cfadbg_print_safe(runtime_core, "Kernel : core %p created\n", &this);
241	}
242
243	void ^?{}(processor & this) with( this ){
244	if( ! __atomic_load_n(&do_terminate, __ATOMIC_ACQUIRE) ) {
245	__cfadbg_print_safe(runtime_core, "Kernel : core %p signaling termination\n", &this);
246
247	__atomic_store_n(&do_terminate, true, __ATOMIC_RELAXED);
248	__wake_proc( &this );
249
250	P( terminated );
251	verify( kernelTLS.this_processor != &this);
252	}
253
254	int err = pthread_join( kernel_thread, 0p );
255	if( err != 0 ) abort("KERNEL ERROR: joining processor %p caused error %s\n", &this, strerror(err));
256
257	free( this.stack );
258	}
259
260	void ?{}(cluster & this, const char name[], Duration preemption_rate, int io_flags) with( this ) {
261	this.name = name;
262	this.preemption_rate = preemption_rate;
263	ready_queue{};
264	ready_lock{};
265
266	#if !defined(__CFA_NO_STATISTICS__)
267	print_stats = false;
268	#endif
269
270	procs{ __get };
271	idles{ __get };
272	threads{ __get };
273
274	__kernel_io_startup( this, io_flags, &this == mainCluster );
275
276	doregister(this);
277	}
278
279	void ^?{}(cluster & this) {
280	__kernel_io_shutdown( this, &this == mainCluster );
281
282	unregister(this);
283	}
284
285	//=============================================================================================
286	// Kernel Scheduling logic
287	//=============================================================================================
288	//Main of the processor contexts
289	void main(processorCtx_t & runner) {
290	// Because of a bug, we couldn't initialized the seed on construction
291	// Do it here
292	kernelTLS.rand_seed ^= rdtscl();
293
294	processor * this = runner.proc;
295	verify(this);
296
297	__cfadbg_print_safe(runtime_core, "Kernel : core %p starting\n", this);
298
299	// register the processor unless it's the main thread which is handled in the boot sequence
300	if(this != mainProcessor) {
301	this->id = doregister(this->cltr, this);
302	ready_queue_grow( this->cltr );
303	}
304
305
306	{
307	// Setup preemption data
308	preemption_scope scope = { this };
309
310	__cfadbg_print_safe(runtime_core, "Kernel : core %p started\n", this);
311
312	$thread * readyThread = 0p;
313	for( unsigned int spin_count = 0; ! __atomic_load_n(&this->do_terminate, __ATOMIC_SEQ_CST); spin_count++ ) {
314	// Try to get the next thread
315	readyThread = __next_thread( this->cltr );
316
317	// If no ready thread
318	if( readyThread == 0p ) {
319	// Block until a thread is ready
320	readyThread = __halt(this);
321	}
322
323	// Check if we actually found a thread
324	if( readyThread ) {
325	/* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
326	/* paranoid */ verifyf( readyThread->state == Ready \|\| readyThread->preempted != __NO_PREEMPTION, "state : %d, preempted %d\n", readyThread->state, readyThread->preempted);
327	/* paranoid */ verifyf( readyThread->next == 0p, "Expected null got %p", readyThread->next );
328
329	// We found a thread run it
330	__run_thread(this, readyThread);
331
332	/* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
333	}
334	}
335
336	__cfadbg_print_safe(runtime_core, "Kernel : core %p stopping\n", this);
337	}
338
339	V( this->terminated );
340
341	// unregister the processor unless it's the main thread which is handled in the boot sequence
342	if(this != mainProcessor) {
343	ready_queue_shrink( this->cltr );
344	unregister(this->cltr, this);
345	}
346	else {
347	// HACK : the coroutine context switch expects this_thread to be set
348	// and it make sense for it to be set in all other cases except here
349	// fake it
350	kernelTLS.this_thread = mainThread;
351	}
352
353	__cfadbg_print_safe(runtime_core, "Kernel : core %p terminated\n", this);
354
355	stats_tls_tally(this->cltr);
356	}
357
358	static int * __volatile_errno() __attribute__((noinline));
359	static int * __volatile_errno() { asm(""); return &errno; }
360
361	// KERNEL ONLY
362	// runThread runs a thread by context switching
363	// from the processor coroutine to the target thread
364	static void __run_thread(processor * this, $thread * thrd_dst) {
365	$coroutine * proc_cor = get_coroutine(this->runner);
366
367	// Update global state
368	kernelTLS.this_thread = thrd_dst;
369
370	// set state of processor coroutine to inactive
371	verify(proc_cor->state == Active);
372	proc_cor->state = Blocked;
373
374	// Actually run the thread
375	RUNNING: while(true) {
376	if(unlikely(thrd_dst->preempted)) {
377	thrd_dst->preempted = __NO_PREEMPTION;
378	verify(thrd_dst->state == Active \|\| thrd_dst->state == Rerun);
379	} else {
380	verify(thrd_dst->state == Blocked \|\| thrd_dst->state == Ready); // Ready means scheduled normally, blocked means rerun
381	thrd_dst->state = Active;
382	}
383
384	__cfaabi_dbg_debug_do(
385	thrd_dst->park_stale = true;
386	thrd_dst->unpark_stale = true;
387	)
388
389	/* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
390	/* paranoid */ verify( kernelTLS.this_thread == thrd_dst );
391	/* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) < ((uintptr_t)__get_stack(thrd_dst->curr_cor)->base ) \|\| thrd_dst->curr_cor == proc_cor, "ERROR : Destination $thread %p has been corrupted.\n StackPointer too small.\n", thrd_dst ); // add escape condition if we are setting up the processor
392	/* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) > ((uintptr_t)__get_stack(thrd_dst->curr_cor)->limit) \|\| thrd_dst->curr_cor == proc_cor, "ERROR : Destination $thread %p has been corrupted.\n StackPointer too large.\n", thrd_dst ); // add escape condition if we are setting up the processor
393
394	// set context switch to the thread that the processor is executing
395	verify( thrd_dst->context.SP );
396	__cfactx_switch( &proc_cor->context, &thrd_dst->context );
397	// when __cfactx_switch returns we are back in the processor coroutine
398
399	/* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) > ((uintptr_t)__get_stack(thrd_dst->curr_cor)->limit), "ERROR : Destination $thread %p has been corrupted.\n StackPointer too large.\n", thrd_dst );
400	/* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) < ((uintptr_t)__get_stack(thrd_dst->curr_cor)->base ), "ERROR : Destination $thread %p has been corrupted.\n StackPointer too small.\n", thrd_dst );
401	/* paranoid */ verify( kernelTLS.this_thread == thrd_dst );
402	/* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
403
404
405	// We just finished running a thread, there are a few things that could have happened.
406	// 1 - Regular case : the thread has blocked and now one has scheduled it yet.
407	// 2 - Racy case : the thread has blocked but someone has already tried to schedule it.
408	// 4 - Preempted
409	// In case 1, we may have won a race so we can't write to the state again.
410	// In case 2, we lost the race so we now own the thread.
411
412	if(unlikely(thrd_dst->preempted != __NO_PREEMPTION)) {
413	// The thread was preempted, reschedule it and reset the flag
414	__schedule_thread( thrd_dst );
415	break RUNNING;
416	}
417
418	// set state of processor coroutine to active and the thread to inactive
419	static_assert(sizeof(thrd_dst->state) == sizeof(int));
420	enum coroutine_state old_state = __atomic_exchange_n(&thrd_dst->state, Blocked, __ATOMIC_SEQ_CST);
421	__cfaabi_dbg_debug_do( thrd_dst->park_result = old_state; )
422	switch(old_state) {
423	case Halted:
424	// The thread has halted, it should never be scheduled/run again, leave it back to Halted and move on
425	thrd_dst->state = Halted;
426
427	// We may need to wake someone up here since
428	unpark( this->destroyer __cfaabi_dbg_ctx2 );
429	this->destroyer = 0p;
430	break RUNNING;
431	case Active:
432	// This is case 1, the regular case, nothing more is needed
433	break RUNNING;
434	case Rerun:
435	// This is case 2, the racy case, someone tried to run this thread before it finished blocking
436	// In this case, just run it again.
437	continue RUNNING;
438	default:
439	// This makes no sense, something is wrong abort
440	abort("Finished running a thread that was Blocked/Start/Primed %d\n", old_state);
441	}
442	}
443
444	// Just before returning to the processor, set the processor coroutine to active
445	proc_cor->state = Active;
446	kernelTLS.this_thread = 0p;
447	}
448
449	// KERNEL_ONLY
450	void returnToKernel() {
451	/* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
452	$coroutine * proc_cor = get_coroutine(kernelTLS.this_processor->runner);
453	$thread * thrd_src = kernelTLS.this_thread;
454
455	// Run the thread on this processor
456	{
457	int local_errno = *__volatile_errno();
458	#if defined( __i386 ) \|\| defined( __x86_64 )
459	__x87_store;
460	#endif
461	verify( proc_cor->context.SP );
462	__cfactx_switch( &thrd_src->context, &proc_cor->context );
463	#if defined( __i386 ) \|\| defined( __x86_64 )
464	__x87_load;
465	#endif
466	*__volatile_errno() = local_errno;
467	}
468
469	/* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
470	/* paranoid */ verifyf( ((uintptr_t)thrd_src->context.SP) < ((uintptr_t)__get_stack(thrd_src->curr_cor)->base ), "ERROR : Returning $thread %p has been corrupted.\n StackPointer too small.\n", thrd_src );
471	/* paranoid */ verifyf( ((uintptr_t)thrd_src->context.SP) > ((uintptr_t)__get_stack(thrd_src->curr_cor)->limit), "ERROR : Returning $thread %p has been corrupted.\n StackPointer too large.\n", thrd_src );
472	}
473
474	// KERNEL_ONLY
475	// Context invoker for processors
476	// This is the entry point for processors (kernel threads)
477	// It effectively constructs a coroutine by stealing the pthread stack
478	static void * __invoke_processor(void * arg) {
479	processor * proc = (processor *) arg;
480	kernelTLS.this_processor = proc;
481	kernelTLS.this_thread = 0p;
482	kernelTLS.preemption_state.[enabled, disable_count] = [false, 1];
483	// SKULLDUGGERY: We want to create a context for the processor coroutine
484	// which is needed for the 2-step context switch. However, there is no reason
485	// to waste the perfectly valid stack create by pthread.
486	current_stack_info_t info;
487	__stack_t ctx;
488	info.storage = &ctx;
489	(proc->runner){ proc, &info };
490
491	__cfaabi_dbg_print_safe("Coroutine : created stack %p\n", get_coroutine(proc->runner)->stack.storage);
492
493	//Set global state
494	kernelTLS.this_thread = 0p;
495
496	//We now have a proper context from which to schedule threads
497	__cfadbg_print_safe(runtime_core, "Kernel : core %p created (%p, %p)\n", proc, &proc->runner, &ctx);
498
499	// SKULLDUGGERY: Since the coroutine doesn't have its own stack, we can't
500	// resume it to start it like it normally would, it will just context switch
501	// back to here. Instead directly call the main since we already are on the
502	// appropriate stack.
503	get_coroutine(proc->runner)->state = Active;
504	main( proc->runner );
505	get_coroutine(proc->runner)->state = Halted;
506
507	// Main routine of the core returned, the core is now fully terminated
508	__cfadbg_print_safe(runtime_core, "Kernel : core %p main ended (%p)\n", proc, &proc->runner);
509
510	return 0p;
511	}
512
513	static void Abort( int ret, const char func[] ) {
514	if ( ret ) { // pthread routines return errno values
515	abort( "%s : internal error, error(%d) %s.", func, ret, strerror( ret ) );
516	} // if
517	} // Abort
518
519	void * __create_pthread( pthread_t * pthread, void * (start)(void ), void * arg ) {
520	pthread_attr_t attr;
521
522	Abort( pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
523
524	size_t stacksize;
525	// default stack size, normally defined by shell limit
526	Abort( pthread_attr_getstacksize( &attr, &stacksize ), "pthread_attr_getstacksize" );
527	assert( stacksize >= PTHREAD_STACK_MIN );
528
529	void * stack;
530	__cfaabi_dbg_debug_do(
531	stack = memalign( __page_size, stacksize + __page_size );
532	// pthread has no mechanism to create the guard page in user supplied stack.
533	if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
534	abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
535	} // if
536	);
537	__cfaabi_dbg_no_debug_do(
538	stack = malloc( stacksize );
539	);
540
541	Abort( pthread_attr_setstack( &attr, stack, stacksize ), "pthread_attr_setstack" );
542
543	Abort( pthread_create( pthread, &attr, start, arg ), "pthread_create" );
544	return stack;
545	}
546
547	// KERNEL_ONLY
548	static void __kernel_first_resume( processor * this ) {
549	$thread * src = mainThread;
550	$coroutine * dst = get_coroutine(this->runner);
551
552	verify( ! kernelTLS.preemption_state.enabled );
553
554	kernelTLS.this_thread->curr_cor = dst;
555	__stack_prepare( &dst->stack, 65000 );
556	__cfactx_start(main, dst, this->runner, __cfactx_invoke_coroutine);
557
558	verify( ! kernelTLS.preemption_state.enabled );
559
560	dst->last = &src->self_cor;
561	dst->starter = dst->starter ? dst->starter : &src->self_cor;
562
563	// make sure the current state is still correct
564	/* paranoid */ verify(src->state == Ready);
565
566	// context switch to specified coroutine
567	verify( dst->context.SP );
568	__cfactx_switch( &src->context, &dst->context );
569	// when __cfactx_switch returns we are back in the src coroutine
570
571	mainThread->curr_cor = &mainThread->self_cor;
572
573	// make sure the current state has been update
574	/* paranoid */ verify(src->state == Active);
575
576	verify( ! kernelTLS.preemption_state.enabled );
577	}
578
579	// KERNEL_ONLY
580	static void __kernel_last_resume( processor * this ) {
581	$coroutine * src = &mainThread->self_cor;
582	$coroutine * dst = get_coroutine(this->runner);
583
584	verify( ! kernelTLS.preemption_state.enabled );
585	verify( dst->starter == src );
586	verify( dst->context.SP );
587
588	// SKULLDUGGERY in debug the processors check that the
589	// stack is still within the limit of the stack limits after running a thread.
590	// that check doesn't make sense if we context switch to the processor using the
591	// coroutine semantics. Since this is a special case, use the current context
592	// info to populate these fields.
593	__cfaabi_dbg_debug_do(
594	__stack_context_t ctx;
595	CtxGet( ctx );
596	mainThread->context.SP = ctx.SP;
597	mainThread->context.FP = ctx.FP;
598	)
599
600	// context switch to the processor
601	__cfactx_switch( &src->context, &dst->context );
602	}
603
604	//-----------------------------------------------------------------------------
605	// Scheduler routines
606	// KERNEL ONLY
607	void __schedule_thread( $thread * thrd ) {
608	/* paranoid */ verify( thrd );
609	/* paranoid */ verify( thrd->state != Halted );
610	/* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
611	/* paranoid */ #if defined( __CFA_WITH_VERIFY__ )
612	/* paranoid */ if( thrd->state == Blocked \|\| thrd->state == Start ) assertf( thrd->preempted == __NO_PREEMPTION,
613	"Error inactive thread marked as preempted, state %d, preemption %d\n", thrd->state, thrd->preempted );
614	/* paranoid */ if( thrd->preempted != __NO_PREEMPTION ) assertf(thrd->state == Active \|\| thrd->state == Rerun,
615	"Error preempted thread marked as not currently running, state %d, preemption %d\n", thrd->state, thrd->preempted );
616	/* paranoid */ #endif
617	/* paranoid */ verifyf( thrd->link.next == 0p, "Expected null got %p", thrd->link.next );
618
619	if (thrd->preempted == __NO_PREEMPTION) thrd->state = Ready;
620
621	ready_schedule_lock(thrd->curr_cluster, kernelTLS.this_processor);
622	bool was_empty = push( thrd->curr_cluster, thrd );
623	ready_schedule_unlock(thrd->curr_cluster, kernelTLS.this_processor);
624
625	__wake_one(thrd->curr_cluster, was_empty);
626
627	/* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
628	}
629
630	// KERNEL ONLY
631	static $thread * __next_thread(cluster * this) with( *this ) {
632	/* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
633
634	ready_schedule_lock(this, kernelTLS.this_processor);
635	$thread * head = pop( this );
636	ready_schedule_unlock(this, kernelTLS.this_processor);
637
638	/* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
639	return head;
640	}
641
642	// KERNEL ONLY unpark with out disabling interrupts
643	void __unpark( $thread * thrd __cfaabi_dbg_ctx_param2 ) {
644	static_assert(sizeof(thrd->state) == sizeof(int));
645
646	// record activity
647	__cfaabi_dbg_record_thrd( *thrd, false, caller );
648
649	enum coroutine_state old_state = __atomic_exchange_n(&thrd->state, Rerun, __ATOMIC_SEQ_CST);
650	__cfaabi_dbg_debug_do( thrd->unpark_result = old_state; )
651	switch(old_state) {
652	case Active:
653	// Wake won the race, the thread will reschedule/rerun itself
654	break;
655	case Blocked:
656	/* paranoid */ verify( ! thrd->preempted != __NO_PREEMPTION );
657
658	// Wake lost the race,
659	thrd->state = Blocked;
660	__schedule_thread( thrd );
661	break;
662	case Rerun:
663	abort("More than one thread attempted to schedule thread %p\n", thrd);
664	break;
665	case Halted:
666	case Start:
667	case Primed:
668	default:
669	// This makes no sense, something is wrong abort
670	abort();
671	}
672	}
673
674	void unpark( $thread * thrd __cfaabi_dbg_ctx_param2 ) {
675	if( !thrd ) return;
676
677	disable_interrupts();
678	__unpark( thrd __cfaabi_dbg_ctx_fwd2 );
679	enable_interrupts( __cfaabi_dbg_ctx );
680	}
681
682	void park( __cfaabi_dbg_ctx_param ) {
683	/* paranoid */ verify( kernelTLS.preemption_state.enabled );
684	disable_interrupts();
685	/* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
686	/* paranoid */ verify( kernelTLS.this_thread->preempted == __NO_PREEMPTION );
687
688	// record activity
689	__cfaabi_dbg_record_thrd( *kernelTLS.this_thread, true, caller );
690
691	returnToKernel();
692
693	/* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
694	enable_interrupts( __cfaabi_dbg_ctx );
695	/* paranoid */ verify( kernelTLS.preemption_state.enabled );
696
697	}
698
699	// KERNEL ONLY
700	void __leave_thread() {
701	/* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
702	returnToKernel();
703	abort();
704	}
705
706	// KERNEL ONLY
707	bool force_yield( __Preemption_Reason reason ) {
708	/* paranoid */ verify( kernelTLS.preemption_state.enabled );
709	disable_interrupts();
710	/* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
711
712	$thread * thrd = kernelTLS.this_thread;
713	/* paranoid */ verify(thrd->state == Active \|\| thrd->state == Rerun);
714
715	// SKULLDUGGERY: It is possible that we are preempting this thread just before
716	// it was going to park itself. If that is the case and it is already using the
717	// intrusive fields then we can't use them to preempt the thread
718	// If that is the case, abandon the preemption.
719	bool preempted = false;
720	if(thrd->next == 0p) {
721	preempted = true;
722	thrd->preempted = reason;
723	returnToKernel();
724	}
725
726	/* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
727	enable_interrupts_noPoll();
728	/* paranoid */ verify( kernelTLS.preemption_state.enabled );
729
730	return preempted;
731	}
732
733	//=============================================================================================
734	// Kernel Setup logic
735	//=============================================================================================
736	//-----------------------------------------------------------------------------
737	// Kernel boot procedures
738	static void __kernel_startup(void) {
739	verify( ! kernelTLS.preemption_state.enabled );
740	__cfadbg_print_safe(runtime_core, "Kernel : Starting\n");
741
742	__page_size = sysconf( _SC_PAGESIZE );
743
744	__cfa_dbg_global_clusters.list{ __get };
745	__cfa_dbg_global_clusters.lock{};
746
747	// Initialize the main cluster
748	mainCluster = (cluster *)&storage_mainCluster;
749	(*mainCluster){"Main Cluster"};
750
751	__cfadbg_print_safe(runtime_core, "Kernel : Main cluster ready\n");
752
753	// Start by initializing the main thread
754	// SKULLDUGGERY: the mainThread steals the process main thread
755	// which will then be scheduled by the mainProcessor normally
756	mainThread = ($thread *)&storage_mainThread;
757	current_stack_info_t info;
758	info.storage = (__stack_t*)&storage_mainThreadCtx;
759	(*mainThread){ &info };
760
761	__cfadbg_print_safe(runtime_core, "Kernel : Main thread ready\n");
762
763
764
765	// Construct the processor context of the main processor
766	void ?{}(processorCtx_t & this, processor * proc) {
767	(this.__cor){ "Processor" };
768	this.__cor.starter = 0p;
769	this.proc = proc;
770	}
771
772	void ?{}(processor & this) with( this ) {
773	name = "Main Processor";
774	cltr = mainCluster;
775	terminated{ 0 };
776	do_terminate = false;
777	preemption_alarm = 0p;
778	pending_preemption = false;
779	kernel_thread = pthread_self();
780	id = -1u;
781
782	runner{ &this };
783	__cfadbg_print_safe(runtime_core, "Kernel : constructed main processor context %p\n", &runner);
784	}
785
786	// Initialize the main processor and the main processor ctx
787	// (the coroutine that contains the processing control flow)
788	mainProcessor = (processor *)&storage_mainProcessor;
789	(*mainProcessor){};
790
791	mainProcessor->id = doregister(mainCluster, mainProcessor);
792
793	//initialize the global state variables
794	kernelTLS.this_processor = mainProcessor;
795	kernelTLS.this_thread = mainThread;
796
797	// Enable preemption
798	kernel_start_preemption();
799
800	// Add the main thread to the ready queue
801	// once resume is called on mainProcessor->runner the mainThread needs to be scheduled like any normal thread
802	__schedule_thread(mainThread);
803
804	// SKULLDUGGERY: Force a context switch to the main processor to set the main thread's context to the current UNIX
805	// context. Hence, the main thread does not begin through __cfactx_invoke_thread, like all other threads. The trick here is that
806	// mainThread is on the ready queue when this call is made.
807	__kernel_first_resume( kernelTLS.this_processor );
808
809
810	// THE SYSTEM IS NOW COMPLETELY RUNNING
811
812
813	// Now that the system is up, finish creating systems that need threading
814	__kernel_io_finish_start( *mainCluster );
815
816
817	__cfadbg_print_safe(runtime_core, "Kernel : Started\n--------------------------------------------------\n\n");
818
819	verify( ! kernelTLS.preemption_state.enabled );
820	enable_interrupts( __cfaabi_dbg_ctx );
821	verify( TL_GET( preemption_state.enabled ) );
822	}
823
824	static void __kernel_shutdown(void) {
825	//Before we start shutting things down, wait for systems that need threading to shutdown
826	__kernel_io_prepare_stop( *mainCluster );
827
828	/* paranoid */ verify( TL_GET( preemption_state.enabled ) );
829	disable_interrupts();
830	/* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
831
832	__cfadbg_print_safe(runtime_core, "\n--------------------------------------------------\nKernel : Shutting down\n");
833
834	// SKULLDUGGERY: Notify the mainProcessor it needs to terminates.
835	// When its coroutine terminates, it return control to the mainThread
836	// which is currently here
837	__atomic_store_n(&mainProcessor->do_terminate, true, __ATOMIC_RELEASE);
838	__kernel_last_resume( kernelTLS.this_processor );
839	mainThread->self_cor.state = Halted;
840
841	// THE SYSTEM IS NOW COMPLETELY STOPPED
842
843	// Disable preemption
844	kernel_stop_preemption();
845
846	unregister(mainCluster, mainProcessor);
847
848	// Destroy the main processor and its context in reverse order of construction
849	// These were manually constructed so we need manually destroy them
850	void ^?{}(processor & this) with( this ){
851	/* paranoid */ verify( this.do_terminate == true );
852	__cfaabi_dbg_print_safe("Kernel : destroyed main processor context %p\n", &runner);
853	}
854
855	^(*mainProcessor){};
856
857	// Final step, destroy the main thread since it is no longer needed
858
859	// Since we provided a stack to this taxk it will not destroy anything
860	/* paranoid / verify(mainThread->self_cor.stack.storage == (__stack_t)(((uintptr_t)&storage_mainThreadCtx)\| 0x1));
861	^(*mainThread){};
862
863	^(*mainCluster){};
864
865	^(__cfa_dbg_global_clusters.list){};
866	^(__cfa_dbg_global_clusters.lock){};
867
868	__cfadbg_print_safe(runtime_core, "Kernel : Shutdown complete\n");
869	}
870
871	//=============================================================================================
872	// Kernel Idle Sleep
873	//=============================================================================================
874	static $thread * __halt(processor * this) with( *this ) {
875	if( do_terminate ) return 0p;
876
877	// First, lock the cluster idle
878	lock( cltr->idle_lock __cfaabi_dbg_ctx2 );
879
880	// Check if we can find a thread
881	if( $thread * found = __next_thread( cltr ) ) {
882	unlock( cltr->idle_lock );
883	return found;
884	}
885
886	// Move this processor from the active list to the idle list
887	move_to_front(cltr->procs, cltr->idles, *this);
888
889	// Unlock the idle lock so we don't go to sleep with a lock
890	unlock (cltr->idle_lock);
891
892	// We are ready to sleep
893	__cfadbg_print_safe(runtime_core, "Kernel : Processor %p ready to sleep\n", this);
894	wait( idle );
895
896	// We have woken up
897	__cfadbg_print_safe(runtime_core, "Kernel : Processor %p woke up and ready to run\n", this);
898
899	// Get ourself off the idle list
900	with( *cltr ) {
901	lock (idle_lock __cfaabi_dbg_ctx2);
902	move_to_front(idles, procs, *this);
903	unlock(idle_lock);
904	}
905
906	// Don't check the ready queue again, we may not be in a position to run a thread
907	return 0p;
908	}
909
910	// Wake a thread from the front if there are any
911	static bool __wake_one(cluster * this, __attribute__((unused)) bool force) {
912	// if we don't want to force check if we know it's false
913	// if( !this->idles.head && !force ) return false;
914
915	// First, lock the cluster idle
916	lock( this->idle_lock __cfaabi_dbg_ctx2 );
917
918	// Check if there is someone to wake up
919	if( !this->idles.head ) {
920	// Nope unlock and return false
921	unlock( this->idle_lock );
922	return false;
923	}
924
925	// Wake them up
926	__cfadbg_print_safe(runtime_core, "Kernel : waking Processor %p\n", this->idles.head);
927	/* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
928	post( this->idles.head->idle );
929
930	// Unlock and return true
931	unlock( this->idle_lock );
932	return true;
933	}
934
935	// Unconditionnaly wake a thread
936	static bool __wake_proc(processor * this) {
937	__cfadbg_print_safe(runtime_core, "Kernel : waking Processor %p\n", this);
938
939	disable_interrupts();
940	/* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
941	bool ret = post( this->idle );
942	enable_interrupts( __cfaabi_dbg_ctx );
943
944	return ret;
945	}
946
947	//=============================================================================================
948	// Unexpected Terminating logic
949	//=============================================================================================
950	static __spinlock_t kernel_abort_lock;
951	static bool kernel_abort_called = false;
952
953	void * kernel_abort(void) __attribute__ ((__nothrow__)) {
954	// abort cannot be recursively entered by the same or different processors because all signal handlers return when
955	// the globalAbort flag is true.
956	lock( kernel_abort_lock __cfaabi_dbg_ctx2 );
957
958	// first task to abort ?
959	if ( kernel_abort_called ) { // not first task to abort ?
960	unlock( kernel_abort_lock );
961
962	sigset_t mask;
963	sigemptyset( &mask );
964	sigaddset( &mask, SIGALRM ); // block SIGALRM signals
965	sigaddset( &mask, SIGUSR1 ); // block SIGALRM signals
966	sigsuspend( &mask ); // block the processor to prevent further damage during abort
967	_exit( EXIT_FAILURE ); // if processor unblocks before it is killed, terminate it
968	}
969	else {
970	kernel_abort_called = true;
971	unlock( kernel_abort_lock );
972	}
973
974	return kernelTLS.this_thread;
975	}
976
977	void kernel_abort_msg( void * kernel_data, char * abort_text, int abort_text_size ) {
978	$thread * thrd = kernel_data;
979
980	if(thrd) {
981	int len = snprintf( abort_text, abort_text_size, "Error occurred while executing thread %.256s (%p)", thrd->self_cor.name, thrd );
982	__cfaabi_bits_write( STDERR_FILENO, abort_text, len );
983
984	if ( &thrd->self_cor != thrd->curr_cor ) {
985	len = snprintf( abort_text, abort_text_size, " in coroutine %.256s (%p).\n", thrd->curr_cor->name, thrd->curr_cor );
986	__cfaabi_bits_write( STDERR_FILENO, abort_text, len );
987	}
988	else {
989	__cfaabi_bits_write( STDERR_FILENO, ".\n", 2 );
990	}
991	}
992	else {
993	int len = snprintf( abort_text, abort_text_size, "Error occurred outside of any thread.\n" );
994	__cfaabi_bits_write( STDERR_FILENO, abort_text, len );
995	}
996	}
997
998	int kernel_abort_lastframe( void ) __attribute__ ((__nothrow__)) {
999	return get_coroutine(kernelTLS.this_thread) == get_coroutine(mainThread) ? 4 : 2;
1000	}
1001
1002	static __spinlock_t kernel_debug_lock;
1003
1004	extern "C" {
1005	void __cfaabi_bits_acquire() {
1006	lock( kernel_debug_lock __cfaabi_dbg_ctx2 );
1007	}
1008
1009	void __cfaabi_bits_release() {
1010	unlock( kernel_debug_lock );
1011	}
1012	}
1013
1014	//=============================================================================================
1015	// Kernel Utilities
1016	//=============================================================================================
1017	//-----------------------------------------------------------------------------
1018	// Locks
1019	void ?{}( semaphore & this, int count = 1 ) {
1020	(this.lock){};
1021	this.count = count;
1022	(this.waiting){};
1023	}
1024	void ^?{}(semaphore & this) {}
1025
1026	bool P(semaphore & this) with( this ){
1027	lock( lock __cfaabi_dbg_ctx2 );
1028	count -= 1;
1029	if ( count < 0 ) {
1030	// queue current task
1031	append( waiting, kernelTLS.this_thread );
1032
1033	// atomically release spin lock and block
1034	unlock( lock );
1035	park( __cfaabi_dbg_ctx );
1036	return true;
1037	}
1038	else {
1039	unlock( lock );
1040	return false;
1041	}
1042	}
1043
1044	bool V(semaphore & this) with( this ) {
1045	$thread * thrd = 0p;
1046	lock( lock __cfaabi_dbg_ctx2 );
1047	count += 1;
1048	if ( count <= 0 ) {
1049	// remove task at head of waiting list
1050	thrd = pop_head( waiting );
1051	}
1052
1053	unlock( lock );
1054
1055	// make new owner
1056	unpark( thrd __cfaabi_dbg_ctx2 );
1057
1058	return thrd != 0p;
1059	}
1060
1061	bool V(semaphore & this, unsigned diff) with( this ) {
1062	$thread * thrd = 0p;
1063	lock( lock __cfaabi_dbg_ctx2 );
1064	int release = max(-count, (int)diff);
1065	count += diff;
1066	for(release) {
1067	unpark( pop_head( waiting ) __cfaabi_dbg_ctx2 );
1068	}
1069
1070	unlock( lock );
1071
1072	return thrd != 0p;
1073	}
1074
1075	//-----------------------------------------------------------------------------
1076	// Global Queues
1077	void doregister( cluster & cltr ) {
1078	lock ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
1079	push_front( __cfa_dbg_global_clusters.list, cltr );
1080	unlock ( __cfa_dbg_global_clusters.lock );
1081	}
1082
1083	void unregister( cluster & cltr ) {
1084	lock ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
1085	remove( __cfa_dbg_global_clusters.list, cltr );
1086	unlock( __cfa_dbg_global_clusters.lock );
1087	}
1088
1089	void doregister( cluster * cltr, $thread & thrd ) {
1090	lock (cltr->thread_list_lock __cfaabi_dbg_ctx2);
1091	cltr->nthreads += 1;
1092	push_front(cltr->threads, thrd);
1093	unlock (cltr->thread_list_lock);
1094	}
1095
1096	void unregister( cluster * cltr, $thread & thrd ) {
1097	lock (cltr->thread_list_lock __cfaabi_dbg_ctx2);
1098	remove(cltr->threads, thrd );
1099	cltr->nthreads -= 1;
1100	unlock(cltr->thread_list_lock);
1101	}
1102
1103	//-----------------------------------------------------------------------------
1104	// Debug
1105	__cfaabi_dbg_debug_do(
1106	extern "C" {
1107	void __cfaabi_dbg_record_lock(__spinlock_t & this, const char prev_name[]) {
1108	this.prev_name = prev_name;
1109	this.prev_thrd = kernelTLS.this_thread;
1110	}
1111
1112	void __cfaabi_dbg_record_thrd($thread & this, bool park, const char prev_name[]) {
1113	if(park) {
1114	this.park_caller = prev_name;
1115	this.park_stale = false;
1116	}
1117	else {
1118	this.unpark_caller = prev_name;
1119	this.unpark_stale = false;
1120	}
1121	}
1122	}
1123	)
1124
1125	//-----------------------------------------------------------------------------
1126	// Debug
1127	bool threading_enabled(void) __attribute__((const)) {
1128	return true;
1129	}
1130	// Local Variables: //
1131	// mode: c //
1132	// tab-width: 4 //
1133	// End: //

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

Original Format