Context Navigation

kernel.cfa@ 442b624

Visit:

ADT arm-eh ast-experimental enum forall-pointer-decay jacob/cs343-translation new-ast-unique-expr pthread-emulation qualifiedEnum

Last change on this file since 442b624 was 74f5c83, checked in by Peter A. Buhr <pabuhr@…>, 5 years ago
assembler for save/restore registers FPSR/FPCR for ARM
Property mode set to `100644`
File size: 20.8 KB

Rev	Line
[8118303]	1	//
	2	// Cforall Version 1.0.0 Copyright (C) 2016 University of Waterloo
	3	//
	4	// The contents of this file are covered under the licence agreement in the
	5	// file "LICENCE" distributed with Cforall.
	6	//
	7	// kernel.c --
	8	//
	9	// Author : Thierry Delisle
[75f3522]	10	// Created On : Tue Jan 17 12:27:26 2017
[6b0b624]	11	// Last Modified By : Peter A. Buhr
[74f5c83]	12	// Last Modified On : Fri Aug 14 15:23:00 2020
	13	// Update Count : 69
[8118303]	14	//
	15
[2026bb6]	16	#define __cforall_thread__
[4069faad]	17	// #define __CFA_DEBUG_PRINT_RUNTIME_CORE__
[2026bb6]	18
[8118303]	19	//C Includes
[214e8da]	20	#include <errno.h>
[9d944b2]	21	#include <stdio.h>
[58b6d1b]	22	#include <signal.h>
[9d944b2]	23	#include <unistd.h>
[8118303]	24
	25	//CFA Includes
[73abe95]	26	#include "kernel_private.hfa"
	27	#include "preemption.hfa"
[8118303]	28
	29	//Private includes
	30	#define __CFA_INVOKE_PRIVATE__
	31	#include "invoke.h"
	32
[4069faad]	33
[deca0f5]	34	//-----------------------------------------------------------------------------
	35	// Some assembly required
[1805b1b]	36	#if defined( __i386 )
[deca0f5]	37	// mxcr : SSE Status and Control bits (control bits are preserved across function calls)
	38	// fcw : X87 FPU control word (preserved across function calls)
	39	#define __x87_store \
	40	uint32_t __mxcr; \
	41	uint16_t __fcw; \
	42	__asm__ volatile ( \
	43	"stmxcsr %0\n" \
	44	"fnstcw %1\n" \
	45	: "=m" (__mxcr),\
	46	"=m" (__fcw) \
	47	)
	48
	49	#define __x87_load \
	50	__asm__ volatile ( \
	51	"fldcw %1\n" \
	52	"ldmxcsr %0\n" \
	53	::"m" (__mxcr),\
	54	"m" (__fcw) \
	55	)
	56
	57	#elif defined( __x86_64 )
	58	#define __x87_store \
	59	uint32_t __mxcr; \
	60	uint16_t __fcw; \
	61	__asm__ volatile ( \
	62	"stmxcsr %0\n" \
	63	"fnstcw %1\n" \
	64	: "=m" (__mxcr),\
	65	"=m" (__fcw) \
	66	)
	67
	68	#define __x87_load \
	69	__asm__ volatile ( \
	70	"fldcw %1\n" \
	71	"ldmxcsr %0\n" \
	72	:: "m" (__mxcr),\
	73	"m" (__fcw) \
	74	)
	75
	76	#elif defined( __ARM_ARCH )
[74f5c83]	77	#define __x87_store \
	78	uint32_t __fpcntl[2]; \
	79	__asm__ volatile ( \
	80	"mrs x9, FPCR\n" \
	81	"mrs x10, FPSR\n" \
	82	"stp x9, x10, %0\n" \
	83	: "=m" (__fpcntl) : : "x9", "x10" \
	84	)
	85
	86	#define __x87_load \
	87	__asm__ volatile ( \
	88	"ldp x9, x10, %0\n" \
	89	"msr FPSR, x10\n" \
	90	"msr FPCR, x9\n" \
	91	: "=m" (__fpcntl) : : "x9", "x10" \
	92	)
	93
[deca0f5]	94	#else
	95	#error unknown hardware architecture
	96	#endif
	97
[e660761]	98	extern $thread * mainThread;
	99	extern processor * mainProcessor;
[2ac095d]	100
[92e7631]	101	//-----------------------------------------------------------------------------
	102	// Kernel Scheduling logic
	103	static $thread * __next_thread(cluster * this);
[64a7146]	104	static bool __has_next_thread(cluster * this);
[92e7631]	105	static void __run_thread(processor * this, $thread * dst);
[64a7146]	106	static bool __wake_one(struct __processor_id_t * id, cluster * cltr);
	107	static void __halt(processor * this);
[e660761]	108	bool __wake_proc(processor *);
[c84e80a]	109
[75f3522]	110	//=============================================================================================
	111	// Kernel Scheduling logic
	112	//=============================================================================================
[8fcbb4c]	113	//Main of the processor contexts
[83a071f9]	114	void main(processorCtx_t & runner) {
[21184e3]	115	// Because of a bug, we couldn't initialized the seed on construction
	116	// Do it here
[7768b8d]	117	kernelTLS.rand_seed ^= rdtscl();
[21184e3]	118
[83a071f9]	119	processor * this = runner.proc;
[094476d]	120	verify(this);
[c81ebf9]	121
[4069faad]	122	__cfadbg_print_safe(runtime_core, "Kernel : core %p starting\n", this);
[28d73c1]	123	#if !defined(__CFA_NO_STATISTICS__)
	124	if( this->print_halts ) {
	125	__cfaabi_bits_print_safe( STDOUT_FILENO, "Processor : %d - %s (%p)\n", this->id, this->name, (void*)this);
	126	}
	127	#endif
[b798713]	128
[75f3522]	129	{
[c81ebf9]	130	// Setup preemption data
	131	preemption_scope scope = { this };
	132
[4069faad]	133	__cfadbg_print_safe(runtime_core, "Kernel : core %p started\n", this);
[8118303]	134
[ac2b598]	135	$thread * readyThread = 0p;
[398e8e9]	136	for( unsigned int spin_count = 0;; spin_count++ ) {
[92e7631]	137	// Try to get the next thread
[8c50aed]	138	readyThread = __next_thread( this->cltr );
[75f3522]	139
[92e7631]	140	// Check if we actually found a thread
	141	if( readyThread ) {
[3381ed7]	142	/* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
[92e7631]	143	/* paranoid */ verifyf( readyThread->state == Ready \|\| readyThread->preempted != __NO_PREEMPTION, "state : %d, preempted %d\n", readyThread->state, readyThread->preempted);
[504a7dc]	144	/* paranoid */ verifyf( readyThread->link.next == 0p, "Expected null got %p", readyThread->link.next );
[04b5cef]	145	__builtin_prefetch( readyThread->context.SP );
[4e6fb8e]	146
[92e7631]	147	// We found a thread run it
[8c50aed]	148	__run_thread(this, readyThread);
[c81ebf9]	149
[3381ed7]	150	/* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
[c81ebf9]	151	}
[398e8e9]	152
	153	if(__atomic_load_n(&this->do_terminate, __ATOMIC_SEQ_CST)) break;
	154
	155	if( !readyThread ) {
[64a7146]	156	// Block until a thread is ready
	157	__halt(this);
	158	}
[c81ebf9]	159	}
	160
[4069faad]	161	__cfadbg_print_safe(runtime_core, "Kernel : core %p stopping\n", this);
[c84e80a]	162	}
[8118303]	163
[4cedd9f]	164	V( this->terminated );
[bdeba0b]	165
[28d73c1]	166	if(this == mainProcessor) {
[6a490b2]	167	// HACK : the coroutine context switch expects this_thread to be set
	168	// and it make sense for it to be set in all other cases except here
	169	// fake it
	170	kernelTLS.this_thread = mainThread;
	171	}
[7768b8d]	172
[4069faad]	173	__cfadbg_print_safe(runtime_core, "Kernel : core %p terminated\n", this);
[c84e80a]	174	}
	175
[5c1a531]	176	static int * __volatile_errno() __attribute__((noinline));
	177	static int * __volatile_errno() { asm(""); return &errno; }
	178
[14a61b5]	179	// KERNEL ONLY
[1c273d0]	180	// runThread runs a thread by context switching
	181	// from the processor coroutine to the target thread
[ac2b598]	182	static void __run_thread(processor * this, $thread * thrd_dst) {
	183	$coroutine * proc_cor = get_coroutine(this->runner);
[8fcbb4c]	184
[14a61b5]	185	// Update global state
[e8e457e]	186	kernelTLS.this_thread = thrd_dst;
	187
[9f575ea]	188	// set state of processor coroutine to inactive
	189	verify(proc_cor->state == Active);
[ae7be7a]	190	proc_cor->state = Blocked;
[e8e457e]	191
[9f575ea]	192	// Actually run the thread
[3381ed7]	193	RUNNING: while(true) {
[ff79d5e]	194	thrd_dst->preempted = __NO_PREEMPTION;
	195	thrd_dst->state = Active;
[e8e457e]	196
[ae66348]	197	__cfaabi_dbg_debug_do(
[276ae57e]	198	thrd_dst->park_stale = true;
	199	thrd_dst->unpark_stale = true;
[ae66348]	200	)
[75f3522]	201
[3381ed7]	202	/* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
[a7b486b]	203	/* paranoid */ verify( kernelTLS.this_thread == thrd_dst );
[210b8b3]	204	/* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) < ((uintptr_t)__get_stack(thrd_dst->curr_cor)->base ) \|\| thrd_dst->curr_cor == proc_cor, "ERROR : Destination $thread %p has been corrupted.\n StackPointer too small.\n", thrd_dst ); // add escape condition if we are setting up the processor
	205	/* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) > ((uintptr_t)__get_stack(thrd_dst->curr_cor)->limit) \|\| thrd_dst->curr_cor == proc_cor, "ERROR : Destination $thread %p has been corrupted.\n StackPointer too large.\n", thrd_dst ); // add escape condition if we are setting up the processor
[3381ed7]	206
[9f575ea]	207	// set context switch to the thread that the processor is executing
	208	verify( thrd_dst->context.SP );
[c7a900a]	209	__cfactx_switch( &proc_cor->context, &thrd_dst->context );
	210	// when __cfactx_switch returns we are back in the processor coroutine
[9f575ea]	211
[210b8b3]	212	/* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) > ((uintptr_t)__get_stack(thrd_dst->curr_cor)->limit), "ERROR : Destination $thread %p has been corrupted.\n StackPointer too large.\n", thrd_dst );
	213	/* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) < ((uintptr_t)__get_stack(thrd_dst->curr_cor)->base ), "ERROR : Destination $thread %p has been corrupted.\n StackPointer too small.\n", thrd_dst );
[a7b486b]	214	/* paranoid */ verify( kernelTLS.this_thread == thrd_dst );
[3381ed7]	215	/* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
[75f3522]	216
[3381ed7]	217
	218	// We just finished running a thread, there are a few things that could have happened.
	219	// 1 - Regular case : the thread has blocked and now one has scheduled it yet.
	220	// 2 - Racy case : the thread has blocked but someone has already tried to schedule it.
	221	// 4 - Preempted
	222	// In case 1, we may have won a race so we can't write to the state again.
	223	// In case 2, we lost the race so we now own the thread.
	224
	225	if(unlikely(thrd_dst->preempted != __NO_PREEMPTION)) {
	226	// The thread was preempted, reschedule it and reset the flag
[9b1dcc2]	227	__schedule_thread( (__processor_id_t*)this, thrd_dst );
[3381ed7]	228	break RUNNING;
	229	}
[75f3522]	230
[ff79d5e]	231	if(unlikely(thrd_dst->state == Halted)) {
	232	// The thread has halted, it should never be scheduled/run again
	233	// We may need to wake someone up here since
	234	unpark( this->destroyer __cfaabi_dbg_ctx2 );
	235	this->destroyer = 0p;
	236	break RUNNING;
	237	}
	238
	239	/* paranoid */ verify( thrd_dst->state == Active );
	240	thrd_dst->state = Blocked;
	241
[3381ed7]	242	// set state of processor coroutine to active and the thread to inactive
[ff79d5e]	243	int old_ticket = __atomic_fetch_sub(&thrd_dst->ticket, 1, __ATOMIC_SEQ_CST);
	244	__cfaabi_dbg_debug_do( thrd_dst->park_result = old_ticket; )
	245	switch(old_ticket) {
	246	case 1:
[3381ed7]	247	// This is case 1, the regular case, nothing more is needed
	248	break RUNNING;
[ff79d5e]	249	case 2:
[3381ed7]	250	// This is case 2, the racy case, someone tried to run this thread before it finished blocking
	251	// In this case, just run it again.
	252	continue RUNNING;
	253	default:
	254	// This makes no sense, something is wrong abort
[ff79d5e]	255	abort();
[3381ed7]	256	}
[9f575ea]	257	}
[e8e457e]	258
[9f575ea]	259	// Just before returning to the processor, set the processor coroutine to active
[e8e457e]	260	proc_cor->state = Active;
[ae7be7a]	261	kernelTLS.this_thread = 0p;
[82c948c]	262	}
	263
[14a61b5]	264	// KERNEL_ONLY
[b0c7419]	265	void returnToKernel() {
[3381ed7]	266	/* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
[ac2b598]	267	$coroutine * proc_cor = get_coroutine(kernelTLS.this_processor->runner);
	268	$thread * thrd_src = kernelTLS.this_thread;
[e8e457e]	269
[29cb302]	270	#if !defined(__CFA_NO_STATISTICS__)
	271	struct processor * last_proc = kernelTLS.this_processor;
	272	#endif
	273
[9f575ea]	274	// Run the thread on this processor
	275	{
	276	int local_errno = *__volatile_errno();
	277	#if defined( __i386 ) \|\| defined( __x86_64 )
	278	__x87_store;
	279	#endif
	280	verify( proc_cor->context.SP );
[c7a900a]	281	__cfactx_switch( &thrd_src->context, &proc_cor->context );
[9f575ea]	282	#if defined( __i386 ) \|\| defined( __x86_64 )
	283	__x87_load;
	284	#endif
	285	*__volatile_errno() = local_errno;
[8fcbb4c]	286	}
[deca0f5]	287
[29cb302]	288	#if !defined(__CFA_NO_STATISTICS__)
	289	if(last_proc != kernelTLS.this_processor) {
	290	__tls_stats()->ready.threads.migration++;
	291	}
	292	#endif
	293
[3381ed7]	294	/* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
[210b8b3]	295	/* paranoid */ verifyf( ((uintptr_t)thrd_src->context.SP) < ((uintptr_t)__get_stack(thrd_src->curr_cor)->base ), "ERROR : Returning $thread %p has been corrupted.\n StackPointer too small.\n", thrd_src );
	296	/* paranoid */ verifyf( ((uintptr_t)thrd_src->context.SP) > ((uintptr_t)__get_stack(thrd_src->curr_cor)->limit), "ERROR : Returning $thread %p has been corrupted.\n StackPointer too large.\n", thrd_src );
[c84e80a]	297	}
	298
[8def349]	299	//-----------------------------------------------------------------------------
	300	// Scheduler routines
[14a61b5]	301	// KERNEL ONLY
[9b1dcc2]	302	void __schedule_thread( struct __processor_id_t * id, $thread * thrd ) {
[6a490b2]	303	/* paranoid */ verify( thrd );
	304	/* paranoid */ verify( thrd->state != Halted );
[9f575ea]	305	/* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
[3381ed7]	306	/* paranoid */ #if defined( __CFA_WITH_VERIFY__ )
[504a7dc]	307	/* paranoid */ if( thrd->state == Blocked \|\| thrd->state == Start ) assertf( thrd->preempted == __NO_PREEMPTION,
	308	"Error inactive thread marked as preempted, state %d, preemption %d\n", thrd->state, thrd->preempted );
[ff79d5e]	309	/* paranoid */ if( thrd->preempted != __NO_PREEMPTION ) assertf(thrd->state == Active,
[504a7dc]	310	"Error preempted thread marked as not currently running, state %d, preemption %d\n", thrd->state, thrd->preempted );
[3381ed7]	311	/* paranoid */ #endif
[6a490b2]	312	/* paranoid */ verifyf( thrd->link.next == 0p, "Expected null got %p", thrd->link.next );
[9f575ea]	313
[ae7be7a]	314	if (thrd->preempted == __NO_PREEMPTION) thrd->state = Ready;
[6b4cdd3]	315
[9b1dcc2]	316	ready_schedule_lock ( id );
[504a7dc]	317	push( thrd->curr_cluster, thrd );
[b798713]	318
[68f36f4]	319	#if !defined(__CFA_NO_STATISTICS__)
	320	bool woke =
	321	#endif
	322	__wake_one(id, thrd->curr_cluster);
	323
	324	#if !defined(__CFA_NO_STATISTICS__)
	325	if(woke) __tls_stats()->ready.sleep.wakes++;
	326	#endif
[9b1dcc2]	327	ready_schedule_unlock( id );
[1c273d0]	328
[9f575ea]	329	/* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
[db6f06a]	330	}
	331
[14a61b5]	332	// KERNEL ONLY
[ac2b598]	333	static $thread * __next_thread(cluster * this) with( *this ) {
[3381ed7]	334	/* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
[7768b8d]	335
[9b1dcc2]	336	ready_schedule_lock ( (__processor_id_t*)kernelTLS.this_processor );
[6a490b2]	337	$thread * head = pop( this );
[9b1dcc2]	338	ready_schedule_unlock( (__processor_id_t*)kernelTLS.this_processor );
[7768b8d]	339
[3381ed7]	340	/* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
[db6f06a]	341	return head;
[eb2e723]	342	}
	343
[64a7146]	344	// KERNEL ONLY
	345	static bool __has_next_thread(cluster * this) with( *this ) {
	346	/* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
	347
	348	ready_schedule_lock ( (__processor_id_t*)kernelTLS.this_processor );
	349	bool not_empty = query( this );
	350	ready_schedule_unlock( (__processor_id_t*)kernelTLS.this_processor );
	351
	352	/* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
	353	return not_empty;
	354	}
	355
[2d8f7b0]	356	// KERNEL ONLY unpark with out disabling interrupts
[9b1dcc2]	357	void __unpark( struct __processor_id_t * id, $thread * thrd __cfaabi_dbg_ctx_param2 ) {
[ae66348]	358	// record activity
	359	__cfaabi_dbg_record_thrd( *thrd, false, caller );
	360
[ff79d5e]	361	int old_ticket = __atomic_fetch_add(&thrd->ticket, 1, __ATOMIC_SEQ_CST);
	362	__cfaabi_dbg_debug_do( thrd->unpark_result = old_ticket; thrd->unpark_state = thrd->state; )
	363	switch(old_ticket) {
	364	case 1:
[3381ed7]	365	// Wake won the race, the thread will reschedule/rerun itself
	366	break;
[ff79d5e]	367	case 0:
[3381ed7]	368	/* paranoid */ verify( ! thrd->preempted != __NO_PREEMPTION );
[ff79d5e]	369	/* paranoid */ verify( thrd->state == Blocked );
[0b33412]	370
[3381ed7]	371	// Wake lost the race,
[9b1dcc2]	372	__schedule_thread( id, thrd );
[3381ed7]	373	break;
	374	default:
	375	// This makes no sense, something is wrong abort
	376	abort();
[de6319f]	377	}
[db6f06a]	378	}
	379
[2d8f7b0]	380	void unpark( $thread * thrd __cfaabi_dbg_ctx_param2 ) {
	381	if( !thrd ) return;
[db6f06a]	382
[82ff5845]	383	disable_interrupts();
[9b1dcc2]	384	__unpark( (__processor_id_t*)kernelTLS.this_processor, thrd __cfaabi_dbg_ctx_fwd2 );
[36982fc]	385	enable_interrupts( __cfaabi_dbg_ctx );
[eb2e723]	386	}
	387
[ae66348]	388	void park( __cfaabi_dbg_ctx_param ) {
[3381ed7]	389	/* paranoid */ verify( kernelTLS.preemption_state.enabled );
[82ff5845]	390	disable_interrupts();
[3381ed7]	391	/* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
	392	/* paranoid */ verify( kernelTLS.this_thread->preempted == __NO_PREEMPTION );
[0b33412]	393
[ae66348]	394	// record activity
	395	__cfaabi_dbg_record_thrd( *kernelTLS.this_thread, true, caller );
[0b33412]	396
[82c948c]	397	returnToKernel();
[0b33412]	398
[3381ed7]	399	/* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
[36982fc]	400	enable_interrupts( __cfaabi_dbg_ctx );
[3381ed7]	401	/* paranoid */ verify( kernelTLS.preemption_state.enabled );
[0c78741]	402
	403	}
[09800e9]	404
[3381ed7]	405	// KERNEL ONLY
[b0c7419]	406	void __leave_thread() {
[3381ed7]	407	/* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
[09800e9]	408	returnToKernel();
[b0c7419]	409	abort();
[09800e9]	410	}
	411
[14a61b5]	412	// KERNEL ONLY
[3381ed7]	413	bool force_yield( __Preemption_Reason reason ) {
	414	/* paranoid */ verify( kernelTLS.preemption_state.enabled );
[09800e9]	415	disable_interrupts();
[3381ed7]	416	/* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
[09800e9]	417
[ac2b598]	418	$thread * thrd = kernelTLS.this_thread;
[ff79d5e]	419	/* paranoid */ verify(thrd->state == Active);
[3381ed7]	420
	421	// SKULLDUGGERY: It is possible that we are preempting this thread just before
	422	// it was going to park itself. If that is the case and it is already using the
	423	// intrusive fields then we can't use them to preempt the thread
	424	// If that is the case, abandon the preemption.
	425	bool preempted = false;
[504a7dc]	426	if(thrd->link.next == 0p) {
[3381ed7]	427	preempted = true;
	428	thrd->preempted = reason;
	429	returnToKernel();
[de6319f]	430	}
[f2b12406]	431
[3381ed7]	432	/* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
	433	enable_interrupts_noPoll();
	434	/* paranoid */ verify( kernelTLS.preemption_state.enabled );
[f2b12406]	435
[3381ed7]	436	return preempted;
[f2b12406]	437	}
	438
[14a61b5]	439	//=============================================================================================
[92e7631]	440	// Kernel Idle Sleep
[14a61b5]	441	//=============================================================================================
[64a7146]	442	// Wake a thread from the front if there are any
	443	static bool __wake_one(struct __processor_id_t * id, cluster * this) {
	444	/* paranoid */ verify( ready_schedule_islocked( id ) );
[14a61b5]	445
[64a7146]	446	// Check if there is a sleeping processor
	447	processor * p = pop(this->idles);
[14a61b5]	448
[64a7146]	449	// If no one is sleeping, we are done
	450	if( 0p == p ) return false;
[14a61b5]	451
[64a7146]	452	// We found a processor, wake it up
	453	post( p->idle );
[14a61b5]	454
[64a7146]	455	return true;
	456	}
[14a61b5]	457
[64a7146]	458	// Unconditionnaly wake a thread
[e660761]	459	bool __wake_proc(processor * this) {
[64a7146]	460	__cfadbg_print_safe(runtime_core, "Kernel : waking Processor %p\n", this);
[14a61b5]	461
[64a7146]	462	disable_interrupts();
	463	/* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
	464	bool ret = post( this->idle );
	465	enable_interrupts( __cfaabi_dbg_ctx );
[92e7631]	466
[64a7146]	467	return ret;
[92e7631]	468	}
	469
[64a7146]	470	static void __halt(processor * this) with( *this ) {
	471	if( do_terminate ) return;
[8e16177]	472
[68f36f4]	473	#if !defined(__CFA_NO_STATISTICS__)
	474	__tls_stats()->ready.sleep.halts++;
	475	#endif
[64a7146]	476	// Push self to queue
	477	push(cltr->idles, *this);
[8834751]	478
[64a7146]	479	// Makre sure we don't miss a thread
	480	if( __has_next_thread(cltr) ) {
	481	// A thread was posted, make sure a processor is woken up
	482	struct __processor_id_t id = (struct __processor_id_t ) this;
	483	ready_schedule_lock ( id );
	484	__wake_one( id, cltr );
	485	ready_schedule_unlock( id );
[68f36f4]	486	#if !defined(__CFA_NO_STATISTICS__)
	487	__tls_stats()->ready.sleep.cancels++;
	488	#endif
[64a7146]	489	}
[8e16177]	490
[c34ebf2]	491	#if !defined(__CFA_NO_STATISTICS__)
	492	if(this->print_halts) {
[f2b18d01]	493	__cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 0\n", this->id, rdtscl());
[c34ebf2]	494	}
	495	#endif
	496
[64a7146]	497	wait( idle );
[c34ebf2]	498
	499	#if !defined(__CFA_NO_STATISTICS__)
	500	if(this->print_halts) {
[f2b18d01]	501	__cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 1\n", this->id, rdtscl());
[c34ebf2]	502	}
	503	#endif
[6b4cdd3]	504	}
	505
[dbe9b08]	506	//=============================================================================================
	507	// Unexpected Terminating logic
	508	//=============================================================================================
[ea7d2b0]	509	static __spinlock_t kernel_abort_lock;
[9d944b2]	510	static bool kernel_abort_called = false;
	511
[afd550c]	512	void * kernel_abort(void) __attribute__ ((__nothrow__)) {
[9d944b2]	513	// abort cannot be recursively entered by the same or different processors because all signal handlers return when
	514	// the globalAbort flag is true.
[36982fc]	515	lock( kernel_abort_lock __cfaabi_dbg_ctx2 );
[9d944b2]	516
	517	// first task to abort ?
[de94a60]	518	if ( kernel_abort_called ) { // not first task to abort ?
[ea7d2b0]	519	unlock( kernel_abort_lock );
[1c273d0]	520
[9d944b2]	521	sigset_t mask;
	522	sigemptyset( &mask );
[de94a60]	523	sigaddset( &mask, SIGALRM ); // block SIGALRM signals
[8a13c47]	524	sigaddset( &mask, SIGUSR1 ); // block SIGALRM signals
	525	sigsuspend( &mask ); // block the processor to prevent further damage during abort
	526	_exit( EXIT_FAILURE ); // if processor unblocks before it is killed, terminate it
[de94a60]	527	}
	528	else {
	529	kernel_abort_called = true;
	530	unlock( kernel_abort_lock );
[9d944b2]	531	}
	532
[14a61b5]	533	return kernelTLS.this_thread;
[9d944b2]	534	}
	535
	536	void kernel_abort_msg( void * kernel_data, char * abort_text, int abort_text_size ) {
[b81fd95]	537	$thread * thrd = ( $thread * ) kernel_data;
[9d944b2]	538
[de94a60]	539	if(thrd) {
	540	int len = snprintf( abort_text, abort_text_size, "Error occurred while executing thread %.256s (%p)", thrd->self_cor.name, thrd );
[1c40091]	541	__cfaabi_bits_write( STDERR_FILENO, abort_text, len );
[de94a60]	542
[212c2187]	543	if ( &thrd->self_cor != thrd->curr_cor ) {
	544	len = snprintf( abort_text, abort_text_size, " in coroutine %.256s (%p).\n", thrd->curr_cor->name, thrd->curr_cor );
[1c40091]	545	__cfaabi_bits_write( STDERR_FILENO, abort_text, len );
[de94a60]	546	}
	547	else {
[1c40091]	548	__cfaabi_bits_write( STDERR_FILENO, ".\n", 2 );
[de94a60]	549	}
[1c273d0]	550	}
[9d944b2]	551	else {
[de94a60]	552	int len = snprintf( abort_text, abort_text_size, "Error occurred outside of any thread.\n" );
[1c40091]	553	__cfaabi_bits_write( STDERR_FILENO, abort_text, len );
[9d944b2]	554	}
	555	}
	556
[2b8bc41]	557	int kernel_abort_lastframe( void ) __attribute__ ((__nothrow__)) {
[14a61b5]	558	return get_coroutine(kernelTLS.this_thread) == get_coroutine(mainThread) ? 4 : 2;
[2b8bc41]	559	}
	560
[de94a60]	561	static __spinlock_t kernel_debug_lock;
	562
[9d944b2]	563	extern "C" {
[1c40091]	564	void __cfaabi_bits_acquire() {
[36982fc]	565	lock( kernel_debug_lock __cfaabi_dbg_ctx2 );
[9d944b2]	566	}
	567
[1c40091]	568	void __cfaabi_bits_release() {
[ea7d2b0]	569	unlock( kernel_debug_lock );
[9d944b2]	570	}
[8118303]	571	}
	572
[fa21ac9]	573	//=============================================================================================
	574	// Kernel Utilities
	575	//=============================================================================================
[bd98b58]	576	//-----------------------------------------------------------------------------
	577	// Locks
[242a902]	578	void ?{}( semaphore & this, int count = 1 ) {
	579	(this.lock){};
	580	this.count = count;
	581	(this.waiting){};
[db6f06a]	582	}
[242a902]	583	void ^?{}(semaphore & this) {}
[db6f06a]	584
[71c8b7e]	585	bool P(semaphore & this) with( this ){
[65deb18]	586	lock( lock __cfaabi_dbg_ctx2 );
	587	count -= 1;
	588	if ( count < 0 ) {
[bdeba0b]	589	// queue current task
[14a61b5]	590	append( waiting, kernelTLS.this_thread );
[bdeba0b]	591
	592	// atomically release spin lock and block
[3381ed7]	593	unlock( lock );
[ae66348]	594	park( __cfaabi_dbg_ctx );
[71c8b7e]	595	return true;
[8def349]	596	}
[4e6fb8e]	597	else {
[65deb18]	598	unlock( lock );
[71c8b7e]	599	return false;
[4e6fb8e]	600	}
[bd98b58]	601	}
	602
[f0ce5f4]	603	bool V(semaphore & this) with( this ) {
[ac2b598]	604	$thread * thrd = 0p;
[65deb18]	605	lock( lock __cfaabi_dbg_ctx2 );
	606	count += 1;
	607	if ( count <= 0 ) {
[bdeba0b]	608	// remove task at head of waiting list
[65deb18]	609	thrd = pop_head( waiting );
[bd98b58]	610	}
[bdeba0b]	611
[65deb18]	612	unlock( lock );
[bdeba0b]	613
	614	// make new owner
[ae66348]	615	unpark( thrd __cfaabi_dbg_ctx2 );
[f0ce5f4]	616
[d384787]	617	return thrd != 0p;
	618	}
	619
	620	bool V(semaphore & this, unsigned diff) with( this ) {
	621	$thread * thrd = 0p;
	622	lock( lock __cfaabi_dbg_ctx2 );
	623	int release = max(-count, (int)diff);
	624	count += diff;
	625	for(release) {
	626	unpark( pop_head( waiting ) __cfaabi_dbg_ctx2 );
	627	}
	628
	629	unlock( lock );
	630
[f0ce5f4]	631	return thrd != 0p;
[bd98b58]	632	}
	633
[de94a60]	634	//-----------------------------------------------------------------------------
	635	// Debug
	636	__cfaabi_dbg_debug_do(
[1997b4e]	637	extern "C" {
[ae66348]	638	void __cfaabi_dbg_record_lock(__spinlock_t & this, const char prev_name[]) {
[1997b4e]	639	this.prev_name = prev_name;
	640	this.prev_thrd = kernelTLS.this_thread;
	641	}
[ae66348]	642
	643	void __cfaabi_dbg_record_thrd($thread & this, bool park, const char prev_name[]) {
	644	if(park) {
[f0ce5f4]	645	this.park_caller = prev_name;
	646	this.park_stale = false;
[ae66348]	647	}
	648	else {
[f0ce5f4]	649	this.unpark_caller = prev_name;
	650	this.unpark_stale = false;
[ae66348]	651	}
	652	}
[9181f1d]	653	}
[f7d6bb0]	654	)
[2026bb6]	655
	656	//-----------------------------------------------------------------------------
	657	// Debug
[8c50aed]	658	bool threading_enabled(void) __attribute__((const)) {
[2026bb6]	659	return true;
	660	}
[c34ebf2]	661
	662	//-----------------------------------------------------------------------------
	663	// Statistics
	664	#if !defined(__CFA_NO_STATISTICS__)
	665	void print_halts( processor & this ) {
	666	this.print_halts = true;
	667	}
	668	#endif
[8118303]	669	// Local Variables: //
	670	// mode: c //
	671	// tab-width: 4 //
	672	// End: //

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

Original Format