Context Navigation

startup.cfa@ dcbfcbc

Visit:

ADT ast-experimental enum forall-pointer-decay jacob/cs343-translation new-ast-unique-expr pthread-emulation qualifiedEnum

Last change on this file since dcbfcbc was 12daa43, checked in by Thierry Delisle <tdelisle@…>, 4 years ago
Added a define switch for using cpu workstealing. Not Fully implemented.
Property mode set to `100644`
File size: 27.5 KB

Rev	Line
[e660761]	1	//
	2	// Cforall Version 1.0.0 Copyright (C) 2020 University of Waterloo
	3	//
	4	// The contents of this file are covered under the licence agreement in the
	5	// file "LICENCE" distributed with Cforall.
	6	//
	7	// kernel/startup.cfa --
	8	//
	9	// Author : Thierry Delisle
	10	// Created On : Thu Jul 30 15:12:54 2020
	11	// Last Modified By :
	12	// Last Modified On :
	13	// Update Count :
	14	//
	15
	16	#define __cforall_thread__
[43784ac]	17	#define _GNU_SOURCE
[e660761]	18
	19	// C Includes
	20	#include <errno.h> // errno
[f558b5f]	21	#include <signal.h>
[e660761]	22	#include <string.h> // strerror
	23	#include <unistd.h> // sysconf
[f558b5f]	24
[e660761]	25	extern "C" {
	26	#include <limits.h> // PTHREAD_STACK_MIN
[f558b5f]	27	#include <unistd.h> // syscall
[dddb3dd0]	28	#include <sys/eventfd.h> // eventfd
[e660761]	29	#include <sys/mman.h> // mprotect
	30	#include <sys/resource.h> // getrlimit
[f558b5f]	31	#include <sys/syscall.h> // __NR_xxx
[e660761]	32	}
	33
	34	// CFA Includes
	35	#include "kernel_private.hfa"
	36	#include "startup.hfa" // STARTUP_PRIORITY_XXX
[bfcf6b9]	37	#include "math.hfa"
[e660761]	38
[97229d6]	39	#define CFA_PROCESSOR_USE_MMAP 0
	40
[e660761]	41	//-----------------------------------------------------------------------------
	42	// Some assembly required
	43	#if defined( __i386 )
[88cafe7]	44	#define CtxGet( ctx ) __asm__ volatile ( \
	45	"movl %%esp,%0\n" \
	46	"movl %%ebp,%1\n" \
	47	: "=rm" (ctx.SP), \
	48	"=rm" (ctx.FP) \
	49	)
[e660761]	50	#elif defined( __x86_64 )
[88cafe7]	51	#define CtxGet( ctx ) __asm__ volatile ( \
	52	"movq %%rsp,%0\n" \
	53	"movq %%rbp,%1\n" \
	54	: "=rm" (ctx.SP), \
	55	"=rm" (ctx.FP) \
	56	)
	57	#elif defined( __aarch64__ )
	58	#define CtxGet( ctx ) __asm__ volatile ( \
	59	"mov %0, sp\n" \
	60	"mov %1, fp\n" \
	61	: "=rm" (ctx.SP), \
	62	"=rm" (ctx.FP) \
	63	)
[e660761]	64	#else
	65	#error unknown hardware architecture
	66	#endif
	67
	68	//-----------------------------------------------------------------------------
	69	// Start and stop routine for the kernel, declared first to make sure they run first
	70	static void __kernel_startup (void) __attribute__(( constructor( STARTUP_PRIORITY_KERNEL ) ));
	71	static void __kernel_shutdown(void) __attribute__(( destructor ( STARTUP_PRIORITY_KERNEL ) ));
	72
	73	//-----------------------------------------------------------------------------
	74	// Static Forward Declarations
	75	struct current_stack_info_t;
	76
	77	static void * __invoke_processor(void * arg);
	78	static void __kernel_first_resume( processor * this );
	79	static void __kernel_last_resume ( processor * this );
[a5e7233]	80	static void init(processor & this, const char name[], cluster & _cltr, $thread * initT);
[e660761]	81	static void deinit(processor & this);
	82	static void doregister( struct cluster & cltr );
	83	static void unregister( struct cluster & cltr );
[c993b15]	84	static void register_tls( processor * this );
	85	static void unregister_tls( processor * this );
[e660761]	86	static void ?{}( $coroutine & this, current_stack_info_t * info);
	87	static void ?{}( $thread & this, current_stack_info_t * info);
	88	static void ?{}(processorCtx_t & this) {}
	89	static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info);
	90
[f2384c9a]	91	#if defined(__CFA_WITH_VERIFY__)
	92	static bool verify_fwd_bck_rng(void);
	93	#endif
	94
[e660761]	95	//-----------------------------------------------------------------------------
	96	// Forward Declarations for other modules
	97	extern void __kernel_alarm_startup(void);
	98	extern void __kernel_alarm_shutdown(void);
	99
	100	//-----------------------------------------------------------------------------
	101	// Other Forward Declarations
[1eb239e4]	102	extern void __wake_proc(processor *);
[e660761]	103
[f558b5f]	104	#if defined(CFA_HAVE_LINUX_LIBRSEQ)
	105	// No forward declaration needed
	106	#define __kernel_rseq_register rseq_register_current_thread
	107	#define __kernel_rseq_unregister rseq_unregister_current_thread
	108	#elif defined(CFA_HAVE_LINUX_RSEQ_H)
	109	void __kernel_raw_rseq_register (void);
	110	void __kernel_raw_rseq_unregister(void);
	111
	112	#define __kernel_rseq_register __kernel_raw_rseq_register
	113	#define __kernel_rseq_unregister __kernel_raw_rseq_unregister
	114	#else
	115	// No forward declaration needed
	116	// No initialization needed
[fde879b3]	117	static inline void noop(void) {}
[f558b5f]	118
	119	#define __kernel_rseq_register noop
	120	#define __kernel_rseq_unregister noop
	121	#endif
	122
[e660761]	123	//-----------------------------------------------------------------------------
	124	// Kernel storage
	125	KERNEL_STORAGE(cluster, mainCluster);
	126	KERNEL_STORAGE(processor, mainProcessor);
	127	KERNEL_STORAGE($thread, mainThread);
	128	KERNEL_STORAGE(__stack_t, mainThreadCtx);
	129	KERNEL_STORAGE(__scheduler_RWLock_t, __scheduler_lock);
	130	#if !defined(__CFA_NO_STATISTICS__)
	131	KERNEL_STORAGE(__stats_t, mainProcStats);
	132	#endif
	133
	134	cluster * mainCluster;
	135	processor * mainProcessor;
	136	$thread * mainThread;
	137	__scheduler_RWLock_t * __scheduler_lock;
	138
	139	extern "C" {
	140	struct { __dllist_t(cluster) list; __spinlock_t lock; } __cfa_dbg_global_clusters;
	141	}
	142
[dd92fe9]	143	extern size_t __page_size;
[28c35e2]	144	extern int __map_prot;
[e660761]	145
	146	//-----------------------------------------------------------------------------
	147	// Global state
[8fc652e0]	148	thread_local struct KernelThreadData __cfaabi_tls __attribute__ ((tls_model ( "initial-exec" ))) @= {
[e660761]	149	NULL, // cannot use 0p
	150	NULL,
[c993b15]	151	false,
[e660761]	152	{ 1, false, false },
[c993b15]	153	0,
	154	{ 0, 0 },
	155	NULL,
	156	#ifdef __CFA_WITH_VERIFY__
	157	false,
	158	0,
	159	#endif
[e660761]	160	};
	161
[3489ea6]	162	#if defined(CFA_HAVE_LINUX_LIBRSEQ)
	163	// No data needed
	164	#elif defined(CFA_HAVE_LINUX_RSEQ_H)
	165	extern "Cforall" {
[f558b5f]	166	__attribute__((aligned(128))) thread_local volatile struct rseq __cfaabi_rseq @= {
	167	.cpu_id : RSEQ_CPU_ID_UNINITIALIZED,
	168	};
[3489ea6]	169	}
	170	#else
	171	// No data needed
	172	#endif
	173
[e660761]	174	//-----------------------------------------------------------------------------
	175	// Struct to steal stack
	176	struct current_stack_info_t {
	177	__stack_t * storage; // pointer to stack object
	178	void * base; // base of stack
	179	void * limit; // stack grows towards stack limit
	180	void * context; // address of cfa_context_t
	181	};
	182
	183	void ?{}( current_stack_info_t & this ) {
	184	__stack_context_t ctx;
	185	CtxGet( ctx );
	186	this.base = ctx.FP;
	187
	188	rlimit r;
	189	getrlimit( RLIMIT_STACK, &r);
	190	size_t size = r.rlim_cur;
	191
	192	this.limit = (void *)(((intptr_t)this.base) - size);
	193	this.context = &storage_mainThreadCtx;
	194	}
	195
	196
	197
	198	//=============================================================================================
	199	// Kernel Setup logic
	200	//=============================================================================================
	201	//-----------------------------------------------------------------------------
	202	// Kernel boot procedures
	203	static void __kernel_startup(void) {
[8fc652e0]	204	/* paranoid */ verify( ! __preemption_enabled() );
[e660761]	205	__cfadbg_print_safe(runtime_core, "Kernel : Starting\n");
	206
	207	__cfa_dbg_global_clusters.list{ __get };
	208	__cfa_dbg_global_clusters.lock{};
	209
[f2384c9a]	210	/* paranoid */ verify( verify_fwd_bck_rng() );
	211
[e660761]	212	// Initialize the global scheduler lock
	213	__scheduler_lock = (__scheduler_RWLock_t*)&storage___scheduler_lock;
	214	(*__scheduler_lock){};
	215
	216	// Initialize the main cluster
	217	mainCluster = (cluster *)&storage_mainCluster;
	218	(*mainCluster){"Main Cluster", 0};
	219
	220	__cfadbg_print_safe(runtime_core, "Kernel : Main cluster ready\n");
	221
	222	// Start by initializing the main thread
	223	// SKULLDUGGERY: the mainThread steals the process main thread
	224	// which will then be scheduled by the mainProcessor normally
	225	mainThread = ($thread *)&storage_mainThread;
	226	current_stack_info_t info;
	227	info.storage = (__stack_t*)&storage_mainThreadCtx;
	228	(*mainThread){ &info };
	229
	230	__cfadbg_print_safe(runtime_core, "Kernel : Main thread ready\n");
	231
	232
	233
	234	// Construct the processor context of the main processor
	235	void ?{}(processorCtx_t & this, processor * proc) {
	236	(this.__cor){ "Processor" };
	237	this.__cor.starter = 0p;
	238	this.proc = proc;
	239	}
	240
	241	void ?{}(processor & this) with( this ) {
[454f478]	242	( this.terminated ){};
[e660761]	243	( this.runner ){};
[a5e7233]	244	init( this, "Main Processor", *mainCluster, 0p );
[e660761]	245	kernel_thread = pthread_self();
	246
	247	runner{ &this };
	248	__cfadbg_print_safe(runtime_core, "Kernel : constructed main processor context %p\n", &runner);
	249	}
	250
	251	// Initialize the main processor and the main processor ctx
	252	// (the coroutine that contains the processing control flow)
	253	mainProcessor = (processor *)&storage_mainProcessor;
	254	(*mainProcessor){};
	255
[c993b15]	256	register_tls( mainProcessor );
	257
[f558b5f]	258	__kernel_rseq_register();
	259
[e660761]	260	//initialize the global state variables
[8fc652e0]	261	__cfaabi_tls.this_processor = mainProcessor;
	262	__cfaabi_tls.this_thread = mainThread;
[e660761]	263
	264	#if !defined( __CFA_NO_STATISTICS__ )
[8fc652e0]	265	__cfaabi_tls.this_stats = (__stats_t *)& storage_mainProcStats;
	266	__init_stats( __cfaabi_tls.this_stats );
[e660761]	267	#endif
[a67c5b6]	268	mainProcessor->local_data = &__cfaabi_tls;
[e660761]	269
	270	// Enable preemption
	271	__kernel_alarm_startup();
	272
	273	// Add the main thread to the ready queue
	274	// once resume is called on mainProcessor->runner the mainThread needs to be scheduled like any normal thread
[254ad1b]	275	schedule_thread$(mainThread);
[e660761]	276
	277	// SKULLDUGGERY: Force a context switch to the main processor to set the main thread's context to the current UNIX
	278	// context. Hence, the main thread does not begin through __cfactx_invoke_thread, like all other threads. The trick here is that
	279	// mainThread is on the ready queue when this call is made.
[8fc652e0]	280	__kernel_first_resume( __cfaabi_tls.this_processor );
[e660761]	281
	282
	283	// THE SYSTEM IS NOW COMPLETELY RUNNING
	284
	285	__cfadbg_print_safe(runtime_core, "Kernel : Started\n--------------------------------------------------\n\n");
	286
[8fc652e0]	287	/* paranoid */ verify( ! __preemption_enabled() );
[a3821fa]	288	enable_interrupts();
[8fc652e0]	289	/* paranoid */ verify( __preemption_enabled() );
	290
[e660761]	291	}
	292
	293	static void __kernel_shutdown(void) {
[8fc652e0]	294	/* paranoid */ verify( __preemption_enabled() );
[e660761]	295	disable_interrupts();
[8fc652e0]	296	/* paranoid */ verify( ! __preemption_enabled() );
[e660761]	297
	298	__cfadbg_print_safe(runtime_core, "\n--------------------------------------------------\nKernel : Shutting down\n");
	299
	300	// SKULLDUGGERY: Notify the mainProcessor it needs to terminates.
	301	// When its coroutine terminates, it return control to the mainThread
	302	// which is currently here
	303	__atomic_store_n(&mainProcessor->do_terminate, true, __ATOMIC_RELEASE);
[8fc652e0]	304	__kernel_last_resume( __cfaabi_tls.this_processor );
[e660761]	305	mainThread->self_cor.state = Halted;
	306
	307	// THE SYSTEM IS NOW COMPLETELY STOPPED
	308
	309	// Disable preemption
	310	__kernel_alarm_shutdown();
	311
[a5a01faa]	312	#if !defined( __CFA_NO_STATISTICS__ )
	313	__stats_t * st = (__stats_t *)& storage_mainProcStats;
	314	__tally_stats(mainCluster->stats, st);
	315	if( 0 != mainProcessor->print_stats ) {
	316	__print_stats( st, mainProcessor->print_stats, "Processor ", mainProcessor->name, (void*)mainProcessor );
	317	}
[73f4d08]	318	#if defined(CFA_STATS_ARRAY)
	319	__flush_stat( st, "Processor", mainProcessor );
	320	#endif
[a5a01faa]	321	#endif
	322
[a67c5b6]	323	mainProcessor->local_data = 0p;
	324
[f558b5f]	325	__kernel_rseq_unregister();
	326
[c993b15]	327	unregister_tls( mainProcessor );
	328
[e660761]	329	// Destroy the main processor and its context in reverse order of construction
	330	// These were manually constructed so we need manually destroy them
	331	void ^?{}(processor & this) with( this ){
	332	deinit( this );
	333
	334	/* paranoid */ verify( this.do_terminate == true );
	335	__cfaabi_dbg_print_safe("Kernel : destroyed main processor context %p\n", &runner);
	336	}
	337
	338	^(*mainProcessor){};
	339
	340	// Final step, destroy the main thread since it is no longer needed
	341
	342	// Since we provided a stack to this taxk it will not destroy anything
	343	/* paranoid / verify(mainThread->self_cor.stack.storage == (__stack_t)(((uintptr_t)&storage_mainThreadCtx)\| 0x1));
	344	^(*mainThread){};
	345
	346	^(*mainCluster){};
	347
	348	^(*__scheduler_lock){};
	349
	350	^(__cfa_dbg_global_clusters.list){};
	351	^(__cfa_dbg_global_clusters.lock){};
	352
	353	__cfadbg_print_safe(runtime_core, "Kernel : Shutdown complete\n");
	354	}
	355
	356	//=============================================================================================
	357	// Kernel Initial Scheduling logic
	358	//=============================================================================================
	359
	360	// Context invoker for processors
	361	// This is the entry point for processors (kernel threads) except for the main processor
	362	// It effectively constructs a coroutine by stealing the pthread stack
	363	static void * __invoke_processor(void * arg) {
	364	#if !defined( __CFA_NO_STATISTICS__ )
	365	__stats_t local_stats;
	366	__init_stats( &local_stats );
[8fc652e0]	367	__cfaabi_tls.this_stats = &local_stats;
[e660761]	368	#endif
	369
	370	processor * proc = (processor *) arg;
[8fc652e0]	371	__cfaabi_tls.this_processor = proc;
	372	__cfaabi_tls.this_thread = 0p;
	373	__cfaabi_tls.preemption_state.[enabled, disable_count] = [false, 1];
[a67c5b6]	374	proc->local_data = &__cfaabi_tls;
[c993b15]	375
	376	register_tls( proc );
	377
[f558b5f]	378	__kernel_rseq_register();
[43784ac]	379
[e660761]	380	// SKULLDUGGERY: We want to create a context for the processor coroutine
	381	// which is needed for the 2-step context switch. However, there is no reason
	382	// to waste the perfectly valid stack create by pthread.
	383	current_stack_info_t info;
	384	__stack_t ctx;
	385	info.storage = &ctx;
	386	(proc->runner){ proc, &info };
	387
	388	__cfaabi_dbg_print_safe("Coroutine : created stack %p\n", get_coroutine(proc->runner)->stack.storage);
	389
	390	//Set global state
[8fc652e0]	391	__cfaabi_tls.this_thread = 0p;
[e660761]	392
	393	//We now have a proper context from which to schedule threads
	394	__cfadbg_print_safe(runtime_core, "Kernel : core %p created (%p, %p)\n", proc, &proc->runner, &ctx);
	395
	396	// SKULLDUGGERY: Since the coroutine doesn't have its own stack, we can't
	397	// resume it to start it like it normally would, it will just context switch
	398	// back to here. Instead directly call the main since we already are on the
	399	// appropriate stack.
	400	get_coroutine(proc->runner)->state = Active;
	401	main( proc->runner );
	402	get_coroutine(proc->runner)->state = Halted;
	403
	404	// Main routine of the core returned, the core is now fully terminated
	405	__cfadbg_print_safe(runtime_core, "Kernel : core %p main ended (%p)\n", proc, &proc->runner);
	406
	407	#if !defined(__CFA_NO_STATISTICS__)
	408	__tally_stats(proc->cltr->stats, &local_stats);
	409	if( 0 != proc->print_stats ) {
[1b033b8]	410	__print_stats( &local_stats, proc->print_stats, "Processor ", proc->name, (void*)proc );
[e660761]	411	}
[73f4d08]	412	#if defined(CFA_STATS_ARRAY)
	413	__flush_stat( &local_stats, "Processor", proc );
	414	#endif
[e660761]	415	#endif
	416
[a67c5b6]	417	proc->local_data = 0p;
	418
[f558b5f]	419	__kernel_rseq_unregister();
	420
[c993b15]	421	unregister_tls( proc );
	422
[e660761]	423	return 0p;
	424	}
	425
	426	static void __kernel_first_resume( processor * this ) {
	427	$thread * src = mainThread;
	428	$coroutine * dst = get_coroutine(this->runner);
	429
[8fc652e0]	430	/* paranoid */ verify( ! __preemption_enabled() );
[e660761]	431
[8fc652e0]	432	__cfaabi_tls.this_thread->curr_cor = dst;
[e660761]	433	__stack_prepare( &dst->stack, 65000 );
	434	__cfactx_start(main, dst, this->runner, __cfactx_invoke_coroutine);
	435
[8fc652e0]	436	/* paranoid */ verify( ! __preemption_enabled() );
[e660761]	437
	438	dst->last = &src->self_cor;
	439	dst->starter = dst->starter ? dst->starter : &src->self_cor;
	440
	441	// make sure the current state is still correct
	442	/* paranoid */ verify(src->state == Ready);
[ab5baab]	443	src->corctx_flag = true;
[e660761]	444
	445	// context switch to specified coroutine
	446	verify( dst->context.SP );
	447	__cfactx_switch( &src->context, &dst->context );
	448	// when __cfactx_switch returns we are back in the src coroutine
	449
	450	mainThread->curr_cor = &mainThread->self_cor;
	451
	452	// make sure the current state has been update
	453	/* paranoid */ verify(src->state == Active);
	454
[8fc652e0]	455	/* paranoid */ verify( ! __preemption_enabled() );
[e660761]	456	}
	457
	458	// KERNEL_ONLY
	459	static void __kernel_last_resume( processor * this ) {
	460	$coroutine * src = &mainThread->self_cor;
	461	$coroutine * dst = get_coroutine(this->runner);
	462
[8fc652e0]	463	/* paranoid */ verify( ! __preemption_enabled() );
	464	/* paranoid */ verify( dst->starter == src );
	465	/* paranoid */ verify( dst->context.SP );
[e660761]	466
	467	// SKULLDUGGERY in debug the processors check that the
	468	// stack is still within the limit of the stack limits after running a thread.
	469	// that check doesn't make sense if we context switch to the processor using the
	470	// coroutine semantics. Since this is a special case, use the current context
	471	// info to populate these fields.
	472	__cfaabi_dbg_debug_do(
	473	__stack_context_t ctx;
	474	CtxGet( ctx );
	475	mainThread->context.SP = ctx.SP;
	476	mainThread->context.FP = ctx.FP;
	477	)
	478
	479	// context switch to the processor
	480	__cfactx_switch( &src->context, &dst->context );
	481	}
	482
	483
	484	//=============================================================================================
	485	// Kernel Object Constructors logic
	486	//=============================================================================================
	487	//-----------------------------------------------------------------------------
	488	// Main thread construction
	489	static void ?{}( $coroutine & this, current_stack_info_t * info) with( this ) {
	490	stack.storage = info->storage;
	491	with(*stack.storage) {
	492	limit = info->limit;
	493	base = info->base;
	494	}
	495	__attribute__((may_alias)) intptr_t * istorage = (intptr_t*) &stack.storage;
	496	*istorage \|= 0x1;
	497	name = "Main Thread";
	498	state = Start;
	499	starter = 0p;
	500	last = 0p;
	501	cancellation = 0p;
	502	}
	503
	504	static void ?{}( $thread & this, current_stack_info_t * info) with( this ) {
[6a77224]	505	ticket = TICKET_RUNNING;
[e660761]	506	state = Start;
	507	self_cor{ info };
	508	curr_cor = &self_cor;
	509	curr_cluster = mainCluster;
	510	self_mon.owner = &this;
	511	self_mon.recursion = 1;
	512	self_mon_p = &self_mon;
	513	link.next = 0p;
[2b96031]	514	link.ts = 0;
[d3ba775]	515	preferred = -1u;
[89eff25]	516	last_proc = 0p;
[b4b63e8]	517	#if defined( __CFA_WITH_VERIFY__ )
[ac12f1f]	518	canary = 0x0D15EA5E0D15EA5Ep;
[b4b63e8]	519	#endif
[e660761]	520
	521	node.next = 0p;
	522	node.prev = 0p;
	523	doregister(curr_cluster, this);
	524
	525	monitors{ &self_mon_p, 1, (fptr_t)0 };
	526	}
	527
	528	//-----------------------------------------------------------------------------
	529	// Processor
	530	// Construct the processor context of non-main processors
	531	static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info) {
	532	(this.__cor){ info };
	533	this.proc = proc;
	534	}
	535
[a5e7233]	536	static void init(processor & this, const char name[], cluster & _cltr, $thread * initT) with( this ) {
[e660761]	537	this.name = name;
	538	this.cltr = &_cltr;
[431cd4f]	539	this.rdq.its = 0;
	540	this.rdq.itr = 0;
	541	this.rdq.id = -1u;
	542	this.rdq.target = -1u;
[12daa43]	543	this.rdq.last = -1u;
[3e1a705]	544	this.rdq.cutoff = 0ull;
[e660761]	545	do_terminate = false;
	546	preemption_alarm = 0p;
	547	pending_preemption = false;
	548
[78da4ab]	549	this.io.ctx = 0p;
[dddb3dd0]	550	this.io.pending = false;
	551	this.io.dirty = false;
	552
[a5e7233]	553	this.init.thrd = initT;
[a1538cd]	554
[a67c5b6]	555	this.local_data = 0p;
	556
[dddb3dd0]	557	this.idle = eventfd(0, 0);
	558	if (idle < 0) {
	559	abort("KERNEL ERROR: PROCESSOR EVENTFD - %s\n", strerror(errno));
	560	}
[78da4ab]	561
[e660761]	562	#if !defined(__CFA_NO_STATISTICS__)
	563	print_stats = 0;
	564	print_halts = false;
	565	#endif
	566
	567	__cfadbg_print_safe(runtime_core, "Kernel : core %p created\n", &this);
	568	}
	569
	570	// Not a ctor, it just preps the destruction but should not destroy members
	571	static void deinit(processor & this) {
[dddb3dd0]	572	close(this.idle);
[e660761]	573	}
	574
[a5e7233]	575	void ?{}(processor & this, const char name[], cluster & _cltr, $thread * initT) {
[454f478]	576	( this.terminated ){};
[e660761]	577	( this.runner ){};
[62502cc4]	578
	579	disable_interrupts();
[a5e7233]	580	init( this, name, _cltr, initT );
[a3821fa]	581	enable_interrupts();
[e660761]	582
	583	__cfadbg_print_safe(runtime_core, "Kernel : Starting core %p\n", &this);
	584
	585	this.stack = __create_pthread( &this.kernel_thread, __invoke_processor, (void *)&this );
[a5e7233]	586	}
[e660761]	587
[a5e7233]	588	void ?{}(processor & this, const char name[], cluster & _cltr) {
	589	(this){name, _cltr, 0p};
[e660761]	590	}
	591
[bfcf6b9]	592	extern size_t __page_size;
[e660761]	593	void ^?{}(processor & this) with( this ){
	594	if( ! __atomic_load_n(&do_terminate, __ATOMIC_ACQUIRE) ) {
	595	__cfadbg_print_safe(runtime_core, "Kernel : core %p signaling termination\n", &this);
	596
	597	__atomic_store_n(&do_terminate, true, __ATOMIC_RELAXED);
	598	__wake_proc( &this );
	599
[454f478]	600	wait( terminated );
[8fc652e0]	601	/* paranoid */ verify( active_processor() != &this);
[e660761]	602	}
	603
[bfcf6b9]	604	__destroy_pthread( kernel_thread, this.stack, 0p );
[e660761]	605
[62502cc4]	606	disable_interrupts();
	607	deinit( this );
[a3821fa]	608	enable_interrupts();
[e660761]	609	}
	610
	611	//-----------------------------------------------------------------------------
	612	// Cluster
[6a9b12b]	613	static void ?{}(__cluster_proc_list & this) {
[1eb239e4]	614	this.lock = 0;
	615	this.idle = 0;
	616	this.total = 0;
	617	}
	618
[e660761]	619	void ?{}(cluster & this, const char name[], Duration preemption_rate, unsigned num_io, const io_context_params & io_params) with( this ) {
	620	this.name = name;
	621	this.preemption_rate = preemption_rate;
	622	ready_queue{};
	623
	624	#if !defined(__CFA_NO_STATISTICS__)
	625	print_stats = 0;
	626	stats = alloc();
	627	__init_stats( stats );
	628	#endif
	629
	630	threads{ __get };
	631
[78da4ab]	632	io.arbiter = create();
	633	io.params = io_params;
	634
[e660761]	635	doregister(this);
	636
	637	// Lock the RWlock so no-one pushes/pops while we are changing the queue
[772411a]	638	disable_interrupts();
[e660761]	639	uint_fast32_t last_size = ready_mutate_lock();
	640
	641	// Adjust the ready queue size
[a017ee7]	642	ready_queue_grow( &this );
[e660761]	643
	644	// Unlock the RWlock
	645	ready_mutate_unlock( last_size );
[a3821fa]	646	enable_interrupts( false ); // Don't poll, could be in main cluster
[e660761]	647	}
	648
	649	void ^?{}(cluster & this) {
[78da4ab]	650	destroy(this.io.arbiter);
[e660761]	651
	652	// Lock the RWlock so no-one pushes/pops while we are changing the queue
[772411a]	653	disable_interrupts();
[e660761]	654	uint_fast32_t last_size = ready_mutate_lock();
	655
	656	// Adjust the ready queue size
[a017ee7]	657	ready_queue_shrink( &this );
[e660761]	658
	659	// Unlock the RWlock
	660	ready_mutate_unlock( last_size );
[a3821fa]	661	enable_interrupts( false ); // Don't poll, could be in main cluster
[e660761]	662
	663	#if !defined(__CFA_NO_STATISTICS__)
	664	if( 0 != this.print_stats ) {
[1b033b8]	665	__print_stats( this.stats, this.print_stats, "Cluster", this.name, (void*)&this );
[e660761]	666	}
[73f4d08]	667	#if defined(CFA_STATS_ARRAY)
	668	__flush_stat( this.stats, "Cluster", &this );
	669	#endif
[e660761]	670	free( this.stats );
	671	#endif
	672
	673	unregister(this);
	674	}
	675
	676	//=============================================================================================
	677	// Miscellaneous Initialization
	678	//=============================================================================================
	679	//-----------------------------------------------------------------------------
	680	// Global Queues
	681	static void doregister( cluster & cltr ) {
	682	lock ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
	683	push_front( __cfa_dbg_global_clusters.list, cltr );
	684	unlock ( __cfa_dbg_global_clusters.lock );
	685	}
	686
	687	static void unregister( cluster & cltr ) {
	688	lock ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
	689	remove( __cfa_dbg_global_clusters.list, cltr );
	690	unlock( __cfa_dbg_global_clusters.lock );
	691	}
	692
	693	void doregister( cluster * cltr, $thread & thrd ) {
	694	lock (cltr->thread_list_lock __cfaabi_dbg_ctx2);
	695	cltr->nthreads += 1;
	696	push_front(cltr->threads, thrd);
	697	unlock (cltr->thread_list_lock);
	698	}
	699
	700	void unregister( cluster * cltr, $thread & thrd ) {
	701	lock (cltr->thread_list_lock __cfaabi_dbg_ctx2);
	702	remove(cltr->threads, thrd );
	703	cltr->nthreads -= 1;
	704	unlock(cltr->thread_list_lock);
	705	}
	706
[c993b15]	707	static void register_tls( processor * this ) {
	708	// Register and Lock the RWlock so no-one pushes/pops while we are changing the queue
	709	uint_fast32_t last_size;
	710	[this->unique_id, last_size] = ready_mutate_register();
	711
	712	this->cltr->procs.total += 1u;
	713	insert_last(this->cltr->procs.actives, *this);
	714
	715	// Adjust the ready queue size
	716	ready_queue_grow( this->cltr );
	717
	718	// Unlock the RWlock
	719	ready_mutate_unlock( last_size );
	720	}
	721
	722
	723	static void unregister_tls( processor * this ) {
	724	// Lock the RWlock so no-one pushes/pops while we are changing the queue
	725	uint_fast32_t last_size = ready_mutate_lock();
	726	this->cltr->procs.total -= 1u;
	727	remove(*this);
	728
	729	// clear the cluster so nothing gets pushed to local queues
	730	cluster * cltr = this->cltr;
	731	this->cltr = 0p;
	732
	733	// Adjust the ready queue size
	734	ready_queue_shrink( cltr );
	735
	736	// Unlock the RWlock and unregister: we don't need the read_lock any more
	737	ready_mutate_unregister( this->unique_id, last_size );
	738	}
	739
[e660761]	740	static void check( int ret, const char func[] ) {
	741	if ( ret ) { // pthread routines return errno values
	742	abort( "%s : internal error, error(%d) %s.", func, ret, strerror( ret ) );
	743	} // if
	744	} // Abort
	745
	746	void * __create_pthread( pthread_t * pthread, void * (start)(void ), void * arg ) {
	747	pthread_attr_t attr;
	748
	749	check( pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
	750
	751	size_t stacksize;
	752	// default stack size, normally defined by shell limit
	753	check( pthread_attr_getstacksize( &attr, &stacksize ), "pthread_attr_getstacksize" );
	754	assert( stacksize >= PTHREAD_STACK_MIN );
	755
	756	void * stack;
[97229d6]	757	#if CFA_PROCESSOR_USE_MMAP
	758	stacksize = ceiling( stacksize, __page_size ) + __page_size;
[dd92fe9]	759	stack = mmap(0p, stacksize, __map_prot, MAP_PRIVATE \| MAP_ANONYMOUS, 0, 0);
[97229d6]	760	if(stack == ((void*)-1)) {
	761	abort( "pthread stack creation : internal error, mmap failure, error(%d) %s.", errno, strerror( errno ) );
	762	}
	763	if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
	764	abort( "pthread stack creation : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
	765	} // if
	766	#else
	767	__cfaabi_dbg_debug_do(
	768	stack = memalign( __page_size, stacksize + __page_size );
	769	// pthread has no mechanism to create the guard page in user supplied stack.
	770	if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
	771	abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
	772	} // if
	773	);
	774	__cfaabi_dbg_no_debug_do(
	775	stack = malloc( stacksize );
	776	);
	777	#endif
	778
[e660761]	779
	780	check( pthread_attr_setstack( &attr, stack, stacksize ), "pthread_attr_setstack" );
	781
	782	check( pthread_create( pthread, &attr, start, arg ), "pthread_create" );
	783	return stack;
[88cafe7]	784	}
[f2384c9a]	785
[bfcf6b9]	786	void __destroy_pthread( pthread_t pthread, void * stack, void ** retval ) {
	787	int err = pthread_join( pthread, retval );
	788	if( err != 0 ) abort("KERNEL ERROR: joining pthread %p caused error %s\n", (void*)pthread, strerror(err));
	789
[97229d6]	790	#if CFA_PROCESSOR_USE_MMAP
	791	pthread_attr_t attr;
[bfcf6b9]	792
[97229d6]	793	check( pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
[bfcf6b9]	794
[97229d6]	795	size_t stacksize;
	796	// default stack size, normally defined by shell limit
	797	check( pthread_attr_getstacksize( &attr, &stacksize ), "pthread_attr_getstacksize" );
	798	assert( stacksize >= PTHREAD_STACK_MIN );
	799	stacksize += __page_size;
[bfcf6b9]	800
[97229d6]	801	if(munmap(stack, stacksize) == -1) {
	802	abort( "pthread stack destruction : internal error, munmap failure, error(%d) %s.", errno, strerror( errno ) );
	803	}
	804	#else
[72a3aff]	805	__cfaabi_dbg_debug_do(
	806	// pthread has no mechanism to create the guard page in user supplied stack.
[28c35e2]	807	if ( mprotect( stack, __page_size, __map_prot ) == -1 ) {
[72a3aff]	808	abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
	809	} // if
	810	);
[97229d6]	811	free( stack );
	812	#endif
[bfcf6b9]	813	}
	814
[f2384c9a]	815	#if defined(__CFA_WITH_VERIFY__)
	816	static bool verify_fwd_bck_rng(void) {
[8fc652e0]	817	__cfaabi_tls.ready_rng.fwd_seed = 25214903917_l64u * (rdtscl() ^ (uintptr_t)&verify_fwd_bck_rng);
[f2384c9a]	818
	819	unsigned values[10];
	820	for(i; 10) {
	821	values[i] = __tls_rand_fwd();
	822	}
	823
	824	__tls_rand_advance_bck();
	825
	826	for ( i; 9 -~= 0 ) {
	827	if(values[i] != __tls_rand_bck()) {
	828	return false;
	829	}
	830	}
	831
	832	return true;
	833	}
[e67a82d]	834	#endif
[f558b5f]	835
	836	#if defined(CFA_HAVE_LINUX_LIBRSEQ)
	837	// No definition needed
	838	#elif defined(CFA_HAVE_LINUX_RSEQ_H)
	839
	840	#if defined( __x86_64 ) \|\| defined( __i386 )
	841	#define RSEQ_SIG 0x53053053
	842	#elif defined( __ARM_ARCH )
	843	#ifdef __ARMEB__
	844	#define RSEQ_SIG 0xf3def5e7 /* udf #24035 ; 0x5de3 (ARMv6+) */
	845	#else
	846	#define RSEQ_SIG 0xe7f5def3 /* udf #24035 ; 0x5de3 */
	847	#endif
	848	#endif
	849
	850	extern void __disable_interrupts_hard();
	851	extern void __enable_interrupts_hard();
	852
	853	void __kernel_raw_rseq_register (void) {
	854	/* paranoid */ verify( __cfaabi_rseq.cpu_id == RSEQ_CPU_ID_UNINITIALIZED );
	855
	856	// int ret = syscall(__NR_rseq, &__cfaabi_rseq, sizeof(struct rseq), 0, (sigset_t *)0p, _NSIG / 8);
	857	int ret = syscall(__NR_rseq, &__cfaabi_rseq, sizeof(struct rseq), 0, RSEQ_SIG);
	858	if(ret != 0) {
	859	int e = errno;
	860	switch(e) {
	861	case EINVAL: abort("KERNEL ERROR: rseq register invalid argument");
	862	case ENOSYS: abort("KERNEL ERROR: rseq register no supported");
	863	case EFAULT: abort("KERNEL ERROR: rseq register with invalid argument");
	864	case EBUSY : abort("KERNEL ERROR: rseq register already registered");
	865	case EPERM : abort("KERNEL ERROR: rseq register sig argument on unregistration does not match the signature received on registration");
	866	default: abort("KERNEL ERROR: rseq register unexpected return %d", e);
	867	}
	868	}
	869	}
	870
	871	void __kernel_raw_rseq_unregister(void) {
	872	/* paranoid */ verify( __cfaabi_rseq.cpu_id >= 0 );
	873
	874	// int ret = syscall(__NR_rseq, &__cfaabi_rseq, sizeof(struct rseq), RSEQ_FLAG_UNREGISTER, (sigset_t *)0p, _NSIG / 8);
	875	int ret = syscall(__NR_rseq, &__cfaabi_rseq, sizeof(struct rseq), RSEQ_FLAG_UNREGISTER, RSEQ_SIG);
	876	if(ret != 0) {
	877	int e = errno;
	878	switch(e) {
	879	case EINVAL: abort("KERNEL ERROR: rseq unregister invalid argument");
	880	case ENOSYS: abort("KERNEL ERROR: rseq unregister no supported");
	881	case EFAULT: abort("KERNEL ERROR: rseq unregister with invalid argument");
	882	case EBUSY : abort("KERNEL ERROR: rseq unregister already registered");
	883	case EPERM : abort("KERNEL ERROR: rseq unregister sig argument on unregistration does not match the signature received on registration");
	884	default: abort("KERNEL ERROR: rseq unregisteunexpected return %d", e);
	885	}
	886	}
	887	}
	888	#else
	889	// No definition needed
	890	#endif

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

Original Format