Context Navigation

startup.cfa@ eeb9f9f

Visit:

ADT arm-eh ast-experimental enum forall-pointer-decay jacob/cs343-translation new-ast-unique-expr pthread-emulation qualifiedEnum

Last change on this file since eeb9f9f was f6fdfb14, checked in by Thierry Delisle <tdelisle@…>, 4 years ago
Removed old sub-queue
Property mode set to `100644`
File size: 24.0 KB

Line
1	//
2	// Cforall Version 1.0.0 Copyright (C) 2020 University of Waterloo
3	//
4	// The contents of this file are covered under the licence agreement in the
5	// file "LICENCE" distributed with Cforall.
6	//
7	// kernel/startup.cfa --
8	//
9	// Author : Thierry Delisle
10	// Created On : Thu Jul 30 15:12:54 2020
11	// Last Modified By :
12	// Last Modified On :
13	// Update Count :
14	//
15
16	#define __cforall_thread__
17
18	// C Includes
19	#include <errno.h> // errno
20	#include <string.h> // strerror
21	#include <unistd.h> // sysconf
22	extern "C" {
23	#include <limits.h> // PTHREAD_STACK_MIN
24	#include <sys/eventfd.h> // eventfd
25	#include <sys/mman.h> // mprotect
26	#include <sys/resource.h> // getrlimit
27	}
28
29	// CFA Includes
30	#include "kernel_private.hfa"
31	#include "startup.hfa" // STARTUP_PRIORITY_XXX
32	#include "math.hfa"
33
34	#define CFA_PROCESSOR_USE_MMAP 0
35
36	//-----------------------------------------------------------------------------
37	// Some assembly required
38	#if defined( __i386 )
39	#define CtxGet( ctx ) __asm__ volatile ( \
40	"movl %%esp,%0\n" \
41	"movl %%ebp,%1\n" \
42	: "=rm" (ctx.SP), \
43	"=rm" (ctx.FP) \
44	)
45	#elif defined( __x86_64 )
46	#define CtxGet( ctx ) __asm__ volatile ( \
47	"movq %%rsp,%0\n" \
48	"movq %%rbp,%1\n" \
49	: "=rm" (ctx.SP), \
50	"=rm" (ctx.FP) \
51	)
52	#elif defined( __aarch64__ )
53	#define CtxGet( ctx ) __asm__ volatile ( \
54	"mov %0, sp\n" \
55	"mov %1, fp\n" \
56	: "=rm" (ctx.SP), \
57	"=rm" (ctx.FP) \
58	)
59	#else
60	#error unknown hardware architecture
61	#endif
62
63	//-----------------------------------------------------------------------------
64	// Start and stop routine for the kernel, declared first to make sure they run first
65	static void __kernel_startup (void) __attribute__(( constructor( STARTUP_PRIORITY_KERNEL ) ));
66	static void __kernel_shutdown(void) __attribute__(( destructor ( STARTUP_PRIORITY_KERNEL ) ));
67
68	//-----------------------------------------------------------------------------
69	// Static Forward Declarations
70	struct current_stack_info_t;
71
72	static void * __invoke_processor(void * arg);
73	static void __kernel_first_resume( processor * this );
74	static void __kernel_last_resume ( processor * this );
75	static void init(processor & this, const char name[], cluster & _cltr, $thread * initT);
76	static void deinit(processor & this);
77	static void doregister( struct cluster & cltr );
78	static void unregister( struct cluster & cltr );
79	static void register_tls( processor * this );
80	static void unregister_tls( processor * this );
81	static void ?{}( $coroutine & this, current_stack_info_t * info);
82	static void ?{}( $thread & this, current_stack_info_t * info);
83	static void ?{}(processorCtx_t & this) {}
84	static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info);
85
86	#if defined(__CFA_WITH_VERIFY__)
87	static bool verify_fwd_bck_rng(void);
88	#endif
89
90	//-----------------------------------------------------------------------------
91	// Forward Declarations for other modules
92	extern void __kernel_alarm_startup(void);
93	extern void __kernel_alarm_shutdown(void);
94
95	//-----------------------------------------------------------------------------
96	// Other Forward Declarations
97	extern void __wake_proc(processor *);
98
99	//-----------------------------------------------------------------------------
100	// Kernel storage
101	KERNEL_STORAGE(cluster, mainCluster);
102	KERNEL_STORAGE(processor, mainProcessor);
103	KERNEL_STORAGE($thread, mainThread);
104	KERNEL_STORAGE(__stack_t, mainThreadCtx);
105	KERNEL_STORAGE(__scheduler_RWLock_t, __scheduler_lock);
106	#if !defined(__CFA_NO_STATISTICS__)
107	KERNEL_STORAGE(__stats_t, mainProcStats);
108	#endif
109
110	cluster * mainCluster;
111	processor * mainProcessor;
112	$thread * mainThread;
113	__scheduler_RWLock_t * __scheduler_lock;
114
115	extern "C" {
116	struct { __dllist_t(cluster) list; __spinlock_t lock; } __cfa_dbg_global_clusters;
117	}
118
119	extern size_t __page_size;
120	extern int __map_prot;
121
122	//-----------------------------------------------------------------------------
123	// Global state
124	thread_local struct KernelThreadData __cfaabi_tls __attribute__ ((tls_model ( "initial-exec" ))) @= {
125	NULL, // cannot use 0p
126	NULL,
127	false,
128	{ 1, false, false },
129	0,
130	{ 0, 0 },
131	NULL,
132	#ifdef __CFA_WITH_VERIFY__
133	false,
134	0,
135	#endif
136	};
137
138	//-----------------------------------------------------------------------------
139	// Struct to steal stack
140	struct current_stack_info_t {
141	__stack_t * storage; // pointer to stack object
142	void * base; // base of stack
143	void * limit; // stack grows towards stack limit
144	void * context; // address of cfa_context_t
145	};
146
147	void ?{}( current_stack_info_t & this ) {
148	__stack_context_t ctx;
149	CtxGet( ctx );
150	this.base = ctx.FP;
151
152	rlimit r;
153	getrlimit( RLIMIT_STACK, &r);
154	size_t size = r.rlim_cur;
155
156	this.limit = (void *)(((intptr_t)this.base) - size);
157	this.context = &storage_mainThreadCtx;
158	}
159
160
161
162	//=============================================================================================
163	// Kernel Setup logic
164	//=============================================================================================
165	//-----------------------------------------------------------------------------
166	// Kernel boot procedures
167	static void __kernel_startup(void) {
168	/* paranoid */ verify( ! __preemption_enabled() );
169	__cfadbg_print_safe(runtime_core, "Kernel : Starting\n");
170
171	__cfa_dbg_global_clusters.list{ __get };
172	__cfa_dbg_global_clusters.lock{};
173
174	/* paranoid */ verify( verify_fwd_bck_rng() );
175
176	// Initialize the global scheduler lock
177	__scheduler_lock = (__scheduler_RWLock_t*)&storage___scheduler_lock;
178	(*__scheduler_lock){};
179
180	// Initialize the main cluster
181	mainCluster = (cluster *)&storage_mainCluster;
182	(*mainCluster){"Main Cluster", 0};
183
184	__cfadbg_print_safe(runtime_core, "Kernel : Main cluster ready\n");
185
186	// Start by initializing the main thread
187	// SKULLDUGGERY: the mainThread steals the process main thread
188	// which will then be scheduled by the mainProcessor normally
189	mainThread = ($thread *)&storage_mainThread;
190	current_stack_info_t info;
191	info.storage = (__stack_t*)&storage_mainThreadCtx;
192	(*mainThread){ &info };
193
194	__cfadbg_print_safe(runtime_core, "Kernel : Main thread ready\n");
195
196
197
198	// Construct the processor context of the main processor
199	void ?{}(processorCtx_t & this, processor * proc) {
200	(this.__cor){ "Processor" };
201	this.__cor.starter = 0p;
202	this.proc = proc;
203	}
204
205	void ?{}(processor & this) with( this ) {
206	( this.terminated ){};
207	( this.runner ){};
208	init( this, "Main Processor", *mainCluster, 0p );
209	kernel_thread = pthread_self();
210
211	runner{ &this };
212	__cfadbg_print_safe(runtime_core, "Kernel : constructed main processor context %p\n", &runner);
213	}
214
215	// Initialize the main processor and the main processor ctx
216	// (the coroutine that contains the processing control flow)
217	mainProcessor = (processor *)&storage_mainProcessor;
218	(*mainProcessor){};
219
220	register_tls( mainProcessor );
221
222	//initialize the global state variables
223	__cfaabi_tls.this_processor = mainProcessor;
224	__cfaabi_tls.this_thread = mainThread;
225
226	#if !defined( __CFA_NO_STATISTICS__ )
227	__cfaabi_tls.this_stats = (__stats_t *)& storage_mainProcStats;
228	__init_stats( __cfaabi_tls.this_stats );
229	#endif
230
231	// Enable preemption
232	__kernel_alarm_startup();
233
234	// Add the main thread to the ready queue
235	// once resume is called on mainProcessor->runner the mainThread needs to be scheduled like any normal thread
236	schedule_thread$(mainThread);
237
238	// SKULLDUGGERY: Force a context switch to the main processor to set the main thread's context to the current UNIX
239	// context. Hence, the main thread does not begin through __cfactx_invoke_thread, like all other threads. The trick here is that
240	// mainThread is on the ready queue when this call is made.
241	__kernel_first_resume( __cfaabi_tls.this_processor );
242
243
244	// THE SYSTEM IS NOW COMPLETELY RUNNING
245
246	__cfadbg_print_safe(runtime_core, "Kernel : Started\n--------------------------------------------------\n\n");
247
248	/* paranoid */ verify( ! __preemption_enabled() );
249	enable_interrupts();
250	/* paranoid */ verify( __preemption_enabled() );
251
252	}
253
254	static void __kernel_shutdown(void) {
255	/* paranoid */ verify( __preemption_enabled() );
256	disable_interrupts();
257	/* paranoid */ verify( ! __preemption_enabled() );
258
259	__cfadbg_print_safe(runtime_core, "\n--------------------------------------------------\nKernel : Shutting down\n");
260
261	// SKULLDUGGERY: Notify the mainProcessor it needs to terminates.
262	// When its coroutine terminates, it return control to the mainThread
263	// which is currently here
264	__atomic_store_n(&mainProcessor->do_terminate, true, __ATOMIC_RELEASE);
265	__kernel_last_resume( __cfaabi_tls.this_processor );
266	mainThread->self_cor.state = Halted;
267
268	// THE SYSTEM IS NOW COMPLETELY STOPPED
269
270	// Disable preemption
271	__kernel_alarm_shutdown();
272
273	#if !defined( __CFA_NO_STATISTICS__ )
274	__stats_t * st = (__stats_t *)& storage_mainProcStats;
275	__tally_stats(mainCluster->stats, st);
276	if( 0 != mainProcessor->print_stats ) {
277	__print_stats( st, mainProcessor->print_stats, "Processor ", mainProcessor->name, (void*)mainProcessor );
278	}
279	#if defined(CFA_STATS_ARRAY)
280	__flush_stat( st, "Processor", mainProcessor );
281	#endif
282	#endif
283
284	unregister_tls( mainProcessor );
285
286	// Destroy the main processor and its context in reverse order of construction
287	// These were manually constructed so we need manually destroy them
288	void ^?{}(processor & this) with( this ){
289	deinit( this );
290
291	/* paranoid */ verify( this.do_terminate == true );
292	__cfaabi_dbg_print_safe("Kernel : destroyed main processor context %p\n", &runner);
293	}
294
295	^(*mainProcessor){};
296
297	// Final step, destroy the main thread since it is no longer needed
298
299	// Since we provided a stack to this taxk it will not destroy anything
300	/* paranoid / verify(mainThread->self_cor.stack.storage == (__stack_t)(((uintptr_t)&storage_mainThreadCtx)\| 0x1));
301	^(*mainThread){};
302
303	^(*mainCluster){};
304
305	^(*__scheduler_lock){};
306
307	^(__cfa_dbg_global_clusters.list){};
308	^(__cfa_dbg_global_clusters.lock){};
309
310	__cfadbg_print_safe(runtime_core, "Kernel : Shutdown complete\n");
311	}
312
313	//=============================================================================================
314	// Kernel Initial Scheduling logic
315	//=============================================================================================
316
317	// Context invoker for processors
318	// This is the entry point for processors (kernel threads) except for the main processor
319	// It effectively constructs a coroutine by stealing the pthread stack
320	static void * __invoke_processor(void * arg) {
321	#if !defined( __CFA_NO_STATISTICS__ )
322	__stats_t local_stats;
323	__init_stats( &local_stats );
324	__cfaabi_tls.this_stats = &local_stats;
325	#endif
326
327	processor * proc = (processor *) arg;
328	__cfaabi_tls.this_processor = proc;
329	__cfaabi_tls.this_thread = 0p;
330	__cfaabi_tls.preemption_state.[enabled, disable_count] = [false, 1];
331
332	register_tls( proc );
333
334	// SKULLDUGGERY: We want to create a context for the processor coroutine
335	// which is needed for the 2-step context switch. However, there is no reason
336	// to waste the perfectly valid stack create by pthread.
337	current_stack_info_t info;
338	__stack_t ctx;
339	info.storage = &ctx;
340	(proc->runner){ proc, &info };
341
342	__cfaabi_dbg_print_safe("Coroutine : created stack %p\n", get_coroutine(proc->runner)->stack.storage);
343
344	//Set global state
345	__cfaabi_tls.this_thread = 0p;
346
347	//We now have a proper context from which to schedule threads
348	__cfadbg_print_safe(runtime_core, "Kernel : core %p created (%p, %p)\n", proc, &proc->runner, &ctx);
349
350	// SKULLDUGGERY: Since the coroutine doesn't have its own stack, we can't
351	// resume it to start it like it normally would, it will just context switch
352	// back to here. Instead directly call the main since we already are on the
353	// appropriate stack.
354	get_coroutine(proc->runner)->state = Active;
355	main( proc->runner );
356	get_coroutine(proc->runner)->state = Halted;
357
358	// Main routine of the core returned, the core is now fully terminated
359	__cfadbg_print_safe(runtime_core, "Kernel : core %p main ended (%p)\n", proc, &proc->runner);
360
361	#if !defined(__CFA_NO_STATISTICS__)
362	__tally_stats(proc->cltr->stats, &local_stats);
363	if( 0 != proc->print_stats ) {
364	__print_stats( &local_stats, proc->print_stats, "Processor ", proc->name, (void*)proc );
365	}
366	#if defined(CFA_STATS_ARRAY)
367	__flush_stat( &local_stats, "Processor", proc );
368	#endif
369	#endif
370
371	unregister_tls( proc );
372
373	return 0p;
374	}
375
376	static void __kernel_first_resume( processor * this ) {
377	$thread * src = mainThread;
378	$coroutine * dst = get_coroutine(this->runner);
379
380	/* paranoid */ verify( ! __preemption_enabled() );
381
382	__cfaabi_tls.this_thread->curr_cor = dst;
383	__stack_prepare( &dst->stack, 65000 );
384	__cfactx_start(main, dst, this->runner, __cfactx_invoke_coroutine);
385
386	/* paranoid */ verify( ! __preemption_enabled() );
387
388	dst->last = &src->self_cor;
389	dst->starter = dst->starter ? dst->starter : &src->self_cor;
390
391	// make sure the current state is still correct
392	/* paranoid */ verify(src->state == Ready);
393
394	// context switch to specified coroutine
395	verify( dst->context.SP );
396	__cfactx_switch( &src->context, &dst->context );
397	// when __cfactx_switch returns we are back in the src coroutine
398
399	mainThread->curr_cor = &mainThread->self_cor;
400
401	// make sure the current state has been update
402	/* paranoid */ verify(src->state == Active);
403
404	/* paranoid */ verify( ! __preemption_enabled() );
405	}
406
407	// KERNEL_ONLY
408	static void __kernel_last_resume( processor * this ) {
409	$coroutine * src = &mainThread->self_cor;
410	$coroutine * dst = get_coroutine(this->runner);
411
412	/* paranoid */ verify( ! __preemption_enabled() );
413	/* paranoid */ verify( dst->starter == src );
414	/* paranoid */ verify( dst->context.SP );
415
416	// SKULLDUGGERY in debug the processors check that the
417	// stack is still within the limit of the stack limits after running a thread.
418	// that check doesn't make sense if we context switch to the processor using the
419	// coroutine semantics. Since this is a special case, use the current context
420	// info to populate these fields.
421	__cfaabi_dbg_debug_do(
422	__stack_context_t ctx;
423	CtxGet( ctx );
424	mainThread->context.SP = ctx.SP;
425	mainThread->context.FP = ctx.FP;
426	)
427
428	// context switch to the processor
429	__cfactx_switch( &src->context, &dst->context );
430	}
431
432
433	//=============================================================================================
434	// Kernel Object Constructors logic
435	//=============================================================================================
436	//-----------------------------------------------------------------------------
437	// Main thread construction
438	static void ?{}( $coroutine & this, current_stack_info_t * info) with( this ) {
439	stack.storage = info->storage;
440	with(*stack.storage) {
441	limit = info->limit;
442	base = info->base;
443	}
444	__attribute__((may_alias)) intptr_t * istorage = (intptr_t*) &stack.storage;
445	*istorage \|= 0x1;
446	name = "Main Thread";
447	state = Start;
448	starter = 0p;
449	last = 0p;
450	cancellation = 0p;
451	}
452
453	static void ?{}( $thread & this, current_stack_info_t * info) with( this ) {
454	ticket = TICKET_RUNNING;
455	state = Start;
456	self_cor{ info };
457	curr_cor = &self_cor;
458	curr_cluster = mainCluster;
459	self_mon.owner = &this;
460	self_mon.recursion = 1;
461	self_mon_p = &self_mon;
462	link.next = 0p;
463	link.ts = 0;
464	link.preferred = -1u;
465	last_proc = 0p;
466	#if defined( __CFA_WITH_VERIFY__ )
467	canary = 0x0D15EA5E0D15EA5Ep;
468	#endif
469
470	node.next = 0p;
471	node.prev = 0p;
472	doregister(curr_cluster, this);
473
474	monitors{ &self_mon_p, 1, (fptr_t)0 };
475	}
476
477	//-----------------------------------------------------------------------------
478	// Processor
479	// Construct the processor context of non-main processors
480	static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info) {
481	(this.__cor){ info };
482	this.proc = proc;
483	}
484
485	static void init(processor & this, const char name[], cluster & _cltr, $thread * initT) with( this ) {
486	this.name = name;
487	this.cltr = &_cltr;
488	this.rdq.its = 0;
489	this.rdq.itr = 0;
490	this.rdq.id = -1u;
491	this.rdq.target = -1u;
492	this.rdq.cutoff = -1ull;
493	do_terminate = false;
494	preemption_alarm = 0p;
495	pending_preemption = false;
496
497	this.io.ctx = 0p;
498	this.io.pending = false;
499	this.io.dirty = false;
500
501	this.init.thrd = initT;
502
503	this.idle = eventfd(0, 0);
504	if (idle < 0) {
505	abort("KERNEL ERROR: PROCESSOR EVENTFD - %s\n", strerror(errno));
506	}
507
508	#if !defined(__CFA_NO_STATISTICS__)
509	print_stats = 0;
510	print_halts = false;
511	#endif
512
513	__cfadbg_print_safe(runtime_core, "Kernel : core %p created\n", &this);
514	}
515
516	// Not a ctor, it just preps the destruction but should not destroy members
517	static void deinit(processor & this) {
518	close(this.idle);
519	}
520
521	void ?{}(processor & this, const char name[], cluster & _cltr, $thread * initT) {
522	( this.terminated ){};
523	( this.runner ){};
524
525	disable_interrupts();
526	init( this, name, _cltr, initT );
527	enable_interrupts();
528
529	__cfadbg_print_safe(runtime_core, "Kernel : Starting core %p\n", &this);
530
531	this.stack = __create_pthread( &this.kernel_thread, __invoke_processor, (void *)&this );
532	}
533
534	void ?{}(processor & this, const char name[], cluster & _cltr) {
535	(this){name, _cltr, 0p};
536	}
537
538	extern size_t __page_size;
539	void ^?{}(processor & this) with( this ){
540	if( ! __atomic_load_n(&do_terminate, __ATOMIC_ACQUIRE) ) {
541	__cfadbg_print_safe(runtime_core, "Kernel : core %p signaling termination\n", &this);
542
543	__atomic_store_n(&do_terminate, true, __ATOMIC_RELAXED);
544	__wake_proc( &this );
545
546	wait( terminated );
547	/* paranoid */ verify( active_processor() != &this);
548	}
549
550	__destroy_pthread( kernel_thread, this.stack, 0p );
551
552	disable_interrupts();
553	deinit( this );
554	enable_interrupts();
555	}
556
557	//-----------------------------------------------------------------------------
558	// Cluster
559	static void ?{}(__cluster_proc_list & this) {
560	this.lock = 0;
561	this.idle = 0;
562	this.total = 0;
563	}
564
565	void ?{}(cluster & this, const char name[], Duration preemption_rate, unsigned num_io, const io_context_params & io_params) with( this ) {
566	this.name = name;
567	this.preemption_rate = preemption_rate;
568	ready_queue{};
569
570	#if !defined(__CFA_NO_STATISTICS__)
571	print_stats = 0;
572	stats = alloc();
573	__init_stats( stats );
574	#endif
575
576	threads{ __get };
577
578	io.arbiter = create();
579	io.params = io_params;
580
581	doregister(this);
582
583	// Lock the RWlock so no-one pushes/pops while we are changing the queue
584	disable_interrupts();
585	uint_fast32_t last_size = ready_mutate_lock();
586
587	// Adjust the ready queue size
588	ready_queue_grow( &this );
589
590	// Unlock the RWlock
591	ready_mutate_unlock( last_size );
592	enable_interrupts( false ); // Don't poll, could be in main cluster
593	}
594
595	void ^?{}(cluster & this) {
596	destroy(this.io.arbiter);
597
598	// Lock the RWlock so no-one pushes/pops while we are changing the queue
599	disable_interrupts();
600	uint_fast32_t last_size = ready_mutate_lock();
601
602	// Adjust the ready queue size
603	ready_queue_shrink( &this );
604
605	// Unlock the RWlock
606	ready_mutate_unlock( last_size );
607	enable_interrupts( false ); // Don't poll, could be in main cluster
608
609	#if !defined(__CFA_NO_STATISTICS__)
610	if( 0 != this.print_stats ) {
611	__print_stats( this.stats, this.print_stats, "Cluster", this.name, (void*)&this );
612	}
613	#if defined(CFA_STATS_ARRAY)
614	__flush_stat( this.stats, "Cluster", &this );
615	#endif
616	free( this.stats );
617	#endif
618
619	unregister(this);
620	}
621
622	//=============================================================================================
623	// Miscellaneous Initialization
624	//=============================================================================================
625	//-----------------------------------------------------------------------------
626	// Global Queues
627	static void doregister( cluster & cltr ) {
628	lock ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
629	push_front( __cfa_dbg_global_clusters.list, cltr );
630	unlock ( __cfa_dbg_global_clusters.lock );
631	}
632
633	static void unregister( cluster & cltr ) {
634	lock ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
635	remove( __cfa_dbg_global_clusters.list, cltr );
636	unlock( __cfa_dbg_global_clusters.lock );
637	}
638
639	void doregister( cluster * cltr, $thread & thrd ) {
640	lock (cltr->thread_list_lock __cfaabi_dbg_ctx2);
641	cltr->nthreads += 1;
642	push_front(cltr->threads, thrd);
643	unlock (cltr->thread_list_lock);
644	}
645
646	void unregister( cluster * cltr, $thread & thrd ) {
647	lock (cltr->thread_list_lock __cfaabi_dbg_ctx2);
648	remove(cltr->threads, thrd );
649	cltr->nthreads -= 1;
650	unlock(cltr->thread_list_lock);
651	}
652
653	static void register_tls( processor * this ) {
654	// Register and Lock the RWlock so no-one pushes/pops while we are changing the queue
655	uint_fast32_t last_size;
656	[this->unique_id, last_size] = ready_mutate_register();
657
658	this->cltr->procs.total += 1u;
659	insert_last(this->cltr->procs.actives, *this);
660
661	// Adjust the ready queue size
662	ready_queue_grow( this->cltr );
663
664	// Unlock the RWlock
665	ready_mutate_unlock( last_size );
666	}
667
668
669	static void unregister_tls( processor * this ) {
670	// Lock the RWlock so no-one pushes/pops while we are changing the queue
671	uint_fast32_t last_size = ready_mutate_lock();
672	this->cltr->procs.total -= 1u;
673	remove(*this);
674
675	// clear the cluster so nothing gets pushed to local queues
676	cluster * cltr = this->cltr;
677	this->cltr = 0p;
678
679	// Adjust the ready queue size
680	ready_queue_shrink( cltr );
681
682	// Unlock the RWlock and unregister: we don't need the read_lock any more
683	ready_mutate_unregister( this->unique_id, last_size );
684	}
685
686	static void check( int ret, const char func[] ) {
687	if ( ret ) { // pthread routines return errno values
688	abort( "%s : internal error, error(%d) %s.", func, ret, strerror( ret ) );
689	} // if
690	} // Abort
691
692	void * __create_pthread( pthread_t * pthread, void * (start)(void ), void * arg ) {
693	pthread_attr_t attr;
694
695	check( pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
696
697	size_t stacksize;
698	// default stack size, normally defined by shell limit
699	check( pthread_attr_getstacksize( &attr, &stacksize ), "pthread_attr_getstacksize" );
700	assert( stacksize >= PTHREAD_STACK_MIN );
701
702	void * stack;
703	#if CFA_PROCESSOR_USE_MMAP
704	stacksize = ceiling( stacksize, __page_size ) + __page_size;
705	stack = mmap(0p, stacksize, __map_prot, MAP_PRIVATE \| MAP_ANONYMOUS, 0, 0);
706	if(stack == ((void*)-1)) {
707	abort( "pthread stack creation : internal error, mmap failure, error(%d) %s.", errno, strerror( errno ) );
708	}
709	if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
710	abort( "pthread stack creation : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
711	} // if
712	#else
713	__cfaabi_dbg_debug_do(
714	stack = memalign( __page_size, stacksize + __page_size );
715	// pthread has no mechanism to create the guard page in user supplied stack.
716	if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
717	abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
718	} // if
719	);
720	__cfaabi_dbg_no_debug_do(
721	stack = malloc( stacksize );
722	);
723	#endif
724
725
726	check( pthread_attr_setstack( &attr, stack, stacksize ), "pthread_attr_setstack" );
727
728	check( pthread_create( pthread, &attr, start, arg ), "pthread_create" );
729	return stack;
730	}
731
732	void __destroy_pthread( pthread_t pthread, void * stack, void ** retval ) {
733	int err = pthread_join( pthread, retval );
734	if( err != 0 ) abort("KERNEL ERROR: joining pthread %p caused error %s\n", (void*)pthread, strerror(err));
735
736	#if CFA_PROCESSOR_USE_MMAP
737	pthread_attr_t attr;
738
739	check( pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
740
741	size_t stacksize;
742	// default stack size, normally defined by shell limit
743	check( pthread_attr_getstacksize( &attr, &stacksize ), "pthread_attr_getstacksize" );
744	assert( stacksize >= PTHREAD_STACK_MIN );
745	stacksize += __page_size;
746
747	if(munmap(stack, stacksize) == -1) {
748	abort( "pthread stack destruction : internal error, munmap failure, error(%d) %s.", errno, strerror( errno ) );
749	}
750	#else
751	__cfaabi_dbg_debug_do(
752	// pthread has no mechanism to create the guard page in user supplied stack.
753	if ( mprotect( stack, __page_size, __map_prot ) == -1 ) {
754	abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
755	} // if
756	);
757	free( stack );
758	#endif
759	}
760
761	#if defined(__CFA_WITH_VERIFY__)
762	static bool verify_fwd_bck_rng(void) {
763	__cfaabi_tls.ready_rng.fwd_seed = 25214903917_l64u * (rdtscl() ^ (uintptr_t)&verify_fwd_bck_rng);
764
765	unsigned values[10];
766	for(i; 10) {
767	values[i] = __tls_rand_fwd();
768	}
769
770	__tls_rand_advance_bck();
771
772	for ( i; 9 -~= 0 ) {
773	if(values[i] != __tls_rand_bck()) {
774	return false;
775	}
776	}
777
778	return true;
779	}
780	#endif

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

Original Format