Context Navigation

startup.cfa@ 43784ac

Visit:

ADT ast-experimental enum forall-pointer-decay jacob/cs343-translation new-ast-unique-expr pthread-emulation qualifiedEnum stuck-waitfor-destruct

Last change on this file since 43784ac was 43784ac, checked in by Thierry Delisle <tdelisle@…>, 5 years ago
Changed libcfathread to consistently define _GNU_SOURCE
Property mode set to `100644`
File size: 24.2 KB

Line
1	//
2	// Cforall Version 1.0.0 Copyright (C) 2020 University of Waterloo
3	//
4	// The contents of this file are covered under the licence agreement in the
5	// file "LICENCE" distributed with Cforall.
6	//
7	// kernel/startup.cfa --
8	//
9	// Author : Thierry Delisle
10	// Created On : Thu Jul 30 15:12:54 2020
11	// Last Modified By :
12	// Last Modified On :
13	// Update Count :
14	//
15
16	#define __cforall_thread__
17	#define _GNU_SOURCE
18
19	// C Includes
20	#include <errno.h> // errno
21	#include <string.h> // strerror
22	#include <unistd.h> // sysconf
23	extern "C" {
24	#include <limits.h> // PTHREAD_STACK_MIN
25	#include <sys/eventfd.h> // eventfd
26	#include <sys/mman.h> // mprotect
27	#include <sys/resource.h> // getrlimit
28	}
29
30	// CFA Includes
31	#include "kernel_private.hfa"
32	#include "startup.hfa" // STARTUP_PRIORITY_XXX
33	#include "math.hfa"
34
35	#define CFA_PROCESSOR_USE_MMAP 0
36
37	//-----------------------------------------------------------------------------
38	// Some assembly required
39	#if defined( __i386 )
40	#define CtxGet( ctx ) __asm__ volatile ( \
41	"movl %%esp,%0\n" \
42	"movl %%ebp,%1\n" \
43	: "=rm" (ctx.SP), \
44	"=rm" (ctx.FP) \
45	)
46	#elif defined( __x86_64 )
47	#define CtxGet( ctx ) __asm__ volatile ( \
48	"movq %%rsp,%0\n" \
49	"movq %%rbp,%1\n" \
50	: "=rm" (ctx.SP), \
51	"=rm" (ctx.FP) \
52	)
53	#elif defined( __aarch64__ )
54	#define CtxGet( ctx ) __asm__ volatile ( \
55	"mov %0, sp\n" \
56	"mov %1, fp\n" \
57	: "=rm" (ctx.SP), \
58	"=rm" (ctx.FP) \
59	)
60	#else
61	#error unknown hardware architecture
62	#endif
63
64	//-----------------------------------------------------------------------------
65	// Start and stop routine for the kernel, declared first to make sure they run first
66	static void __kernel_startup (void) __attribute__(( constructor( STARTUP_PRIORITY_KERNEL ) ));
67	static void __kernel_shutdown(void) __attribute__(( destructor ( STARTUP_PRIORITY_KERNEL ) ));
68
69	//-----------------------------------------------------------------------------
70	// Static Forward Declarations
71	struct current_stack_info_t;
72
73	static void * __invoke_processor(void * arg);
74	static void __kernel_first_resume( processor * this );
75	static void __kernel_last_resume ( processor * this );
76	static void init(processor & this, const char name[], cluster & _cltr, $thread * initT);
77	static void deinit(processor & this);
78	static void doregister( struct cluster & cltr );
79	static void unregister( struct cluster & cltr );
80	static void register_tls( processor * this );
81	static void unregister_tls( processor * this );
82	static void ?{}( $coroutine & this, current_stack_info_t * info);
83	static void ?{}( $thread & this, current_stack_info_t * info);
84	static void ?{}(processorCtx_t & this) {}
85	static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info);
86
87	#if defined(__CFA_WITH_VERIFY__)
88	static bool verify_fwd_bck_rng(void);
89	#endif
90
91	//-----------------------------------------------------------------------------
92	// Forward Declarations for other modules
93	extern void __kernel_alarm_startup(void);
94	extern void __kernel_alarm_shutdown(void);
95
96	//-----------------------------------------------------------------------------
97	// Other Forward Declarations
98	extern void __wake_proc(processor *);
99
100	//-----------------------------------------------------------------------------
101	// Kernel storage
102	KERNEL_STORAGE(cluster, mainCluster);
103	KERNEL_STORAGE(processor, mainProcessor);
104	KERNEL_STORAGE($thread, mainThread);
105	KERNEL_STORAGE(__stack_t, mainThreadCtx);
106	KERNEL_STORAGE(__scheduler_RWLock_t, __scheduler_lock);
107	#if !defined(__CFA_NO_STATISTICS__)
108	KERNEL_STORAGE(__stats_t, mainProcStats);
109	#endif
110
111	cluster * mainCluster;
112	processor * mainProcessor;
113	$thread * mainThread;
114	__scheduler_RWLock_t * __scheduler_lock;
115
116	extern "C" {
117	struct { __dllist_t(cluster) list; __spinlock_t lock; } __cfa_dbg_global_clusters;
118	}
119
120	extern size_t __page_size;
121	extern int __map_prot;
122
123	//-----------------------------------------------------------------------------
124	// Global state
125	thread_local struct KernelThreadData __cfaabi_tls __attribute__ ((tls_model ( "initial-exec" ))) @= {
126	NULL, // cannot use 0p
127	NULL,
128	false,
129	{ 1, false, false },
130	0,
131	{ 0, 0 },
132	NULL,
133	#ifdef __CFA_WITH_VERIFY__
134	false,
135	0,
136	#endif
137	};
138
139	//-----------------------------------------------------------------------------
140	// Struct to steal stack
141	struct current_stack_info_t {
142	__stack_t * storage; // pointer to stack object
143	void * base; // base of stack
144	void * limit; // stack grows towards stack limit
145	void * context; // address of cfa_context_t
146	};
147
148	void ?{}( current_stack_info_t & this ) {
149	__stack_context_t ctx;
150	CtxGet( ctx );
151	this.base = ctx.FP;
152
153	rlimit r;
154	getrlimit( RLIMIT_STACK, &r);
155	size_t size = r.rlim_cur;
156
157	this.limit = (void *)(((intptr_t)this.base) - size);
158	this.context = &storage_mainThreadCtx;
159	}
160
161
162
163	//=============================================================================================
164	// Kernel Setup logic
165	//=============================================================================================
166	//-----------------------------------------------------------------------------
167	// Kernel boot procedures
168	static void __kernel_startup(void) {
169	/* paranoid */ verify( ! __preemption_enabled() );
170	__cfadbg_print_safe(runtime_core, "Kernel : Starting\n");
171
172	__cfa_dbg_global_clusters.list{ __get };
173	__cfa_dbg_global_clusters.lock{};
174
175	/* paranoid */ verify( verify_fwd_bck_rng() );
176
177	// Initialize the global scheduler lock
178	__scheduler_lock = (__scheduler_RWLock_t*)&storage___scheduler_lock;
179	(*__scheduler_lock){};
180
181	// Initialize the main cluster
182	mainCluster = (cluster *)&storage_mainCluster;
183	(*mainCluster){"Main Cluster", 0};
184
185	__cfadbg_print_safe(runtime_core, "Kernel : Main cluster ready\n");
186
187	// Start by initializing the main thread
188	// SKULLDUGGERY: the mainThread steals the process main thread
189	// which will then be scheduled by the mainProcessor normally
190	mainThread = ($thread *)&storage_mainThread;
191	current_stack_info_t info;
192	info.storage = (__stack_t*)&storage_mainThreadCtx;
193	(*mainThread){ &info };
194
195	__cfadbg_print_safe(runtime_core, "Kernel : Main thread ready\n");
196
197
198
199	// Construct the processor context of the main processor
200	void ?{}(processorCtx_t & this, processor * proc) {
201	(this.__cor){ "Processor" };
202	this.__cor.starter = 0p;
203	this.proc = proc;
204	}
205
206	void ?{}(processor & this) with( this ) {
207	( this.terminated ){};
208	( this.runner ){};
209	init( this, "Main Processor", *mainCluster, 0p );
210	kernel_thread = pthread_self();
211
212	runner{ &this };
213	__cfadbg_print_safe(runtime_core, "Kernel : constructed main processor context %p\n", &runner);
214	}
215
216	// Initialize the main processor and the main processor ctx
217	// (the coroutine that contains the processing control flow)
218	mainProcessor = (processor *)&storage_mainProcessor;
219	(*mainProcessor){};
220
221	register_tls( mainProcessor );
222
223	//initialize the global state variables
224	__cfaabi_tls.this_processor = mainProcessor;
225	__cfaabi_tls.this_thread = mainThread;
226
227	#if !defined( __CFA_NO_STATISTICS__ )
228	__cfaabi_tls.this_stats = (__stats_t *)& storage_mainProcStats;
229	__init_stats( __cfaabi_tls.this_stats );
230	#endif
231	mainProcessor->local_data = &__cfaabi_tls;
232
233	// Enable preemption
234	__kernel_alarm_startup();
235
236	// Add the main thread to the ready queue
237	// once resume is called on mainProcessor->runner the mainThread needs to be scheduled like any normal thread
238	schedule_thread$(mainThread);
239
240	// SKULLDUGGERY: Force a context switch to the main processor to set the main thread's context to the current UNIX
241	// context. Hence, the main thread does not begin through __cfactx_invoke_thread, like all other threads. The trick here is that
242	// mainThread is on the ready queue when this call is made.
243	__kernel_first_resume( __cfaabi_tls.this_processor );
244
245
246	// THE SYSTEM IS NOW COMPLETELY RUNNING
247
248	__cfadbg_print_safe(runtime_core, "Kernel : Started\n--------------------------------------------------\n\n");
249
250	/* paranoid */ verify( ! __preemption_enabled() );
251	enable_interrupts();
252	/* paranoid */ verify( __preemption_enabled() );
253
254	}
255
256	static void __kernel_shutdown(void) {
257	/* paranoid */ verify( __preemption_enabled() );
258	disable_interrupts();
259	/* paranoid */ verify( ! __preemption_enabled() );
260
261	__cfadbg_print_safe(runtime_core, "\n--------------------------------------------------\nKernel : Shutting down\n");
262
263	// SKULLDUGGERY: Notify the mainProcessor it needs to terminates.
264	// When its coroutine terminates, it return control to the mainThread
265	// which is currently here
266	__atomic_store_n(&mainProcessor->do_terminate, true, __ATOMIC_RELEASE);
267	__kernel_last_resume( __cfaabi_tls.this_processor );
268	mainThread->self_cor.state = Halted;
269
270	// THE SYSTEM IS NOW COMPLETELY STOPPED
271
272	// Disable preemption
273	__kernel_alarm_shutdown();
274
275	#if !defined( __CFA_NO_STATISTICS__ )
276	__stats_t * st = (__stats_t *)& storage_mainProcStats;
277	__tally_stats(mainCluster->stats, st);
278	if( 0 != mainProcessor->print_stats ) {
279	__print_stats( st, mainProcessor->print_stats, "Processor ", mainProcessor->name, (void*)mainProcessor );
280	}
281	#if defined(CFA_STATS_ARRAY)
282	__flush_stat( st, "Processor", mainProcessor );
283	#endif
284	#endif
285
286	mainProcessor->local_data = 0p;
287
288	unregister_tls( mainProcessor );
289
290	// Destroy the main processor and its context in reverse order of construction
291	// These were manually constructed so we need manually destroy them
292	void ^?{}(processor & this) with( this ){
293	deinit( this );
294
295	/* paranoid */ verify( this.do_terminate == true );
296	__cfaabi_dbg_print_safe("Kernel : destroyed main processor context %p\n", &runner);
297	}
298
299	^(*mainProcessor){};
300
301	// Final step, destroy the main thread since it is no longer needed
302
303	// Since we provided a stack to this taxk it will not destroy anything
304	/* paranoid / verify(mainThread->self_cor.stack.storage == (__stack_t)(((uintptr_t)&storage_mainThreadCtx)\| 0x1));
305	^(*mainThread){};
306
307	^(*mainCluster){};
308
309	^(*__scheduler_lock){};
310
311	^(__cfa_dbg_global_clusters.list){};
312	^(__cfa_dbg_global_clusters.lock){};
313
314	__cfadbg_print_safe(runtime_core, "Kernel : Shutdown complete\n");
315	}
316
317	//=============================================================================================
318	// Kernel Initial Scheduling logic
319	//=============================================================================================
320
321	// Context invoker for processors
322	// This is the entry point for processors (kernel threads) except for the main processor
323	// It effectively constructs a coroutine by stealing the pthread stack
324	static void * __invoke_processor(void * arg) {
325	#if !defined( __CFA_NO_STATISTICS__ )
326	__stats_t local_stats;
327	__init_stats( &local_stats );
328	__cfaabi_tls.this_stats = &local_stats;
329	#endif
330
331	processor * proc = (processor *) arg;
332	__cfaabi_tls.this_processor = proc;
333	__cfaabi_tls.this_thread = 0p;
334	__cfaabi_tls.preemption_state.[enabled, disable_count] = [false, 1];
335	proc->local_data = &__cfaabi_tls;
336
337	register_tls( proc );
338
339	#if defined(CFA_HAVE_LINUX_LIBRSEQ)
340	rseq_register_current_thread();
341	#endif
342
343	// SKULLDUGGERY: We want to create a context for the processor coroutine
344	// which is needed for the 2-step context switch. However, there is no reason
345	// to waste the perfectly valid stack create by pthread.
346	current_stack_info_t info;
347	__stack_t ctx;
348	info.storage = &ctx;
349	(proc->runner){ proc, &info };
350
351	__cfaabi_dbg_print_safe("Coroutine : created stack %p\n", get_coroutine(proc->runner)->stack.storage);
352
353	//Set global state
354	__cfaabi_tls.this_thread = 0p;
355
356	//We now have a proper context from which to schedule threads
357	__cfadbg_print_safe(runtime_core, "Kernel : core %p created (%p, %p)\n", proc, &proc->runner, &ctx);
358
359	// SKULLDUGGERY: Since the coroutine doesn't have its own stack, we can't
360	// resume it to start it like it normally would, it will just context switch
361	// back to here. Instead directly call the main since we already are on the
362	// appropriate stack.
363	get_coroutine(proc->runner)->state = Active;
364	main( proc->runner );
365	get_coroutine(proc->runner)->state = Halted;
366
367	// Main routine of the core returned, the core is now fully terminated
368	__cfadbg_print_safe(runtime_core, "Kernel : core %p main ended (%p)\n", proc, &proc->runner);
369
370	#if !defined(__CFA_NO_STATISTICS__)
371	__tally_stats(proc->cltr->stats, &local_stats);
372	if( 0 != proc->print_stats ) {
373	__print_stats( &local_stats, proc->print_stats, "Processor ", proc->name, (void*)proc );
374	}
375	#if defined(CFA_STATS_ARRAY)
376	__flush_stat( &local_stats, "Processor", proc );
377	#endif
378	#endif
379
380	proc->local_data = 0p;
381
382	unregister_tls( proc );
383
384	return 0p;
385	}
386
387	static void __kernel_first_resume( processor * this ) {
388	$thread * src = mainThread;
389	$coroutine * dst = get_coroutine(this->runner);
390
391	/* paranoid */ verify( ! __preemption_enabled() );
392
393	__cfaabi_tls.this_thread->curr_cor = dst;
394	__stack_prepare( &dst->stack, 65000 );
395	__cfactx_start(main, dst, this->runner, __cfactx_invoke_coroutine);
396
397	/* paranoid */ verify( ! __preemption_enabled() );
398
399	dst->last = &src->self_cor;
400	dst->starter = dst->starter ? dst->starter : &src->self_cor;
401
402	// make sure the current state is still correct
403	/* paranoid */ verify(src->state == Ready);
404	src->corctx_flag = true;
405
406	// context switch to specified coroutine
407	verify( dst->context.SP );
408	__cfactx_switch( &src->context, &dst->context );
409	// when __cfactx_switch returns we are back in the src coroutine
410
411	mainThread->curr_cor = &mainThread->self_cor;
412
413	// make sure the current state has been update
414	/* paranoid */ verify(src->state == Active);
415
416	/* paranoid */ verify( ! __preemption_enabled() );
417	}
418
419	// KERNEL_ONLY
420	static void __kernel_last_resume( processor * this ) {
421	$coroutine * src = &mainThread->self_cor;
422	$coroutine * dst = get_coroutine(this->runner);
423
424	/* paranoid */ verify( ! __preemption_enabled() );
425	/* paranoid */ verify( dst->starter == src );
426	/* paranoid */ verify( dst->context.SP );
427
428	// SKULLDUGGERY in debug the processors check that the
429	// stack is still within the limit of the stack limits after running a thread.
430	// that check doesn't make sense if we context switch to the processor using the
431	// coroutine semantics. Since this is a special case, use the current context
432	// info to populate these fields.
433	__cfaabi_dbg_debug_do(
434	__stack_context_t ctx;
435	CtxGet( ctx );
436	mainThread->context.SP = ctx.SP;
437	mainThread->context.FP = ctx.FP;
438	)
439
440	// context switch to the processor
441	__cfactx_switch( &src->context, &dst->context );
442	}
443
444
445	//=============================================================================================
446	// Kernel Object Constructors logic
447	//=============================================================================================
448	//-----------------------------------------------------------------------------
449	// Main thread construction
450	static void ?{}( $coroutine & this, current_stack_info_t * info) with( this ) {
451	stack.storage = info->storage;
452	with(*stack.storage) {
453	limit = info->limit;
454	base = info->base;
455	}
456	__attribute__((may_alias)) intptr_t * istorage = (intptr_t*) &stack.storage;
457	*istorage \|= 0x1;
458	name = "Main Thread";
459	state = Start;
460	starter = 0p;
461	last = 0p;
462	cancellation = 0p;
463	}
464
465	static void ?{}( $thread & this, current_stack_info_t * info) with( this ) {
466	ticket = TICKET_RUNNING;
467	state = Start;
468	self_cor{ info };
469	curr_cor = &self_cor;
470	curr_cluster = mainCluster;
471	self_mon.owner = &this;
472	self_mon.recursion = 1;
473	self_mon_p = &self_mon;
474	link.next = 0p;
475	link.ts = 0;
476	preferred = -1u;
477	last_proc = 0p;
478	#if defined( __CFA_WITH_VERIFY__ )
479	canary = 0x0D15EA5E0D15EA5Ep;
480	#endif
481
482	node.next = 0p;
483	node.prev = 0p;
484	doregister(curr_cluster, this);
485
486	monitors{ &self_mon_p, 1, (fptr_t)0 };
487	}
488
489	//-----------------------------------------------------------------------------
490	// Processor
491	// Construct the processor context of non-main processors
492	static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info) {
493	(this.__cor){ info };
494	this.proc = proc;
495	}
496
497	static void init(processor & this, const char name[], cluster & _cltr, $thread * initT) with( this ) {
498	this.name = name;
499	this.cltr = &_cltr;
500	this.rdq.its = 0;
501	this.rdq.itr = 0;
502	this.rdq.id = -1u;
503	this.rdq.target = -1u;
504	this.rdq.cutoff = 0ull;
505	do_terminate = false;
506	preemption_alarm = 0p;
507	pending_preemption = false;
508
509	this.io.ctx = 0p;
510	this.io.pending = false;
511	this.io.dirty = false;
512
513	this.init.thrd = initT;
514
515	this.local_data = 0p;
516
517	this.idle = eventfd(0, 0);
518	if (idle < 0) {
519	abort("KERNEL ERROR: PROCESSOR EVENTFD - %s\n", strerror(errno));
520	}
521
522	#if !defined(__CFA_NO_STATISTICS__)
523	print_stats = 0;
524	print_halts = false;
525	#endif
526
527	__cfadbg_print_safe(runtime_core, "Kernel : core %p created\n", &this);
528	}
529
530	// Not a ctor, it just preps the destruction but should not destroy members
531	static void deinit(processor & this) {
532	close(this.idle);
533	}
534
535	void ?{}(processor & this, const char name[], cluster & _cltr, $thread * initT) {
536	( this.terminated ){};
537	( this.runner ){};
538
539	disable_interrupts();
540	init( this, name, _cltr, initT );
541	enable_interrupts();
542
543	__cfadbg_print_safe(runtime_core, "Kernel : Starting core %p\n", &this);
544
545	this.stack = __create_pthread( &this.kernel_thread, __invoke_processor, (void *)&this );
546	}
547
548	void ?{}(processor & this, const char name[], cluster & _cltr) {
549	(this){name, _cltr, 0p};
550	}
551
552	extern size_t __page_size;
553	void ^?{}(processor & this) with( this ){
554	if( ! __atomic_load_n(&do_terminate, __ATOMIC_ACQUIRE) ) {
555	__cfadbg_print_safe(runtime_core, "Kernel : core %p signaling termination\n", &this);
556
557	__atomic_store_n(&do_terminate, true, __ATOMIC_RELAXED);
558	__wake_proc( &this );
559
560	wait( terminated );
561	/* paranoid */ verify( active_processor() != &this);
562	}
563
564	__destroy_pthread( kernel_thread, this.stack, 0p );
565
566	disable_interrupts();
567	deinit( this );
568	enable_interrupts();
569	}
570
571	//-----------------------------------------------------------------------------
572	// Cluster
573	static void ?{}(__cluster_proc_list & this) {
574	this.lock = 0;
575	this.idle = 0;
576	this.total = 0;
577	}
578
579	void ?{}(cluster & this, const char name[], Duration preemption_rate, unsigned num_io, const io_context_params & io_params) with( this ) {
580	this.name = name;
581	this.preemption_rate = preemption_rate;
582	ready_queue{};
583
584	#if !defined(__CFA_NO_STATISTICS__)
585	print_stats = 0;
586	stats = alloc();
587	__init_stats( stats );
588	#endif
589
590	threads{ __get };
591
592	io.arbiter = create();
593	io.params = io_params;
594
595	doregister(this);
596
597	// Lock the RWlock so no-one pushes/pops while we are changing the queue
598	disable_interrupts();
599	uint_fast32_t last_size = ready_mutate_lock();
600
601	// Adjust the ready queue size
602	ready_queue_grow( &this );
603
604	// Unlock the RWlock
605	ready_mutate_unlock( last_size );
606	enable_interrupts( false ); // Don't poll, could be in main cluster
607	}
608
609	void ^?{}(cluster & this) {
610	destroy(this.io.arbiter);
611
612	// Lock the RWlock so no-one pushes/pops while we are changing the queue
613	disable_interrupts();
614	uint_fast32_t last_size = ready_mutate_lock();
615
616	// Adjust the ready queue size
617	ready_queue_shrink( &this );
618
619	// Unlock the RWlock
620	ready_mutate_unlock( last_size );
621	enable_interrupts( false ); // Don't poll, could be in main cluster
622
623	#if !defined(__CFA_NO_STATISTICS__)
624	if( 0 != this.print_stats ) {
625	__print_stats( this.stats, this.print_stats, "Cluster", this.name, (void*)&this );
626	}
627	#if defined(CFA_STATS_ARRAY)
628	__flush_stat( this.stats, "Cluster", &this );
629	#endif
630	free( this.stats );
631	#endif
632
633	unregister(this);
634	}
635
636	//=============================================================================================
637	// Miscellaneous Initialization
638	//=============================================================================================
639	//-----------------------------------------------------------------------------
640	// Global Queues
641	static void doregister( cluster & cltr ) {
642	lock ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
643	push_front( __cfa_dbg_global_clusters.list, cltr );
644	unlock ( __cfa_dbg_global_clusters.lock );
645	}
646
647	static void unregister( cluster & cltr ) {
648	lock ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
649	remove( __cfa_dbg_global_clusters.list, cltr );
650	unlock( __cfa_dbg_global_clusters.lock );
651	}
652
653	void doregister( cluster * cltr, $thread & thrd ) {
654	lock (cltr->thread_list_lock __cfaabi_dbg_ctx2);
655	cltr->nthreads += 1;
656	push_front(cltr->threads, thrd);
657	unlock (cltr->thread_list_lock);
658	}
659
660	void unregister( cluster * cltr, $thread & thrd ) {
661	lock (cltr->thread_list_lock __cfaabi_dbg_ctx2);
662	remove(cltr->threads, thrd );
663	cltr->nthreads -= 1;
664	unlock(cltr->thread_list_lock);
665	}
666
667	static void register_tls( processor * this ) {
668	// Register and Lock the RWlock so no-one pushes/pops while we are changing the queue
669	uint_fast32_t last_size;
670	[this->unique_id, last_size] = ready_mutate_register();
671
672	this->cltr->procs.total += 1u;
673	insert_last(this->cltr->procs.actives, *this);
674
675	// Adjust the ready queue size
676	ready_queue_grow( this->cltr );
677
678	// Unlock the RWlock
679	ready_mutate_unlock( last_size );
680	}
681
682
683	static void unregister_tls( processor * this ) {
684	// Lock the RWlock so no-one pushes/pops while we are changing the queue
685	uint_fast32_t last_size = ready_mutate_lock();
686	this->cltr->procs.total -= 1u;
687	remove(*this);
688
689	// clear the cluster so nothing gets pushed to local queues
690	cluster * cltr = this->cltr;
691	this->cltr = 0p;
692
693	// Adjust the ready queue size
694	ready_queue_shrink( cltr );
695
696	// Unlock the RWlock and unregister: we don't need the read_lock any more
697	ready_mutate_unregister( this->unique_id, last_size );
698	}
699
700	static void check( int ret, const char func[] ) {
701	if ( ret ) { // pthread routines return errno values
702	abort( "%s : internal error, error(%d) %s.", func, ret, strerror( ret ) );
703	} // if
704	} // Abort
705
706	void * __create_pthread( pthread_t * pthread, void * (start)(void ), void * arg ) {
707	pthread_attr_t attr;
708
709	check( pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
710
711	size_t stacksize;
712	// default stack size, normally defined by shell limit
713	check( pthread_attr_getstacksize( &attr, &stacksize ), "pthread_attr_getstacksize" );
714	assert( stacksize >= PTHREAD_STACK_MIN );
715
716	void * stack;
717	#if CFA_PROCESSOR_USE_MMAP
718	stacksize = ceiling( stacksize, __page_size ) + __page_size;
719	stack = mmap(0p, stacksize, __map_prot, MAP_PRIVATE \| MAP_ANONYMOUS, 0, 0);
720	if(stack == ((void*)-1)) {
721	abort( "pthread stack creation : internal error, mmap failure, error(%d) %s.", errno, strerror( errno ) );
722	}
723	if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
724	abort( "pthread stack creation : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
725	} // if
726	#else
727	__cfaabi_dbg_debug_do(
728	stack = memalign( __page_size, stacksize + __page_size );
729	// pthread has no mechanism to create the guard page in user supplied stack.
730	if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
731	abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
732	} // if
733	);
734	__cfaabi_dbg_no_debug_do(
735	stack = malloc( stacksize );
736	);
737	#endif
738
739
740	check( pthread_attr_setstack( &attr, stack, stacksize ), "pthread_attr_setstack" );
741
742	check( pthread_create( pthread, &attr, start, arg ), "pthread_create" );
743	return stack;
744	}
745
746	void __destroy_pthread( pthread_t pthread, void * stack, void ** retval ) {
747	int err = pthread_join( pthread, retval );
748	if( err != 0 ) abort("KERNEL ERROR: joining pthread %p caused error %s\n", (void*)pthread, strerror(err));
749
750	#if CFA_PROCESSOR_USE_MMAP
751	pthread_attr_t attr;
752
753	check( pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
754
755	size_t stacksize;
756	// default stack size, normally defined by shell limit
757	check( pthread_attr_getstacksize( &attr, &stacksize ), "pthread_attr_getstacksize" );
758	assert( stacksize >= PTHREAD_STACK_MIN );
759	stacksize += __page_size;
760
761	if(munmap(stack, stacksize) == -1) {
762	abort( "pthread stack destruction : internal error, munmap failure, error(%d) %s.", errno, strerror( errno ) );
763	}
764	#else
765	__cfaabi_dbg_debug_do(
766	// pthread has no mechanism to create the guard page in user supplied stack.
767	if ( mprotect( stack, __page_size, __map_prot ) == -1 ) {
768	abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
769	} // if
770	);
771	free( stack );
772	#endif
773	}
774
775	#if defined(__CFA_WITH_VERIFY__)
776	static bool verify_fwd_bck_rng(void) {
777	__cfaabi_tls.ready_rng.fwd_seed = 25214903917_l64u * (rdtscl() ^ (uintptr_t)&verify_fwd_bck_rng);
778
779	unsigned values[10];
780	for(i; 10) {
781	values[i] = __tls_rand_fwd();
782	}
783
784	__tls_rand_advance_bck();
785
786	for ( i; 9 -~= 0 ) {
787	if(values[i] != __tls_rand_bck()) {
788	return false;
789	}
790	}
791
792	return true;
793	}
794	#endif

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

Original Format