Context Navigation

startup.cfa@ fe18b46

Visit:

ADT ast-experimental enum pthread-emulation qualifiedEnum

Last change on this file since fe18b46 was 24e321c, checked in by Thierry Delisle <tdelisle@…>, 4 years ago
Unpark now takes a hint on locality.
Property mode set to `100644`
File size: 24.6 KB

Line
1	//
2	// Cforall Version 1.0.0 Copyright (C) 2020 University of Waterloo
3	//
4	// The contents of this file are covered under the licence agreement in the
5	// file "LICENCE" distributed with Cforall.
6	//
7	// kernel/startup.cfa --
8	//
9	// Author : Thierry Delisle
10	// Created On : Thu Jul 30 15:12:54 2020
11	// Last Modified By :
12	// Last Modified On :
13	// Update Count :
14	//
15
16	#define __cforall_thread__
17	#define _GNU_SOURCE
18
19	// C Includes
20	#include <errno.h> // errno
21	#include <signal.h>
22	#include <string.h> // strerror
23	#include <unistd.h> // sysconf
24
25	extern "C" {
26	#include <limits.h> // PTHREAD_STACK_MIN
27	#include <unistd.h> // syscall
28	#include <sys/eventfd.h> // eventfd
29	#include <sys/mman.h> // mprotect
30	#include <sys/resource.h> // getrlimit
31	}
32
33	// CFA Includes
34	#include "kernel_private.hfa"
35	#include "startup.hfa" // STARTUP_PRIORITY_XXX
36	#include "math.hfa"
37
38	#define CFA_PROCESSOR_USE_MMAP 0
39
40	//-----------------------------------------------------------------------------
41	// Some assembly required
42	#if defined( __i386 )
43	#define CtxGet( ctx ) __asm__ volatile ( \
44	"movl %%esp,%0\n" \
45	"movl %%ebp,%1\n" \
46	: "=rm" (ctx.SP), \
47	"=rm" (ctx.FP) \
48	)
49	#elif defined( __x86_64 )
50	#define CtxGet( ctx ) __asm__ volatile ( \
51	"movq %%rsp,%0\n" \
52	"movq %%rbp,%1\n" \
53	: "=rm" (ctx.SP), \
54	"=rm" (ctx.FP) \
55	)
56	#elif defined( __aarch64__ )
57	#define CtxGet( ctx ) __asm__ volatile ( \
58	"mov %0, sp\n" \
59	"mov %1, fp\n" \
60	: "=rm" (ctx.SP), \
61	"=rm" (ctx.FP) \
62	)
63	#else
64	#error unknown hardware architecture
65	#endif
66
67	//-----------------------------------------------------------------------------
68	// Start and stop routine for the kernel, declared first to make sure they run first
69	static void __kernel_startup (void) __attribute__(( constructor( STARTUP_PRIORITY_KERNEL ) ));
70	static void __kernel_shutdown(void) __attribute__(( destructor ( STARTUP_PRIORITY_KERNEL ) ));
71
72	//-----------------------------------------------------------------------------
73	// Static Forward Declarations
74	struct current_stack_info_t;
75
76	static void * __invoke_processor(void * arg);
77	static void __kernel_first_resume( processor * this );
78	static void __kernel_last_resume ( processor * this );
79	static void init(processor & this, const char name[], cluster & _cltr, thread$ * initT);
80	static void deinit(processor & this);
81	static void doregister( struct cluster & cltr );
82	static void unregister( struct cluster & cltr );
83	static void register_tls( processor * this );
84	static void unregister_tls( processor * this );
85	static void ?{}( coroutine$ & this, current_stack_info_t * info);
86	static void ?{}( thread$ & this, current_stack_info_t * info);
87	static void ?{}(processorCtx_t & this) {}
88	static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info);
89
90	#if defined(__CFA_WITH_VERIFY__)
91	static bool verify_fwd_bck_rng(void);
92	#endif
93
94	//-----------------------------------------------------------------------------
95	// Forward Declarations for other modules
96	extern void __kernel_alarm_startup(void);
97	extern void __kernel_alarm_shutdown(void);
98
99	//-----------------------------------------------------------------------------
100	// Other Forward Declarations
101	extern void __wake_proc(processor *);
102
103	//-----------------------------------------------------------------------------
104	// Kernel storage
105	KERNEL_STORAGE(cluster, mainCluster);
106	KERNEL_STORAGE(processor, mainProcessor);
107	KERNEL_STORAGE(thread$, mainThread);
108	KERNEL_STORAGE(__stack_t, mainThreadCtx);
109	KERNEL_STORAGE(__scheduler_RWLock_t, __scheduler_lock);
110	#if !defined(__CFA_NO_STATISTICS__)
111	KERNEL_STORAGE(__stats_t, mainProcStats);
112	#endif
113
114	cluster * mainCluster;
115	processor * mainProcessor;
116	thread$ * mainThread;
117	__scheduler_RWLock_t * __scheduler_lock;
118
119	extern "C" {
120	struct { __dllist_t(cluster) list; __spinlock_t lock; } __cfa_dbg_global_clusters;
121	}
122
123	extern size_t __page_size;
124	extern int __map_prot;
125
126	//-----------------------------------------------------------------------------
127	// Global state
128	thread_local struct KernelThreadData __cfaabi_tls __attribute__ ((tls_model ( "initial-exec" ))) @= {
129	NULL, // cannot use 0p
130	NULL,
131	false,
132	{ 1, false, false },
133	0,
134	{ 0, 0 },
135	NULL,
136	#ifdef __CFA_WITH_VERIFY__
137	false,
138	0,
139	#endif
140	};
141
142	#if defined(CFA_HAVE_LINUX_LIBRSEQ)
143	// No data needed
144	#elif defined(CFA_HAVE_LINUX_RSEQ_H)
145	extern "Cforall" {
146	__attribute__((aligned(128))) thread_local volatile struct rseq __cfaabi_rseq @= {
147	.cpu_id : RSEQ_CPU_ID_UNINITIALIZED,
148	};
149	}
150	#else
151	// No data needed
152	#endif
153
154	//-----------------------------------------------------------------------------
155	// Struct to steal stack
156	struct current_stack_info_t {
157	__stack_t * storage; // pointer to stack object
158	void * base; // base of stack
159	void * limit; // stack grows towards stack limit
160	void * context; // address of cfa_context_t
161	};
162
163	void ?{}( current_stack_info_t & this ) {
164	__stack_context_t ctx;
165	CtxGet( ctx );
166	this.base = ctx.FP;
167
168	rlimit r;
169	getrlimit( RLIMIT_STACK, &r);
170	size_t size = r.rlim_cur;
171
172	this.limit = (void *)(((intptr_t)this.base) - size);
173	this.context = &storage_mainThreadCtx;
174	}
175
176
177
178	//=============================================================================================
179	// Kernel Setup logic
180	//=============================================================================================
181	//-----------------------------------------------------------------------------
182	// Kernel boot procedures
183	static void __kernel_startup(void) {
184	/* paranoid */ verify( ! __preemption_enabled() );
185	__cfadbg_print_safe(runtime_core, "Kernel : Starting\n");
186
187	__cfa_dbg_global_clusters.list{ __get };
188	__cfa_dbg_global_clusters.lock{};
189
190	/* paranoid */ verify( verify_fwd_bck_rng() );
191
192	// Initialize the global scheduler lock
193	__scheduler_lock = (__scheduler_RWLock_t*)&storage___scheduler_lock;
194	(*__scheduler_lock){};
195
196	// Initialize the main cluster
197	mainCluster = (cluster *)&storage_mainCluster;
198	(*mainCluster){"Main Cluster", 0};
199
200	__cfadbg_print_safe(runtime_core, "Kernel : Main cluster ready\n");
201
202	// Construct the processor context of the main processor
203	void ?{}(processorCtx_t & this, processor * proc) {
204	(this.__cor){ "Processor" };
205	this.__cor.starter = 0p;
206	this.proc = proc;
207	}
208
209	void ?{}(processor & this) with( this ) {
210	( this.terminated ){};
211	( this.runner ){};
212	init( this, "Main Processor", *mainCluster, 0p );
213	kernel_thread = pthread_self();
214
215	runner{ &this };
216	__cfadbg_print_safe(runtime_core, "Kernel : constructed main processor context %p\n", &runner);
217	}
218
219	// Initialize the main processor and the main processor ctx
220	// (the coroutine that contains the processing control flow)
221	mainProcessor = (processor *)&storage_mainProcessor;
222	(*mainProcessor){};
223
224	register_tls( mainProcessor );
225
226	// Start by initializing the main thread
227	// SKULLDUGGERY: the mainThread steals the process main thread
228	// which will then be scheduled by the mainProcessor normally
229	mainThread = (thread$ *)&storage_mainThread;
230	current_stack_info_t info;
231	info.storage = (__stack_t*)&storage_mainThreadCtx;
232	(*mainThread){ &info };
233
234	__cfadbg_print_safe(runtime_core, "Kernel : Main thread ready\n");
235
236	//initialize the global state variables
237	__cfaabi_tls.this_processor = mainProcessor;
238	__cfaabi_tls.this_thread = mainThread;
239
240	#if !defined( __CFA_NO_STATISTICS__ )
241	__cfaabi_tls.this_stats = (__stats_t *)& storage_mainProcStats;
242	__init_stats( __cfaabi_tls.this_stats );
243	#endif
244	mainProcessor->local_data = &__cfaabi_tls;
245
246	// Enable preemption
247	__kernel_alarm_startup();
248
249	// Add the main thread to the ready queue
250	// once resume is called on mainProcessor->runner the mainThread needs to be scheduled like any normal thread
251	schedule_thread$(mainThread, UNPARK_LOCAL);
252
253	// SKULLDUGGERY: Force a context switch to the main processor to set the main thread's context to the current UNIX
254	// context. Hence, the main thread does not begin through __cfactx_invoke_thread, like all other threads. The trick here is that
255	// mainThread is on the ready queue when this call is made.
256	__kernel_first_resume( __cfaabi_tls.this_processor );
257
258
259	// THE SYSTEM IS NOW COMPLETELY RUNNING
260
261	__cfadbg_print_safe(runtime_core, "Kernel : Started\n--------------------------------------------------\n\n");
262
263	/* paranoid */ verify( ! __preemption_enabled() );
264	enable_interrupts();
265	/* paranoid */ verify( __preemption_enabled() );
266
267	}
268
269	static void __kernel_shutdown(void) {
270	/* paranoid */ verify( __preemption_enabled() );
271	disable_interrupts();
272	/* paranoid */ verify( ! __preemption_enabled() );
273
274	__cfadbg_print_safe(runtime_core, "\n--------------------------------------------------\nKernel : Shutting down\n");
275
276	// SKULLDUGGERY: Notify the mainProcessor it needs to terminates.
277	// When its coroutine terminates, it return control to the mainThread
278	// which is currently here
279	__atomic_store_n(&mainProcessor->do_terminate, true, __ATOMIC_RELEASE);
280	__kernel_last_resume( __cfaabi_tls.this_processor );
281	mainThread->self_cor.state = Halted;
282
283	// THE SYSTEM IS NOW COMPLETELY STOPPED
284
285	// Disable preemption
286	__kernel_alarm_shutdown();
287
288	#if !defined( __CFA_NO_STATISTICS__ )
289	__stats_t * st = (__stats_t *)& storage_mainProcStats;
290	__tally_stats(mainCluster->stats, st);
291	if( 0 != mainProcessor->print_stats ) {
292	__print_stats( st, mainProcessor->print_stats, "Processor ", mainProcessor->name, (void*)mainProcessor );
293	}
294	#if defined(CFA_STATS_ARRAY)
295	__flush_stat( st, "Processor", mainProcessor );
296	#endif
297	#endif
298
299	mainProcessor->local_data = 0p;
300
301	unregister_tls( mainProcessor );
302
303	// Destroy the main processor and its context in reverse order of construction
304	// These were manually constructed so we need manually destroy them
305	void ^?{}(processor & this) with( this ){
306	deinit( this );
307
308	/* paranoid */ verify( this.do_terminate == true );
309	__cfaabi_dbg_print_safe("Kernel : destroyed main processor context %p\n", &runner);
310	}
311
312	^(*mainProcessor){};
313
314	// Final step, destroy the main thread since it is no longer needed
315
316	// Since we provided a stack to this taxk it will not destroy anything
317	/* paranoid / verify(mainThread->self_cor.stack.storage == (__stack_t)(((uintptr_t)&storage_mainThreadCtx)\| 0x1));
318	^(*mainThread){};
319
320	^(*mainCluster){};
321
322	^(*__scheduler_lock){};
323
324	^(__cfa_dbg_global_clusters.list){};
325	^(__cfa_dbg_global_clusters.lock){};
326
327	__cfadbg_print_safe(runtime_core, "Kernel : Shutdown complete\n");
328	}
329
330	//=============================================================================================
331	// Kernel Initial Scheduling logic
332	//=============================================================================================
333
334	// Context invoker for processors
335	// This is the entry point for processors (kernel threads) except for the main processor
336	// It effectively constructs a coroutine by stealing the pthread stack
337	static void * __invoke_processor(void * arg) {
338	#if !defined( __CFA_NO_STATISTICS__ )
339	__stats_t local_stats;
340	__init_stats( &local_stats );
341	__cfaabi_tls.this_stats = &local_stats;
342	#endif
343
344	processor * proc = (processor *) arg;
345	__cfaabi_tls.this_processor = proc;
346	__cfaabi_tls.this_thread = 0p;
347	__cfaabi_tls.preemption_state.[enabled, disable_count] = [false, 1];
348	proc->local_data = &__cfaabi_tls;
349
350	register_tls( proc );
351
352	// SKULLDUGGERY: We want to create a context for the processor coroutine
353	// which is needed for the 2-step context switch. However, there is no reason
354	// to waste the perfectly valid stack create by pthread.
355	current_stack_info_t info;
356	__stack_t ctx;
357	info.storage = &ctx;
358	(proc->runner){ proc, &info };
359
360	__cfaabi_dbg_print_safe("Coroutine : created stack %p\n", get_coroutine(proc->runner)->stack.storage);
361
362	//Set global state
363	__cfaabi_tls.this_thread = 0p;
364
365	//We now have a proper context from which to schedule threads
366	__cfadbg_print_safe(runtime_core, "Kernel : core %p created (%p, %p)\n", proc, &proc->runner, &ctx);
367
368	// SKULLDUGGERY: Since the coroutine doesn't have its own stack, we can't
369	// resume it to start it like it normally would, it will just context switch
370	// back to here. Instead directly call the main since we already are on the
371	// appropriate stack.
372	get_coroutine(proc->runner)->state = Active;
373	main( proc->runner );
374	get_coroutine(proc->runner)->state = Halted;
375
376	// Main routine of the core returned, the core is now fully terminated
377	__cfadbg_print_safe(runtime_core, "Kernel : core %p main ended (%p)\n", proc, &proc->runner);
378
379	#if !defined(__CFA_NO_STATISTICS__)
380	__tally_stats(proc->cltr->stats, &local_stats);
381	if( 0 != proc->print_stats ) {
382	__print_stats( &local_stats, proc->print_stats, "Processor ", proc->name, (void*)proc );
383	}
384	#if defined(CFA_STATS_ARRAY)
385	__flush_stat( &local_stats, "Processor", proc );
386	#endif
387	#endif
388
389	proc->local_data = 0p;
390
391	unregister_tls( proc );
392
393	return 0p;
394	}
395
396	static void __kernel_first_resume( processor * this ) {
397	thread$ * src = mainThread;
398	coroutine$ * dst = get_coroutine(this->runner);
399
400	/* paranoid */ verify( ! __preemption_enabled() );
401
402	__cfaabi_tls.this_thread->curr_cor = dst;
403	__stack_prepare( &dst->stack, 65000 );
404	__cfactx_start(main, dst, this->runner, __cfactx_invoke_coroutine);
405
406	/* paranoid */ verify( ! __preemption_enabled() );
407
408	dst->last = &src->self_cor;
409	dst->starter = dst->starter ? dst->starter : &src->self_cor;
410
411	// make sure the current state is still correct
412	/* paranoid */ verify(src->state == Ready);
413	src->corctx_flag = true;
414
415	// context switch to specified coroutine
416	verify( dst->context.SP );
417	__cfactx_switch( &src->context, &dst->context );
418	// when __cfactx_switch returns we are back in the src coroutine
419
420	mainThread->curr_cor = &mainThread->self_cor;
421
422	// make sure the current state has been update
423	/* paranoid */ verify(src->state == Active);
424
425	/* paranoid */ verify( ! __preemption_enabled() );
426	}
427
428	// KERNEL_ONLY
429	static void __kernel_last_resume( processor * this ) {
430	coroutine$ * src = &mainThread->self_cor;
431	coroutine$ * dst = get_coroutine(this->runner);
432
433	/* paranoid */ verify( ! __preemption_enabled() );
434	/* paranoid */ verify( dst->starter == src );
435	/* paranoid */ verify( dst->context.SP );
436
437	// SKULLDUGGERY in debug the processors check that the
438	// stack is still within the limit of the stack limits after running a thread.
439	// that check doesn't make sense if we context switch to the processor using the
440	// coroutine semantics. Since this is a special case, use the current context
441	// info to populate these fields.
442	__cfaabi_dbg_debug_do(
443	__stack_context_t ctx;
444	CtxGet( ctx );
445	mainThread->context.SP = ctx.SP;
446	mainThread->context.FP = ctx.FP;
447	)
448
449	// context switch to the processor
450	__cfactx_switch( &src->context, &dst->context );
451	}
452
453
454	//=============================================================================================
455	// Kernel Object Constructors logic
456	//=============================================================================================
457	//-----------------------------------------------------------------------------
458	// Main thread construction
459	static void ?{}( coroutine$ & this, current_stack_info_t * info) with( this ) {
460	stack.storage = info->storage;
461	with(*stack.storage) {
462	limit = info->limit;
463	base = info->base;
464	}
465	__attribute__((may_alias)) intptr_t * istorage = (intptr_t*) &stack.storage;
466	*istorage \|= 0x1;
467	name = "Main Thread";
468	state = Start;
469	starter = 0p;
470	last = 0p;
471	cancellation = 0p;
472	}
473
474	static void ?{}( thread$ & this, current_stack_info_t * info) with( this ) {
475	ticket = TICKET_RUNNING;
476	state = Start;
477	self_cor{ info };
478	curr_cor = &self_cor;
479	curr_cluster = mainCluster;
480	self_mon.owner = &this;
481	self_mon.recursion = 1;
482	self_mon_p = &self_mon;
483	link.next = 0p;
484	link.ts = -1llu;
485	preferred = ready_queue_new_preferred();
486	last_proc = 0p;
487	#if defined( __CFA_WITH_VERIFY__ )
488	canary = 0x0D15EA5E0D15EA5Ep;
489	#endif
490
491	node.next = 0p;
492	node.prev = 0p;
493	doregister(curr_cluster, this);
494
495	monitors{ &self_mon_p, 1, (fptr_t)0 };
496	}
497
498	//-----------------------------------------------------------------------------
499	// Processor
500	// Construct the processor context of non-main processors
501	static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info) {
502	(this.__cor){ info };
503	this.proc = proc;
504	}
505
506	static void init(processor & this, const char name[], cluster & _cltr, thread$ * initT) with( this ) {
507	this.name = name;
508	this.cltr = &_cltr;
509	this.rdq.its = 0;
510	this.rdq.itr = 0;
511	this.rdq.id = -1u;
512	this.rdq.target = -1u;
513	this.rdq.last = -1u;
514	this.rdq.cutoff = 0ull;
515	do_terminate = false;
516	preemption_alarm = 0p;
517	pending_preemption = false;
518
519	this.io.ctx = 0p;
520	this.io.pending = false;
521	this.io.dirty = false;
522
523	this.init.thrd = initT;
524
525	this.local_data = 0p;
526
527	this.idle = eventfd(0, 0);
528	if (idle < 0) {
529	abort("KERNEL ERROR: PROCESSOR EVENTFD - %s\n", strerror(errno));
530	}
531
532	#if !defined(__CFA_NO_STATISTICS__)
533	print_stats = 0;
534	print_halts = false;
535	#endif
536
537	__cfadbg_print_safe(runtime_core, "Kernel : core %p created\n", &this);
538	}
539
540	// Not a ctor, it just preps the destruction but should not destroy members
541	static void deinit(processor & this) {
542	close(this.idle);
543	}
544
545	void ?{}(processor & this, const char name[], cluster & _cltr, thread$ * initT) {
546	( this.terminated ){};
547	( this.runner ){};
548
549	disable_interrupts();
550	init( this, name, _cltr, initT );
551	enable_interrupts();
552
553	__cfadbg_print_safe(runtime_core, "Kernel : Starting core %p\n", &this);
554
555	this.stack = __create_pthread( &this.kernel_thread, __invoke_processor, (void *)&this );
556	}
557
558	void ?{}(processor & this, const char name[], cluster & _cltr) {
559	(this){name, _cltr, 0p};
560	}
561
562	extern size_t __page_size;
563	void ^?{}(processor & this) with( this ){
564	if( ! __atomic_load_n(&do_terminate, __ATOMIC_ACQUIRE) ) {
565	__cfadbg_print_safe(runtime_core, "Kernel : core %p signaling termination\n", &this);
566
567	__atomic_store_n(&do_terminate, true, __ATOMIC_RELAXED);
568	__wake_proc( &this );
569
570	wait( terminated );
571	/* paranoid */ verify( active_processor() != &this);
572	}
573
574	__destroy_pthread( kernel_thread, this.stack, 0p );
575
576	disable_interrupts();
577	deinit( this );
578	enable_interrupts();
579	}
580
581	//-----------------------------------------------------------------------------
582	// Cluster
583	static void ?{}(__cluster_proc_list & this) {
584	this.lock = 0;
585	this.idle = 0;
586	this.total = 0;
587	}
588
589	void ?{}(cluster & this, const char name[], Duration preemption_rate, unsigned num_io, const io_context_params & io_params) with( this ) {
590	this.name = name;
591	this.preemption_rate = preemption_rate;
592	ready_queue{};
593
594	#if !defined(__CFA_NO_STATISTICS__)
595	print_stats = 0;
596	stats = alloc();
597	__init_stats( stats );
598	#endif
599
600	threads{ __get };
601
602	io.arbiter = create();
603	io.params = io_params;
604
605	doregister(this);
606
607	// Lock the RWlock so no-one pushes/pops while we are changing the queue
608	disable_interrupts();
609	uint_fast32_t last_size = ready_mutate_lock();
610
611	// Adjust the ready queue size
612	ready_queue_grow( &this );
613
614	// Unlock the RWlock
615	ready_mutate_unlock( last_size );
616	enable_interrupts( false ); // Don't poll, could be in main cluster
617	}
618
619	void ^?{}(cluster & this) {
620	destroy(this.io.arbiter);
621
622	// Lock the RWlock so no-one pushes/pops while we are changing the queue
623	disable_interrupts();
624	uint_fast32_t last_size = ready_mutate_lock();
625
626	// Adjust the ready queue size
627	ready_queue_shrink( &this );
628
629	// Unlock the RWlock
630	ready_mutate_unlock( last_size );
631	enable_interrupts( false ); // Don't poll, could be in main cluster
632
633	#if !defined(__CFA_NO_STATISTICS__)
634	if( 0 != this.print_stats ) {
635	__print_stats( this.stats, this.print_stats, "Cluster", this.name, (void*)&this );
636	}
637	#if defined(CFA_STATS_ARRAY)
638	__flush_stat( this.stats, "Cluster", &this );
639	#endif
640	free( this.stats );
641	#endif
642
643	unregister(this);
644	}
645
646	//=============================================================================================
647	// Miscellaneous Initialization
648	//=============================================================================================
649	//-----------------------------------------------------------------------------
650	// Global Queues
651	static void doregister( cluster & cltr ) {
652	lock ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
653	push_front( __cfa_dbg_global_clusters.list, cltr );
654	unlock ( __cfa_dbg_global_clusters.lock );
655	}
656
657	static void unregister( cluster & cltr ) {
658	lock ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
659	remove( __cfa_dbg_global_clusters.list, cltr );
660	unlock( __cfa_dbg_global_clusters.lock );
661	}
662
663	void doregister( cluster * cltr, thread$ & thrd ) {
664	lock (cltr->thread_list_lock __cfaabi_dbg_ctx2);
665	cltr->nthreads += 1;
666	push_front(cltr->threads, thrd);
667	unlock (cltr->thread_list_lock);
668	}
669
670	void unregister( cluster * cltr, thread$ & thrd ) {
671	lock (cltr->thread_list_lock __cfaabi_dbg_ctx2);
672	remove(cltr->threads, thrd );
673	cltr->nthreads -= 1;
674	unlock(cltr->thread_list_lock);
675	}
676
677	static void register_tls( processor * this ) {
678	// Register and Lock the RWlock so no-one pushes/pops while we are changing the queue
679	uint_fast32_t last_size;
680	[this->unique_id, last_size] = ready_mutate_register();
681
682	this->cltr->procs.total += 1u;
683	insert_last(this->cltr->procs.actives, *this);
684
685	// Adjust the ready queue size
686	ready_queue_grow( this->cltr );
687
688	// Unlock the RWlock
689	ready_mutate_unlock( last_size );
690	}
691
692
693	static void unregister_tls( processor * this ) {
694	// Lock the RWlock so no-one pushes/pops while we are changing the queue
695	uint_fast32_t last_size = ready_mutate_lock();
696	this->cltr->procs.total -= 1u;
697	remove(*this);
698
699	// clear the cluster so nothing gets pushed to local queues
700	cluster * cltr = this->cltr;
701	this->cltr = 0p;
702
703	// Adjust the ready queue size
704	ready_queue_shrink( cltr );
705
706	// Unlock the RWlock and unregister: we don't need the read_lock any more
707	ready_mutate_unregister( this->unique_id, last_size );
708	}
709
710	static void check( int ret, const char func[] ) {
711	if ( ret ) { // pthread routines return errno values
712	abort( "%s : internal error, error(%d) %s.", func, ret, strerror( ret ) );
713	} // if
714	} // Abort
715
716	void * __create_pthread( pthread_t * pthread, void * (start)(void ), void * arg ) {
717	pthread_attr_t attr;
718
719	check( pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
720
721	size_t stacksize;
722	// default stack size, normally defined by shell limit
723	check( pthread_attr_getstacksize( &attr, &stacksize ), "pthread_attr_getstacksize" );
724	assert( stacksize >= PTHREAD_STACK_MIN );
725
726	void * stack;
727	#if CFA_PROCESSOR_USE_MMAP
728	stacksize = ceiling( stacksize, __page_size ) + __page_size;
729	stack = mmap(0p, stacksize, __map_prot, MAP_PRIVATE \| MAP_ANONYMOUS, 0, 0);
730	if(stack == ((void*)-1)) {
731	abort( "pthread stack creation : internal error, mmap failure, error(%d) %s.", errno, strerror( errno ) );
732	}
733	if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
734	abort( "pthread stack creation : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
735	} // if
736	#else
737	__cfaabi_dbg_debug_do(
738	stack = memalign( __page_size, stacksize + __page_size );
739	// pthread has no mechanism to create the guard page in user supplied stack.
740	if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
741	abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
742	} // if
743	);
744	__cfaabi_dbg_no_debug_do(
745	stack = malloc( stacksize );
746	);
747	#endif
748
749
750	check( pthread_attr_setstack( &attr, stack, stacksize ), "pthread_attr_setstack" );
751
752	check( pthread_create( pthread, &attr, start, arg ), "pthread_create" );
753	return stack;
754	}
755
756	void __destroy_pthread( pthread_t pthread, void * stack, void ** retval ) {
757	int err = pthread_join( pthread, retval );
758	if( err != 0 ) abort("KERNEL ERROR: joining pthread %p caused error %s\n", (void*)pthread, strerror(err));
759
760	#if CFA_PROCESSOR_USE_MMAP
761	pthread_attr_t attr;
762
763	check( pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
764
765	size_t stacksize;
766	// default stack size, normally defined by shell limit
767	check( pthread_attr_getstacksize( &attr, &stacksize ), "pthread_attr_getstacksize" );
768	assert( stacksize >= PTHREAD_STACK_MIN );
769	stacksize += __page_size;
770
771	if(munmap(stack, stacksize) == -1) {
772	abort( "pthread stack destruction : internal error, munmap failure, error(%d) %s.", errno, strerror( errno ) );
773	}
774	#else
775	__cfaabi_dbg_debug_do(
776	// pthread has no mechanism to create the guard page in user supplied stack.
777	if ( mprotect( stack, __page_size, __map_prot ) == -1 ) {
778	abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
779	} // if
780	);
781	free( stack );
782	#endif
783	}
784
785	#if defined(__CFA_WITH_VERIFY__)
786	static bool verify_fwd_bck_rng(void) {
787	__cfaabi_tls.ready_rng.fwd_seed = 25214903917_l64u * (rdtscl() ^ (uintptr_t)&verify_fwd_bck_rng);
788
789	unsigned values[10];
790	for(i; 10) {
791	values[i] = __tls_rand_fwd();
792	}
793
794	__tls_rand_advance_bck();
795
796	for ( i; 9 -~= 0 ) {
797	if(values[i] != __tls_rand_bck()) {
798	return false;
799	}
800	}
801
802	return true;
803	}
804	#endif

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

Original Format