Context Navigation

startup.cfa@ db1ebed

Visit:

ADT ast-experimental enum forall-pointer-decay pthread-emulation qualifiedEnum

Last change on this file since db1ebed was 1757f98, checked in by Thierry Delisle <tdelisle@…>, 4 years ago
Refactoring idle sleep to try and help the change from idle on read to idle on io_uring_enter.
Property mode set to `100644`
File size: 24.7 KB

Line
1	//
2	// Cforall Version 1.0.0 Copyright (C) 2020 University of Waterloo
3	//
4	// The contents of this file are covered under the licence agreement in the
5	// file "LICENCE" distributed with Cforall.
6	//
7	// kernel/startup.cfa --
8	//
9	// Author : Thierry Delisle
10	// Created On : Thu Jul 30 15:12:54 2020
11	// Last Modified By :
12	// Last Modified On :
13	// Update Count :
14	//
15
16	#define __cforall_thread__
17	#define _GNU_SOURCE
18
19	// C Includes
20	#include <errno.h> // errno
21	#include <signal.h>
22	#include <string.h> // strerror
23	#include <unistd.h> // sysconf
24
25	extern "C" {
26	#include <limits.h> // PTHREAD_STACK_MIN
27	#include <unistd.h> // syscall
28	#include <sys/eventfd.h> // eventfd
29	#include <sys/mman.h> // mprotect
30	#include <sys/resource.h> // getrlimit
31	}
32
33	// CFA Includes
34	#include "kernel_private.hfa"
35	#include "startup.hfa" // STARTUP_PRIORITY_XXX
36	#include "math.hfa"
37
38	#define CFA_PROCESSOR_USE_MMAP 0
39
40	//-----------------------------------------------------------------------------
41	// Some assembly required
42	#if defined( __i386 )
43	#define CtxGet( ctx ) __asm__ volatile ( \
44	"movl %%esp,%0\n" \
45	"movl %%ebp,%1\n" \
46	: "=rm" (ctx.SP), \
47	"=rm" (ctx.FP) \
48	)
49	#elif defined( __x86_64 )
50	#define CtxGet( ctx ) __asm__ volatile ( \
51	"movq %%rsp,%0\n" \
52	"movq %%rbp,%1\n" \
53	: "=rm" (ctx.SP), \
54	"=rm" (ctx.FP) \
55	)
56	#elif defined( __aarch64__ )
57	#define CtxGet( ctx ) __asm__ volatile ( \
58	"mov %0, sp\n" \
59	"mov %1, fp\n" \
60	: "=rm" (ctx.SP), \
61	"=rm" (ctx.FP) \
62	)
63	#else
64	#error unknown hardware architecture
65	#endif
66
67	//-----------------------------------------------------------------------------
68	// Start and stop routine for the kernel, declared first to make sure they run first
69	static void __kernel_startup (void) __attribute__(( constructor( STARTUP_PRIORITY_KERNEL ) ));
70	static void __kernel_shutdown(void) __attribute__(( destructor ( STARTUP_PRIORITY_KERNEL ) ));
71
72	//-----------------------------------------------------------------------------
73	// Static Forward Declarations
74	struct current_stack_info_t;
75
76	static void * __invoke_processor(void * arg);
77	static void __kernel_first_resume( processor * this );
78	static void __kernel_last_resume ( processor * this );
79	static void init(processor & this, const char name[], cluster & _cltr, thread$ * initT);
80	static void deinit(processor & this);
81	static void doregister( struct cluster & cltr );
82	static void unregister( struct cluster & cltr );
83	static void register_tls( processor * this );
84	static void unregister_tls( processor * this );
85	static void ?{}( coroutine$ & this, current_stack_info_t * info);
86	static void ?{}( thread$ & this, current_stack_info_t * info);
87	static void ?{}(processorCtx_t & this) {}
88	static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info);
89
90	#if defined(__CFA_WITH_VERIFY__)
91	static bool verify_fwd_bck_rng(void);
92	#endif
93
94	//-----------------------------------------------------------------------------
95	// Forward Declarations for other modules
96	extern void __kernel_alarm_startup(void);
97	extern void __kernel_alarm_shutdown(void);
98
99	//-----------------------------------------------------------------------------
100	// Other Forward Declarations
101	extern void __wake_proc(processor *);
102	extern int cfa_main_returned; // from interpose.cfa
103
104	//-----------------------------------------------------------------------------
105	// Kernel storage
106	KERNEL_STORAGE(cluster, mainCluster);
107	KERNEL_STORAGE(processor, mainProcessor);
108	KERNEL_STORAGE(thread$, mainThread);
109	KERNEL_STORAGE(__stack_t, mainThreadCtx);
110	KERNEL_STORAGE(__scheduler_RWLock_t, __scheduler_lock);
111	#if !defined(__CFA_NO_STATISTICS__)
112	KERNEL_STORAGE(__stats_t, mainProcStats);
113	#endif
114
115	cluster * mainCluster;
116	processor * mainProcessor;
117	thread$ * mainThread;
118	__scheduler_RWLock_t * __scheduler_lock;
119
120	extern "C" {
121	struct { __dllist_t(cluster) list; __spinlock_t lock; } __cfa_dbg_global_clusters;
122	}
123
124	extern size_t __page_size;
125	extern int __map_prot;
126
127	//-----------------------------------------------------------------------------
128	// Global state
129	thread_local struct KernelThreadData __cfaabi_tls __attribute__ ((tls_model ( "initial-exec" ))) @= {
130	NULL, // cannot use 0p
131	NULL,
132	false,
133	{ 1, false, false },
134	0,
135	{ 0, 0 },
136	NULL,
137	#ifdef __CFA_WITH_VERIFY__
138	false,
139	0,
140	#endif
141	};
142
143	#if defined(CFA_HAVE_LINUX_LIBRSEQ)
144	// No data needed
145	#elif defined(CFA_HAVE_LINUX_RSEQ_H)
146	extern "Cforall" {
147	__attribute__((aligned(128))) thread_local volatile struct rseq __cfaabi_rseq @= {
148	.cpu_id : RSEQ_CPU_ID_UNINITIALIZED,
149	};
150	}
151	#else
152	// No data needed
153	#endif
154
155	//-----------------------------------------------------------------------------
156	// Struct to steal stack
157	struct current_stack_info_t {
158	__stack_t * storage; // pointer to stack object
159	void * base; // base of stack
160	void * limit; // stack grows towards stack limit
161	void * context; // address of cfa_context_t
162	};
163
164	void ?{}( current_stack_info_t & this ) {
165	__stack_context_t ctx;
166	CtxGet( ctx );
167	this.base = ctx.FP;
168
169	rlimit r;
170	getrlimit( RLIMIT_STACK, &r);
171	size_t size = r.rlim_cur;
172
173	this.limit = (void *)(((intptr_t)this.base) - size);
174	this.context = &storage_mainThreadCtx;
175	}
176
177
178
179	//=============================================================================================
180	// Kernel Setup logic
181	//=============================================================================================
182	//-----------------------------------------------------------------------------
183	// Kernel boot procedures
184	static void __kernel_startup(void) {
185	/* paranoid */ verify( ! __preemption_enabled() );
186	__cfadbg_print_safe(runtime_core, "Kernel : Starting\n");
187
188	__cfa_dbg_global_clusters.list{ __get };
189	__cfa_dbg_global_clusters.lock{};
190
191	/* paranoid */ verify( verify_fwd_bck_rng() );
192
193	// Initialize the global scheduler lock
194	__scheduler_lock = (__scheduler_RWLock_t*)&storage___scheduler_lock;
195	(*__scheduler_lock){};
196
197	// Initialize the main cluster
198	mainCluster = (cluster *)&storage_mainCluster;
199	(*mainCluster){"Main Cluster", 0};
200
201	__cfadbg_print_safe(runtime_core, "Kernel : Main cluster ready\n");
202
203	// Construct the processor context of the main processor
204	void ?{}(processorCtx_t & this, processor * proc) {
205	(this.__cor){ "Processor" };
206	this.__cor.starter = 0p;
207	this.proc = proc;
208	}
209
210	void ?{}(processor & this) with( this ) {
211	( this.terminated ){};
212	( this.runner ){};
213	init( this, "Main Processor", *mainCluster, 0p );
214	kernel_thread = pthread_self();
215
216	runner{ &this };
217	__cfadbg_print_safe(runtime_core, "Kernel : constructed main processor context %p\n", &runner);
218	}
219
220	// Initialize the main processor and the main processor ctx
221	// (the coroutine that contains the processing control flow)
222	mainProcessor = (processor *)&storage_mainProcessor;
223	(*mainProcessor){};
224
225	register_tls( mainProcessor );
226
227	// Start by initializing the main thread
228	// SKULLDUGGERY: the mainThread steals the process main thread
229	// which will then be scheduled by the mainProcessor normally
230	mainThread = (thread$ *)&storage_mainThread;
231	current_stack_info_t info;
232	info.storage = (__stack_t*)&storage_mainThreadCtx;
233	(*mainThread){ &info };
234
235	__cfadbg_print_safe(runtime_core, "Kernel : Main thread ready\n");
236
237	//initialize the global state variables
238	__cfaabi_tls.this_processor = mainProcessor;
239	__cfaabi_tls.this_thread = mainThread;
240
241	#if !defined( __CFA_NO_STATISTICS__ )
242	__cfaabi_tls.this_stats = (__stats_t *)& storage_mainProcStats;
243	__init_stats( __cfaabi_tls.this_stats );
244	#endif
245	mainProcessor->local_data = &__cfaabi_tls;
246
247	// Enable preemption
248	__kernel_alarm_startup();
249
250	// Add the main thread to the ready queue
251	// once resume is called on mainProcessor->runner the mainThread needs to be scheduled like any normal thread
252	schedule_thread$(mainThread, UNPARK_LOCAL);
253
254	// SKULLDUGGERY: Force a context switch to the main processor to set the main thread's context to the current UNIX
255	// context. Hence, the main thread does not begin through __cfactx_invoke_thread, like all other threads. The trick here is that
256	// mainThread is on the ready queue when this call is made.
257	__kernel_first_resume( __cfaabi_tls.this_processor );
258
259
260	// THE SYSTEM IS NOW COMPLETELY RUNNING
261
262	__cfadbg_print_safe(runtime_core, "Kernel : Started\n--------------------------------------------------\n\n");
263
264	/* paranoid */ verify( ! __preemption_enabled() );
265	enable_interrupts();
266	/* paranoid */ verify( __preemption_enabled() );
267
268	}
269
270	static void __kernel_shutdown(void) {
271	if(!cfa_main_returned) return;
272	/* paranoid */ verify( __preemption_enabled() );
273	disable_interrupts();
274	/* paranoid */ verify( ! __preemption_enabled() );
275
276	__cfadbg_print_safe(runtime_core, "\n--------------------------------------------------\nKernel : Shutting down\n");
277
278	// SKULLDUGGERY: Notify the mainProcessor it needs to terminates.
279	// When its coroutine terminates, it return control to the mainThread
280	// which is currently here
281	__atomic_store_n(&mainProcessor->do_terminate, true, __ATOMIC_RELEASE);
282	__kernel_last_resume( __cfaabi_tls.this_processor );
283	mainThread->self_cor.state = Halted;
284
285	// THE SYSTEM IS NOW COMPLETELY STOPPED
286
287	// Disable preemption
288	__kernel_alarm_shutdown();
289
290	#if !defined( __CFA_NO_STATISTICS__ )
291	__stats_t * st = (__stats_t *)& storage_mainProcStats;
292	__tally_stats(mainCluster->stats, st);
293	if( 0 != mainProcessor->print_stats ) {
294	__print_stats( st, mainProcessor->print_stats, "Processor ", mainProcessor->name, (void*)mainProcessor );
295	}
296	#if defined(CFA_STATS_ARRAY)
297	__flush_stat( st, "Processor", mainProcessor );
298	#endif
299	#endif
300
301	mainProcessor->local_data = 0p;
302
303	unregister_tls( mainProcessor );
304
305	// Destroy the main processor and its context in reverse order of construction
306	// These were manually constructed so we need manually destroy them
307	void ^?{}(processor & this) with( this ){
308	deinit( this );
309
310	/* paranoid */ verify( this.do_terminate == true );
311	__cfaabi_dbg_print_safe("Kernel : destroyed main processor context %p\n", &runner);
312	}
313
314	^(*mainProcessor){};
315
316	// Final step, destroy the main thread since it is no longer needed
317
318	// Since we provided a stack to this taxk it will not destroy anything
319	/* paranoid / verify(mainThread->self_cor.stack.storage == (__stack_t)(((uintptr_t)&storage_mainThreadCtx)\| 0x1));
320	^(*mainThread){};
321
322	^(*mainCluster){};
323
324	^(*__scheduler_lock){};
325
326	^(__cfa_dbg_global_clusters.list){};
327	^(__cfa_dbg_global_clusters.lock){};
328
329	__cfadbg_print_safe(runtime_core, "Kernel : Shutdown complete\n");
330	}
331
332	//=============================================================================================
333	// Kernel Initial Scheduling logic
334	//=============================================================================================
335
336	// Context invoker for processors
337	// This is the entry point for processors (kernel threads) except for the main processor
338	// It effectively constructs a coroutine by stealing the pthread stack
339	static void * __invoke_processor(void * arg) {
340	#if !defined( __CFA_NO_STATISTICS__ )
341	__stats_t local_stats;
342	__init_stats( &local_stats );
343	__cfaabi_tls.this_stats = &local_stats;
344	#endif
345
346	processor * proc = (processor *) arg;
347	__cfaabi_tls.this_processor = proc;
348	__cfaabi_tls.this_thread = 0p;
349	__cfaabi_tls.preemption_state.[enabled, disable_count] = [false, 1];
350	proc->local_data = &__cfaabi_tls;
351
352	register_tls( proc );
353
354	// SKULLDUGGERY: We want to create a context for the processor coroutine
355	// which is needed for the 2-step context switch. However, there is no reason
356	// to waste the perfectly valid stack create by pthread.
357	current_stack_info_t info;
358	__stack_t ctx;
359	info.storage = &ctx;
360	(proc->runner){ proc, &info };
361
362	__cfaabi_dbg_print_safe("Coroutine : created stack %p\n", get_coroutine(proc->runner)->stack.storage);
363
364	//Set global state
365	__cfaabi_tls.this_thread = 0p;
366
367	//We now have a proper context from which to schedule threads
368	__cfadbg_print_safe(runtime_core, "Kernel : core %p created (%p, %p)\n", proc, &proc->runner, &ctx);
369
370	// SKULLDUGGERY: Since the coroutine doesn't have its own stack, we can't
371	// resume it to start it like it normally would, it will just context switch
372	// back to here. Instead directly call the main since we already are on the
373	// appropriate stack.
374	get_coroutine(proc->runner)->state = Active;
375	main( proc->runner );
376	get_coroutine(proc->runner)->state = Halted;
377
378	// Main routine of the core returned, the core is now fully terminated
379	__cfadbg_print_safe(runtime_core, "Kernel : core %p main ended (%p)\n", proc, &proc->runner);
380
381	#if !defined(__CFA_NO_STATISTICS__)
382	__tally_stats(proc->cltr->stats, &local_stats);
383	if( 0 != proc->print_stats ) {
384	__print_stats( &local_stats, proc->print_stats, "Processor ", proc->name, (void*)proc );
385	}
386	#if defined(CFA_STATS_ARRAY)
387	__flush_stat( &local_stats, "Processor", proc );
388	#endif
389	#endif
390
391	proc->local_data = 0p;
392
393	unregister_tls( proc );
394
395	return 0p;
396	}
397
398	static void __kernel_first_resume( processor * this ) {
399	thread$ * src = mainThread;
400	coroutine$ * dst = get_coroutine(this->runner);
401
402	/* paranoid */ verify( ! __preemption_enabled() );
403
404	__cfaabi_tls.this_thread->curr_cor = dst;
405	__stack_prepare( &dst->stack, 65000 );
406	__cfactx_start(main, dst, this->runner, __cfactx_invoke_coroutine);
407
408	/* paranoid */ verify( ! __preemption_enabled() );
409
410	dst->last = &src->self_cor;
411	dst->starter = dst->starter ? dst->starter : &src->self_cor;
412
413	// make sure the current state is still correct
414	/* paranoid */ verify(src->state == Ready);
415	src->corctx_flag = true;
416
417	// context switch to specified coroutine
418	verify( dst->context.SP );
419	__cfactx_switch( &src->context, &dst->context );
420	// when __cfactx_switch returns we are back in the src coroutine
421
422	mainThread->curr_cor = &mainThread->self_cor;
423
424	// make sure the current state has been update
425	/* paranoid */ verify(src->state == Active);
426
427	/* paranoid */ verify( ! __preemption_enabled() );
428	}
429
430	// KERNEL_ONLY
431	static void __kernel_last_resume( processor * this ) {
432	coroutine$ * src = &mainThread->self_cor;
433	coroutine$ * dst = get_coroutine(this->runner);
434
435	/* paranoid */ verify( ! __preemption_enabled() );
436	/* paranoid */ verify( dst->starter == src );
437	/* paranoid */ verify( dst->context.SP );
438
439	// SKULLDUGGERY in debug the processors check that the
440	// stack is still within the limit of the stack limits after running a thread.
441	// that check doesn't make sense if we context switch to the processor using the
442	// coroutine semantics. Since this is a special case, use the current context
443	// info to populate these fields.
444	__cfaabi_dbg_debug_do(
445	__stack_context_t ctx;
446	CtxGet( ctx );
447	mainThread->context.SP = ctx.SP;
448	mainThread->context.FP = ctx.FP;
449	)
450
451	// context switch to the processor
452	__cfactx_switch( &src->context, &dst->context );
453	}
454
455
456	//=============================================================================================
457	// Kernel Object Constructors logic
458	//=============================================================================================
459	//-----------------------------------------------------------------------------
460	// Main thread construction
461	static void ?{}( coroutine$ & this, current_stack_info_t * info) with( this ) {
462	stack.storage = info->storage;
463	with(*stack.storage) {
464	limit = info->limit;
465	base = info->base;
466	}
467	__attribute__((may_alias)) intptr_t * istorage = (intptr_t*) &stack.storage;
468	*istorage \|= 0x1;
469	name = "Main Thread";
470	state = Start;
471	starter = 0p;
472	last = 0p;
473	cancellation = 0p;
474	}
475
476	static void ?{}( thread$ & this, current_stack_info_t * info) with( this ) {
477	ticket = TICKET_RUNNING;
478	state = Start;
479	self_cor{ info };
480	curr_cor = &self_cor;
481	curr_cluster = mainCluster;
482	self_mon.owner = &this;
483	self_mon.recursion = 1;
484	self_mon_p = &self_mon;
485	link.next = 0p;
486	link.ts = -1llu;
487	preferred = ready_queue_new_preferred();
488	last_proc = 0p;
489	#if defined( __CFA_WITH_VERIFY__ )
490	canary = 0x0D15EA5E0D15EA5Ep;
491	#endif
492
493	node.next = 0p;
494	node.prev = 0p;
495	doregister(curr_cluster, this);
496
497	monitors{ &self_mon_p, 1, (fptr_t)0 };
498	}
499
500	//-----------------------------------------------------------------------------
501	// Processor
502	// Construct the processor context of non-main processors
503	static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info) {
504	(this.__cor){ info };
505	this.proc = proc;
506	}
507
508	static void init(processor & this, const char name[], cluster & _cltr, thread$ * initT) with( this ) {
509	this.name = name;
510	this.cltr = &_cltr;
511	this.rdq.its = 0;
512	this.rdq.itr = 0;
513	this.rdq.id = -1u;
514	this.rdq.target = -1u;
515	this.rdq.last = -1u;
516	this.rdq.cutoff = 0ull;
517	do_terminate = false;
518	preemption_alarm = 0p;
519	pending_preemption = false;
520
521	this.io.ctx = 0p;
522	this.io.pending = false;
523	this.io.dirty = false;
524
525	this.init.thrd = initT;
526
527	this.local_data = 0p;
528
529	this.idle_fd = eventfd(0, 0);
530	if (idle_fd < 0) {
531	abort("KERNEL ERROR: PROCESSOR EVENTFD - %s\n", strerror(errno));
532	}
533
534	#if !defined(__CFA_NO_STATISTICS__)
535	print_stats = 0;
536	print_halts = false;
537	#endif
538
539	__cfadbg_print_safe(runtime_core, "Kernel : core %p created\n", &this);
540	}
541
542	// Not a ctor, it just preps the destruction but should not destroy members
543	static void deinit(processor & this) {
544	close(this.idle_fd);
545	}
546
547	void ?{}(processor & this, const char name[], cluster & _cltr, thread$ * initT) {
548	( this.terminated ){};
549	( this.runner ){};
550
551	disable_interrupts();
552	init( this, name, _cltr, initT );
553	enable_interrupts();
554
555	__cfadbg_print_safe(runtime_core, "Kernel : Starting core %p\n", &this);
556
557	this.stack = __create_pthread( &this.kernel_thread, __invoke_processor, (void *)&this );
558	}
559
560	void ?{}(processor & this, const char name[], cluster & _cltr) {
561	(this){name, _cltr, 0p};
562	}
563
564	extern size_t __page_size;
565	void ^?{}(processor & this) with( this ){
566	if( ! __atomic_load_n(&do_terminate, __ATOMIC_ACQUIRE) ) {
567	__cfadbg_print_safe(runtime_core, "Kernel : core %p signaling termination\n", &this);
568
569	__atomic_store_n(&do_terminate, true, __ATOMIC_RELAXED);
570	__wake_proc( &this );
571
572	wait( terminated );
573	/* paranoid */ verify( active_processor() != &this);
574	}
575
576	__destroy_pthread( kernel_thread, this.stack, 0p );
577
578	disable_interrupts();
579	deinit( this );
580	enable_interrupts();
581	}
582
583	//-----------------------------------------------------------------------------
584	// Cluster
585	static void ?{}(__cluster_proc_list & this) {
586	this.fd = 0;
587	this.idle = 0;
588	this.total = 0;
589	}
590
591	void ?{}(cluster & this, const char name[], Duration preemption_rate, unsigned num_io, const io_context_params & io_params) with( this ) {
592	this.name = name;
593	this.preemption_rate = preemption_rate;
594	ready_queue{};
595
596	#if !defined(__CFA_NO_STATISTICS__)
597	print_stats = 0;
598	stats = alloc();
599	__init_stats( stats );
600	#endif
601
602	threads{ __get };
603
604	io.arbiter = create();
605	io.params = io_params;
606
607	doregister(this);
608
609	// Lock the RWlock so no-one pushes/pops while we are changing the queue
610	disable_interrupts();
611	uint_fast32_t last_size = ready_mutate_lock();
612
613	// Adjust the ready queue size
614	ready_queue_grow( &this );
615
616	// Unlock the RWlock
617	ready_mutate_unlock( last_size );
618	enable_interrupts( false ); // Don't poll, could be in main cluster
619	}
620
621	void ^?{}(cluster & this) {
622	destroy(this.io.arbiter);
623
624	// Lock the RWlock so no-one pushes/pops while we are changing the queue
625	disable_interrupts();
626	uint_fast32_t last_size = ready_mutate_lock();
627
628	// Adjust the ready queue size
629	ready_queue_shrink( &this );
630
631	// Unlock the RWlock
632	ready_mutate_unlock( last_size );
633	enable_interrupts( false ); // Don't poll, could be in main cluster
634
635	#if !defined(__CFA_NO_STATISTICS__)
636	if( 0 != this.print_stats ) {
637	__print_stats( this.stats, this.print_stats, "Cluster", this.name, (void*)&this );
638	}
639	#if defined(CFA_STATS_ARRAY)
640	__flush_stat( this.stats, "Cluster", &this );
641	#endif
642	free( this.stats );
643	#endif
644
645	unregister(this);
646	}
647
648	//=============================================================================================
649	// Miscellaneous Initialization
650	//=============================================================================================
651	//-----------------------------------------------------------------------------
652	// Global Queues
653	static void doregister( cluster & cltr ) {
654	lock ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
655	push_front( __cfa_dbg_global_clusters.list, cltr );
656	unlock ( __cfa_dbg_global_clusters.lock );
657	}
658
659	static void unregister( cluster & cltr ) {
660	lock ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
661	remove( __cfa_dbg_global_clusters.list, cltr );
662	unlock( __cfa_dbg_global_clusters.lock );
663	}
664
665	void doregister( cluster * cltr, thread$ & thrd ) {
666	lock (cltr->thread_list_lock __cfaabi_dbg_ctx2);
667	cltr->nthreads += 1;
668	push_front(cltr->threads, thrd);
669	unlock (cltr->thread_list_lock);
670	}
671
672	void unregister( cluster * cltr, thread$ & thrd ) {
673	lock (cltr->thread_list_lock __cfaabi_dbg_ctx2);
674	remove(cltr->threads, thrd );
675	cltr->nthreads -= 1;
676	unlock(cltr->thread_list_lock);
677	}
678
679	static void register_tls( processor * this ) {
680	// Register and Lock the RWlock so no-one pushes/pops while we are changing the queue
681	uint_fast32_t last_size;
682	[this->unique_id, last_size] = ready_mutate_register();
683
684	this->cltr->procs.total += 1u;
685	insert_last(this->cltr->procs.actives, *this);
686
687	// Adjust the ready queue size
688	ready_queue_grow( this->cltr );
689
690	// Unlock the RWlock
691	ready_mutate_unlock( last_size );
692	}
693
694
695	static void unregister_tls( processor * this ) {
696	// Lock the RWlock so no-one pushes/pops while we are changing the queue
697	uint_fast32_t last_size = ready_mutate_lock();
698	this->cltr->procs.total -= 1u;
699	remove(*this);
700
701	// clear the cluster so nothing gets pushed to local queues
702	cluster * cltr = this->cltr;
703	this->cltr = 0p;
704
705	// Adjust the ready queue size
706	ready_queue_shrink( cltr );
707
708	// Unlock the RWlock and unregister: we don't need the read_lock any more
709	ready_mutate_unregister( this->unique_id, last_size );
710	}
711
712	static void check( int ret, const char func[] ) {
713	if ( ret ) { // pthread routines return errno values
714	abort( "%s : internal error, error(%d) %s.", func, ret, strerror( ret ) );
715	} // if
716	} // Abort
717
718	void * __create_pthread( pthread_t * pthread, void * (start)(void ), void * arg ) {
719	pthread_attr_t attr;
720
721	check( pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
722
723	size_t stacksize;
724	// default stack size, normally defined by shell limit
725	check( pthread_attr_getstacksize( &attr, &stacksize ), "pthread_attr_getstacksize" );
726	assert( stacksize >= PTHREAD_STACK_MIN );
727
728	void * stack;
729	#if CFA_PROCESSOR_USE_MMAP
730	stacksize = ceiling( stacksize, __page_size ) + __page_size;
731	stack = mmap(0p, stacksize, __map_prot, MAP_PRIVATE \| MAP_ANONYMOUS, 0, 0);
732	if(stack == ((void*)-1)) {
733	abort( "pthread stack creation : internal error, mmap failure, error(%d) %s.", errno, strerror( errno ) );
734	}
735	if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
736	abort( "pthread stack creation : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
737	} // if
738	#else
739	__cfaabi_dbg_debug_do(
740	stack = memalign( __page_size, stacksize + __page_size );
741	// pthread has no mechanism to create the guard page in user supplied stack.
742	if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
743	abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
744	} // if
745	);
746	__cfaabi_dbg_no_debug_do(
747	stack = malloc( stacksize );
748	);
749	#endif
750
751
752	check( pthread_attr_setstack( &attr, stack, stacksize ), "pthread_attr_setstack" );
753
754	check( pthread_create( pthread, &attr, start, arg ), "pthread_create" );
755	return stack;
756	}
757
758	void __destroy_pthread( pthread_t pthread, void * stack, void ** retval ) {
759	int err = pthread_join( pthread, retval );
760	if( err != 0 ) abort("KERNEL ERROR: joining pthread %p caused error %s\n", (void*)pthread, strerror(err));
761
762	#if CFA_PROCESSOR_USE_MMAP
763	pthread_attr_t attr;
764
765	check( pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
766
767	size_t stacksize;
768	// default stack size, normally defined by shell limit
769	check( pthread_attr_getstacksize( &attr, &stacksize ), "pthread_attr_getstacksize" );
770	assert( stacksize >= PTHREAD_STACK_MIN );
771	stacksize += __page_size;
772
773	if(munmap(stack, stacksize) == -1) {
774	abort( "pthread stack destruction : internal error, munmap failure, error(%d) %s.", errno, strerror( errno ) );
775	}
776	#else
777	__cfaabi_dbg_debug_do(
778	// pthread has no mechanism to create the guard page in user supplied stack.
779	if ( mprotect( stack, __page_size, __map_prot ) == -1 ) {
780	abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
781	} // if
782	);
783	free( stack );
784	#endif
785	}
786
787	#if defined(__CFA_WITH_VERIFY__)
788	static bool verify_fwd_bck_rng(void) {
789	__cfaabi_tls.ready_rng.fwd_seed = 25214903917_l64u * (rdtscl() ^ (uintptr_t)&verify_fwd_bck_rng);
790
791	unsigned values[10];
792	for(i; 10) {
793	values[i] = __tls_rand_fwd();
794	}
795
796	__tls_rand_advance_bck();
797
798	for ( i; 9 -~= 0 ) {
799	if(values[i] != __tls_rand_bck()) {
800	return false;
801	}
802	}
803
804	return true;
805	}
806	#endif

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

Original Format