Context Navigation

startup.cfa@ f0567a8

Visit:

ADT ast-experimental enum pthread-emulation qualifiedEnum

Last change on this file since f0567a8 was 5614a191, checked in by m3zulfiq <m3zulfiq@…>, 4 years ago
removed old memory allocator and replaced it with the concurrent allocator
Property mode set to `100644`
File size: 25.2 KB

Line
1	//
2	// Cforall Version 1.0.0 Copyright (C) 2020 University of Waterloo
3	//
4	// The contents of this file are covered under the licence agreement in the
5	// file "LICENCE" distributed with Cforall.
6	//
7	// kernel/startup.cfa --
8	//
9	// Author : Thierry Delisle
10	// Created On : Thu Jul 30 15:12:54 2020
11	// Last Modified By :
12	// Last Modified On :
13	// Update Count :
14	//
15
16	#define __cforall_thread__
17	#define _GNU_SOURCE
18
19	// C Includes
20	#include <errno.h> // errno
21	#include <signal.h>
22	#include <string.h> // strerror
23	#include <unistd.h> // sysconf
24
25	extern "C" {
26	#include <limits.h> // PTHREAD_STACK_MIN
27	#include <unistd.h> // syscall
28	#include <sys/eventfd.h> // eventfd
29	#include <sys/mman.h> // mprotect
30	#include <sys/resource.h> // getrlimit
31	}
32
33	// CFA Includes
34	#include "kernel_private.hfa"
35	#include "startup.hfa" // STARTUP_PRIORITY_XXX
36	#include "limits.hfa"
37	#include "math.hfa"
38
39	#define CFA_PROCESSOR_USE_MMAP 0
40
41	//-----------------------------------------------------------------------------
42	// Some assembly required
43	#if defined( __i386 )
44	#define CtxGet( ctx ) __asm__ volatile ( \
45	"movl %%esp,%0\n" \
46	"movl %%ebp,%1\n" \
47	: "=rm" (ctx.SP), \
48	"=rm" (ctx.FP) \
49	)
50	#elif defined( __x86_64 )
51	#define CtxGet( ctx ) __asm__ volatile ( \
52	"movq %%rsp,%0\n" \
53	"movq %%rbp,%1\n" \
54	: "=rm" (ctx.SP), \
55	"=rm" (ctx.FP) \
56	)
57	#elif defined( __aarch64__ )
58	#define CtxGet( ctx ) __asm__ volatile ( \
59	"mov %0, sp\n" \
60	"mov %1, fp\n" \
61	: "=rm" (ctx.SP), \
62	"=rm" (ctx.FP) \
63	)
64	#else
65	#error unknown hardware architecture
66	#endif
67
68	//-----------------------------------------------------------------------------
69	// Start and stop routine for the kernel, declared first to make sure they run first
70	static void __kernel_startup (void) __attribute__(( constructor( STARTUP_PRIORITY_KERNEL ) ));
71	static void __kernel_shutdown(void) __attribute__(( destructor ( STARTUP_PRIORITY_KERNEL ) ));
72
73	//-----------------------------------------------------------------------------
74	// Static Forward Declarations
75	struct current_stack_info_t;
76
77	static void * __invoke_processor(void * arg);
78	static void __kernel_first_resume( processor * this );
79	static void __kernel_last_resume ( processor * this );
80	static void init(processor & this, const char name[], cluster & _cltr, thread$ * initT);
81	static void deinit(processor & this);
82	static void doregister( struct cluster & cltr );
83	static void unregister( struct cluster & cltr );
84	static void register_tls( processor * this );
85	static void unregister_tls( processor * this );
86	static void ?{}( coroutine$ & this, current_stack_info_t * info);
87	static void ?{}( thread$ & this, current_stack_info_t * info);
88	static void ?{}(processorCtx_t & this) {}
89	static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info);
90
91	#if defined(__CFA_WITH_VERIFY__)
92	static bool verify_fwd_bck_rng(void);
93	#endif
94
95	//-----------------------------------------------------------------------------
96	// Forward Declarations for other modules
97	extern void __kernel_alarm_startup(void);
98	extern void __kernel_alarm_shutdown(void);
99
100	//-----------------------------------------------------------------------------
101	// Other Forward Declarations
102	extern void __wake_proc(processor *);
103	extern int cfa_main_returned; // from interpose.cfa
104	uint32_t __global_random_prime = 4_294_967_291u, __global_random_mask = false;
105
106	//-----------------------------------------------------------------------------
107	// Kernel storage
108	KERNEL_STORAGE(cluster, mainCluster);
109	KERNEL_STORAGE(processor, mainProcessor);
110	KERNEL_STORAGE(thread$, mainThread);
111	KERNEL_STORAGE(__stack_t, mainThreadCtx);
112	KERNEL_STORAGE(__scheduler_RWLock_t, __scheduler_lock);
113	#if !defined(__CFA_NO_STATISTICS__)
114	KERNEL_STORAGE(__stats_t, mainProcStats);
115	#endif
116
117	cluster * mainCluster;
118	processor * mainProcessor;
119	thread$ * mainThread;
120	__scheduler_RWLock_t * __scheduler_lock;
121
122	extern "C" {
123	struct { __dllist_t(cluster) list; __spinlock_t lock; } __cfa_dbg_global_clusters;
124	extern size_t __cfa_page_size;
125	extern int __map_prot;
126	}
127
128	//-----------------------------------------------------------------------------
129	// Global state
130	thread_local struct KernelThreadData __cfaabi_tls __attribute__ ((tls_model ( "initial-exec" ))) @= {
131	NULL, // cannot use 0p
132	NULL,
133	false,
134	{ 1, false, false },
135	0,
136	{ 0, 0 },
137	NULL,
138	#ifdef __CFA_WITH_VERIFY__
139	false,
140	0,
141	#endif
142	};
143
144	#if defined(CFA_HAVE_LINUX_LIBRSEQ)
145	// No data needed
146	#elif defined(CFA_HAVE_LINUX_RSEQ_H)
147	extern "Cforall" {
148	__attribute__((aligned(128))) thread_local volatile struct rseq __cfaabi_rseq @= {
149	.cpu_id : RSEQ_CPU_ID_UNINITIALIZED,
150	};
151	}
152	#else
153	// No data needed
154	#endif
155
156	//-----------------------------------------------------------------------------
157	// Struct to steal stack
158	struct current_stack_info_t {
159	__stack_t * storage; // pointer to stack object
160	void * base; // base of stack
161	void * limit; // stack grows towards stack limit
162	void * context; // address of cfa_context_t
163	};
164
165	void ?{}( current_stack_info_t & this ) {
166	__stack_context_t ctx;
167	CtxGet( ctx );
168	this.base = ctx.FP;
169
170	rlimit r;
171	getrlimit( RLIMIT_STACK, &r);
172	size_t size = r.rlim_cur;
173
174	this.limit = (void *)(((intptr_t)this.base) - size);
175	this.context = &storage_mainThreadCtx;
176	}
177
178
179	//=============================================================================================
180	// Kernel Setup logic
181	//=============================================================================================
182	//-----------------------------------------------------------------------------
183	// Kernel boot procedures
184	static void __kernel_startup(void) {
185	/* paranoid */ verify( ! __preemption_enabled() );
186	__cfadbg_print_safe(runtime_core, "Kernel : Starting\n");
187
188	__cfa_dbg_global_clusters.list{ __get };
189	__cfa_dbg_global_clusters.lock{};
190
191	/* paranoid */ verify( verify_fwd_bck_rng() );
192
193	// Initialize the global scheduler lock
194	__scheduler_lock = (__scheduler_RWLock_t*)&storage___scheduler_lock;
195	(*__scheduler_lock){};
196
197	// Initialize the main cluster
198	mainCluster = (cluster *)&storage_mainCluster;
199	(*mainCluster){"Main Cluster", 0};
200
201	__cfadbg_print_safe(runtime_core, "Kernel : Main cluster ready\n");
202
203	// Construct the processor context of the main processor
204	void ?{}(processorCtx_t & this, processor * proc) {
205	(this.__cor){ "Processor" };
206	this.__cor.starter = 0p;
207	this.proc = proc;
208	}
209
210	void ?{}(processor & this) with( this ) {
211	( this.terminated ){};
212	( this.runner ){};
213	init( this, "Main Processor", *mainCluster, 0p );
214	kernel_thread = pthread_self();
215
216	runner{ &this };
217	__cfadbg_print_safe(runtime_core, "Kernel : constructed main processor context %p\n", &runner);
218	}
219
220	// Initialize the main processor and the main processor ctx
221	// (the coroutine that contains the processing control flow)
222	mainProcessor = (processor *)&storage_mainProcessor;
223	(*mainProcessor){};
224
225	register_tls( mainProcessor );
226
227	// Start by initializing the main thread
228	// SKULLDUGGERY: the mainThread steals the process main thread
229	// which will then be scheduled by the mainProcessor normally
230	mainThread = (thread$ *)&storage_mainThread;
231	current_stack_info_t info;
232	info.storage = (__stack_t*)&storage_mainThreadCtx;
233	(*mainThread){ &info };
234
235	__cfadbg_print_safe(runtime_core, "Kernel : Main thread ready\n");
236
237	//initialize the global state variables
238	__cfaabi_tls.this_processor = mainProcessor;
239	__cfaabi_tls.this_thread = mainThread;
240
241	#if !defined( __CFA_NO_STATISTICS__ )
242	__cfaabi_tls.this_stats = (__stats_t *)& storage_mainProcStats;
243	__init_stats( __cfaabi_tls.this_stats );
244	#endif
245	mainProcessor->local_data = &__cfaabi_tls;
246
247	// Enable preemption
248	__kernel_alarm_startup();
249
250	// Add the main thread to the ready queue
251	// once resume is called on mainProcessor->runner the mainThread needs to be scheduled like any normal thread
252	schedule_thread$(mainThread, UNPARK_LOCAL);
253
254	// SKULLDUGGERY: Force a context switch to the main processor to set the main thread's context to the current UNIX
255	// context. Hence, the main thread does not begin through __cfactx_invoke_thread, like all other threads. The trick here is that
256	// mainThread is on the ready queue when this call is made.
257	__kernel_first_resume( __cfaabi_tls.this_processor );
258
259
260	// THE SYSTEM IS NOW COMPLETELY RUNNING
261
262	__cfadbg_print_safe(runtime_core, "Kernel : Started\n--------------------------------------------------\n\n");
263
264	/* paranoid */ verify( ! __preemption_enabled() );
265	enable_interrupts();
266	/* paranoid */ verify( __preemption_enabled() );
267
268	}
269
270	static void __kernel_shutdown(void) {
271	if(!cfa_main_returned) return;
272	/* paranoid */ verify( __preemption_enabled() );
273	disable_interrupts();
274	/* paranoid */ verify( ! __preemption_enabled() );
275
276	__cfadbg_print_safe(runtime_core, "\n--------------------------------------------------\nKernel : Shutting down\n");
277
278	// SKULLDUGGERY: Notify the mainProcessor it needs to terminates.
279	// When its coroutine terminates, it return control to the mainThread
280	// which is currently here
281	/* paranoid */ verify( !__atomic_load_n(&mainProcessor->do_terminate, __ATOMIC_ACQUIRE) );
282	__atomic_store_n(&mainProcessor->do_terminate, true, __ATOMIC_RELEASE);
283	__wake_proc( mainProcessor );
284	__kernel_last_resume( __cfaabi_tls.this_processor );
285	mainThread->self_cor.state = Halted;
286
287	// THE SYSTEM IS NOW COMPLETELY STOPPED
288
289	// Disable preemption
290	__kernel_alarm_shutdown();
291
292	#if !defined( __CFA_NO_STATISTICS__ )
293	__stats_t * st = (__stats_t *)& storage_mainProcStats;
294	__tally_stats(mainCluster->stats, st);
295	if( 0 != mainProcessor->print_stats ) {
296	__print_stats( st, mainProcessor->print_stats, "Processor ", mainProcessor->name, (void*)mainProcessor );
297	}
298	#if defined(CFA_STATS_ARRAY)
299	__flush_stat( st, "Processor", mainProcessor );
300	#endif
301	#endif
302
303	mainProcessor->local_data = 0p;
304
305	unregister_tls( mainProcessor );
306
307	// Destroy the main processor and its context in reverse order of construction
308	// These were manually constructed so we need manually destroy them
309	void ^?{}(processor & this) with( this ){
310	deinit( this );
311
312	/* paranoid */ verify( this.do_terminate == true );
313	__cfaabi_dbg_print_safe("Kernel : destroyed main processor context %p\n", &runner);
314	}
315
316	^(*mainProcessor){};
317
318	// Final step, destroy the main thread since it is no longer needed
319
320	// Since we provided a stack to this taxk it will not destroy anything
321	/* paranoid / verify(mainThread->self_cor.stack.storage == (__stack_t)(((uintptr_t)&storage_mainThreadCtx)\| 0x1));
322	^(*mainThread){};
323
324	^(*mainCluster){};
325
326	^(*__scheduler_lock){};
327
328	^(__cfa_dbg_global_clusters.list){};
329	^(__cfa_dbg_global_clusters.lock){};
330
331	__cfadbg_print_safe(runtime_core, "Kernel : Shutdown complete\n");
332	}
333
334	//=============================================================================================
335	// Kernel Initial Scheduling logic
336	//=============================================================================================
337
338	// Context invoker for processors
339	// This is the entry point for processors (kernel threads) except for the main processor
340	// It effectively constructs a coroutine by stealing the pthread stack
341	static void * __invoke_processor(void * arg) {
342	#if !defined( __CFA_NO_STATISTICS__ )
343	__stats_t local_stats;
344	__init_stats( &local_stats );
345	__cfaabi_tls.this_stats = &local_stats;
346	#endif
347
348	processor * proc = (processor *) arg;
349	__cfaabi_tls.this_processor = proc;
350	__cfaabi_tls.this_thread = 0p;
351	__cfaabi_tls.preemption_state.[enabled, disable_count] = [false, 1];
352	proc->local_data = &__cfaabi_tls;
353
354	register_tls( proc );
355
356	// SKULLDUGGERY: We want to create a context for the processor coroutine
357	// which is needed for the 2-step context switch. However, there is no reason
358	// to waste the perfectly valid stack create by pthread.
359	current_stack_info_t info;
360	__stack_t ctx;
361	info.storage = &ctx;
362	(proc->runner){ proc, &info };
363
364	__cfaabi_dbg_print_safe("Coroutine : created stack %p\n", get_coroutine(proc->runner)->stack.storage);
365
366	//Set global state
367	__cfaabi_tls.this_thread = 0p;
368
369	//We now have a proper context from which to schedule threads
370	__cfadbg_print_safe(runtime_core, "Kernel : core %p created (%p, %p)\n", proc, &proc->runner, &ctx);
371
372	// SKULLDUGGERY: Since the coroutine doesn't have its own stack, we can't
373	// resume it to start it like it normally would, it will just context switch
374	// back to here. Instead directly call the main since we already are on the
375	// appropriate stack.
376	get_coroutine(proc->runner)->state = Active;
377	main( proc->runner );
378	get_coroutine(proc->runner)->state = Halted;
379
380	// Main routine of the core returned, the core is now fully terminated
381	__cfadbg_print_safe(runtime_core, "Kernel : core %p main ended (%p)\n", proc, &proc->runner);
382
383	#if !defined(__CFA_NO_STATISTICS__)
384	__tally_stats(proc->cltr->stats, &local_stats);
385	if( 0 != proc->print_stats ) {
386	__print_stats( &local_stats, proc->print_stats, "Processor ", proc->name, (void*)proc );
387	}
388	#if defined(CFA_STATS_ARRAY)
389	__flush_stat( &local_stats, "Processor", proc );
390	#endif
391	#endif
392
393	proc->local_data = 0p;
394
395	unregister_tls( proc );
396
397	return 0p;
398	}
399
400	static void __kernel_first_resume( processor * this ) {
401	thread$ * src = mainThread;
402	coroutine$ * dst = get_coroutine(this->runner);
403
404	/* paranoid */ verify( ! __preemption_enabled() );
405
406	__cfaabi_tls.this_thread->curr_cor = dst;
407	__stack_prepare( &dst->stack, DEFAULT_STACK_SIZE );
408	__cfactx_start(main, dst, this->runner, __cfactx_invoke_coroutine);
409
410	/* paranoid */ verify( ! __preemption_enabled() );
411
412	dst->last = &src->self_cor;
413	dst->starter = dst->starter ? dst->starter : &src->self_cor;
414
415	// make sure the current state is still correct
416	/* paranoid */ verify(src->state == Ready);
417	src->corctx_flag = true;
418
419	// context switch to specified coroutine
420	verify( dst->context.SP );
421	__cfactx_switch( &src->context, &dst->context );
422	// when __cfactx_switch returns we are back in the src coroutine
423
424	mainThread->curr_cor = &mainThread->self_cor;
425
426	// make sure the current state has been update
427	/* paranoid */ verify(src->state == Active);
428
429	/* paranoid */ verify( ! __preemption_enabled() );
430	}
431
432	// KERNEL_ONLY
433	static void __kernel_last_resume( processor * this ) {
434	coroutine$ * src = &mainThread->self_cor;
435	coroutine$ * dst = get_coroutine(this->runner);
436
437	/* paranoid */ verify( ! __preemption_enabled() );
438	/* paranoid */ verify( dst->starter == src );
439	/* paranoid */ verify( dst->context.SP );
440
441	// SKULLDUGGERY in debug the processors check that the
442	// stack is still within the limit of the stack limits after running a thread.
443	// that check doesn't make sense if we context switch to the processor using the
444	// coroutine semantics. Since this is a special case, use the current context
445	// info to populate these fields.
446	__cfaabi_dbg_debug_do(
447	__stack_context_t ctx;
448	CtxGet( ctx );
449	mainThread->context.SP = ctx.SP;
450	mainThread->context.FP = ctx.FP;
451	)
452
453	// context switch to the processor
454	__cfactx_switch( &src->context, &dst->context );
455	}
456
457
458	//=============================================================================================
459	// Kernel Object Constructors logic
460	//=============================================================================================
461	//-----------------------------------------------------------------------------
462	// Main thread construction
463	static void ?{}( coroutine$ & this, current_stack_info_t * info) with( this ) {
464	stack.storage = info->storage;
465	with(*stack.storage) {
466	limit = info->limit;
467	base = info->base;
468	}
469	__attribute__((may_alias)) intptr_t * istorage = (intptr_t*) &stack.storage;
470	*istorage \|= 0x1;
471	name = "Main Thread";
472	state = Start;
473	starter = 0p;
474	last = 0p;
475	cancellation = 0p;
476	}
477
478	static void ?{}( thread$ & this, current_stack_info_t * info) with( this ) {
479	ticket = TICKET_RUNNING;
480	state = Start;
481	self_cor{ info };
482	curr_cor = &self_cor;
483	curr_cluster = mainCluster;
484	self_mon.owner = &this;
485	self_mon.recursion = 1;
486	self_mon_p = &self_mon;
487	link.next = 0p;
488	link.ts = -1llu;
489	preferred = ready_queue_new_preferred();
490	last_proc = 0p;
491	random_state = __global_random_mask ? __global_random_prime : __global_random_prime ^ rdtscl();
492	#if defined( __CFA_WITH_VERIFY__ )
493	canary = 0x0D15EA5E0D15EA5Ep;
494	#endif
495
496	node.next = 0p;
497	node.prev = 0p;
498	doregister(curr_cluster, this);
499
500	monitors{ &self_mon_p, 1, (fptr_t)0 };
501	}
502
503	//-----------------------------------------------------------------------------
504	// Processor
505	// Construct the processor context of non-main processors
506	static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info) {
507	(this.__cor){ info };
508	this.proc = proc;
509	}
510
511	static void init(processor & this, const char name[], cluster & _cltr, thread$ * initT) with( this ) {
512	this.name = name;
513	this.cltr = &_cltr;
514	this.rdq.its = 0;
515	this.rdq.itr = 0;
516	this.rdq.id = MAX;
517	this.rdq.target = MAX;
518	this.rdq.last = MAX;
519	this.rdq.cpu = 0;
520	// this.rdq.cutoff = 0ull;
521	do_terminate = false;
522	preemption_alarm = 0p;
523	pending_preemption = false;
524
525	this.io.ctx = 0p;
526	this.io.pending = false;
527	this.io.dirty = false;
528
529	this.init.thrd = initT;
530
531	this.local_data = 0p;
532
533	this.idle_fd = eventfd(0, 0);
534	if (idle_fd < 0) {
535	abort("KERNEL ERROR: PROCESSOR EVENTFD - %s\n", strerror(errno));
536	}
537
538	this.idle_wctx.fd = 0;
539
540	// I'm assuming these two are reserved for standard input and output
541	// so I'm using them as sentinels with idle_wctx.
542	/* paranoid */ verify( this.idle_fd != 0 );
543	/* paranoid */ verify( this.idle_fd != 1 );
544
545	#if !defined(__CFA_NO_STATISTICS__)
546	print_stats = 0;
547	print_halts = false;
548	#endif
549
550	__cfadbg_print_safe(runtime_core, "Kernel : core %p created\n", &this);
551	}
552
553	// Not a ctor, it just preps the destruction but should not destroy members
554	static void deinit(processor & this) {
555	close(this.idle_fd);
556	}
557
558	void ?{}(processor & this, const char name[], cluster & _cltr, thread$ * initT) {
559	( this.terminated ){};
560	( this.runner ){};
561
562	disable_interrupts();
563	init( this, name, _cltr, initT );
564	enable_interrupts();
565
566	__cfadbg_print_safe(runtime_core, "Kernel : Starting core %p\n", &this);
567
568	this.stack = __create_pthread( &this.kernel_thread, __invoke_processor, (void *)&this );
569	}
570
571	void ?{}(processor & this, const char name[], cluster & _cltr) {
572	(this){name, _cltr, 0p};
573	}
574
575	void ^?{}(processor & this) with( this ){
576	/* paranoid */ verify( !__atomic_load_n(&do_terminate, __ATOMIC_ACQUIRE) );
577	__cfadbg_print_safe(runtime_core, "Kernel : core %p signaling termination\n", &this);
578
579	__atomic_store_n(&do_terminate, true, __ATOMIC_RELAXED);
580	__disable_interrupts_checked();
581	__wake_proc( &this );
582	__enable_interrupts_checked();
583
584	wait( terminated );
585	/* paranoid */ verify( active_processor() != &this);
586
587	__destroy_pthread( kernel_thread, this.stack, 0p );
588
589	disable_interrupts();
590	deinit( this );
591	enable_interrupts();
592	}
593
594	//-----------------------------------------------------------------------------
595	// Cluster
596	static void ?{}(__cluster_proc_list & this) {
597	this.fdw = 0p;
598	this.idle = 0;
599	this.total = 0;
600	}
601
602	void ?{}(cluster & this, const char name[], Duration preemption_rate, unsigned num_io, const io_context_params & io_params) with( this ) {
603	this.name = name;
604	this.preemption_rate = preemption_rate;
605	ready_queue{};
606
607	#if !defined(__CFA_NO_STATISTICS__)
608	print_stats = 0;
609	stats = alloc();
610	__init_stats( stats );
611	#endif
612
613	threads{ __get };
614
615	io.arbiter = create();
616	io.params = io_params;
617
618	doregister(this);
619
620	// Lock the RWlock so no-one pushes/pops while we are changing the queue
621	disable_interrupts();
622	uint_fast32_t last_size = ready_mutate_lock();
623
624	// Adjust the ready queue size
625	ready_queue_grow( &this );
626
627	// Unlock the RWlock
628	ready_mutate_unlock( last_size );
629	enable_interrupts( false ); // Don't poll, could be in main cluster
630	}
631
632	void ^?{}(cluster & this) {
633	destroy(this.io.arbiter);
634
635	// Lock the RWlock so no-one pushes/pops while we are changing the queue
636	disable_interrupts();
637	uint_fast32_t last_size = ready_mutate_lock();
638
639	// Adjust the ready queue size
640	ready_queue_shrink( &this );
641
642	// Unlock the RWlock
643	ready_mutate_unlock( last_size );
644	enable_interrupts( false ); // Don't poll, could be in main cluster
645
646	#if !defined(__CFA_NO_STATISTICS__)
647	if( 0 != this.print_stats ) {
648	__print_stats( this.stats, this.print_stats, "Cluster", this.name, (void*)&this );
649	}
650	#if defined(CFA_STATS_ARRAY)
651	__flush_stat( this.stats, "Cluster", &this );
652	#endif
653	free( this.stats );
654	#endif
655
656	unregister(this);
657	}
658
659	//=============================================================================================
660	// Miscellaneous Initialization
661	//=============================================================================================
662	//-----------------------------------------------------------------------------
663	// Global Queues
664	static void doregister( cluster & cltr ) {
665	lock ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
666	push_front( __cfa_dbg_global_clusters.list, cltr );
667	unlock ( __cfa_dbg_global_clusters.lock );
668	}
669
670	static void unregister( cluster & cltr ) {
671	lock ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
672	remove( __cfa_dbg_global_clusters.list, cltr );
673	unlock( __cfa_dbg_global_clusters.lock );
674	}
675
676	void doregister( cluster * cltr, thread$ & thrd ) {
677	lock (cltr->thread_list_lock __cfaabi_dbg_ctx2);
678	cltr->nthreads += 1;
679	push_front(cltr->threads, thrd);
680	unlock (cltr->thread_list_lock);
681	}
682
683	void unregister( cluster * cltr, thread$ & thrd ) {
684	lock (cltr->thread_list_lock __cfaabi_dbg_ctx2);
685	remove(cltr->threads, thrd );
686	cltr->nthreads -= 1;
687	unlock(cltr->thread_list_lock);
688	}
689
690	static void register_tls( processor * this ) {
691	// Register and Lock the RWlock so no-one pushes/pops while we are changing the queue
692	uint_fast32_t last_size;
693	[this->unique_id, last_size] = ready_mutate_register();
694
695	this->rdq.cpu = __kernel_getcpu();
696
697	this->cltr->procs.total += 1u;
698	insert_last(this->cltr->procs.actives, *this);
699
700	// Adjust the ready queue size
701	ready_queue_grow( this->cltr );
702
703	// Unlock the RWlock
704	ready_mutate_unlock( last_size );
705	}
706
707
708	static void unregister_tls( processor * this ) {
709	// Lock the RWlock so no-one pushes/pops while we are changing the queue
710	uint_fast32_t last_size = ready_mutate_lock();
711	this->cltr->procs.total -= 1u;
712	remove(*this);
713
714	// clear the cluster so nothing gets pushed to local queues
715	cluster * cltr = this->cltr;
716	this->cltr = 0p;
717
718	// Adjust the ready queue size
719	ready_queue_shrink( cltr );
720
721	// Unlock the RWlock and unregister: we don't need the read_lock any more
722	ready_mutate_unregister( this->unique_id, last_size );
723	}
724
725	static void check( int ret, const char func[] ) {
726	if ( ret ) { // pthread routines return errno values
727	abort( "%s : internal error, error(%d) %s.", func, ret, strerror( ret ) );
728	} // if
729	} // Abort
730
731	void * __create_pthread( pthread_t * pthread, void * (start)(void ), void * arg ) {
732	pthread_attr_t attr;
733
734	check( pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
735
736	size_t stacksize = DEFAULT_STACK_SIZE;
737
738	void * stack;
739	#if CFA_PROCESSOR_USE_MMAP
740	stacksize = ceiling( stacksize, __cfa_page_size ) + __cfa_page_size;
741	stack = mmap(0p, stacksize, __map_prot, MAP_PRIVATE \| MAP_ANONYMOUS, 0, 0);
742	if(stack == ((void*)-1)) {
743	abort( "pthread stack creation : internal error, mmap failure, error(%d) %s.", errno, strerror( errno ) );
744	}
745	if ( mprotect( stack, __cfa_page_size, PROT_NONE ) == -1 ) {
746	abort( "pthread stack creation : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
747	} // if
748	#else
749	__cfaabi_dbg_debug_do(
750	stack = memalign( __cfa_page_size, stacksize + __cfa_page_size );
751	// pthread has no mechanism to create the guard page in user supplied stack.
752	if ( mprotect( stack, __cfa_page_size, PROT_NONE ) == -1 ) {
753	abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
754	} // if
755	);
756	__cfaabi_dbg_no_debug_do(
757	stack = malloc( stacksize );
758	);
759	#endif
760
761	check( pthread_attr_setstack( &attr, stack, stacksize ), "pthread_attr_setstack" );
762	check( pthread_create( pthread, &attr, start, arg ), "pthread_create" );
763	return stack;
764	}
765
766	void __destroy_pthread( pthread_t pthread, void * stack, void ** retval ) {
767	int err = pthread_join( pthread, retval );
768	if( err != 0 ) abort("KERNEL ERROR: joining pthread %p caused error %s\n", (void*)pthread, strerror(err));
769
770	#if CFA_PROCESSOR_USE_MMAP
771	pthread_attr_t attr;
772
773	check( pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
774
775	size_t stacksize;
776	// default stack size, normally defined by shell limit
777	check( pthread_attr_getstacksize( &attr, &stacksize ), "pthread_attr_getstacksize" );
778	assert( stacksize >= PTHREAD_STACK_MIN );
779	stacksize += __cfa_page_size;
780
781	if(munmap(stack, stacksize) == -1) {
782	abort( "pthread stack destruction : internal error, munmap failure, error(%d) %s.", errno, strerror( errno ) );
783	}
784	#else
785	__cfaabi_dbg_debug_do(
786	// pthread has no mechanism to create the guard page in user supplied stack.
787	if ( mprotect( stack, __cfa_page_size, __map_prot ) == -1 ) {
788	abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
789	} // if
790	);
791	free( stack );
792	#endif
793	}
794
795	#if defined(__CFA_WITH_VERIFY__)
796	static bool verify_fwd_bck_rng(void) {
797	__cfaabi_tls.ready_rng.fwd_seed = 25214903917_l64u * (rdtscl() ^ (uintptr_t)&verify_fwd_bck_rng);
798
799	unsigned values[10];
800	for(i; 10) {
801	values[i] = __tls_rand_fwd();
802	}
803
804	__tls_rand_advance_bck();
805
806	for ( i; 9 -~= 0 ) {
807	if(values[i] != __tls_rand_bck()) {
808	return false;
809	}
810	}
811
812	return true;
813	}
814	#endif

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

Original Format