Context Navigation

startup.cfa @ 8110bc3

ADTast-experimentalenumforall-pointer-decayjacob/cs343-translationnew-ast-unique-exprpthread-emulationqualifiedEnum

Last change on this file since 8110bc3 was 12daa43, checked in by Thierry Delisle <tdelisle@…>, 3 years ago
Added a define switch for using cpu workstealing. Not Fully implemented.
Property mode set to `100644`
File size: 27.5 KB

Line
1	//
2	// Cforall Version 1.0.0 Copyright (C) 2020 University of Waterloo
3	//
4	// The contents of this file are covered under the licence agreement in the
5	// file "LICENCE" distributed with Cforall.
6	//
7	// kernel/startup.cfa --
8	//
9	// Author : Thierry Delisle
10	// Created On : Thu Jul 30 15:12:54 2020
11	// Last Modified By :
12	// Last Modified On :
13	// Update Count :
14	//
15
16	#define __cforall_thread__
17	#define _GNU_SOURCE
18
19	// C Includes
20	#include <errno.h> // errno
21	#include <signal.h>
22	#include <string.h> // strerror
23	#include <unistd.h> // sysconf
24
25	extern "C" {
26	#include <limits.h> // PTHREAD_STACK_MIN
27	#include <unistd.h> // syscall
28	#include <sys/eventfd.h> // eventfd
29	#include <sys/mman.h> // mprotect
30	#include <sys/resource.h> // getrlimit
31	#include <sys/syscall.h> // __NR_xxx
32	}
33
34	// CFA Includes
35	#include "kernel_private.hfa"
36	#include "startup.hfa" // STARTUP_PRIORITY_XXX
37	#include "math.hfa"
38
39	#define CFA_PROCESSOR_USE_MMAP 0
40
41	//-----------------------------------------------------------------------------
42	// Some assembly required
43	#if defined( __i386 )
44	#define CtxGet( ctx ) __asm__ volatile ( \
45	"movl %%esp,%0\n" \
46	"movl %%ebp,%1\n" \
47	: "=rm" (ctx.SP), \
48	"=rm" (ctx.FP) \
49	)
50	#elif defined( __x86_64 )
51	#define CtxGet( ctx ) __asm__ volatile ( \
52	"movq %%rsp,%0\n" \
53	"movq %%rbp,%1\n" \
54	: "=rm" (ctx.SP), \
55	"=rm" (ctx.FP) \
56	)
57	#elif defined( __aarch64__ )
58	#define CtxGet( ctx ) __asm__ volatile ( \
59	"mov %0, sp\n" \
60	"mov %1, fp\n" \
61	: "=rm" (ctx.SP), \
62	"=rm" (ctx.FP) \
63	)
64	#else
65	#error unknown hardware architecture
66	#endif
67
68	//-----------------------------------------------------------------------------
69	// Start and stop routine for the kernel, declared first to make sure they run first
70	static void __kernel_startup (void) __attribute__(( constructor( STARTUP_PRIORITY_KERNEL ) ));
71	static void __kernel_shutdown(void) __attribute__(( destructor ( STARTUP_PRIORITY_KERNEL ) ));
72
73	//-----------------------------------------------------------------------------
74	// Static Forward Declarations
75	struct current_stack_info_t;
76
77	static void * __invoke_processor(void * arg);
78	static void __kernel_first_resume( processor * this );
79	static void __kernel_last_resume ( processor * this );
80	static void init(processor & this, const char name[], cluster & _cltr, $thread * initT);
81	static void deinit(processor & this);
82	static void doregister( struct cluster & cltr );
83	static void unregister( struct cluster & cltr );
84	static void register_tls( processor * this );
85	static void unregister_tls( processor * this );
86	static void ?{}( $coroutine & this, current_stack_info_t * info);
87	static void ?{}( $thread & this, current_stack_info_t * info);
88	static void ?{}(processorCtx_t & this) {}
89	static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info);
90
91	#if defined(__CFA_WITH_VERIFY__)
92	static bool verify_fwd_bck_rng(void);
93	#endif
94
95	//-----------------------------------------------------------------------------
96	// Forward Declarations for other modules
97	extern void __kernel_alarm_startup(void);
98	extern void __kernel_alarm_shutdown(void);
99
100	//-----------------------------------------------------------------------------
101	// Other Forward Declarations
102	extern void __wake_proc(processor *);
103
104	#if defined(CFA_HAVE_LINUX_LIBRSEQ)
105	// No forward declaration needed
106	#define __kernel_rseq_register rseq_register_current_thread
107	#define __kernel_rseq_unregister rseq_unregister_current_thread
108	#elif defined(CFA_HAVE_LINUX_RSEQ_H)
109	void __kernel_raw_rseq_register (void);
110	void __kernel_raw_rseq_unregister(void);
111
112	#define __kernel_rseq_register __kernel_raw_rseq_register
113	#define __kernel_rseq_unregister __kernel_raw_rseq_unregister
114	#else
115	// No forward declaration needed
116	// No initialization needed
117	static inline void noop(void) {}
118
119	#define __kernel_rseq_register noop
120	#define __kernel_rseq_unregister noop
121	#endif
122
123	//-----------------------------------------------------------------------------
124	// Kernel storage
125	KERNEL_STORAGE(cluster, mainCluster);
126	KERNEL_STORAGE(processor, mainProcessor);
127	KERNEL_STORAGE($thread, mainThread);
128	KERNEL_STORAGE(__stack_t, mainThreadCtx);
129	KERNEL_STORAGE(__scheduler_RWLock_t, __scheduler_lock);
130	#if !defined(__CFA_NO_STATISTICS__)
131	KERNEL_STORAGE(__stats_t, mainProcStats);
132	#endif
133
134	cluster * mainCluster;
135	processor * mainProcessor;
136	$thread * mainThread;
137	__scheduler_RWLock_t * __scheduler_lock;
138
139	extern "C" {
140	struct { __dllist_t(cluster) list; __spinlock_t lock; } __cfa_dbg_global_clusters;
141	}
142
143	extern size_t __page_size;
144	extern int __map_prot;
145
146	//-----------------------------------------------------------------------------
147	// Global state
148	thread_local struct KernelThreadData __cfaabi_tls __attribute__ ((tls_model ( "initial-exec" ))) @= {
149	NULL, // cannot use 0p
150	NULL,
151	false,
152	{ 1, false, false },
153	0,
154	{ 0, 0 },
155	NULL,
156	#ifdef __CFA_WITH_VERIFY__
157	false,
158	0,
159	#endif
160	};
161
162	#if defined(CFA_HAVE_LINUX_LIBRSEQ)
163	// No data needed
164	#elif defined(CFA_HAVE_LINUX_RSEQ_H)
165	extern "Cforall" {
166	__attribute__((aligned(128))) thread_local volatile struct rseq __cfaabi_rseq @= {
167	.cpu_id : RSEQ_CPU_ID_UNINITIALIZED,
168	};
169	}
170	#else
171	// No data needed
172	#endif
173
174	//-----------------------------------------------------------------------------
175	// Struct to steal stack
176	struct current_stack_info_t {
177	__stack_t * storage; // pointer to stack object
178	void * base; // base of stack
179	void * limit; // stack grows towards stack limit
180	void * context; // address of cfa_context_t
181	};
182
183	void ?{}( current_stack_info_t & this ) {
184	__stack_context_t ctx;
185	CtxGet( ctx );
186	this.base = ctx.FP;
187
188	rlimit r;
189	getrlimit( RLIMIT_STACK, &r);
190	size_t size = r.rlim_cur;
191
192	this.limit = (void *)(((intptr_t)this.base) - size);
193	this.context = &storage_mainThreadCtx;
194	}
195
196
197
198	//=============================================================================================
199	// Kernel Setup logic
200	//=============================================================================================
201	//-----------------------------------------------------------------------------
202	// Kernel boot procedures
203	static void __kernel_startup(void) {
204	/* paranoid */ verify( ! __preemption_enabled() );
205	__cfadbg_print_safe(runtime_core, "Kernel : Starting\n");
206
207	__cfa_dbg_global_clusters.list{ __get };
208	__cfa_dbg_global_clusters.lock{};
209
210	/* paranoid */ verify( verify_fwd_bck_rng() );
211
212	// Initialize the global scheduler lock
213	__scheduler_lock = (__scheduler_RWLock_t*)&storage___scheduler_lock;
214	(*__scheduler_lock){};
215
216	// Initialize the main cluster
217	mainCluster = (cluster *)&storage_mainCluster;
218	(*mainCluster){"Main Cluster", 0};
219
220	__cfadbg_print_safe(runtime_core, "Kernel : Main cluster ready\n");
221
222	// Start by initializing the main thread
223	// SKULLDUGGERY: the mainThread steals the process main thread
224	// which will then be scheduled by the mainProcessor normally
225	mainThread = ($thread *)&storage_mainThread;
226	current_stack_info_t info;
227	info.storage = (__stack_t*)&storage_mainThreadCtx;
228	(*mainThread){ &info };
229
230	__cfadbg_print_safe(runtime_core, "Kernel : Main thread ready\n");
231
232
233
234	// Construct the processor context of the main processor
235	void ?{}(processorCtx_t & this, processor * proc) {
236	(this.__cor){ "Processor" };
237	this.__cor.starter = 0p;
238	this.proc = proc;
239	}
240
241	void ?{}(processor & this) with( this ) {
242	( this.terminated ){};
243	( this.runner ){};
244	init( this, "Main Processor", *mainCluster, 0p );
245	kernel_thread = pthread_self();
246
247	runner{ &this };
248	__cfadbg_print_safe(runtime_core, "Kernel : constructed main processor context %p\n", &runner);
249	}
250
251	// Initialize the main processor and the main processor ctx
252	// (the coroutine that contains the processing control flow)
253	mainProcessor = (processor *)&storage_mainProcessor;
254	(*mainProcessor){};
255
256	register_tls( mainProcessor );
257
258	__kernel_rseq_register();
259
260	//initialize the global state variables
261	__cfaabi_tls.this_processor = mainProcessor;
262	__cfaabi_tls.this_thread = mainThread;
263
264	#if !defined( __CFA_NO_STATISTICS__ )
265	__cfaabi_tls.this_stats = (__stats_t *)& storage_mainProcStats;
266	__init_stats( __cfaabi_tls.this_stats );
267	#endif
268	mainProcessor->local_data = &__cfaabi_tls;
269
270	// Enable preemption
271	__kernel_alarm_startup();
272
273	// Add the main thread to the ready queue
274	// once resume is called on mainProcessor->runner the mainThread needs to be scheduled like any normal thread
275	schedule_thread$(mainThread);
276
277	// SKULLDUGGERY: Force a context switch to the main processor to set the main thread's context to the current UNIX
278	// context. Hence, the main thread does not begin through __cfactx_invoke_thread, like all other threads. The trick here is that
279	// mainThread is on the ready queue when this call is made.
280	__kernel_first_resume( __cfaabi_tls.this_processor );
281
282
283	// THE SYSTEM IS NOW COMPLETELY RUNNING
284
285	__cfadbg_print_safe(runtime_core, "Kernel : Started\n--------------------------------------------------\n\n");
286
287	/* paranoid */ verify( ! __preemption_enabled() );
288	enable_interrupts();
289	/* paranoid */ verify( __preemption_enabled() );
290
291	}
292
293	static void __kernel_shutdown(void) {
294	/* paranoid */ verify( __preemption_enabled() );
295	disable_interrupts();
296	/* paranoid */ verify( ! __preemption_enabled() );
297
298	__cfadbg_print_safe(runtime_core, "\n--------------------------------------------------\nKernel : Shutting down\n");
299
300	// SKULLDUGGERY: Notify the mainProcessor it needs to terminates.
301	// When its coroutine terminates, it return control to the mainThread
302	// which is currently here
303	__atomic_store_n(&mainProcessor->do_terminate, true, __ATOMIC_RELEASE);
304	__kernel_last_resume( __cfaabi_tls.this_processor );
305	mainThread->self_cor.state = Halted;
306
307	// THE SYSTEM IS NOW COMPLETELY STOPPED
308
309	// Disable preemption
310	__kernel_alarm_shutdown();
311
312	#if !defined( __CFA_NO_STATISTICS__ )
313	__stats_t * st = (__stats_t *)& storage_mainProcStats;
314	__tally_stats(mainCluster->stats, st);
315	if( 0 != mainProcessor->print_stats ) {
316	__print_stats( st, mainProcessor->print_stats, "Processor ", mainProcessor->name, (void*)mainProcessor );
317	}
318	#if defined(CFA_STATS_ARRAY)
319	__flush_stat( st, "Processor", mainProcessor );
320	#endif
321	#endif
322
323	mainProcessor->local_data = 0p;
324
325	__kernel_rseq_unregister();
326
327	unregister_tls( mainProcessor );
328
329	// Destroy the main processor and its context in reverse order of construction
330	// These were manually constructed so we need manually destroy them
331	void ^?{}(processor & this) with( this ){
332	deinit( this );
333
334	/* paranoid */ verify( this.do_terminate == true );
335	__cfaabi_dbg_print_safe("Kernel : destroyed main processor context %p\n", &runner);
336	}
337
338	^(*mainProcessor){};
339
340	// Final step, destroy the main thread since it is no longer needed
341
342	// Since we provided a stack to this taxk it will not destroy anything
343	/* paranoid / verify(mainThread->self_cor.stack.storage == (__stack_t)(((uintptr_t)&storage_mainThreadCtx)\| 0x1));
344	^(*mainThread){};
345
346	^(*mainCluster){};
347
348	^(*__scheduler_lock){};
349
350	^(__cfa_dbg_global_clusters.list){};
351	^(__cfa_dbg_global_clusters.lock){};
352
353	__cfadbg_print_safe(runtime_core, "Kernel : Shutdown complete\n");
354	}
355
356	//=============================================================================================
357	// Kernel Initial Scheduling logic
358	//=============================================================================================
359
360	// Context invoker for processors
361	// This is the entry point for processors (kernel threads) except for the main processor
362	// It effectively constructs a coroutine by stealing the pthread stack
363	static void * __invoke_processor(void * arg) {
364	#if !defined( __CFA_NO_STATISTICS__ )
365	__stats_t local_stats;
366	__init_stats( &local_stats );
367	__cfaabi_tls.this_stats = &local_stats;
368	#endif
369
370	processor * proc = (processor *) arg;
371	__cfaabi_tls.this_processor = proc;
372	__cfaabi_tls.this_thread = 0p;
373	__cfaabi_tls.preemption_state.[enabled, disable_count] = [false, 1];
374	proc->local_data = &__cfaabi_tls;
375
376	register_tls( proc );
377
378	__kernel_rseq_register();
379
380	// SKULLDUGGERY: We want to create a context for the processor coroutine
381	// which is needed for the 2-step context switch. However, there is no reason
382	// to waste the perfectly valid stack create by pthread.
383	current_stack_info_t info;
384	__stack_t ctx;
385	info.storage = &ctx;
386	(proc->runner){ proc, &info };
387
388	__cfaabi_dbg_print_safe("Coroutine : created stack %p\n", get_coroutine(proc->runner)->stack.storage);
389
390	//Set global state
391	__cfaabi_tls.this_thread = 0p;
392
393	//We now have a proper context from which to schedule threads
394	__cfadbg_print_safe(runtime_core, "Kernel : core %p created (%p, %p)\n", proc, &proc->runner, &ctx);
395
396	// SKULLDUGGERY: Since the coroutine doesn't have its own stack, we can't
397	// resume it to start it like it normally would, it will just context switch
398	// back to here. Instead directly call the main since we already are on the
399	// appropriate stack.
400	get_coroutine(proc->runner)->state = Active;
401	main( proc->runner );
402	get_coroutine(proc->runner)->state = Halted;
403
404	// Main routine of the core returned, the core is now fully terminated
405	__cfadbg_print_safe(runtime_core, "Kernel : core %p main ended (%p)\n", proc, &proc->runner);
406
407	#if !defined(__CFA_NO_STATISTICS__)
408	__tally_stats(proc->cltr->stats, &local_stats);
409	if( 0 != proc->print_stats ) {
410	__print_stats( &local_stats, proc->print_stats, "Processor ", proc->name, (void*)proc );
411	}
412	#if defined(CFA_STATS_ARRAY)
413	__flush_stat( &local_stats, "Processor", proc );
414	#endif
415	#endif
416
417	proc->local_data = 0p;
418
419	__kernel_rseq_unregister();
420
421	unregister_tls( proc );
422
423	return 0p;
424	}
425
426	static void __kernel_first_resume( processor * this ) {
427	$thread * src = mainThread;
428	$coroutine * dst = get_coroutine(this->runner);
429
430	/* paranoid */ verify( ! __preemption_enabled() );
431
432	__cfaabi_tls.this_thread->curr_cor = dst;
433	__stack_prepare( &dst->stack, 65000 );
434	__cfactx_start(main, dst, this->runner, __cfactx_invoke_coroutine);
435
436	/* paranoid */ verify( ! __preemption_enabled() );
437
438	dst->last = &src->self_cor;
439	dst->starter = dst->starter ? dst->starter : &src->self_cor;
440
441	// make sure the current state is still correct
442	/* paranoid */ verify(src->state == Ready);
443	src->corctx_flag = true;
444
445	// context switch to specified coroutine
446	verify( dst->context.SP );
447	__cfactx_switch( &src->context, &dst->context );
448	// when __cfactx_switch returns we are back in the src coroutine
449
450	mainThread->curr_cor = &mainThread->self_cor;
451
452	// make sure the current state has been update
453	/* paranoid */ verify(src->state == Active);
454
455	/* paranoid */ verify( ! __preemption_enabled() );
456	}
457
458	// KERNEL_ONLY
459	static void __kernel_last_resume( processor * this ) {
460	$coroutine * src = &mainThread->self_cor;
461	$coroutine * dst = get_coroutine(this->runner);
462
463	/* paranoid */ verify( ! __preemption_enabled() );
464	/* paranoid */ verify( dst->starter == src );
465	/* paranoid */ verify( dst->context.SP );
466
467	// SKULLDUGGERY in debug the processors check that the
468	// stack is still within the limit of the stack limits after running a thread.
469	// that check doesn't make sense if we context switch to the processor using the
470	// coroutine semantics. Since this is a special case, use the current context
471	// info to populate these fields.
472	__cfaabi_dbg_debug_do(
473	__stack_context_t ctx;
474	CtxGet( ctx );
475	mainThread->context.SP = ctx.SP;
476	mainThread->context.FP = ctx.FP;
477	)
478
479	// context switch to the processor
480	__cfactx_switch( &src->context, &dst->context );
481	}
482
483
484	//=============================================================================================
485	// Kernel Object Constructors logic
486	//=============================================================================================
487	//-----------------------------------------------------------------------------
488	// Main thread construction
489	static void ?{}( $coroutine & this, current_stack_info_t * info) with( this ) {
490	stack.storage = info->storage;
491	with(*stack.storage) {
492	limit = info->limit;
493	base = info->base;
494	}
495	__attribute__((may_alias)) intptr_t * istorage = (intptr_t*) &stack.storage;
496	*istorage \|= 0x1;
497	name = "Main Thread";
498	state = Start;
499	starter = 0p;
500	last = 0p;
501	cancellation = 0p;
502	}
503
504	static void ?{}( $thread & this, current_stack_info_t * info) with( this ) {
505	ticket = TICKET_RUNNING;
506	state = Start;
507	self_cor{ info };
508	curr_cor = &self_cor;
509	curr_cluster = mainCluster;
510	self_mon.owner = &this;
511	self_mon.recursion = 1;
512	self_mon_p = &self_mon;
513	link.next = 0p;
514	link.ts = 0;
515	preferred = -1u;
516	last_proc = 0p;
517	#if defined( __CFA_WITH_VERIFY__ )
518	canary = 0x0D15EA5E0D15EA5Ep;
519	#endif
520
521	node.next = 0p;
522	node.prev = 0p;
523	doregister(curr_cluster, this);
524
525	monitors{ &self_mon_p, 1, (fptr_t)0 };
526	}
527
528	//-----------------------------------------------------------------------------
529	// Processor
530	// Construct the processor context of non-main processors
531	static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info) {
532	(this.__cor){ info };
533	this.proc = proc;
534	}
535
536	static void init(processor & this, const char name[], cluster & _cltr, $thread * initT) with( this ) {
537	this.name = name;
538	this.cltr = &_cltr;
539	this.rdq.its = 0;
540	this.rdq.itr = 0;
541	this.rdq.id = -1u;
542	this.rdq.target = -1u;
543	this.rdq.last = -1u;
544	this.rdq.cutoff = 0ull;
545	do_terminate = false;
546	preemption_alarm = 0p;
547	pending_preemption = false;
548
549	this.io.ctx = 0p;
550	this.io.pending = false;
551	this.io.dirty = false;
552
553	this.init.thrd = initT;
554
555	this.local_data = 0p;
556
557	this.idle = eventfd(0, 0);
558	if (idle < 0) {
559	abort("KERNEL ERROR: PROCESSOR EVENTFD - %s\n", strerror(errno));
560	}
561
562	#if !defined(__CFA_NO_STATISTICS__)
563	print_stats = 0;
564	print_halts = false;
565	#endif
566
567	__cfadbg_print_safe(runtime_core, "Kernel : core %p created\n", &this);
568	}
569
570	// Not a ctor, it just preps the destruction but should not destroy members
571	static void deinit(processor & this) {
572	close(this.idle);
573	}
574
575	void ?{}(processor & this, const char name[], cluster & _cltr, $thread * initT) {
576	( this.terminated ){};
577	( this.runner ){};
578
579	disable_interrupts();
580	init( this, name, _cltr, initT );
581	enable_interrupts();
582
583	__cfadbg_print_safe(runtime_core, "Kernel : Starting core %p\n", &this);
584
585	this.stack = __create_pthread( &this.kernel_thread, __invoke_processor, (void *)&this );
586	}
587
588	void ?{}(processor & this, const char name[], cluster & _cltr) {
589	(this){name, _cltr, 0p};
590	}
591
592	extern size_t __page_size;
593	void ^?{}(processor & this) with( this ){
594	if( ! __atomic_load_n(&do_terminate, __ATOMIC_ACQUIRE) ) {
595	__cfadbg_print_safe(runtime_core, "Kernel : core %p signaling termination\n", &this);
596
597	__atomic_store_n(&do_terminate, true, __ATOMIC_RELAXED);
598	__wake_proc( &this );
599
600	wait( terminated );
601	/* paranoid */ verify( active_processor() != &this);
602	}
603
604	__destroy_pthread( kernel_thread, this.stack, 0p );
605
606	disable_interrupts();
607	deinit( this );
608	enable_interrupts();
609	}
610
611	//-----------------------------------------------------------------------------
612	// Cluster
613	static void ?{}(__cluster_proc_list & this) {
614	this.lock = 0;
615	this.idle = 0;
616	this.total = 0;
617	}
618
619	void ?{}(cluster & this, const char name[], Duration preemption_rate, unsigned num_io, const io_context_params & io_params) with( this ) {
620	this.name = name;
621	this.preemption_rate = preemption_rate;
622	ready_queue{};
623
624	#if !defined(__CFA_NO_STATISTICS__)
625	print_stats = 0;
626	stats = alloc();
627	__init_stats( stats );
628	#endif
629
630	threads{ __get };
631
632	io.arbiter = create();
633	io.params = io_params;
634
635	doregister(this);
636
637	// Lock the RWlock so no-one pushes/pops while we are changing the queue
638	disable_interrupts();
639	uint_fast32_t last_size = ready_mutate_lock();
640
641	// Adjust the ready queue size
642	ready_queue_grow( &this );
643
644	// Unlock the RWlock
645	ready_mutate_unlock( last_size );
646	enable_interrupts( false ); // Don't poll, could be in main cluster
647	}
648
649	void ^?{}(cluster & this) {
650	destroy(this.io.arbiter);
651
652	// Lock the RWlock so no-one pushes/pops while we are changing the queue
653	disable_interrupts();
654	uint_fast32_t last_size = ready_mutate_lock();
655
656	// Adjust the ready queue size
657	ready_queue_shrink( &this );
658
659	// Unlock the RWlock
660	ready_mutate_unlock( last_size );
661	enable_interrupts( false ); // Don't poll, could be in main cluster
662
663	#if !defined(__CFA_NO_STATISTICS__)
664	if( 0 != this.print_stats ) {
665	__print_stats( this.stats, this.print_stats, "Cluster", this.name, (void*)&this );
666	}
667	#if defined(CFA_STATS_ARRAY)
668	__flush_stat( this.stats, "Cluster", &this );
669	#endif
670	free( this.stats );
671	#endif
672
673	unregister(this);
674	}
675
676	//=============================================================================================
677	// Miscellaneous Initialization
678	//=============================================================================================
679	//-----------------------------------------------------------------------------
680	// Global Queues
681	static void doregister( cluster & cltr ) {
682	lock ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
683	push_front( __cfa_dbg_global_clusters.list, cltr );
684	unlock ( __cfa_dbg_global_clusters.lock );
685	}
686
687	static void unregister( cluster & cltr ) {
688	lock ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
689	remove( __cfa_dbg_global_clusters.list, cltr );
690	unlock( __cfa_dbg_global_clusters.lock );
691	}
692
693	void doregister( cluster * cltr, $thread & thrd ) {
694	lock (cltr->thread_list_lock __cfaabi_dbg_ctx2);
695	cltr->nthreads += 1;
696	push_front(cltr->threads, thrd);
697	unlock (cltr->thread_list_lock);
698	}
699
700	void unregister( cluster * cltr, $thread & thrd ) {
701	lock (cltr->thread_list_lock __cfaabi_dbg_ctx2);
702	remove(cltr->threads, thrd );
703	cltr->nthreads -= 1;
704	unlock(cltr->thread_list_lock);
705	}
706
707	static void register_tls( processor * this ) {
708	// Register and Lock the RWlock so no-one pushes/pops while we are changing the queue
709	uint_fast32_t last_size;
710	[this->unique_id, last_size] = ready_mutate_register();
711
712	this->cltr->procs.total += 1u;
713	insert_last(this->cltr->procs.actives, *this);
714
715	// Adjust the ready queue size
716	ready_queue_grow( this->cltr );
717
718	// Unlock the RWlock
719	ready_mutate_unlock( last_size );
720	}
721
722
723	static void unregister_tls( processor * this ) {
724	// Lock the RWlock so no-one pushes/pops while we are changing the queue
725	uint_fast32_t last_size = ready_mutate_lock();
726	this->cltr->procs.total -= 1u;
727	remove(*this);
728
729	// clear the cluster so nothing gets pushed to local queues
730	cluster * cltr = this->cltr;
731	this->cltr = 0p;
732
733	// Adjust the ready queue size
734	ready_queue_shrink( cltr );
735
736	// Unlock the RWlock and unregister: we don't need the read_lock any more
737	ready_mutate_unregister( this->unique_id, last_size );
738	}
739
740	static void check( int ret, const char func[] ) {
741	if ( ret ) { // pthread routines return errno values
742	abort( "%s : internal error, error(%d) %s.", func, ret, strerror( ret ) );
743	} // if
744	} // Abort
745
746	void * __create_pthread( pthread_t * pthread, void * (start)(void ), void * arg ) {
747	pthread_attr_t attr;
748
749	check( pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
750
751	size_t stacksize;
752	// default stack size, normally defined by shell limit
753	check( pthread_attr_getstacksize( &attr, &stacksize ), "pthread_attr_getstacksize" );
754	assert( stacksize >= PTHREAD_STACK_MIN );
755
756	void * stack;
757	#if CFA_PROCESSOR_USE_MMAP
758	stacksize = ceiling( stacksize, __page_size ) + __page_size;
759	stack = mmap(0p, stacksize, __map_prot, MAP_PRIVATE \| MAP_ANONYMOUS, 0, 0);
760	if(stack == ((void*)-1)) {
761	abort( "pthread stack creation : internal error, mmap failure, error(%d) %s.", errno, strerror( errno ) );
762	}
763	if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
764	abort( "pthread stack creation : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
765	} // if
766	#else
767	__cfaabi_dbg_debug_do(
768	stack = memalign( __page_size, stacksize + __page_size );
769	// pthread has no mechanism to create the guard page in user supplied stack.
770	if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
771	abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
772	} // if
773	);
774	__cfaabi_dbg_no_debug_do(
775	stack = malloc( stacksize );
776	);
777	#endif
778
779
780	check( pthread_attr_setstack( &attr, stack, stacksize ), "pthread_attr_setstack" );
781
782	check( pthread_create( pthread, &attr, start, arg ), "pthread_create" );
783	return stack;
784	}
785
786	void __destroy_pthread( pthread_t pthread, void * stack, void ** retval ) {
787	int err = pthread_join( pthread, retval );
788	if( err != 0 ) abort("KERNEL ERROR: joining pthread %p caused error %s\n", (void*)pthread, strerror(err));
789
790	#if CFA_PROCESSOR_USE_MMAP
791	pthread_attr_t attr;
792
793	check( pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
794
795	size_t stacksize;
796	// default stack size, normally defined by shell limit
797	check( pthread_attr_getstacksize( &attr, &stacksize ), "pthread_attr_getstacksize" );
798	assert( stacksize >= PTHREAD_STACK_MIN );
799	stacksize += __page_size;
800
801	if(munmap(stack, stacksize) == -1) {
802	abort( "pthread stack destruction : internal error, munmap failure, error(%d) %s.", errno, strerror( errno ) );
803	}
804	#else
805	__cfaabi_dbg_debug_do(
806	// pthread has no mechanism to create the guard page in user supplied stack.
807	if ( mprotect( stack, __page_size, __map_prot ) == -1 ) {
808	abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
809	} // if
810	);
811	free( stack );
812	#endif
813	}
814
815	#if defined(__CFA_WITH_VERIFY__)
816	static bool verify_fwd_bck_rng(void) {
817	__cfaabi_tls.ready_rng.fwd_seed = 25214903917_l64u * (rdtscl() ^ (uintptr_t)&verify_fwd_bck_rng);
818
819	unsigned values[10];
820	for(i; 10) {
821	values[i] = __tls_rand_fwd();
822	}
823
824	__tls_rand_advance_bck();
825
826	for ( i; 9 -~= 0 ) {
827	if(values[i] != __tls_rand_bck()) {
828	return false;
829	}
830	}
831
832	return true;
833	}
834	#endif
835
836	#if defined(CFA_HAVE_LINUX_LIBRSEQ)
837	// No definition needed
838	#elif defined(CFA_HAVE_LINUX_RSEQ_H)
839
840	#if defined( __x86_64 ) \|\| defined( __i386 )
841	#define RSEQ_SIG 0x53053053
842	#elif defined( __ARM_ARCH )
843	#ifdef __ARMEB__
844	#define RSEQ_SIG 0xf3def5e7 /* udf #24035 ; 0x5de3 (ARMv6+) */
845	#else
846	#define RSEQ_SIG 0xe7f5def3 /* udf #24035 ; 0x5de3 */
847	#endif
848	#endif
849
850	extern void __disable_interrupts_hard();
851	extern void __enable_interrupts_hard();
852
853	void __kernel_raw_rseq_register (void) {
854	/* paranoid */ verify( __cfaabi_rseq.cpu_id == RSEQ_CPU_ID_UNINITIALIZED );
855
856	// int ret = syscall(__NR_rseq, &__cfaabi_rseq, sizeof(struct rseq), 0, (sigset_t *)0p, _NSIG / 8);
857	int ret = syscall(__NR_rseq, &__cfaabi_rseq, sizeof(struct rseq), 0, RSEQ_SIG);
858	if(ret != 0) {
859	int e = errno;
860	switch(e) {
861	case EINVAL: abort("KERNEL ERROR: rseq register invalid argument");
862	case ENOSYS: abort("KERNEL ERROR: rseq register no supported");
863	case EFAULT: abort("KERNEL ERROR: rseq register with invalid argument");
864	case EBUSY : abort("KERNEL ERROR: rseq register already registered");
865	case EPERM : abort("KERNEL ERROR: rseq register sig argument on unregistration does not match the signature received on registration");
866	default: abort("KERNEL ERROR: rseq register unexpected return %d", e);
867	}
868	}
869	}
870
871	void __kernel_raw_rseq_unregister(void) {
872	/* paranoid */ verify( __cfaabi_rseq.cpu_id >= 0 );
873
874	// int ret = syscall(__NR_rseq, &__cfaabi_rseq, sizeof(struct rseq), RSEQ_FLAG_UNREGISTER, (sigset_t *)0p, _NSIG / 8);
875	int ret = syscall(__NR_rseq, &__cfaabi_rseq, sizeof(struct rseq), RSEQ_FLAG_UNREGISTER, RSEQ_SIG);
876	if(ret != 0) {
877	int e = errno;
878	switch(e) {
879	case EINVAL: abort("KERNEL ERROR: rseq unregister invalid argument");
880	case ENOSYS: abort("KERNEL ERROR: rseq unregister no supported");
881	case EFAULT: abort("KERNEL ERROR: rseq unregister with invalid argument");
882	case EBUSY : abort("KERNEL ERROR: rseq unregister already registered");
883	case EPERM : abort("KERNEL ERROR: rseq unregister sig argument on unregistration does not match the signature received on registration");
884	default: abort("KERNEL ERROR: rseq unregisteunexpected return %d", e);
885	}
886	}
887	}
888	#else
889	// No definition needed
890	#endif

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

Original Format