Context Navigation

startup.cfa@ 878cfcc

Visit:

ADT ast-experimental

Last change on this file since 878cfcc was 878cfcc, checked in by Thierry Delisle <tdelisle@…>, 3 years ago
Added extra check to make sure threads is never double executed
Property mode set to `100644`
File size: 26.8 KB

Line
1	//
2	// Cforall Version 1.0.0 Copyright (C) 2020 University of Waterloo
3	//
4	// The contents of this file are covered under the licence agreement in the
5	// file "LICENCE" distributed with Cforall.
6	//
7	// kernel/startup.cfa --
8	//
9	// Author : Thierry Delisle
10	// Created On : Thu Jul 30 15:12:54 2020
11	// Last Modified By :
12	// Last Modified On :
13	// Update Count :
14	//
15
16	#define __cforall_thread__
17	#define _GNU_SOURCE
18
19	// C Includes
20	#include <errno.h> // errno
21	#include <signal.h>
22	#include <string.h> // strerror
23	#include <unistd.h> // sysconf
24
25	extern "C" {
26	#include <limits.h> // PTHREAD_STACK_MIN
27	#include <unistd.h> // syscall
28	#include <sys/eventfd.h> // eventfd
29	#include <sys/mman.h> // mprotect
30	#include <sys/resource.h> // getrlimit
31	}
32
33	// CFA Includes
34	#include "kernel/private.hfa"
35	#include "iofwd.hfa"
36	#include "startup.hfa" // STARTUP_PRIORITY_XXX
37	#include "limits.hfa"
38	#include "math.hfa"
39
40	#define CFA_PROCESSOR_USE_MMAP 0
41
42	//-----------------------------------------------------------------------------
43	// Some assembly required
44	#if defined( __i386 )
45	#define CtxGet( ctx ) __asm__ volatile ( \
46	"movl %%esp,%0\n" \
47	"movl %%ebp,%1\n" \
48	: "=rm" (ctx.SP), \
49	"=rm" (ctx.FP) \
50	)
51	#elif defined( __x86_64 )
52	#define CtxGet( ctx ) __asm__ volatile ( \
53	"movq %%rsp,%0\n" \
54	"movq %%rbp,%1\n" \
55	: "=rm" (ctx.SP), \
56	"=rm" (ctx.FP) \
57	)
58	#elif defined( __aarch64__ )
59	#define CtxGet( ctx ) __asm__ volatile ( \
60	"mov %0, sp\n" \
61	"mov %1, fp\n" \
62	: "=rm" (ctx.SP), \
63	"=rm" (ctx.FP) \
64	)
65	#else
66	#error unknown hardware architecture
67	#endif
68
69	//-----------------------------------------------------------------------------
70	// Start and stop routine for the kernel, declared first to make sure they run first
71	static void __kernel_startup (void) __attribute__(( constructor( STARTUP_PRIORITY_KERNEL ) ));
72	static void __kernel_shutdown(void) __attribute__(( destructor ( STARTUP_PRIORITY_KERNEL ) ));
73
74	//-----------------------------------------------------------------------------
75	// Static Forward Declarations
76	struct current_stack_info_t;
77
78	static void * __invoke_processor(void * arg);
79	static void __kernel_first_resume( processor * this );
80	static void __kernel_last_resume ( processor * this );
81	static void init(processor & this, const char name[], cluster & _cltr, thread$ * initT);
82	static void deinit(processor & this);
83	static void doregister( struct cluster & cltr );
84	static void unregister( struct cluster & cltr );
85	static void register_tls( processor * this );
86	static void unregister_tls( processor * this );
87	static void ?{}( coroutine$ & this, current_stack_info_t * info);
88	static void ?{}( thread$ & this, current_stack_info_t * info);
89	static void ?{}(processorCtx_t & this) {}
90	static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info);
91
92	#if defined(__CFA_WITH_VERIFY__)
93	static bool verify_fwd_bck_rng(void);
94	#endif
95
96	//-----------------------------------------------------------------------------
97	// Forward Declarations for other modules
98	extern void __kernel_alarm_startup(void);
99	extern void __kernel_alarm_shutdown(void);
100	extern void __cfa_io_start( processor * );
101	extern void __cfa_io_stop ( processor * );
102
103	//-----------------------------------------------------------------------------
104	// Other Forward Declarations
105	extern void __wake_proc(processor *);
106	extern int cfa_main_returned; // from interpose.cfa
107	uint32_t __global_random_prime = 4_294_967_291u, __global_random_mask = false;
108
109	//-----------------------------------------------------------------------------
110	// Kernel storage
111	KERNEL_STORAGE(cluster, mainCluster);
112	KERNEL_STORAGE(processor, mainProcessor);
113	KERNEL_STORAGE(thread$, mainThread);
114	KERNEL_STORAGE(__stack_t, mainThreadCtx);
115	// KERNEL_STORAGE(__scheduler_RWLock_t, __scheduler_lock);
116	KERNEL_STORAGE(eventfd_t, mainIdleEventFd);
117	KERNEL_STORAGE(io_future_t, mainIdleFuture);
118	#if !defined(__CFA_NO_STATISTICS__)
119	KERNEL_STORAGE(__stats_t, mainProcStats);
120	#endif
121
122	cluster * mainCluster libcfa_public;
123	processor * mainProcessor;
124	thread$ * mainThread;
125
126	extern "C" {
127	struct { __dllist_t(cluster) list; __spinlock_t lock; } __cfa_dbg_global_clusters;
128	}
129
130	extern size_t __page_size;
131	extern int __map_prot;
132
133	//-----------------------------------------------------------------------------
134	// Global state
135	__thread struct KernelThreadData __cfaabi_tls __attribute__ ((tls_model ( "initial-exec" ))) @= {
136	NULL, // cannot use 0p
137	NULL,
138	false,
139	{ 1, false, false },
140	0,
141	{ 0, 0 },
142	NULL,
143	#ifdef __CFA_WITH_VERIFY__
144	false,
145	0,
146	#endif
147	};
148
149	__scheduler_RWLock_t __scheduler_lock @= { 0 };
150
151	#if defined(CFA_HAVE_LINUX_LIBRSEQ)
152	// No data needed
153	#elif defined(CFA_HAVE_LINUX_RSEQ_H)
154	extern "Cforall" {
155	__attribute__((aligned(64))) __thread volatile struct rseq __cfaabi_rseq @= {
156	.cpu_id : RSEQ_CPU_ID_UNINITIALIZED,
157	};
158	}
159	#else
160	// No data needed
161	#endif
162
163	//-----------------------------------------------------------------------------
164	// Struct to steal stack
165	struct current_stack_info_t {
166	__stack_t * storage; // pointer to stack object
167	void * base; // base of stack
168	void * limit; // stack grows towards stack limit
169	void * context; // address of cfa_context_t
170	};
171
172	static void ?{}( current_stack_info_t & this ) {
173	__stack_context_t ctx;
174	CtxGet( ctx );
175	this.base = ctx.FP;
176
177	rlimit r;
178	getrlimit( RLIMIT_STACK, &r);
179	size_t size = r.rlim_cur;
180
181	this.limit = (void *)(((intptr_t)this.base) - size);
182	this.context = &storage_mainThreadCtx;
183	}
184
185
186	extern void heapManagerCtor();
187	extern void heapManagerDtor();
188
189	//=============================================================================================
190	// Kernel Setup logic
191	//=============================================================================================
192	//-----------------------------------------------------------------------------
193	// Kernel boot procedures
194	static void __kernel_startup(void) {
195	/* paranoid */ verify( ! __preemption_enabled() );
196	__cfadbg_print_safe(runtime_core, "Kernel : Starting\n");
197
198	__cfa_dbg_global_clusters.list{ __get };
199	__cfa_dbg_global_clusters.lock{};
200
201	/* paranoid */ verify( verify_fwd_bck_rng() );
202
203	// Initialize the global scheduler lock
204	// __scheduler_lock = (__scheduler_RWLock_t*)&storage___scheduler_lock;
205	(__scheduler_lock){};
206
207	// Initialize the main cluster
208	mainCluster = (cluster *)&storage_mainCluster;
209	(*mainCluster){"Main Cluster", 0};
210
211	__cfadbg_print_safe(runtime_core, "Kernel : Main cluster ready\n");
212
213	// Construct the processor context of the main processor
214	void ?{}(processorCtx_t & this, processor * proc) {
215	(this.self){ "Processor" };
216	this.self.starter = 0p;
217	this.proc = proc;
218	}
219
220	void ?{}(processor & this) with( this ) {
221	( this.terminated ){};
222	( this.runner ){};
223	init( this, "Main Processor", *mainCluster, 0p );
224	kernel_thread = __cfaabi_pthread_self();
225
226	runner{ &this };
227	__cfadbg_print_safe(runtime_core, "Kernel : constructed main processor context %p\n", &runner);
228	}
229
230	// Initialize the main processor and the main processor ctx
231	// (the coroutine that contains the processing control flow)
232	mainProcessor = (processor *)&storage_mainProcessor;
233	(*mainProcessor){};
234
235	mainProcessor->idle_wctx.rdbuf = &storage_mainIdleEventFd;
236	mainProcessor->idle_wctx.ftr = (io_future_t*)&storage_mainIdleFuture;
237	/* paranoid */ verify( sizeof(storage_mainIdleEventFd) == sizeof(eventfd_t) );
238
239	__cfa_io_start( mainProcessor );
240	register_tls( mainProcessor );
241
242	// Start by initializing the main thread
243	// SKULLDUGGERY: the mainThread steals the process main thread
244	// which will then be scheduled by the mainProcessor normally
245	mainThread = (thread$ *)&storage_mainThread;
246	current_stack_info_t info;
247	info.storage = (__stack_t*)&storage_mainThreadCtx;
248	(*mainThread){ &info };
249
250	__cfadbg_print_safe(runtime_core, "Kernel : Main thread ready\n");
251
252	//initialize the global state variables
253	__cfaabi_tls.this_processor = mainProcessor;
254	__cfaabi_tls.this_thread = mainThread;
255
256	#if !defined( __CFA_NO_STATISTICS__ )
257	__cfaabi_tls.this_stats = (__stats_t *)& storage_mainProcStats;
258	__init_stats( __cfaabi_tls.this_stats );
259	#endif
260	mainProcessor->local_data = &__cfaabi_tls;
261
262	// Enable preemption
263	__kernel_alarm_startup();
264
265	// Add the main thread to the ready queue
266	// once resume is called on mainProcessor->runner the mainThread needs to be scheduled like any normal thread
267	schedule_thread$(mainThread, UNPARK_LOCAL);
268
269	// SKULLDUGGERY: Force a context switch to the main processor to set the main thread's context to the current UNIX
270	// context. Hence, the main thread does not begin through __cfactx_invoke_thread, like all other threads. The trick here is that
271	// mainThread is on the ready queue when this call is made.
272	__kernel_first_resume( __cfaabi_tls.this_processor );
273
274
275	// THE SYSTEM IS NOW COMPLETELY RUNNING
276
277	__cfadbg_print_safe(runtime_core, "Kernel : Started\n--------------------------------------------------\n\n");
278
279	/* paranoid */ verify( ! __preemption_enabled() );
280	enable_interrupts();
281	/* paranoid */ verify( __preemption_enabled() );
282
283	}
284
285	extern "C"{
286	void pthread_delete_kernel_threads_();
287	}
288
289
290	static void __kernel_shutdown(void) {
291	if(!cfa_main_returned) return;
292
293	//delete kernel threads for pthread_concurrency
294	pthread_delete_kernel_threads_();
295
296	/* paranoid */ verify( __preemption_enabled() );
297	disable_interrupts();
298	/* paranoid */ verify( ! __preemption_enabled() );
299
300	__cfadbg_print_safe(runtime_core, "\n--------------------------------------------------\nKernel : Shutting down\n");
301
302	// SKULLDUGGERY: Notify the mainProcessor it needs to terminates.
303	// When its coroutine terminates, it return control to the mainThread
304	// which is currently here
305	/* paranoid */ verify( !__atomic_load_n(&mainProcessor->do_terminate, __ATOMIC_ACQUIRE) );
306	__atomic_store_n(&mainProcessor->do_terminate, true, __ATOMIC_RELEASE);
307	__wake_proc( mainProcessor );
308	__kernel_last_resume( __cfaabi_tls.this_processor );
309	mainThread->self_cor.state = Halted;
310
311	// THE SYSTEM IS NOW COMPLETELY STOPPED
312
313	// Disable preemption
314	__kernel_alarm_shutdown();
315
316	#if !defined( __CFA_NO_STATISTICS__ )
317	__stats_t * st = (__stats_t *)& storage_mainProcStats;
318	__tally_stats(mainCluster->stats, st);
319	if( 0 != mainProcessor->print_stats ) {
320	__print_stats( st, mainProcessor->print_stats, "Processor ", mainProcessor->name, (void*)mainProcessor );
321	}
322	#if defined(CFA_STATS_ARRAY)
323	__flush_stat( st, "Processor", mainProcessor );
324	#endif
325	#endif
326
327	mainProcessor->local_data = 0p;
328
329	unregister_tls( mainProcessor );
330	__cfa_io_stop( mainProcessor );
331
332	// Destroy the main processor and its context in reverse order of construction
333	// These were manually constructed so we need manually destroy them
334	void ^?{}(processor & this) with( this ){
335	deinit( this );
336
337	/* paranoid */ verify( this.do_terminate == true );
338	__cfaabi_dbg_print_safe("Kernel : destroyed main processor context %p\n", &runner);
339	}
340
341	^(*mainProcessor){};
342
343	// Final step, destroy the main thread since it is no longer needed
344
345	// Since we provided a stack to this taxk it will not destroy anything
346	/* paranoid / verify(mainThread->self_cor.stack.storage == (__stack_t)(((uintptr_t)&storage_mainThreadCtx)\| 0x1));
347	^(*mainThread){};
348
349	^(*mainCluster){};
350
351	^(__scheduler_lock){};
352
353	^(__cfa_dbg_global_clusters.list){};
354	^(__cfa_dbg_global_clusters.lock){};
355
356	__cfadbg_print_safe(runtime_core, "Kernel : Shutdown complete\n");
357	}
358
359	//=============================================================================================
360	// Kernel Initial Scheduling logic
361	//=============================================================================================
362
363	// Context invoker for processors
364	// This is the entry point for processors (kernel threads) except for the main processor
365	// It effectively constructs a coroutine by stealing the pthread stack
366	static void * __invoke_processor(void * arg) {
367	#if !defined( __CFA_NO_STATISTICS__ )
368	__stats_t local_stats;
369	__init_stats( &local_stats );
370	__cfaabi_tls.this_stats = &local_stats;
371	#endif
372
373	processor * proc = (processor *) arg;
374	__cfaabi_tls.this_processor = proc;
375	__cfaabi_tls.this_thread = 0p;
376	__cfaabi_tls.preemption_state.[enabled, disable_count] = [false, 1];
377	proc->local_data = &__cfaabi_tls;
378
379	heapManagerCtor(); // initialize heap
380
381	__cfa_io_start( proc );
382	register_tls( proc );
383
384	// used for idle sleep when io_uring is present
385	io_future_t future;
386	eventfd_t idle_buf;
387	proc->idle_wctx.ftr = &future;
388	proc->idle_wctx.rdbuf = &idle_buf;
389
390
391	// SKULLDUGGERY: We want to create a context for the processor coroutine
392	// which is needed for the 2-step context switch. However, there is no reason
393	// to waste the perfectly valid stack create by pthread.
394	current_stack_info_t info;
395	__stack_t ctx;
396	info.storage = &ctx;
397	(proc->runner){ proc, &info };
398
399	__cfaabi_dbg_print_safe("Coroutine : created stack %p\n", get_coroutine(proc->runner)->stack.storage);
400
401	//Set global state
402	__cfaabi_tls.this_thread = 0p;
403
404	//We now have a proper context from which to schedule threads
405	__cfadbg_print_safe(runtime_core, "Kernel : core %p created (%p, %p)\n", proc, &proc->runner, &ctx);
406
407	// SKULLDUGGERY: Since the coroutine doesn't have its own stack, we can't
408	// resume it to start it like it normally would, it will just context switch
409	// back to here. Instead directly call the main since we already are on the
410	// appropriate stack.
411	get_coroutine(proc->runner)->state = Active;
412	main( proc->runner );
413	get_coroutine(proc->runner)->state = Halted;
414
415	// Main routine of the core returned, the core is now fully terminated
416	__cfadbg_print_safe(runtime_core, "Kernel : core %p main ended (%p)\n", proc, &proc->runner);
417
418	#if !defined(__CFA_NO_STATISTICS__)
419	__tally_stats(proc->cltr->stats, &local_stats);
420	if( 0 != proc->print_stats ) {
421	__print_stats( &local_stats, proc->print_stats, "Processor ", proc->name, (void*)proc );
422	}
423	#if defined(CFA_STATS_ARRAY)
424	__flush_stat( &local_stats, "Processor", proc );
425	#endif
426	#endif
427
428	proc->local_data = 0p;
429
430	unregister_tls( proc );
431	__cfa_io_stop( proc );
432
433	heapManagerDtor(); // de-initialize heap
434
435	return 0p;
436	}
437
438	static void __kernel_first_resume( processor * this ) {
439	thread$ * src = mainThread;
440	coroutine$ * dst = get_coroutine(this->runner);
441
442	/* paranoid */ verify( ! __preemption_enabled() );
443
444	__cfaabi_tls.this_thread->curr_cor = dst;
445	__stack_prepare( &dst->stack, DEFAULT_STACK_SIZE );
446	__cfactx_start(main, dst, this->runner, __cfactx_invoke_coroutine);
447
448	/* paranoid */ verify( ! __preemption_enabled() );
449
450	dst->last = &src->self_cor;
451	dst->starter = dst->starter ? dst->starter : &src->self_cor;
452
453	// make sure the current state is still correct
454	/* paranoid */ verify(src->state == Ready);
455	src->corctx_flag = true;
456
457	// context switch to specified coroutine
458	verify( dst->context.SP );
459	__cfactx_switch( &src->context, &dst->context );
460	// when __cfactx_switch returns we are back in the src coroutine
461
462	mainThread->curr_cor = &mainThread->self_cor;
463
464	// make sure the current state has been update
465	/* paranoid */ verify(src->state == Active);
466
467	/* paranoid */ verify( ! __preemption_enabled() );
468	}
469
470	// KERNEL_ONLY
471	static void __kernel_last_resume( processor * this ) {
472	coroutine$ * src = &mainThread->self_cor;
473	coroutine$ * dst = get_coroutine(this->runner);
474
475	/* paranoid */ verify( ! __preemption_enabled() );
476	/* paranoid */ verify( dst->starter == src );
477	/* paranoid */ verify( dst->context.SP );
478
479	// SKULLDUGGERY in debug the processors check that the
480	// stack is still within the limit of the stack limits after running a thread.
481	// that check doesn't make sense if we context switch to the processor using the
482	// coroutine semantics. Since this is a special case, use the current context
483	// info to populate these fields.
484	__cfaabi_dbg_debug_do(
485	__stack_context_t ctx;
486	CtxGet( ctx );
487	mainThread->context.SP = ctx.SP;
488	mainThread->context.FP = ctx.FP;
489	)
490
491	// context switch to the processor
492	__cfactx_switch( &src->context, &dst->context );
493	}
494
495
496	//=============================================================================================
497	// Kernel Object Constructors logic
498	//=============================================================================================
499	//-----------------------------------------------------------------------------
500	// Main thread construction
501	static void ?{}( coroutine$ & this, current_stack_info_t * info) with( this ) {
502	stack.storage = info->storage;
503	with(*stack.storage) {
504	limit = info->limit;
505	base = info->base;
506	}
507	__attribute__((may_alias)) intptr_t * istorage = (intptr_t*) &stack.storage;
508	*istorage \|= 0x1;
509	name = "Main Thread";
510	state = Start;
511	starter = 0p;
512	last = 0p;
513	cancellation = 0p;
514	}
515
516	static void ?{}( thread$ & this, current_stack_info_t * info) with( this ) {
517	ticket = TICKET_RUNNING;
518	state = Start;
519	self_cor{ info };
520	curr_cor = &self_cor;
521	curr_cluster = mainCluster;
522	self_mon.owner = &this;
523	self_mon.recursion = 1;
524	self_mon_p = &self_mon;
525	link.next = 0p;
526	link.ts = MAX;
527	preferred = ready_queue_new_preferred();
528	last_proc = 0p;
529	random_state = __global_random_mask ? __global_random_prime : __global_random_prime ^ rdtscl();
530	#if defined( __CFA_WITH_VERIFY__ )
531	executing = 0p;
532	canary = 0x0D15EA5E0D15EA5Ep;
533	#endif
534
535	node.next = 0p;
536	node.prev = 0p;
537	doregister(curr_cluster, this);
538
539	monitors{ &self_mon_p, 1, (fptr_t)0 };
540	}
541
542	//-----------------------------------------------------------------------------
543	// Processor
544	// Construct the processor context of non-main processors
545	static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info) {
546	(this.self){ info };
547	this.proc = proc;
548	}
549
550	static void init(processor & this, const char name[], cluster & _cltr, thread$ * initT) with( this ) {
551	this.name = name;
552	this.cltr = &_cltr;
553	this.rdq.its = 0;
554	this.rdq.itr = 0;
555	this.rdq.id = 0;
556	this.rdq.target = MAX;
557	this.rdq.last = MAX;
558	this.rdq.cpu = 0;
559	// this.rdq.cutoff = 0ull;
560	do_terminate = false;
561	preemption_alarm = 0p;
562	pending_preemption = false;
563
564	this.io.ctx = 0p;
565	this.io.pending = false;
566	this.io.dirty = false;
567
568	this.init.thrd = initT;
569
570	this.local_data = 0p;
571
572	idle_wctx.evfd = eventfd(0, 0);
573	if (idle_wctx.evfd < 0) {
574	abort("KERNEL ERROR: PROCESSOR EVENTFD - %s\n", strerror(errno));
575	}
576
577	idle_wctx.sem = 0;
578	idle_wctx.wake__time = 0;
579
580	// I'm assuming these two are reserved for standard input and output
581	// so I'm using them as sentinels with idle_wctx.
582	/* paranoid */ verify( idle_wctx.evfd != 0 );
583	/* paranoid */ verify( idle_wctx.evfd != 1 );
584
585	#if !defined(__CFA_NO_STATISTICS__)
586	print_stats = 0;
587	print_halts = false;
588	#endif
589
590	__cfadbg_print_safe(runtime_core, "Kernel : core %p created\n", &this);
591	}
592
593	// Not a ctor, it just preps the destruction but should not destroy members
594	static void deinit(processor & this) {
595	close(this.idle_wctx.evfd);
596	}
597
598	void ?{}(processor & this, const char name[], cluster & _cltr, thread$ * initT) libcfa_public {
599	( this.terminated ){};
600	( this.runner ){};
601
602	disable_interrupts();
603	init( this, name, _cltr, initT );
604	enable_interrupts();
605
606	__cfadbg_print_safe(runtime_core, "Kernel : Starting core %p\n", &this);
607
608	this.stack = __create_pthread( &this.kernel_thread, __invoke_processor, (void *)&this );
609	}
610
611	void ?{}(processor & this, const char name[], cluster & _cltr) libcfa_public {
612	(this){name, _cltr, 0p};
613	}
614
615	extern size_t __page_size;
616	void ^?{}(processor & this) libcfa_public with( this ) {
617	/* paranoid */ verify( !__atomic_load_n(&do_terminate, __ATOMIC_ACQUIRE) );
618	__cfadbg_print_safe(runtime_core, "Kernel : core %p signaling termination\n", &this);
619
620	__atomic_store_n(&do_terminate, true, __ATOMIC_RELAXED);
621	__disable_interrupts_checked();
622	__wake_proc( &this );
623	__enable_interrupts_checked();
624
625	wait( terminated );
626	/* paranoid */ verify( active_processor() != &this);
627
628	__destroy_pthread( kernel_thread, this.stack, 0p );
629
630	disable_interrupts();
631	deinit( this );
632	enable_interrupts();
633	}
634
635	//-----------------------------------------------------------------------------
636	// Cluster
637	static void ?{}(__cluster_proc_list & this) {
638	this.fdw = 0p;
639	this.idle = 0;
640	this.total = 0;
641	}
642
643	void ?{}(cluster & this, const char name[], Duration preemption_rate, unsigned num_io, const io_context_params & io_params) libcfa_public with( this ) {
644	this.name = name;
645	this.preemption_rate = preemption_rate;
646	this.sched.readyQ.data = 0p;
647	this.sched.readyQ.tscs = 0p;
648	this.sched.readyQ.count = 0;
649	this.sched.io.tscs = 0p;
650	this.sched.io.data = 0p;
651	this.sched.caches = 0p;
652
653	#if !defined(__CFA_NO_STATISTICS__)
654	print_stats = 0;
655	stats = alloc();
656	__init_stats( stats );
657	#endif
658
659	threads{ __get };
660
661	io.arbiter = create();
662	io.params = io_params;
663
664	doregister(this);
665
666	// Lock the RWlock so no-one pushes/pops while we are changing the queue
667	disable_interrupts();
668	uint_fast32_t last_size = ready_mutate_lock();
669
670	// Adjust the ready queue size
671	ready_queue_grow( &this );
672
673	// Unlock the RWlock
674	ready_mutate_unlock( last_size );
675	enable_interrupts( false ); // Don't poll, could be in main cluster
676	}
677
678	void ^?{}(cluster & this) libcfa_public {
679	destroy(this.io.arbiter);
680
681	// Lock the RWlock so no-one pushes/pops while we are changing the queue
682	disable_interrupts();
683	uint_fast32_t last_size = ready_mutate_lock();
684
685	// Adjust the ready queue size
686	ready_queue_shrink( &this );
687
688	// Unlock the RWlock
689	ready_mutate_unlock( last_size );
690
691	ready_queue_close( &this );
692	/* paranoid */ verify( this.sched.readyQ.data == 0p );
693	/* paranoid */ verify( this.sched.readyQ.tscs == 0p );
694	/* paranoid */ verify( this.sched.readyQ.count == 0 );
695	/* paranoid */ verify( this.sched.io.tscs == 0p );
696	/* paranoid */ verify( this.sched.caches == 0p );
697
698	enable_interrupts( false ); // Don't poll, could be in main cluster
699
700
701	#if !defined(__CFA_NO_STATISTICS__)
702	if( 0 != this.print_stats ) {
703	__print_stats( this.stats, this.print_stats, "Cluster", this.name, (void*)&this );
704	}
705	#if defined(CFA_STATS_ARRAY)
706	__flush_stat( this.stats, "Cluster", &this );
707	#endif
708	free( this.stats );
709	#endif
710
711	unregister(this);
712	}
713
714	//=============================================================================================
715	// Miscellaneous Initialization
716	//=============================================================================================
717	//-----------------------------------------------------------------------------
718	// Global Queues
719	static void doregister( cluster & cltr ) {
720	lock ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
721	push_front( __cfa_dbg_global_clusters.list, cltr );
722	unlock ( __cfa_dbg_global_clusters.lock );
723	}
724
725	static void unregister( cluster & cltr ) {
726	lock ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
727	remove( __cfa_dbg_global_clusters.list, cltr );
728	unlock( __cfa_dbg_global_clusters.lock );
729	}
730
731	void doregister( cluster * cltr, thread$ & thrd ) {
732	lock (cltr->thread_list_lock __cfaabi_dbg_ctx2);
733	cltr->nthreads += 1;
734	push_front(cltr->threads, thrd);
735	unlock (cltr->thread_list_lock);
736	}
737
738	void unregister( cluster * cltr, thread$ & thrd ) {
739	lock (cltr->thread_list_lock __cfaabi_dbg_ctx2);
740	remove(cltr->threads, thrd );
741	cltr->nthreads -= 1;
742	unlock(cltr->thread_list_lock);
743	}
744
745	static void register_tls( processor * this ) {
746	// Register and Lock the RWlock so no-one pushes/pops while we are changing the queue
747	uint_fast32_t last_size;
748	[this->unique_id, last_size] = ready_mutate_register();
749
750	this->rdq.cpu = __kernel_getcpu();
751
752	this->cltr->procs.total += 1u;
753	insert_last(this->cltr->procs.actives, *this);
754
755	// Adjust the ready queue size
756	ready_queue_grow( this->cltr );
757
758	// Unlock the RWlock
759	ready_mutate_unlock( last_size );
760	}
761
762
763	static void unregister_tls( processor * this ) {
764	// Lock the RWlock so no-one pushes/pops while we are changing the queue
765	uint_fast32_t last_size = ready_mutate_lock();
766	this->cltr->procs.total -= 1u;
767	remove(*this);
768
769	// clear the cluster so nothing gets pushed to local queues
770	cluster * cltr = this->cltr;
771	this->cltr = 0p;
772
773	// Adjust the ready queue size
774	ready_queue_shrink( cltr );
775
776	// Unlock the RWlock and unregister: we don't need the read_lock any more
777	ready_mutate_unregister( this->unique_id, last_size );
778	}
779
780	static void check( int ret, const char func[] ) {
781	if ( ret ) { // pthread routines return errno values
782	abort( "%s : internal error, error(%d) %s.", func, ret, strerror( ret ) );
783	} // if
784	} // Abort
785
786	void * __create_pthread( pthread_t * pthread, void * (start)(void ), void * arg ) {
787	pthread_attr_t attr;
788
789	check( __cfaabi_pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
790
791	size_t stacksize = max( PTHREAD_STACK_MIN, DEFAULT_STACK_SIZE );
792
793	void * stack;
794	#if CFA_PROCESSOR_USE_MMAP
795	stacksize = ceiling( stacksize, __page_size ) + __page_size;
796	stack = mmap(0p, stacksize, __map_prot, MAP_PRIVATE \| MAP_ANONYMOUS, 0, 0);
797	if(stack == ((void*)-1)) {
798	abort( "pthread stack creation : internal error, mmap failure, error(%d) %s.", errno, strerror( errno ) );
799	}
800	if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
801	abort( "pthread stack creation : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
802	} // if
803	#else
804	__cfaabi_dbg_debug_do(
805	stack = memalign( __page_size, stacksize + __page_size );
806	// pthread has no mechanism to create the guard page in user supplied stack.
807	if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
808	abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
809	} // if
810	);
811	__cfaabi_dbg_no_debug_do(
812	stack = malloc( stacksize );
813	);
814	#endif
815
816	check( __cfaabi_pthread_attr_setstack( &attr, stack, stacksize ), "pthread_attr_setstack" );
817	check( __cfaabi_pthread_create( pthread, &attr, start, arg ), "pthread_create" );
818	return stack;
819	}
820
821	void __destroy_pthread( pthread_t pthread, void * stack, void ** retval ) {
822	int err = __cfaabi_pthread_join( pthread, retval );
823	if( err != 0 ) abort("KERNEL ERROR: joining pthread %p caused error %s\n", (void*)pthread, strerror(err));
824
825	#if CFA_PROCESSOR_USE_MMAP
826	pthread_attr_t attr;
827
828	check( __cfaabi_pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
829
830	size_t stacksize;
831	// default stack size, normally defined by shell limit
832	check( __cfaabi_pthread_attr_getstacksize( &attr, &stacksize ), "pthread_attr_getstacksize" );
833	assert( stacksize >= PTHREAD_STACK_MIN );
834	stacksize += __page_size;
835
836	if(munmap(stack, stacksize) == -1) {
837	abort( "pthread stack destruction : internal error, munmap failure, error(%d) %s.", errno, strerror( errno ) );
838	}
839	#else
840	__cfaabi_dbg_debug_do(
841	// pthread has no mechanism to create the guard page in user supplied stack.
842	if ( mprotect( stack, __page_size, __map_prot ) == -1 ) {
843	abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
844	} // if
845	);
846	free( stack );
847	#endif
848	}
849
850	#if defined(__CFA_WITH_VERIFY__)
851	static bool verify_fwd_bck_rng(void) {
852	__cfaabi_tls.ready_rng.fwd_seed = 25214903917_l64u * (rdtscl() ^ (uintptr_t)&verify_fwd_bck_rng);
853
854	unsigned values[10];
855	for(i; 10) {
856	values[i] = __tls_rand_fwd();
857	}
858
859	__tls_rand_advance_bck();
860
861	for ( i; 9 -~= 0 ) {
862	if(values[i] != __tls_rand_bck()) {
863	return false;
864	}
865	}
866
867	return true;
868	}
869	#endif

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

Original Format