Context Navigation

startup.cfa

Last change on this file was 6dd4091, checked in by Peter A. Buhr <pabuhr@…>, 11 days ago
comment out asserts that fail when malloc(0) returns non-null
Property mode set to `100644`
File size: 27.6 KB

Line
1	//
2	// Cforall Version 1.0.0 Copyright (C) 2020 University of Waterloo
3	//
4	// The contents of this file are covered under the licence agreement in the
5	// file "LICENCE" distributed with Cforall.
6	//
7	// kernel/startup.cfa --
8	//
9	// Author : Thierry Delisle
10	// Created On : Thu Jul 30 15:12:54 2020
11	// Last Modified By :
12	// Last Modified On :
13	// Update Count :
14	//
15
16	#define __cforall_thread__
17
18	// #define __CFA_DEBUG_PRINT_RUNTIME_CORE__
19
20	// C Includes
21	#include <errno.h> // errno
22	#include <signal.h>
23	#include <string.h> // strerror
24	#include <unistd.h>
25	#include <limits.h> // PTHREAD_STACK_MIN
26	extern "C" {
27	#include <sys/eventfd.h> // eventfd
28	#include <sys/mman.h> // mprotect
29	#include <sys/resource.h> // getrlimit
30	}
31
32	// CFA Includes
33	#include "kernel/private.hfa"
34	#include "iofwd.hfa"
35	#include "startup.hfa" // STARTUP_PRIORITY_XXX
36	#include "limits.hfa"
37	#include "math.hfa"
38	#include "bits/random.hfa" // prng
39
40	#define CFA_PROCESSOR_USE_MMAP 0
41
42	//-----------------------------------------------------------------------------
43	// Some assembly required
44	#if defined( __i386 )
45	#define CtxGet( ctx ) __asm__ volatile ( \
46	"movl %%esp,%0\n" \
47	"movl %%ebp,%1\n" \
48	: "=rm" (ctx.SP), \
49	"=rm" (ctx.FP) \
50	)
51	#elif defined( __x86_64 )
52	#define CtxGet( ctx ) __asm__ volatile ( \
53	"movq %%rsp,%0\n" \
54	"movq %%rbp,%1\n" \
55	: "=rm" (ctx.SP), \
56	"=rm" (ctx.FP) \
57	)
58	#elif defined( __aarch64__ )
59	#define CtxGet( ctx ) __asm__ volatile ( \
60	"mov %0, sp\n" \
61	"mov %1, fp\n" \
62	: "=rm" (ctx.SP), \
63	"=rm" (ctx.FP) \
64	)
65	#else
66	#error unknown hardware architecture
67	#endif
68
69	//-----------------------------------------------------------------------------
70	// Start and stop routine for the kernel, declared first to make sure they run first
71	static void __kernel_startup (void) __attribute__(( constructor( STARTUP_PRIORITY_KERNEL ) ));
72	static void __kernel_shutdown(void) __attribute__(( destructor ( STARTUP_PRIORITY_KERNEL ) ));
73
74	//-----------------------------------------------------------------------------
75	// Static Forward Declarations
76	struct current_stack_info_t;
77
78	static void * __invoke_processor(void * arg);
79	static void __kernel_first_resume( processor * this );
80	static void __kernel_last_resume ( processor * this );
81	static void init(processor & this, const char name[], cluster & _cltr, thread$ * initT);
82	static void deinit(processor & this);
83	static void doregister( struct cluster & cltr );
84	static void unregister( struct cluster & cltr );
85	static void register_tls( processor * this );
86	static void unregister_tls( processor * this );
87	static void ?{}( coroutine$ & this, current_stack_info_t * info);
88	static void ?{}( thread$ & this, current_stack_info_t * info);
89	static void ?{}(processorCtx_t & this) {}
90	static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info);
91
92	#if defined(__CFA_WITH_VERIFY__)
93	static bool verify_fwd_bck_rng(void);
94	#endif
95
96	//-----------------------------------------------------------------------------
97	// Forward Declarations for other modules
98	extern void __kernel_alarm_startup(void);
99	extern void __kernel_alarm_shutdown(void);
100	extern void __cfa_io_start( processor * );
101	extern void __cfa_io_stop ( processor * );
102
103	//-----------------------------------------------------------------------------
104	// Other Forward Declarations
105	extern void __wake_proc(processor *);
106	extern int cfa_main_returned; // from interpose.cfa
107	size_t __global_random_prime = 4_294_967_291u;
108	bool __global_random_mask = false;
109
110	//-----------------------------------------------------------------------------
111	// Kernel storage
112	KERNEL_STORAGE(cluster, mainCluster);
113	KERNEL_STORAGE(processor, mainProcessor);
114	KERNEL_STORAGE(thread$, mainThread);
115	KERNEL_STORAGE(__stack_t, mainThreadCtx);
116	#if !defined(__CFA_NO_STATISTICS__)
117	KERNEL_STORAGE(__stats_t, mainProcStats);
118	#endif
119
120	cluster * mainCluster libcfa_public;
121	processor * mainProcessor;
122	thread$ * mainThread;
123
124	extern "C" {
125	struct { __dllist_t(cluster) list; __spinlock_t lock; } __cfa_dbg_global_clusters;
126	}
127
128	extern size_t __page_size;
129	extern int __map_prot;
130
131	//-----------------------------------------------------------------------------
132	// Global state
133	__thread struct KernelThreadData __cfaabi_tls __attribute__ ((tls_model ( "initial-exec" ))) @= {
134	.this_thread : NULL, // cannot use 0p
135	.this_processor : NULL,
136	.sched_lock : false,
137	.preemption_state : { .disable_count : 1, .enabled : false, .in_progress : false },
138	// random_state uninitialized
139	.ready_rng : { .fwd_seed : 0, .bck_seed : 0 },
140	.this_stats : NULL,
141	#ifdef __CFA_WITH_VERIFY__
142	.in_sched_lock : false,
143	.sched_id : 0,
144	#endif
145	};
146
147	__scheduler_RWLock_t __scheduler_lock @= { 0 };
148
149	//-----------------------------------------------------------------------------
150	// Struct to steal stack
151	struct current_stack_info_t {
152	__stack_t * storage; // pointer to stack object
153	void * base; // base of stack
154	void * limit; // stack grows towards stack limit
155	void * context; // address of cfa_context_t
156	};
157
158	static void ?{}( current_stack_info_t & this ) {
159	__stack_context_t ctx;
160	CtxGet( ctx );
161	this.base = ctx.FP;
162
163	rlimit r;
164	getrlimit( RLIMIT_STACK, &r);
165	size_t size = r.rlim_cur;
166
167	this.limit = (void *)(((intptr_t)this.base) - size);
168	this.context = &storage_mainThreadCtx;
169	}
170
171
172	extern void heapManagerCtor();
173	extern void heapManagerDtor();
174
175	//=============================================================================================
176	// Kernel Setup logic
177	//=============================================================================================
178	//-----------------------------------------------------------------------------
179	// Kernel boot procedures
180	static void __kernel_startup(void) {
181	/* paranoid */ verify( ! __preemption_enabled() );
182	__cfadbg_print_safe(runtime_core, "Kernel : Starting\n");
183
184	__cfa_dbg_global_clusters.list{ __get };
185	__cfa_dbg_global_clusters.lock{};
186
187	/* paranoid */ verify( verify_fwd_bck_rng() );
188
189	// Initialize the global scheduler lock
190	// __scheduler_lock = (__scheduler_RWLock_t*)&storage___scheduler_lock;
191	(__scheduler_lock){};
192
193	// Initialize the main cluster
194	mainCluster = (cluster *)&storage_mainCluster;
195	(*mainCluster){"Main Cluster", 0};
196
197	__cfadbg_print_safe(runtime_core, "Kernel : Main cluster ready\n");
198
199	// Construct the processor context of the main processor
200	void ?{}(processorCtx_t & this, processor * proc) {
201	(this.self){ "Processor" };
202	this.self.starter = 0p;
203	this.proc = proc;
204	}
205
206	void ?{}(processor & this) with( this ) {
207	( this.terminated ){};
208	( this.runner ){};
209	init( this, "Main Processor", *mainCluster, 0p );
210	kernel_thread = __cfaabi_pthread_self();
211
212	runner{ &this };
213	__cfadbg_print_safe(runtime_core, "Kernel : constructed main processor context %p\n", &runner);
214	}
215
216	// Initialize the main processor and the main processor ctx
217	// (the coroutine that contains the processing control flow)
218	mainProcessor = (processor *)&storage_mainProcessor;
219	(*mainProcessor){};
220
221	__cfa_io_start( mainProcessor );
222	register_tls( mainProcessor );
223
224	// Start by initializing the main thread
225	// SKULLDUGGERY: the mainThread steals the process main thread
226	// which will then be scheduled by the mainProcessor normally
227	mainThread = (thread$ *)&storage_mainThread;
228	current_stack_info_t info;
229	info.storage = (__stack_t*)&storage_mainThreadCtx;
230	(*mainThread){ &info };
231
232	__cfadbg_print_safe(runtime_core, "Kernel : Main thread ready\n");
233
234	//initialize the global state variables
235	__cfaabi_tls.this_processor = mainProcessor;
236	__cfaabi_tls.this_thread = mainThread;
237
238	#if !defined( __CFA_NO_STATISTICS__ )
239	__cfaabi_tls.this_stats = (__stats_t *)& storage_mainProcStats;
240	__init_stats( __cfaabi_tls.this_stats );
241	#endif
242	mainProcessor->local_data = &__cfaabi_tls;
243
244	// Enable preemption
245	__kernel_alarm_startup();
246
247	// Add the main thread to the ready queue
248	// once resume is called on mainProcessor->runner the mainThread needs to be scheduled like any normal thread
249	schedule_thread$(mainThread, UNPARK_LOCAL);
250
251	// SKULLDUGGERY: Force a context switch to the main processor to set the main thread's context to the current UNIX
252	// context. Hence, the main thread does not begin through __cfactx_invoke_thread, like all other threads. The trick here is that
253	// mainThread is on the ready queue when this call is made.
254	__kernel_first_resume( __cfaabi_tls.this_processor );
255
256
257	// THE SYSTEM IS NOW COMPLETELY RUNNING
258
259	__cfadbg_print_safe(runtime_core, "Kernel : Started\n--------------------------------------------------\n\n");
260
261	/* paranoid */ verify( ! __preemption_enabled() );
262	enable_interrupts();
263	/* paranoid */ verify( __preemption_enabled() );
264
265	}
266
267	extern "C"{
268	void pthread_delete_kernel_threads_();
269	}
270
271
272	static void __kernel_shutdown(void) {
273	if(!cfa_main_returned) return;
274
275	//delete kernel threads for pthread_concurrency
276	pthread_delete_kernel_threads_();
277
278	/* paranoid */ verify( __preemption_enabled() );
279	disable_interrupts();
280	/* paranoid */ verify( ! __preemption_enabled() );
281
282	__cfadbg_print_safe(runtime_core, "\n--------------------------------------------------\nKernel : Shutting down\n");
283
284	// SKULLDUGGERY: Notify the mainProcessor it needs to terminates.
285	// When its coroutine terminates, it return control to the mainThread
286	// which is currently here
287	/* paranoid */ verify( !__atomic_load_n(&mainProcessor->do_terminate, __ATOMIC_ACQUIRE) );
288	__atomic_store_n(&mainProcessor->do_terminate, true, __ATOMIC_RELEASE);
289	__wake_proc( mainProcessor );
290	__kernel_last_resume( __cfaabi_tls.this_processor );
291	mainThread->self_cor.state = Halted;
292
293	// THE SYSTEM IS NOW COMPLETELY STOPPED
294
295	// Disable preemption
296	__kernel_alarm_shutdown();
297
298	#if !defined( __CFA_NO_STATISTICS__ )
299	__stats_t * st = (__stats_t *)& storage_mainProcStats;
300	__tally_stats(mainCluster->stats, st);
301	if( 0 != mainProcessor->print_stats ) {
302	__print_stats( st, mainProcessor->print_stats, "Processor ", mainProcessor->name, (void*)mainProcessor );
303	}
304	#if defined(CFA_STATS_ARRAY)
305	__flush_stat( st, "Processor", mainProcessor );
306	#endif
307	#endif
308
309	mainProcessor->local_data = 0p;
310
311	unregister_tls( mainProcessor );
312	__cfa_io_stop( mainProcessor );
313
314	// Destroy the main processor and its context in reverse order of construction
315	// These were manually constructed so we need manually destroy them
316	void ^?{}(processor & this) with( this ){
317	deinit( this );
318
319	/* paranoid */ verify( this.do_terminate == true );
320	__cfadbg_print_safe(runtime_core, "Kernel : destroyed main processor context %p\n", &runner);
321	}
322
323	^(*mainProcessor){};
324
325	// Final step, destroy the main thread since it is no longer needed
326
327	// Since we provided a stack to this taxk it will not destroy anything
328	/* paranoid / verify(mainThread->self_cor.stack.storage == (__stack_t)(((uintptr_t)&storage_mainThreadCtx)\| 0x1));
329	^(*mainThread){};
330
331	^(*mainCluster){};
332
333	^(__scheduler_lock){};
334
335	^(__cfa_dbg_global_clusters.list){};
336	^(__cfa_dbg_global_clusters.lock){};
337
338	__cfadbg_print_safe(runtime_core, "Kernel : Shutdown complete\n");
339	}
340
341	//=============================================================================================
342	// Kernel Initial Scheduling logic
343	//=============================================================================================
344
345	// Context invoker for processors
346	// This is the entry point for processors (kernel threads) except for the main processor
347	// It effectively constructs a coroutine by stealing the pthread stack
348	static void * __invoke_processor(void * arg) {
349	#if !defined( __CFA_NO_STATISTICS__ )
350	__stats_t local_stats;
351	__init_stats( &local_stats );
352	__cfaabi_tls.this_stats = &local_stats;
353	#endif
354
355	processor * proc = (processor *) arg;
356	__cfaabi_tls.this_processor = proc;
357	__cfaabi_tls.this_thread = 0p;
358	__cfaabi_tls.preemption_state.[enabled, disable_count] = [false, 1];
359	proc->local_data = &__cfaabi_tls;
360
361	heapManagerCtor(); // initialize heap
362
363	__cfa_io_start( proc );
364	register_tls( proc );
365
366	// SKULLDUGGERY: We want to create a context for the processor coroutine
367	// which is needed for the 2-step context switch. However, there is no reason
368	// to waste the perfectly valid stack create by pthread.
369	current_stack_info_t info;
370	__stack_t ctx;
371	info.storage = &ctx;
372	(proc->runner){ proc, &info };
373
374	__cfadbg_print_safe(runtime_core, "Coroutine : created stack %p\n", get_coroutine(proc->runner)->stack.storage);
375
376	//Set global state
377	__cfaabi_tls.this_thread = 0p;
378
379	//We now have a proper context from which to schedule threads
380	__cfadbg_print_safe(runtime_core, "Kernel : core %p created (%p, %p)\n", proc, &proc->runner, &ctx);
381
382	// SKULLDUGGERY: Since the coroutine doesn't have its own stack, we can't
383	// resume it to start it like it normally would, it will just context switch
384	// back to here. Instead directly call the main since we already are on the
385	// appropriate stack.
386	get_coroutine(proc->runner)->state = Active;
387	main( proc->runner );
388	get_coroutine(proc->runner)->state = Halted;
389
390	// Main routine of the core returned, the core is now fully terminated
391	__cfadbg_print_safe(runtime_core, "Kernel : core %p main ended (%p)\n", proc, &proc->runner);
392
393	#if !defined(__CFA_NO_STATISTICS__)
394	__tally_stats(proc->cltr->stats, &local_stats);
395	if( 0 != proc->print_stats ) {
396	__print_stats( &local_stats, proc->print_stats, "Processor ", proc->name, (void*)proc );
397	}
398	#if defined(CFA_STATS_ARRAY)
399	__flush_stat( &local_stats, "Processor", proc );
400	#endif
401	#endif
402
403	proc->local_data = 0p;
404
405	unregister_tls( proc );
406	__cfa_io_stop( proc );
407
408	heapManagerDtor(); // de-initialize heap
409
410	return 0p;
411	}
412
413	static void __kernel_first_resume( processor * this ) {
414	thread$ * src = mainThread;
415	coroutine$ * dst = get_coroutine(this->runner);
416
417	/* paranoid */ verify( ! __preemption_enabled() );
418
419	__cfaabi_tls.this_thread->curr_cor = dst;
420	__stack_prepare( &dst->stack, DEFAULT_STACK_SIZE );
421	__cfactx_start(main, dst, this->runner, __cfactx_invoke_coroutine);
422
423	/* paranoid */ verify( ! __preemption_enabled() );
424
425	dst->last = &src->self_cor;
426	dst->starter = dst->starter ? dst->starter : &src->self_cor;
427
428	// make sure the current state is still correct
429	/* paranoid */ verify(src->state == Ready);
430	src->corctx_flag = true;
431
432	// context switch to specified coroutine
433	verify( dst->context.SP );
434	__cfactx_switch( &src->context, &dst->context );
435	// when __cfactx_switch returns we are back in the src coroutine
436
437	mainThread->curr_cor = &mainThread->self_cor;
438
439	// make sure the current state has been update
440	/* paranoid */ verify(src->state == Active);
441
442	/* paranoid */ verify( ! __preemption_enabled() );
443	}
444
445	// KERNEL_ONLY
446	static void __kernel_last_resume( processor * this ) {
447	coroutine$ * src = &mainThread->self_cor;
448	coroutine$ * dst = get_coroutine(this->runner);
449
450	/* paranoid */ verify( ! __preemption_enabled() );
451	/* paranoid */ verify( dst->starter == src );
452	/* paranoid */ verify( dst->context.SP );
453
454	// SKULLDUGGERY in debug the processors check that the
455	// stack is still within the limit of the stack limits after running a thread.
456	// that check doesn't make sense if we context switch to the processor using the
457	// coroutine semantics. Since this is a special case, use the current context
458	// info to populate these fields.
459	__cfaabi_dbg_debug_do(
460	__stack_context_t ctx;
461	CtxGet( ctx );
462	mainThread->context.SP = ctx.SP;
463	mainThread->context.FP = ctx.FP;
464	)
465
466	// context switch to the processor
467	__cfactx_switch( &src->context, &dst->context );
468	}
469
470
471	//=============================================================================================
472	// Kernel Object Constructors logic
473	//=============================================================================================
474	//-----------------------------------------------------------------------------
475	// Main thread construction
476	static void ?{}( coroutine$ & this, current_stack_info_t * info) with( this ) {
477	stack.storage = info->storage;
478	with(*stack.storage) {
479	limit = info->limit;
480	base = info->base;
481	}
482	__attribute__((may_alias)) intptr_t * istorage = (intptr_t*) &stack.storage;
483	*istorage \|= 0x1;
484	name = "Main Thread";
485	state = Start;
486	starter = 0p;
487	last = 0p;
488	cancellation = 0p;
489	ehm_state.ehm_buffer{};
490	ehm_state.buffer_lock{};
491	ehm_state.ehm_enabled = false;
492	}
493
494	static void ?{}( thread$ & this, current_stack_info_t * info) with( this ) {
495	ticket = TICKET_RUNNING;
496	state = Start;
497	self_cor{ info };
498	curr_cor = &self_cor;
499	curr_cluster = mainCluster;
500	self_mon.owner = &this;
501	self_mon.recursion = 1;
502	self_mon_p = &self_mon;
503	rdy_link.next = 0p;
504	rdy_link.ts = MAX;
505	user_link.next = 0p;
506	user_link.prev = 0p;
507	cltr_link.next = 0p;
508	cltr_link.prev = 0p;
509	preferred = ready_queue_new_preferred();
510	last_proc = 0p;
511	PRNG_SET_SEED( random_state, __global_random_mask ? __global_random_prime : __global_random_prime ^ rdtscl() );
512	#if defined( __CFA_WITH_VERIFY__ )
513	executing = 0p;
514	canary = 0x0D15EA5E0D15EA5Ep;
515	#endif
516
517	doregister(curr_cluster, this);
518
519	monitors{ &self_mon_p, 1, (fptr_t)0 };
520	}
521
522	//-----------------------------------------------------------------------------
523	// Processor
524	// Construct the processor context of non-main processors
525	static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info) {
526	(this.self){ info };
527	this.proc = proc;
528	}
529
530	static void init(processor & this, const char name[], cluster & _cltr, thread$ * initT) with( this ) {
531	this.name = name;
532	this.cltr = &_cltr;
533	__atomic_add_fetch( &_cltr.procs.constructed, 1u, __ATOMIC_RELAXED );
534	this.rdq.its = 0;
535	this.rdq.itr = 0;
536	this.rdq.id = 0;
537	this.rdq.target = MAX;
538	this.rdq.last = MAX;
539	this.rdq.cpu = 0;
540	// this.rdq.cutoff = 0ull;
541	do_terminate = false;
542	preemption_alarm = 0p;
543	pending_preemption = false;
544
545	this.io.ctx = 0p;
546	this.io.pending = false;
547	this.io.dirty = false;
548
549	this.init.thrd = initT;
550
551	this.local_data = 0p;
552
553	idle_wctx.evfd = eventfd(0, 0);
554	if (idle_wctx.evfd < 0) {
555	abort("KERNEL ERROR: PROCESSOR EVENTFD - %s\n", strerror(errno));
556	}
557
558	idle_wctx.sem = 0;
559	idle_wctx.wake__time = 0;
560
561	// I'm assuming these two are reserved for standard input and output
562	// so I'm using them as sentinels with idle_wctx.
563	/* paranoid */ verify( idle_wctx.evfd != 0 );
564	/* paranoid */ verify( idle_wctx.evfd != 1 );
565
566	#if !defined(__CFA_NO_STATISTICS__)
567	print_stats = 0;
568	print_halts = false;
569	#endif
570
571	__cfadbg_print_safe(runtime_core, "Kernel : core %p created\n", &this);
572	}
573
574	// Not a ctor, it just preps the destruction but should not destroy members
575	static void deinit(processor & this) {
576	close(this.idle_wctx.evfd);
577	}
578
579	void ?{}(processor & this, const char name[], cluster & _cltr, thread$ * initT) libcfa_public {
580	( this.terminated ){};
581	( this.runner ){};
582
583	disable_interrupts();
584	init( this, name, _cltr, initT );
585	enable_interrupts();
586
587	__cfadbg_print_safe(runtime_core, "Kernel : Starting core %p\n", &this);
588
589	this.stack = __create_pthread( &this.kernel_thread, __invoke_processor, (void *)&this );
590	}
591
592	void ?{}(processor & this, const char name[], cluster & _cltr) libcfa_public {
593	(this){name, _cltr, 0p};
594	}
595
596	extern size_t __page_size;
597	void ^?{}(processor & this) libcfa_public with( this ) {
598	/* paranoid */ verify( !__atomic_load_n(&do_terminate, __ATOMIC_ACQUIRE) );
599	__cfadbg_print_safe(runtime_core, "Kernel : core %p signaling termination\n", &this);
600
601	__atomic_sub_fetch( &this.cltr->procs.constructed, 1u, __ATOMIC_RELAXED );
602
603	__atomic_store_n(&do_terminate, true, __ATOMIC_RELAXED);
604	__disable_interrupts_checked();
605	__wake_proc( &this );
606	__enable_interrupts_checked();
607
608	wait( terminated );
609	/* paranoid */ verify( active_processor() != &this);
610
611	__destroy_pthread( kernel_thread, this.stack, 0p );
612
613	disable_interrupts();
614	deinit( this );
615	enable_interrupts();
616	}
617
618	//-----------------------------------------------------------------------------
619	// Cluster
620	static void ?{}(__cluster_proc_list & this) {
621	this.fdw = 0p;
622	this.idle = 0;
623	this.constructed = 0;
624	this.total = 0;
625	}
626
627	void ?{}(cluster & this, const char name[], Duration preemption_rate, unsigned num_io, const io_context_params & io_params) libcfa_public with( this ) {
628	this.name = name;
629	this.preemption_rate = preemption_rate;
630	this.sched.readyQ.data = 0p;
631	this.sched.readyQ.tscs = 0p;
632	this.sched.readyQ.count = 0;
633	this.sched.io.tscs = 0p;
634	this.sched.io.data = 0p;
635	this.sched.caches = 0p;
636
637	#if !defined(__CFA_NO_STATISTICS__)
638	print_stats = 0;
639	stats = alloc();
640	__init_stats( stats );
641	#endif
642
643	threads{};
644
645	io.arbiter = create();
646	io.params = io_params;
647
648	managed.procs = 0p;
649	managed.cnt = 0;
650
651	doregister(this);
652
653	// Lock the RWlock so no-one pushes/pops while we are changing the queue
654	disable_interrupts();
655	uint_fast32_t last_size = ready_mutate_lock();
656
657	// Adjust the ready queue size
658	ready_queue_grow( &this );
659
660	// Unlock the RWlock
661	ready_mutate_unlock( last_size );
662	enable_interrupts( false ); // Don't poll, could be in main cluster
663	}
664
665	void ^?{}(cluster & this) libcfa_public {
666	set_concurrency( this, 0 );
667
668	destroy(this.io.arbiter);
669
670	// Lock the RWlock so no-one pushes/pops while we are changing the queue
671	disable_interrupts();
672	uint_fast32_t last_size = ready_mutate_lock();
673
674	// Adjust the ready queue size
675	ready_queue_shrink( &this );
676
677	// Unlock the RWlock
678	ready_mutate_unlock( last_size );
679
680	ready_queue_close( &this );
681	/* paranoid */ verify( this.sched.readyQ.data == 0p );
682	/* paranoid */ verify( this.sched.readyQ.tscs == 0p );
683	/* paranoid */ verify( this.sched.readyQ.count == 0 );
684	// /* paranoid */ verify( this.sched.io.tscs == 0p );
685	// /* paranoid */ verify( this.sched.caches == 0p );
686
687	enable_interrupts( false ); // Don't poll, could be in main cluster
688
689
690	#if !defined(__CFA_NO_STATISTICS__)
691	if( 0 != this.print_stats ) {
692	__print_stats( this.stats, this.print_stats, "Cluster", this.name, (void*)&this );
693	}
694	#if defined(CFA_STATS_ARRAY)
695	__flush_stat( this.stats, "Cluster", &this );
696	#endif
697	free( this.stats );
698	#endif
699
700	unregister(this);
701	}
702
703	//=============================================================================================
704	// Miscellaneous Initialization
705	//=============================================================================================
706	//-----------------------------------------------------------------------------
707	// Global Queues
708	static void doregister( cluster & cltr ) {
709	lock ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
710	push_front( __cfa_dbg_global_clusters.list, cltr );
711	unlock ( __cfa_dbg_global_clusters.lock );
712	}
713
714	static void unregister( cluster & cltr ) {
715	lock ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
716	remove( __cfa_dbg_global_clusters.list, cltr );
717	unlock( __cfa_dbg_global_clusters.lock );
718	}
719
720	void doregister( cluster * cltr, thread$ & thrd ) {
721	lock (cltr->thread_list_lock __cfaabi_dbg_ctx2);
722	cltr->nthreads += 1;
723	insert_first(cltr->threads, thrd);
724	unlock (cltr->thread_list_lock);
725	}
726
727	void unregister( cluster * cltr, thread$ & thrd ) {
728	lock (cltr->thread_list_lock __cfaabi_dbg_ctx2);
729	{
730	tytagref( dlink(thread$), dlink(thread$) ) ?`inner( thread$ & this ) = void;
731	with( DLINK_VIA( thread$, struct __thread_user_link ) )
732	remove( thrd );
733	cltr->nthreads -= 1;
734	}
735	unlock(cltr->thread_list_lock);
736	}
737
738	static void register_tls( processor * this ) {
739	// Register and Lock the RWlock so no-one pushes/pops while we are changing the queue
740	uint_fast32_t last_size;
741	[this->unique_id, last_size] = ready_mutate_register();
742
743	this->rdq.cpu = __kernel_getcpu();
744
745	this->cltr->procs.total += 1u;
746	insert_last(this->cltr->procs.actives, *this);
747
748	// Adjust the ready queue size
749	ready_queue_grow( this->cltr );
750
751	// Unlock the RWlock
752	ready_mutate_unlock( last_size );
753	}
754
755
756	static void unregister_tls( processor * this ) {
757	// Lock the RWlock so no-one pushes/pops while we are changing the queue
758	uint_fast32_t last_size = ready_mutate_lock();
759	this->cltr->procs.total -= 1u;
760	remove(*this);
761
762	// clear the cluster so nothing gets pushed to local queues
763	cluster * cltr = this->cltr;
764	this->cltr = 0p;
765
766	// Adjust the ready queue size
767	ready_queue_shrink( cltr );
768
769	// Unlock the RWlock and unregister: we don't need the read_lock any more
770	ready_mutate_unregister( this->unique_id, last_size );
771	}
772
773	static void check( int ret, const char func[] ) {
774	if ( ret ) { // pthread routines return errno values
775	abort( "%s : internal error, error(%d) %s.", func, ret, strerror( ret ) );
776	} // if
777	} // Abort
778
779	void * __create_pthread( pthread_t * pthread, void * (start)(void ), void * arg ) {
780	pthread_attr_t attr;
781
782	check( __cfaabi_pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
783
784	size_t stacksize = max( PTHREAD_STACK_MIN, DEFAULT_STACK_SIZE );
785
786	void * stack;
787	#if CFA_PROCESSOR_USE_MMAP
788	stacksize = ceiling( stacksize, __page_size ) + __page_size;
789	stack = mmap(0p, stacksize, __map_prot, MAP_PRIVATE \| MAP_ANONYMOUS, 0, 0);
790	if(stack == ((void*)-1)) {
791	abort( "pthread stack creation : internal error, mmap failure, error(%d) %s.", errno, strerror( errno ) );
792	}
793	if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
794	abort( "pthread stack creation : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
795	} // if
796	#else
797	__cfaabi_dbg_debug_do(
798	stack = memalign( __page_size, stacksize + __page_size );
799	// pthread has no mechanism to create the guard page in user supplied stack.
800	if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
801	abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
802	} // if
803	);
804	__cfaabi_dbg_no_debug_do(
805	stack = malloc( stacksize );
806	);
807	#endif
808
809	check( __cfaabi_pthread_attr_setstack( &attr, stack, stacksize ), "pthread_attr_setstack" );
810	check( __cfaabi_pthread_create( pthread, &attr, start, arg ), "pthread_create" );
811	return stack;
812	}
813
814	void __destroy_pthread( pthread_t pthread, void * stack, void ** retval ) {
815	int err = __cfaabi_pthread_join( pthread, retval );
816	if( err != 0 ) abort("KERNEL ERROR: joining pthread %p caused error %s\n", (void*)pthread, strerror(err));
817
818	#if CFA_PROCESSOR_USE_MMAP
819	pthread_attr_t attr;
820
821	check( __cfaabi_pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
822
823	size_t stacksize;
824	// default stack size, normally defined by shell limit
825	check( __cfaabi_pthread_attr_getstacksize( &attr, &stacksize ), "pthread_attr_getstacksize" );
826	assert( stacksize >= PTHREAD_STACK_MIN );
827	stacksize += __page_size;
828
829	if(munmap(stack, stacksize) == -1) {
830	abort( "pthread stack destruction : internal error, munmap failure, error(%d) %s.", errno, strerror( errno ) );
831	}
832	#else
833	__cfaabi_dbg_debug_do(
834	// pthread has no mechanism to create the guard page in user supplied stack.
835	if ( mprotect( stack, __page_size, __map_prot ) == -1 ) {
836	abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
837	} // if
838	);
839	free( stack );
840	#endif
841	}
842
843	unsigned set_concurrency( cluster & this, unsigned new ) libcfa_public {
844	unsigned old = this.managed.cnt;
845
846	__cfadbg_print_safe(runtime_core, "Kernel : resizing cluster from %u to %u\n", old, (unsigned)new);
847
848	// Delete all the old unneeded procs
849	if(old > new) for(i; (unsigned)new ~ old) {
850	__cfadbg_print_safe(runtime_core, "Kernel : destroying %u\n", i);
851	delete( this.managed.procs[i] );
852	}
853
854	// Allocate new array (uses realloc and memcpies the data)
855	this.managed.procs = alloc( new, this.managed.procs`realloc );
856	this.managed.cnt = new;
857
858	// Create the desired new procs
859	if(old < new) for(i; old ~ new) {
860	__cfadbg_print_safe(runtime_core, "Kernel : constructing %u\n", i);
861	(*(this.managed.procs[i] = alloc())){ this };
862	}
863
864	// return the old count
865	return old;
866	}
867
868	#if defined(__CFA_WITH_VERIFY__)
869	static bool verify_fwd_bck_rng(void) {
870	__cfaabi_tls.ready_rng.fwd_seed = 25214903917_l64u * (rdtscl() ^ (uintptr_t)&verify_fwd_bck_rng);
871
872	unsigned values[10];
873	for(i; 10) {
874	values[i] = __tls_rand_fwd();
875	}
876
877	__tls_rand_advance_bck();
878
879	for ( i; 9 -~= 0 ) {
880	if(values[i] != __tls_rand_bck()) {
881	return false;
882	}
883	}
884
885	return true;
886	}
887	#endif

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

Original Format