Context Navigation

startup.cfa@ c1fb3903

Visit:

ADT ast-experimental

Last change on this file since c1fb3903 was 639e4fc, checked in by Thierry Delisle <tdelisle@…>, 3 years ago
Changed cluster link to use explicit type to avoid anonymous names in symbols gdb cares about
Property mode set to `100644`
File size: 27.3 KB

Line
1	//
2	// Cforall Version 1.0.0 Copyright (C) 2020 University of Waterloo
3	//
4	// The contents of this file are covered under the licence agreement in the
5	// file "LICENCE" distributed with Cforall.
6	//
7	// kernel/startup.cfa --
8	//
9	// Author : Thierry Delisle
10	// Created On : Thu Jul 30 15:12:54 2020
11	// Last Modified By :
12	// Last Modified On :
13	// Update Count :
14	//
15
16	#define __cforall_thread__
17	#define _GNU_SOURCE
18
19	// #define __CFA_DEBUG_PRINT_RUNTIME_CORE__
20
21	// C Includes
22	#include <errno.h> // errno
23	#include <signal.h>
24	#include <string.h> // strerror
25	#include <unistd.h> // sysconf
26
27	extern "C" {
28	#include <limits.h> // PTHREAD_STACK_MIN
29	#include <unistd.h> // syscall
30	#include <sys/eventfd.h> // eventfd
31	#include <sys/mman.h> // mprotect
32	#include <sys/resource.h> // getrlimit
33	}
34
35	// CFA Includes
36	#include "kernel/private.hfa"
37	#include "iofwd.hfa"
38	#include "startup.hfa" // STARTUP_PRIORITY_XXX
39	#include "limits.hfa"
40	#include "math.hfa"
41
42	#define CFA_PROCESSOR_USE_MMAP 0
43
44	//-----------------------------------------------------------------------------
45	// Some assembly required
46	#if defined( __i386 )
47	#define CtxGet( ctx ) __asm__ volatile ( \
48	"movl %%esp,%0\n" \
49	"movl %%ebp,%1\n" \
50	: "=rm" (ctx.SP), \
51	"=rm" (ctx.FP) \
52	)
53	#elif defined( __x86_64 )
54	#define CtxGet( ctx ) __asm__ volatile ( \
55	"movq %%rsp,%0\n" \
56	"movq %%rbp,%1\n" \
57	: "=rm" (ctx.SP), \
58	"=rm" (ctx.FP) \
59	)
60	#elif defined( __aarch64__ )
61	#define CtxGet( ctx ) __asm__ volatile ( \
62	"mov %0, sp\n" \
63	"mov %1, fp\n" \
64	: "=rm" (ctx.SP), \
65	"=rm" (ctx.FP) \
66	)
67	#else
68	#error unknown hardware architecture
69	#endif
70
71	//-----------------------------------------------------------------------------
72	// Start and stop routine for the kernel, declared first to make sure they run first
73	static void __kernel_startup (void) __attribute__(( constructor( STARTUP_PRIORITY_KERNEL ) ));
74	static void __kernel_shutdown(void) __attribute__(( destructor ( STARTUP_PRIORITY_KERNEL ) ));
75
76	//-----------------------------------------------------------------------------
77	// Static Forward Declarations
78	struct current_stack_info_t;
79
80	static void * __invoke_processor(void * arg);
81	static void __kernel_first_resume( processor * this );
82	static void __kernel_last_resume ( processor * this );
83	static void init(processor & this, const char name[], cluster & _cltr, thread$ * initT);
84	static void deinit(processor & this);
85	static void doregister( struct cluster & cltr );
86	static void unregister( struct cluster & cltr );
87	static void register_tls( processor * this );
88	static void unregister_tls( processor * this );
89	static void ?{}( coroutine$ & this, current_stack_info_t * info);
90	static void ?{}( thread$ & this, current_stack_info_t * info);
91	static void ?{}(processorCtx_t & this) {}
92	static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info);
93
94	#if defined(__CFA_WITH_VERIFY__)
95	static bool verify_fwd_bck_rng(void);
96	#endif
97
98	//-----------------------------------------------------------------------------
99	// Forward Declarations for other modules
100	extern void __kernel_alarm_startup(void);
101	extern void __kernel_alarm_shutdown(void);
102	extern void __cfa_io_start( processor * );
103	extern void __cfa_io_stop ( processor * );
104
105	//-----------------------------------------------------------------------------
106	// Other Forward Declarations
107	extern void __wake_proc(processor *);
108	extern int cfa_main_returned; // from interpose.cfa
109	uint32_t __global_random_prime = 4_294_967_291u, __global_random_mask = false;
110
111	//-----------------------------------------------------------------------------
112	// Kernel storage
113	KERNEL_STORAGE(cluster, mainCluster);
114	KERNEL_STORAGE(processor, mainProcessor);
115	KERNEL_STORAGE(thread$, mainThread);
116	KERNEL_STORAGE(__stack_t, mainThreadCtx);
117	#if !defined(__CFA_NO_STATISTICS__)
118	KERNEL_STORAGE(__stats_t, mainProcStats);
119	#endif
120
121	cluster * mainCluster libcfa_public;
122	processor * mainProcessor;
123	thread$ * mainThread;
124
125	extern "C" {
126	struct { __dllist_t(cluster) list; __spinlock_t lock; } __cfa_dbg_global_clusters;
127	}
128
129	extern size_t __page_size;
130	extern int __map_prot;
131
132	//-----------------------------------------------------------------------------
133	// Global state
134	__thread struct KernelThreadData __cfaabi_tls __attribute__ ((tls_model ( "initial-exec" ))) @= {
135	NULL, // cannot use 0p
136	NULL,
137	false,
138	{ 1, false, false },
139	0,
140	{ 0, 0 },
141	NULL,
142	#ifdef __CFA_WITH_VERIFY__
143	false,
144	0,
145	#endif
146	};
147
148	__scheduler_RWLock_t __scheduler_lock @= { 0 };
149
150	#if defined(CFA_HAVE_LINUX_LIBRSEQ)
151	// No data needed
152	#elif defined(CFA_HAVE_LINUX_RSEQ_H)
153	extern "Cforall" {
154	__attribute__((aligned(64))) __thread volatile struct rseq __cfaabi_rseq @= {
155	.cpu_id : RSEQ_CPU_ID_UNINITIALIZED,
156	};
157	}
158	#else
159	// No data needed
160	#endif
161
162	//-----------------------------------------------------------------------------
163	// Struct to steal stack
164	struct current_stack_info_t {
165	__stack_t * storage; // pointer to stack object
166	void * base; // base of stack
167	void * limit; // stack grows towards stack limit
168	void * context; // address of cfa_context_t
169	};
170
171	static void ?{}( current_stack_info_t & this ) {
172	__stack_context_t ctx;
173	CtxGet( ctx );
174	this.base = ctx.FP;
175
176	rlimit r;
177	getrlimit( RLIMIT_STACK, &r);
178	size_t size = r.rlim_cur;
179
180	this.limit = (void *)(((intptr_t)this.base) - size);
181	this.context = &storage_mainThreadCtx;
182	}
183
184
185	extern void heapManagerCtor();
186	extern void heapManagerDtor();
187
188	//=============================================================================================
189	// Kernel Setup logic
190	//=============================================================================================
191	//-----------------------------------------------------------------------------
192	// Kernel boot procedures
193	static void __kernel_startup(void) {
194	/* paranoid */ verify( ! __preemption_enabled() );
195	__cfadbg_print_safe(runtime_core, "Kernel : Starting\n");
196
197	__cfa_dbg_global_clusters.list{ __get };
198	__cfa_dbg_global_clusters.lock{};
199
200	/* paranoid */ verify( verify_fwd_bck_rng() );
201
202	// Initialize the global scheduler lock
203	// __scheduler_lock = (__scheduler_RWLock_t*)&storage___scheduler_lock;
204	(__scheduler_lock){};
205
206	// Initialize the main cluster
207	mainCluster = (cluster *)&storage_mainCluster;
208	(*mainCluster){"Main Cluster", 0};
209
210	__cfadbg_print_safe(runtime_core, "Kernel : Main cluster ready\n");
211
212	// Construct the processor context of the main processor
213	void ?{}(processorCtx_t & this, processor * proc) {
214	(this.self){ "Processor" };
215	this.self.starter = 0p;
216	this.proc = proc;
217	}
218
219	void ?{}(processor & this) with( this ) {
220	( this.terminated ){};
221	( this.runner ){};
222	init( this, "Main Processor", *mainCluster, 0p );
223	kernel_thread = __cfaabi_pthread_self();
224
225	runner{ &this };
226	__cfadbg_print_safe(runtime_core, "Kernel : constructed main processor context %p\n", &runner);
227	}
228
229	// Initialize the main processor and the main processor ctx
230	// (the coroutine that contains the processing control flow)
231	mainProcessor = (processor *)&storage_mainProcessor;
232	(*mainProcessor){};
233
234	__cfa_io_start( mainProcessor );
235	register_tls( mainProcessor );
236
237	// Start by initializing the main thread
238	// SKULLDUGGERY: the mainThread steals the process main thread
239	// which will then be scheduled by the mainProcessor normally
240	mainThread = (thread$ *)&storage_mainThread;
241	current_stack_info_t info;
242	info.storage = (__stack_t*)&storage_mainThreadCtx;
243	(*mainThread){ &info };
244
245	__cfadbg_print_safe(runtime_core, "Kernel : Main thread ready\n");
246
247	//initialize the global state variables
248	__cfaabi_tls.this_processor = mainProcessor;
249	__cfaabi_tls.this_thread = mainThread;
250
251	#if !defined( __CFA_NO_STATISTICS__ )
252	__cfaabi_tls.this_stats = (__stats_t *)& storage_mainProcStats;
253	__init_stats( __cfaabi_tls.this_stats );
254	#endif
255	mainProcessor->local_data = &__cfaabi_tls;
256
257	// Enable preemption
258	__kernel_alarm_startup();
259
260	// Add the main thread to the ready queue
261	// once resume is called on mainProcessor->runner the mainThread needs to be scheduled like any normal thread
262	schedule_thread$(mainThread, UNPARK_LOCAL);
263
264	// SKULLDUGGERY: Force a context switch to the main processor to set the main thread's context to the current UNIX
265	// context. Hence, the main thread does not begin through __cfactx_invoke_thread, like all other threads. The trick here is that
266	// mainThread is on the ready queue when this call is made.
267	__kernel_first_resume( __cfaabi_tls.this_processor );
268
269
270	// THE SYSTEM IS NOW COMPLETELY RUNNING
271
272	__cfadbg_print_safe(runtime_core, "Kernel : Started\n--------------------------------------------------\n\n");
273
274	/* paranoid */ verify( ! __preemption_enabled() );
275	enable_interrupts();
276	/* paranoid */ verify( __preemption_enabled() );
277
278	}
279
280	extern "C"{
281	void pthread_delete_kernel_threads_();
282	}
283
284
285	static void __kernel_shutdown(void) {
286	if(!cfa_main_returned) return;
287
288	//delete kernel threads for pthread_concurrency
289	pthread_delete_kernel_threads_();
290
291	/* paranoid */ verify( __preemption_enabled() );
292	disable_interrupts();
293	/* paranoid */ verify( ! __preemption_enabled() );
294
295	__cfadbg_print_safe(runtime_core, "\n--------------------------------------------------\nKernel : Shutting down\n");
296
297	// SKULLDUGGERY: Notify the mainProcessor it needs to terminates.
298	// When its coroutine terminates, it return control to the mainThread
299	// which is currently here
300	/* paranoid */ verify( !__atomic_load_n(&mainProcessor->do_terminate, __ATOMIC_ACQUIRE) );
301	__atomic_store_n(&mainProcessor->do_terminate, true, __ATOMIC_RELEASE);
302	__wake_proc( mainProcessor );
303	__kernel_last_resume( __cfaabi_tls.this_processor );
304	mainThread->self_cor.state = Halted;
305
306	// THE SYSTEM IS NOW COMPLETELY STOPPED
307
308	// Disable preemption
309	__kernel_alarm_shutdown();
310
311	#if !defined( __CFA_NO_STATISTICS__ )
312	__stats_t * st = (__stats_t *)& storage_mainProcStats;
313	__tally_stats(mainCluster->stats, st);
314	if( 0 != mainProcessor->print_stats ) {
315	__print_stats( st, mainProcessor->print_stats, "Processor ", mainProcessor->name, (void*)mainProcessor );
316	}
317	#if defined(CFA_STATS_ARRAY)
318	__flush_stat( st, "Processor", mainProcessor );
319	#endif
320	#endif
321
322	mainProcessor->local_data = 0p;
323
324	unregister_tls( mainProcessor );
325	__cfa_io_stop( mainProcessor );
326
327	// Destroy the main processor and its context in reverse order of construction
328	// These were manually constructed so we need manually destroy them
329	void ^?{}(processor & this) with( this ){
330	deinit( this );
331
332	/* paranoid */ verify( this.do_terminate == true );
333	__cfadbg_print_safe(runtime_core, "Kernel : destroyed main processor context %p\n", &runner);
334	}
335
336	^(*mainProcessor){};
337
338	// Final step, destroy the main thread since it is no longer needed
339
340	// Since we provided a stack to this taxk it will not destroy anything
341	/* paranoid / verify(mainThread->self_cor.stack.storage == (__stack_t)(((uintptr_t)&storage_mainThreadCtx)\| 0x1));
342	^(*mainThread){};
343
344	^(*mainCluster){};
345
346	^(__scheduler_lock){};
347
348	^(__cfa_dbg_global_clusters.list){};
349	^(__cfa_dbg_global_clusters.lock){};
350
351	__cfadbg_print_safe(runtime_core, "Kernel : Shutdown complete\n");
352	}
353
354	//=============================================================================================
355	// Kernel Initial Scheduling logic
356	//=============================================================================================
357
358	// Context invoker for processors
359	// This is the entry point for processors (kernel threads) except for the main processor
360	// It effectively constructs a coroutine by stealing the pthread stack
361	static void * __invoke_processor(void * arg) {
362	#if !defined( __CFA_NO_STATISTICS__ )
363	__stats_t local_stats;
364	__init_stats( &local_stats );
365	__cfaabi_tls.this_stats = &local_stats;
366	#endif
367
368	processor * proc = (processor *) arg;
369	__cfaabi_tls.this_processor = proc;
370	__cfaabi_tls.this_thread = 0p;
371	__cfaabi_tls.preemption_state.[enabled, disable_count] = [false, 1];
372	proc->local_data = &__cfaabi_tls;
373
374	heapManagerCtor(); // initialize heap
375
376	__cfa_io_start( proc );
377	register_tls( proc );
378
379	// SKULLDUGGERY: We want to create a context for the processor coroutine
380	// which is needed for the 2-step context switch. However, there is no reason
381	// to waste the perfectly valid stack create by pthread.
382	current_stack_info_t info;
383	__stack_t ctx;
384	info.storage = &ctx;
385	(proc->runner){ proc, &info };
386
387	__cfadbg_print_safe(runtime_core, "Coroutine : created stack %p\n", get_coroutine(proc->runner)->stack.storage);
388
389	//Set global state
390	__cfaabi_tls.this_thread = 0p;
391
392	//We now have a proper context from which to schedule threads
393	__cfadbg_print_safe(runtime_core, "Kernel : core %p created (%p, %p)\n", proc, &proc->runner, &ctx);
394
395	// SKULLDUGGERY: Since the coroutine doesn't have its own stack, we can't
396	// resume it to start it like it normally would, it will just context switch
397	// back to here. Instead directly call the main since we already are on the
398	// appropriate stack.
399	get_coroutine(proc->runner)->state = Active;
400	main( proc->runner );
401	get_coroutine(proc->runner)->state = Halted;
402
403	// Main routine of the core returned, the core is now fully terminated
404	__cfadbg_print_safe(runtime_core, "Kernel : core %p main ended (%p)\n", proc, &proc->runner);
405
406	#if !defined(__CFA_NO_STATISTICS__)
407	__tally_stats(proc->cltr->stats, &local_stats);
408	if( 0 != proc->print_stats ) {
409	__print_stats( &local_stats, proc->print_stats, "Processor ", proc->name, (void*)proc );
410	}
411	#if defined(CFA_STATS_ARRAY)
412	__flush_stat( &local_stats, "Processor", proc );
413	#endif
414	#endif
415
416	proc->local_data = 0p;
417
418	unregister_tls( proc );
419	__cfa_io_stop( proc );
420
421	heapManagerDtor(); // de-initialize heap
422
423	return 0p;
424	}
425
426	static void __kernel_first_resume( processor * this ) {
427	thread$ * src = mainThread;
428	coroutine$ * dst = get_coroutine(this->runner);
429
430	/* paranoid */ verify( ! __preemption_enabled() );
431
432	__cfaabi_tls.this_thread->curr_cor = dst;
433	__stack_prepare( &dst->stack, DEFAULT_STACK_SIZE );
434	__cfactx_start(main, dst, this->runner, __cfactx_invoke_coroutine);
435
436	/* paranoid */ verify( ! __preemption_enabled() );
437
438	dst->last = &src->self_cor;
439	dst->starter = dst->starter ? dst->starter : &src->self_cor;
440
441	// make sure the current state is still correct
442	/* paranoid */ verify(src->state == Ready);
443	src->corctx_flag = true;
444
445	// context switch to specified coroutine
446	verify( dst->context.SP );
447	__cfactx_switch( &src->context, &dst->context );
448	// when __cfactx_switch returns we are back in the src coroutine
449
450	mainThread->curr_cor = &mainThread->self_cor;
451
452	// make sure the current state has been update
453	/* paranoid */ verify(src->state == Active);
454
455	/* paranoid */ verify( ! __preemption_enabled() );
456	}
457
458	// KERNEL_ONLY
459	static void __kernel_last_resume( processor * this ) {
460	coroutine$ * src = &mainThread->self_cor;
461	coroutine$ * dst = get_coroutine(this->runner);
462
463	/* paranoid */ verify( ! __preemption_enabled() );
464	/* paranoid */ verify( dst->starter == src );
465	/* paranoid */ verify( dst->context.SP );
466
467	// SKULLDUGGERY in debug the processors check that the
468	// stack is still within the limit of the stack limits after running a thread.
469	// that check doesn't make sense if we context switch to the processor using the
470	// coroutine semantics. Since this is a special case, use the current context
471	// info to populate these fields.
472	__cfaabi_dbg_debug_do(
473	__stack_context_t ctx;
474	CtxGet( ctx );
475	mainThread->context.SP = ctx.SP;
476	mainThread->context.FP = ctx.FP;
477	)
478
479	// context switch to the processor
480	__cfactx_switch( &src->context, &dst->context );
481	}
482
483
484	//=============================================================================================
485	// Kernel Object Constructors logic
486	//=============================================================================================
487	//-----------------------------------------------------------------------------
488	// Main thread construction
489	static void ?{}( coroutine$ & this, current_stack_info_t * info) with( this ) {
490	stack.storage = info->storage;
491	with(*stack.storage) {
492	limit = info->limit;
493	base = info->base;
494	}
495	__attribute__((may_alias)) intptr_t * istorage = (intptr_t*) &stack.storage;
496	*istorage \|= 0x1;
497	name = "Main Thread";
498	state = Start;
499	starter = 0p;
500	last = 0p;
501	cancellation = 0p;
502	}
503
504	static void ?{}( thread$ & this, current_stack_info_t * info) with( this ) {
505	ticket = TICKET_RUNNING;
506	state = Start;
507	self_cor{ info };
508	curr_cor = &self_cor;
509	curr_cluster = mainCluster;
510	self_mon.owner = &this;
511	self_mon.recursion = 1;
512	self_mon_p = &self_mon;
513	rdy_link.next = 0p;
514	rdy_link.ts = MAX;
515	preferred = ready_queue_new_preferred();
516	last_proc = 0p;
517	random_state = __global_random_mask ? __global_random_prime : __global_random_prime ^ rdtscl();
518	#if defined( __CFA_WITH_VERIFY__ )
519	executing = 0p;
520	canary = 0x0D15EA5E0D15EA5Ep;
521	#endif
522
523	doregister(curr_cluster, this);
524
525	monitors{ &self_mon_p, 1, (fptr_t)0 };
526	}
527
528	//-----------------------------------------------------------------------------
529	// Processor
530	// Construct the processor context of non-main processors
531	static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info) {
532	(this.self){ info };
533	this.proc = proc;
534	}
535
536	static void init(processor & this, const char name[], cluster & _cltr, thread$ * initT) with( this ) {
537	this.name = name;
538	this.cltr = &_cltr;
539	this.rdq.its = 0;
540	this.rdq.itr = 0;
541	this.rdq.id = 0;
542	this.rdq.target = MAX;
543	this.rdq.last = MAX;
544	this.rdq.cpu = 0;
545	// this.rdq.cutoff = 0ull;
546	do_terminate = false;
547	preemption_alarm = 0p;
548	pending_preemption = false;
549
550	this.io.ctx = 0p;
551	this.io.pending = false;
552	this.io.dirty = false;
553
554	this.init.thrd = initT;
555
556	this.local_data = 0p;
557
558	idle_wctx.evfd = eventfd(0, 0);
559	if (idle_wctx.evfd < 0) {
560	abort("KERNEL ERROR: PROCESSOR EVENTFD - %s\n", strerror(errno));
561	}
562
563	idle_wctx.sem = 0;
564	idle_wctx.wake__time = 0;
565
566	// I'm assuming these two are reserved for standard input and output
567	// so I'm using them as sentinels with idle_wctx.
568	/* paranoid */ verify( idle_wctx.evfd != 0 );
569	/* paranoid */ verify( idle_wctx.evfd != 1 );
570
571	#if !defined(__CFA_NO_STATISTICS__)
572	print_stats = 0;
573	print_halts = false;
574	#endif
575
576	__cfadbg_print_safe(runtime_core, "Kernel : core %p created\n", &this);
577	}
578
579	// Not a ctor, it just preps the destruction but should not destroy members
580	static void deinit(processor & this) {
581	close(this.idle_wctx.evfd);
582	}
583
584	void ?{}(processor & this, const char name[], cluster & _cltr, thread$ * initT) libcfa_public {
585	( this.terminated ){};
586	( this.runner ){};
587
588	disable_interrupts();
589	init( this, name, _cltr, initT );
590	enable_interrupts();
591
592	__cfadbg_print_safe(runtime_core, "Kernel : Starting core %p\n", &this);
593
594	this.stack = __create_pthread( &this.kernel_thread, __invoke_processor, (void *)&this );
595	}
596
597	void ?{}(processor & this, const char name[], cluster & _cltr) libcfa_public {
598	(this){name, _cltr, 0p};
599	}
600
601	extern size_t __page_size;
602	void ^?{}(processor & this) libcfa_public with( this ) {
603	/* paranoid */ verify( !__atomic_load_n(&do_terminate, __ATOMIC_ACQUIRE) );
604	__cfadbg_print_safe(runtime_core, "Kernel : core %p signaling termination\n", &this);
605
606	__atomic_store_n(&do_terminate, true, __ATOMIC_RELAXED);
607	__disable_interrupts_checked();
608	__wake_proc( &this );
609	__enable_interrupts_checked();
610
611	wait( terminated );
612	/* paranoid */ verify( active_processor() != &this);
613
614	__destroy_pthread( kernel_thread, this.stack, 0p );
615
616	disable_interrupts();
617	deinit( this );
618	enable_interrupts();
619	}
620
621	//-----------------------------------------------------------------------------
622	// Cluster
623	static void ?{}(__cluster_proc_list & this) {
624	this.fdw = 0p;
625	this.idle = 0;
626	this.total = 0;
627	}
628
629	void ?{}(cluster & this, const char name[], Duration preemption_rate, unsigned num_io, const io_context_params & io_params) libcfa_public with( this ) {
630	this.name = name;
631	this.preemption_rate = preemption_rate;
632	this.sched.readyQ.data = 0p;
633	this.sched.readyQ.tscs = 0p;
634	this.sched.readyQ.count = 0;
635	this.sched.io.tscs = 0p;
636	this.sched.io.data = 0p;
637	this.sched.caches = 0p;
638
639	#if !defined(__CFA_NO_STATISTICS__)
640	print_stats = 0;
641	stats = alloc();
642	__init_stats( stats );
643	#endif
644
645	threads{};
646
647	io.arbiter = create();
648	io.params = io_params;
649
650	managed.procs = 0p;
651	managed.cnt = 0;
652
653	doregister(this);
654
655	// Lock the RWlock so no-one pushes/pops while we are changing the queue
656	disable_interrupts();
657	uint_fast32_t last_size = ready_mutate_lock();
658
659	// Adjust the ready queue size
660	ready_queue_grow( &this );
661
662	// Unlock the RWlock
663	ready_mutate_unlock( last_size );
664	enable_interrupts( false ); // Don't poll, could be in main cluster
665	}
666
667	void ^?{}(cluster & this) libcfa_public {
668	set_concurrency( this, 0 );
669
670	destroy(this.io.arbiter);
671
672	// Lock the RWlock so no-one pushes/pops while we are changing the queue
673	disable_interrupts();
674	uint_fast32_t last_size = ready_mutate_lock();
675
676	// Adjust the ready queue size
677	ready_queue_shrink( &this );
678
679	// Unlock the RWlock
680	ready_mutate_unlock( last_size );
681
682	ready_queue_close( &this );
683	/* paranoid */ verify( this.sched.readyQ.data == 0p );
684	/* paranoid */ verify( this.sched.readyQ.tscs == 0p );
685	/* paranoid */ verify( this.sched.readyQ.count == 0 );
686	/* paranoid */ verify( this.sched.io.tscs == 0p );
687	/* paranoid */ verify( this.sched.caches == 0p );
688
689	enable_interrupts( false ); // Don't poll, could be in main cluster
690
691
692	#if !defined(__CFA_NO_STATISTICS__)
693	if( 0 != this.print_stats ) {
694	__print_stats( this.stats, this.print_stats, "Cluster", this.name, (void*)&this );
695	}
696	#if defined(CFA_STATS_ARRAY)
697	__flush_stat( this.stats, "Cluster", &this );
698	#endif
699	free( this.stats );
700	#endif
701
702	unregister(this);
703	}
704
705	//=============================================================================================
706	// Miscellaneous Initialization
707	//=============================================================================================
708	//-----------------------------------------------------------------------------
709	// Global Queues
710	static void doregister( cluster & cltr ) {
711	lock ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
712	push_front( __cfa_dbg_global_clusters.list, cltr );
713	unlock ( __cfa_dbg_global_clusters.lock );
714	}
715
716	static void unregister( cluster & cltr ) {
717	lock ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);
718	remove( __cfa_dbg_global_clusters.list, cltr );
719	unlock( __cfa_dbg_global_clusters.lock );
720	}
721
722	void doregister( cluster * cltr, thread$ & thrd ) {
723	lock (cltr->thread_list_lock __cfaabi_dbg_ctx2);
724	cltr->nthreads += 1;
725	insert_first(cltr->threads, thrd);
726	unlock (cltr->thread_list_lock);
727	}
728
729	void unregister( cluster * cltr, thread$ & thrd ) {
730	lock (cltr->thread_list_lock __cfaabi_dbg_ctx2);
731	{
732	tytagref( dlink(thread$), dlink(thread$) ) ?`inner( thread$ & this ) = void;
733	with( DLINK_VIA( thread$, struct __thread_user_link ) )
734	remove( thrd );
735	cltr->nthreads -= 1;
736	}
737	unlock(cltr->thread_list_lock);
738	}
739
740	static void register_tls( processor * this ) {
741	// Register and Lock the RWlock so no-one pushes/pops while we are changing the queue
742	uint_fast32_t last_size;
743	[this->unique_id, last_size] = ready_mutate_register();
744
745	this->rdq.cpu = __kernel_getcpu();
746
747	this->cltr->procs.total += 1u;
748	insert_last(this->cltr->procs.actives, *this);
749
750	// Adjust the ready queue size
751	ready_queue_grow( this->cltr );
752
753	// Unlock the RWlock
754	ready_mutate_unlock( last_size );
755	}
756
757
758	static void unregister_tls( processor * this ) {
759	// Lock the RWlock so no-one pushes/pops while we are changing the queue
760	uint_fast32_t last_size = ready_mutate_lock();
761	this->cltr->procs.total -= 1u;
762	remove(*this);
763
764	// clear the cluster so nothing gets pushed to local queues
765	cluster * cltr = this->cltr;
766	this->cltr = 0p;
767
768	// Adjust the ready queue size
769	ready_queue_shrink( cltr );
770
771	// Unlock the RWlock and unregister: we don't need the read_lock any more
772	ready_mutate_unregister( this->unique_id, last_size );
773	}
774
775	static void check( int ret, const char func[] ) {
776	if ( ret ) { // pthread routines return errno values
777	abort( "%s : internal error, error(%d) %s.", func, ret, strerror( ret ) );
778	} // if
779	} // Abort
780
781	void * __create_pthread( pthread_t * pthread, void * (start)(void ), void * arg ) {
782	pthread_attr_t attr;
783
784	check( __cfaabi_pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
785
786	size_t stacksize = max( PTHREAD_STACK_MIN, DEFAULT_STACK_SIZE );
787
788	void * stack;
789	#if CFA_PROCESSOR_USE_MMAP
790	stacksize = ceiling( stacksize, __page_size ) + __page_size;
791	stack = mmap(0p, stacksize, __map_prot, MAP_PRIVATE \| MAP_ANONYMOUS, 0, 0);
792	if(stack == ((void*)-1)) {
793	abort( "pthread stack creation : internal error, mmap failure, error(%d) %s.", errno, strerror( errno ) );
794	}
795	if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
796	abort( "pthread stack creation : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
797	} // if
798	#else
799	__cfaabi_dbg_debug_do(
800	stack = memalign( __page_size, stacksize + __page_size );
801	// pthread has no mechanism to create the guard page in user supplied stack.
802	if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {
803	abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
804	} // if
805	);
806	__cfaabi_dbg_no_debug_do(
807	stack = malloc( stacksize );
808	);
809	#endif
810
811	check( __cfaabi_pthread_attr_setstack( &attr, stack, stacksize ), "pthread_attr_setstack" );
812	check( __cfaabi_pthread_create( pthread, &attr, start, arg ), "pthread_create" );
813	return stack;
814	}
815
816	void __destroy_pthread( pthread_t pthread, void * stack, void ** retval ) {
817	int err = __cfaabi_pthread_join( pthread, retval );
818	if( err != 0 ) abort("KERNEL ERROR: joining pthread %p caused error %s\n", (void*)pthread, strerror(err));
819
820	#if CFA_PROCESSOR_USE_MMAP
821	pthread_attr_t attr;
822
823	check( __cfaabi_pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute
824
825	size_t stacksize;
826	// default stack size, normally defined by shell limit
827	check( __cfaabi_pthread_attr_getstacksize( &attr, &stacksize ), "pthread_attr_getstacksize" );
828	assert( stacksize >= PTHREAD_STACK_MIN );
829	stacksize += __page_size;
830
831	if(munmap(stack, stacksize) == -1) {
832	abort( "pthread stack destruction : internal error, munmap failure, error(%d) %s.", errno, strerror( errno ) );
833	}
834	#else
835	__cfaabi_dbg_debug_do(
836	// pthread has no mechanism to create the guard page in user supplied stack.
837	if ( mprotect( stack, __page_size, __map_prot ) == -1 ) {
838	abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
839	} // if
840	);
841	free( stack );
842	#endif
843	}
844
845	unsigned set_concurrency( cluster & this, unsigned new ) libcfa_public {
846	unsigned old = this.managed.cnt;
847
848	__cfadbg_print_safe(runtime_core, "Kernel : resizing cluster from %u to %u\n", old, (unsigned)new);
849
850	// Delete all the old unneeded procs
851	if(old > new) for(i; (unsigned)new ~ old) {
852	__cfadbg_print_safe(runtime_core, "Kernel : destroying %u\n", i);
853	delete( this.managed.procs[i] );
854	}
855
856	// Allocate new array (uses realloc and memcpies the data)
857	this.managed.procs = alloc( new, this.managed.procs`realloc );
858	this.managed.cnt = new;
859
860	// Create the desired new procs
861	if(old < new) for(i; old ~ new) {
862	__cfadbg_print_safe(runtime_core, "Kernel : constructing %u\n", i);
863	(*(this.managed.procs[i] = alloc())){ this };
864	}
865
866	// return the old count
867	return old;
868	}
869
870	#if defined(__CFA_WITH_VERIFY__)
871	static bool verify_fwd_bck_rng(void) {
872	__cfaabi_tls.ready_rng.fwd_seed = 25214903917_l64u * (rdtscl() ^ (uintptr_t)&verify_fwd_bck_rng);
873
874	unsigned values[10];
875	for(i; 10) {
876	values[i] = __tls_rand_fwd();
877	}
878
879	__tls_rand_advance_bck();
880
881	for ( i; 9 -~= 0 ) {
882	if(values[i] != __tls_rand_bck()) {
883	return false;
884	}
885	}
886
887	return true;
888	}
889	#endif

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

Original Format