source: libcfa/src/concurrency/kernel.cfa

Last change on this file was dd46fd3, checked in by Peter A. Buhr <pabuhr@…>, 9 days ago

generalization of PRNG

  • Property mode set to 100644
File size: 28.6 KB
Line 
1//
2// Cforall Version 1.0.0 Copyright (C) 2016 University of Waterloo
3//
4// The contents of this file are covered under the licence agreement in the
5// file "LICENCE" distributed with Cforall.
6//
7// kernel.c --
8//
9// Author           : Thierry Delisle
10// Created On       : Tue Jan 17 12:27:26 2017
11// Last Modified By : Peter A. Buhr
12// Last Modified On : Wed Nov 30 18:14:08 2022
13// Update Count     : 76
14//
15
16#define __cforall_thread__
17#define _GNU_SOURCE
18
19// #define __CFA_DEBUG_PRINT_RUNTIME_CORE__
20
21#pragma GCC diagnostic push
22#pragma GCC diagnostic ignored "-Waddress-of-packed-member"
23
24//C Includes
25#include <errno.h>
26#include <stdio.h>
27#include <string.h>
28#include <signal.h>
29#include <unistd.h>
30
31extern "C" {
32        #include <sys/eventfd.h>
33        #include <sys/uio.h>
34}
35
36//CFA Includes
37#include "kernel/private.hfa"
38#include "preemption.hfa"
39#include "strstream.hfa"
40#include "device/cpu.hfa"
41#include "io/types.hfa"
42
43//Private includes
44#define __CFA_INVOKE_PRIVATE__
45#include "invoke.h"
46#pragma GCC diagnostic pop
47
48#if !defined(__CFA_NO_STATISTICS__)
49        #define __STATS_DEF( ...) __VA_ARGS__
50#else
51        #define __STATS_DEF( ...)
52#endif
53
54//-----------------------------------------------------------------------------
55// Some assembly required
56#if defined( __i386 )
57        // mxcr : SSE Status and Control bits (control bits are preserved across function calls)
58        // fcw  : X87 FPU control word (preserved across function calls)
59        #define __x87_store         \
60                uint32_t __mxcr;      \
61                uint16_t __fcw;       \
62                __asm__ volatile (    \
63                        "stmxcsr %0\n"  \
64                        "fnstcw  %1\n"  \
65                        : "=m" (__mxcr),\
66                                "=m" (__fcw)  \
67                )
68
69        #define __x87_load         \
70                __asm__ volatile (   \
71                        "fldcw  %1\n"  \
72                        "ldmxcsr %0\n" \
73                        ::"m" (__mxcr),\
74                                "m" (__fcw)  \
75                )
76
77#elif defined( __x86_64 )
78        #define __x87_store         \
79                uint32_t __mxcr;      \
80                uint16_t __fcw;       \
81                __asm__ volatile (    \
82                        "stmxcsr %0\n"  \
83                        "fnstcw  %1\n"  \
84                        : "=m" (__mxcr),\
85                                "=m" (__fcw)  \
86                )
87
88        #define __x87_load          \
89                __asm__ volatile (    \
90                        "fldcw  %1\n"   \
91                        "ldmxcsr %0\n"  \
92                        :: "m" (__mxcr),\
93                                "m" (__fcw)  \
94                )
95
96#elif defined( __arm__ )
97        #define __x87_store
98        #define __x87_load
99
100#elif defined( __aarch64__ )
101        #define __x87_store              \
102                uint32_t __fpcntl[2];    \
103                __asm__ volatile (    \
104                        "mrs x9, FPCR\n" \
105                        "mrs x10, FPSR\n"  \
106                        "stp x9, x10, %0\n"  \
107                        : "=m" (__fpcntl) : : "x9", "x10" \
108                )
109
110        #define __x87_load         \
111                __asm__ volatile (    \
112                        "ldp x9, x10, %0\n"  \
113                        "msr FPSR, x10\n"  \
114                        "msr FPCR, x9\n" \
115                : "=m" (__fpcntl) : : "x9", "x10" \
116                )
117
118#else
119        #error unsupported hardware architecture
120#endif
121
122extern thread$ * mainThread;
123extern processor * mainProcessor;
124
125//-----------------------------------------------------------------------------
126// Kernel Scheduling logic
127static thread$ * __next_thread(cluster * this);
128static thread$ * __next_thread_slow(cluster * this);
129static thread$ * __next_thread_search(cluster * this);
130static inline bool __must_unpark( thread$ * thrd ) __attribute((nonnull(1)));
131static void __run_thread(processor * this, thread$ * dst);
132static void __wake_one(cluster * cltr);
133
134static void idle_sleep(processor * proc);
135static bool mark_idle (__cluster_proc_list & idles, processor & proc);
136static void mark_awake(__cluster_proc_list & idles, processor & proc);
137
138extern bool __cfa_io_drain( processor * proc ) __attribute__((nonnull (1)));
139extern bool __cfa_io_flush( processor * ) __attribute__((nonnull (1)));
140
141
142extern void __disable_interrupts_hard();
143extern void __enable_interrupts_hard();
144
145
146//=============================================================================================
147// Kernel Scheduling logic
148//=============================================================================================
149//Main of the processor contexts
150void main(processorCtx_t & runner) {
151        // Because of a bug, we couldn't initialized the seed on construction
152        // Do it here
153        PRNG_SET_SEED( __cfaabi_tls.random_state, rdtscl() );
154        __cfaabi_tls.ready_rng.fwd_seed = 25214903917_l64u * (rdtscl() ^ (uintptr_t)&runner);
155        __tls_rand_advance_bck();
156
157        processor * this = runner.proc;
158        verify(this);
159
160        __cfadbg_print_safe(runtime_core, "Kernel : core %p starting\n", this);
161        #if !defined(__CFA_NO_STATISTICS__)
162                if( this->print_halts ) {
163                        __cfaabi_bits_print_safe( STDOUT_FILENO, "Processor : %d - %s (%p)\n", this->unique_id, this->name, (void*)this);
164                }
165        #endif
166
167        {
168                // Setup preemption data
169                preemption_scope scope = { this };
170
171                // if we need to run some special setup, now is the time to do it.
172                if(this->init.thrd) {
173                        this->init.thrd->curr_cluster = this->cltr;
174                        __run_thread(this, this->init.thrd);
175                }
176
177                __cfadbg_print_safe(runtime_core, "Kernel : core %p started\n", this);
178
179                thread$ * readyThread = 0p;
180                MAIN_LOOP:
181                for() {
182                        // Check if there is pending io
183                        __cfa_io_drain( this );
184
185                        // Try to get the next thread
186                        readyThread = __next_thread( this->cltr );
187
188                        if( !readyThread ) {
189                                // there is no point in holding submissions if we are idle
190                                __IO_STATS__(true, io.flush.idle++; )
191                                __cfa_io_flush( this );
192
193                                // drain again in case something showed up
194                                __cfa_io_drain( this );
195
196                                readyThread = __next_thread( this->cltr );
197                        }
198
199                        if( !readyThread ) for(5) {
200                                readyThread = __next_thread_slow( this->cltr );
201
202                                if( readyThread ) break;
203
204                                // It's unlikely we still I/O to submit, but the arbiter could
205                                __IO_STATS__(true, io.flush.idle++; )
206                                __cfa_io_flush( this );
207
208                                // drain again in case something showed up
209                                __cfa_io_drain( this );
210                        }
211
212                        HALT:
213                        if( !readyThread ) {
214                                // Don't block if we are done
215                                if( __atomic_load_n(&this->do_terminate, __ATOMIC_SEQ_CST) ) break MAIN_LOOP;
216
217                                // Push self to idle stack
218                                if(!mark_idle(this->cltr->procs, * this)) continue MAIN_LOOP;
219
220                                // Confirm the ready-queue is empty
221                                readyThread = __next_thread_search( this->cltr );
222                                if( readyThread ) {
223                                        // A thread was found, cancel the halt
224                                        mark_awake(this->cltr->procs, * this);
225
226                                        __STATS__(true, ready.sleep.cancels++; )
227
228                                        // continue the mai loop
229                                        break HALT;
230                                }
231
232                                idle_sleep( this );
233
234                                // We were woken up, remove self from idle
235                                mark_awake(this->cltr->procs, * this);
236
237                                // DON'T just proceed, start looking again
238                                continue MAIN_LOOP;
239                        }
240
241                        /* paranoid */ verify( readyThread );
242
243                        // Reset io dirty bit
244                        this->io.dirty = false;
245
246                        // We found a thread run it
247                        __run_thread(this, readyThread);
248
249                        // Are we done?
250                        if( __atomic_load_n(&this->do_terminate, __ATOMIC_SEQ_CST) ) break MAIN_LOOP;
251
252                        if(__atomic_load_n(&this->io.pending, __ATOMIC_RELAXED) && !__atomic_load_n(&this->io.dirty, __ATOMIC_RELAXED)) {
253                                __IO_STATS__(true, io.flush.dirty++; )
254                                __cfa_io_flush( this );
255                        }
256                }
257
258                __cfadbg_print_safe(runtime_core, "Kernel : core %p stopping\n", this);
259        }
260
261        post( this->terminated );
262
263        if(this == mainProcessor) {
264                // HACK : the coroutine context switch expects this_thread to be set
265                // and it make sense for it to be set in all other cases except here
266                // fake it
267                __cfaabi_tls.this_thread = mainThread;
268        }
269
270        __cfadbg_print_safe(runtime_core, "Kernel : core %p terminated\n", this);
271}
272
273static int * __volatile_errno() __attribute__((noinline));
274static int * __volatile_errno() { asm(""); return &errno; }
275
276// KERNEL ONLY
277// runThread runs a thread by context switching
278// from the processor coroutine to the target thread
279static void __run_thread(processor * this, thread$ * thrd_dst) {
280        /* paranoid */ verify( ! __preemption_enabled() );
281        /* paranoid */ verifyf( thrd_dst->state == Ready || thrd_dst->preempted != __NO_PREEMPTION, "state : %d, preempted %d\n", thrd_dst->state, thrd_dst->preempted);
282        /* paranoid */ verifyf( thrd_dst->rdy_link.next == 0p, "Expected null got %p", thrd_dst->rdy_link.next );
283        __builtin_prefetch( thrd_dst->context.SP );
284
285        __cfadbg_print_safe(runtime_core, "Kernel : core %p running thread %p (%s)\n", this, thrd_dst, thrd_dst->self_cor.name);
286
287        coroutine$ * proc_cor = get_coroutine(this->runner);
288
289        // set state of processor coroutine to inactive
290        verify(proc_cor->state == Active);
291        proc_cor->state = Blocked;
292
293        // Actually run the thread
294        RUNNING:  while(true) {
295                thrd_dst->preempted = __NO_PREEMPTION;
296
297                // Update global state
298                kernelTLS().this_thread = thrd_dst;
299
300                // Update the state after setting this_thread
301                // so that the debugger can find all active threads
302                // in tls storage
303                thrd_dst->state = Active;
304
305                /* paranoid */ verify( ! __preemption_enabled() );
306                /* paranoid */ verify( kernelTLS().this_thread == thrd_dst );
307                /* paranoid */ verify( thrd_dst->curr_cluster == this->cltr );
308                /* paranoid */ verify( thrd_dst->context.SP );
309                /* paranoid */ verify( thrd_dst->state != Halted );
310                /* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) < ((uintptr_t)__get_stack(thrd_dst->curr_cor)->base ) || thrd_dst->curr_cor == proc_cor || thrd_dst->corctx_flag, "ERROR : Destination thread$ %p has been corrupted.\n StackPointer too small.\n", thrd_dst ); // add escape condition if we are setting up the processor
311                /* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) > ((uintptr_t)__get_stack(thrd_dst->curr_cor)->limit) || thrd_dst->curr_cor == proc_cor || thrd_dst->corctx_flag, "ERROR : Destination thread$ %p has been corrupted.\n StackPointer too large.\n", thrd_dst ); // add escape condition if we are setting up the processor
312                /* paranoid */ verify( __atomic_exchange_n( &thrd_dst->executing, this, __ATOMIC_SEQ_CST) == 0p );
313                /* paranoid */ verify( 0x0D15EA5E0D15EA5Ep == thrd_dst->canary );
314
315
316
317                // set context switch to the thread that the processor is executing
318                __cfactx_switch( &proc_cor->context, &thrd_dst->context );
319                // when __cfactx_switch returns we are back in the processor coroutine
320
321
322
323                /* paranoid */ verify( 0x0D15EA5E0D15EA5Ep == thrd_dst->canary );
324                /* paranoid */ verify( __atomic_exchange_n( &thrd_dst->executing, 0p, __ATOMIC_SEQ_CST) == this );
325                /* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) > ((uintptr_t)__get_stack(thrd_dst->curr_cor)->limit) || thrd_dst->corctx_flag, "ERROR : Destination thread$ %p has been corrupted.\n StackPointer too large.\n", thrd_dst );
326                /* paranoid */ verifyf( ((uintptr_t)thrd_dst->context.SP) < ((uintptr_t)__get_stack(thrd_dst->curr_cor)->base ) || thrd_dst->corctx_flag, "ERROR : Destination thread$ %p has been corrupted.\n StackPointer too small.\n", thrd_dst );
327                /* paranoid */ verify( thrd_dst->state != Halted );
328                /* paranoid */ verify( thrd_dst->context.SP );
329                /* paranoid */ verify( kernelTLS().this_thread == thrd_dst );
330                /* paranoid */ verify( ! __preemption_enabled() );
331
332                // We just finished running a thread, there are a few things that could have happened.
333                // 1 - Regular case : the thread has blocked and now one has scheduled it yet.
334                // 2 - Racy case    : the thread has blocked but someone has already tried to schedule it.
335                // 4 - Preempted
336                // In case 1, we may have won a race so we can't write to the state again.
337                // In case 2, we lost the race so we now own the thread.
338
339                if(unlikely(thrd_dst->preempted != __NO_PREEMPTION)) {
340                        // Reset the this_thread now that we know
341                        // the state isn't active anymore
342                        kernelTLS().this_thread = 0p;
343
344                        // The thread was preempted, reschedule it and reset the flag
345                        schedule_thread$( thrd_dst, UNPARK_LOCAL );
346                        break RUNNING;
347                }
348
349                if(unlikely(thrd_dst->state == Halting)) {
350                        // Reset the this_thread now that we know
351                        // the state isn't active anymore
352                        kernelTLS().this_thread = 0p;
353
354                        // The thread has halted, it should never be scheduled/run again
355                        // finish the thread
356                        __thread_finish( thrd_dst );
357                        break RUNNING;
358                }
359
360                /* paranoid */ verify( thrd_dst->state == Active );
361                thrd_dst->state = Blocked;
362
363                // Reset the this_thread now that we know
364                // the state isn't active anymore
365                kernelTLS().this_thread = 0p;
366
367                // set state of processor coroutine to active and the thread to inactive
368                int old_ticket = __atomic_fetch_sub(&thrd_dst->ticket, 1, __ATOMIC_SEQ_CST);
369                switch(old_ticket) {
370                        case TICKET_RUNNING:
371                                // This is case 1, the regular case, nothing more is needed
372                                break RUNNING;
373                        case TICKET_UNBLOCK:
374                                __STATS__(true, ready.threads.threads++; )
375                                // This is case 2, the racy case, someone tried to run this thread before it finished blocking
376                                // In this case, just run it again.
377                                continue RUNNING;
378                        default:
379                                // This makes no sense, something is wrong abort
380                                abort();
381                }
382        }
383
384        // Just before returning to the processor, set the processor coroutine to active
385        proc_cor->state = Active;
386
387        __cfadbg_print_safe(runtime_core, "Kernel : core %p finished running thread %p\n", this, thrd_dst);
388
389        __STATS__(true, ready.threads.threads--; )
390
391        /* paranoid */ verify( ! __preemption_enabled() );
392}
393
394// KERNEL_ONLY
395static void returnToKernel() {
396        /* paranoid */ verify( ! __preemption_enabled() );
397        coroutine$ * proc_cor = get_coroutine(kernelTLS().this_processor->runner);
398        thread$ * thrd_src = kernelTLS().this_thread;
399
400        __STATS_DEF( thrd_src->last_proc = kernelTLS().this_processor; )
401
402        // Run the thread on this processor
403        {
404                int local_errno = *__volatile_errno();
405                #if defined( __i386 ) || defined( __x86_64 )
406                        __x87_store;
407                #endif
408                /* paranoid */ verify( proc_cor->context.SP );
409                /* paranoid */ verify( 0x0D15EA5E0D15EA5Ep == thrd_src->canary );
410                __cfactx_switch( &thrd_src->context, &proc_cor->context );
411                /* paranoid */ verify( 0x0D15EA5E0D15EA5Ep == thrd_src->canary );
412                #if defined( __i386 ) || defined( __x86_64 )
413                        __x87_load;
414                #endif
415                *__volatile_errno() = local_errno;
416        }
417
418        #if !defined(__CFA_NO_STATISTICS__)
419                /* paranoid */ verify( thrd_src->last_proc != 0p );
420                if(thrd_src->last_proc != kernelTLS().this_processor) {
421                        __tls_stats()->ready.threads.migration++;
422                }
423        #endif
424
425        /* paranoid */ verify( ! __preemption_enabled() );
426        /* paranoid */ verifyf( ((uintptr_t)thrd_src->context.SP) < ((uintptr_t)__get_stack(thrd_src->curr_cor)->base ) || thrd_src->corctx_flag, "ERROR : Returning thread$ %p has been corrupted.\n StackPointer too small.\n", thrd_src );
427        /* paranoid */ verifyf( ((uintptr_t)thrd_src->context.SP) > ((uintptr_t)__get_stack(thrd_src->curr_cor)->limit) || thrd_src->corctx_flag, "ERROR : Returning thread$ %p has been corrupted.\n StackPointer too large.\n", thrd_src );
428}
429
430//-----------------------------------------------------------------------------
431// Scheduler routines
432// KERNEL ONLY
433static void __schedule_thread( thread$ * thrd, unpark_hint hint ) {
434        /* paranoid */ verify( ! __preemption_enabled() );
435        /* paranoid */ verify( ready_schedule_islocked());
436        /* paranoid */ verify( thrd );
437        /* paranoid */ verify( thrd->state != Halted );
438        /* paranoid */ verify( thrd->curr_cluster );
439        /* paranoid */ #if defined( __CFA_WITH_VERIFY__ )
440        /* paranoid */  if( thrd->state == Blocked || thrd->state == Start ) assertf( thrd->preempted == __NO_PREEMPTION,
441                                        "Error inactive thread marked as preempted, state %d, preemption %d\n", thrd->state, thrd->preempted );
442        /* paranoid */  if( thrd->preempted != __NO_PREEMPTION ) assertf(thrd->state == Active,
443                                        "Error preempted thread marked as not currently running, state %d, preemption %d\n", thrd->state, thrd->preempted );
444        /* paranoid */ #endif
445        /* paranoid */ verifyf( thrd->rdy_link.next == 0p, "Expected null got %p", thrd->rdy_link.next );
446        /* paranoid */ verify( 0x0D15EA5E0D15EA5Ep == thrd->canary );
447
448        if (thrd->preempted == __NO_PREEMPTION) thrd->state = Ready;
449
450        // Dereference the thread now because once we push it, there is not guaranteed it's still valid.
451        struct cluster * cl = thrd->curr_cluster;
452        __STATS_DEF(bool outside = hint == UNPARK_LOCAL && thrd->last_proc && thrd->last_proc != kernelTLS().this_processor; )
453
454        // push the thread to the cluster ready-queue
455        push( cl, thrd, hint );
456
457        // variable thrd is no longer safe to use
458        thrd = 0xdeaddeaddeaddeadp;
459
460        // wake the cluster using the save variable.
461        __wake_one( cl );
462
463        #if !defined(__CFA_NO_STATISTICS__)
464                if( kernelTLS().this_stats ) {
465                        __tls_stats()->ready.threads.threads++;
466                        if(outside) {
467                                __tls_stats()->ready.threads.extunpark++;
468                        }
469                }
470                else {
471                        __atomic_fetch_add(&cl->stats->ready.threads.threads, 1, __ATOMIC_RELAXED);
472                        __atomic_fetch_add(&cl->stats->ready.threads.extunpark, 1, __ATOMIC_RELAXED);
473                }
474        #endif
475
476        /* paranoid */ verify( ready_schedule_islocked());
477        /* paranoid */ verify( ! __preemption_enabled() );
478}
479
480void schedule_thread$( thread$ * thrd, unpark_hint hint ) {
481        ready_schedule_lock();
482                __schedule_thread( thrd, hint );
483        ready_schedule_unlock();
484}
485
486// KERNEL ONLY
487static inline thread$ * __next_thread(cluster * this) with( *this ) {
488        /* paranoid */ verify( ! __preemption_enabled() );
489
490        ready_schedule_lock();
491                thread$ * thrd = pop_fast( this );
492        ready_schedule_unlock();
493
494        /* paranoid */ verify( ! __preemption_enabled() );
495        return thrd;
496}
497
498// KERNEL ONLY
499static inline thread$ * __next_thread_slow(cluster * this) with( *this ) {
500        /* paranoid */ verify( ! __preemption_enabled() );
501
502        ready_schedule_lock();
503                thread$ * thrd = pop_slow( this );
504        ready_schedule_unlock();
505
506        /* paranoid */ verify( ! __preemption_enabled() );
507        return thrd;
508}
509
510// KERNEL ONLY
511static inline thread$ * __next_thread_search(cluster * this) with( *this ) {
512        /* paranoid */ verify( ! __preemption_enabled() );
513
514        ready_schedule_lock();
515                thread$ * thrd = pop_search( this );
516        ready_schedule_unlock();
517
518        /* paranoid */ verify( ! __preemption_enabled() );
519        return thrd;
520}
521
522static inline bool __must_unpark( thread$ * thrd ) {
523        int old_ticket = __atomic_fetch_add(&thrd->ticket, 1, __ATOMIC_SEQ_CST);
524        switch(old_ticket) {
525                case TICKET_RUNNING:
526                        // Wake won the race, the thread will reschedule/rerun itself
527                        return false;
528                case TICKET_BLOCKED:
529                        /* paranoid */ verify( ! thrd->preempted != __NO_PREEMPTION );
530                        /* paranoid */ verify( thrd->state == Blocked );
531                        return true;
532                default:
533                        // This makes no sense, something is wrong abort
534                        abort("Thread %p (%s) has mismatch park/unpark\n", thrd, thrd->self_cor.name);
535        }
536}
537
538void __kernel_unpark( thread$ * thrd, unpark_hint hint ) {
539        /* paranoid */ verify( ! __preemption_enabled() );
540        /* paranoid */ verify( ready_schedule_islocked());
541
542        if( !thrd ) return;
543
544        if(__must_unpark(thrd)) {
545                // Wake lost the race,
546                __schedule_thread( thrd, hint );
547        }
548
549        /* paranoid */ verify( ready_schedule_islocked());
550        /* paranoid */ verify( ! __preemption_enabled() );
551}
552
553void unpark( thread$ * thrd, unpark_hint hint ) libcfa_public {
554        if( !thrd ) return;
555
556        if(__must_unpark(thrd)) {
557                disable_interrupts();
558                        // Wake lost the race,
559                        schedule_thread$( thrd, hint );
560                enable_interrupts(false);
561        }
562}
563
564void park( void ) libcfa_public {
565        __disable_interrupts_checked();
566                /* paranoid */ verify( kernelTLS().this_thread->preempted == __NO_PREEMPTION );
567                returnToKernel();
568        __enable_interrupts_checked();
569
570}
571
572extern "C" {
573        // Leave the thread monitor
574        // last routine called by a thread.
575        // Should never return
576        void __cfactx_thrd_leave() {
577                thread$ * thrd = active_thread();
578                monitor$ * this = &thrd->self_mon;
579
580                // Lock the monitor now
581                lock( this->lock __cfaabi_dbg_ctx2 );
582
583                disable_interrupts();
584
585                /* paranoid */ verify( ! __preemption_enabled() );
586                /* paranoid */ verify( thrd->state == Active );
587                /* paranoid */ verify( 0x0D15EA5E0D15EA5Ep == thrd->canary );
588                /* paranoid */ verify( kernelTLS().this_thread == thrd );
589                /* paranoid */ verify( thrd->context.SP );
590                /* paranoid */ verifyf( ((uintptr_t)thrd->context.SP) > ((uintptr_t)__get_stack(thrd->curr_cor)->limit), "ERROR : thread$ %p has been corrupted.\n StackPointer too large.\n", thrd );
591                /* paranoid */ verifyf( ((uintptr_t)thrd->context.SP) < ((uintptr_t)__get_stack(thrd->curr_cor)->base ), "ERROR : thread$ %p has been corrupted.\n StackPointer too small.\n", thrd );
592
593                if( TICKET_RUNNING != thrd->ticket ) { abort( "Thread terminated with pending unpark" ); }
594                if( thrd != this->owner ) { abort( "Thread internal monitor has incorrect owner" ); }
595                if( this->recursion != 1) { abort( "Thread internal monitor has unbalanced recursion" ); }
596
597                thrd->state = Halting;
598                thrd->ticket = TICKET_DEAD;
599
600                // Leave the thread
601                returnToKernel();
602
603                // Control flow should never reach here!
604                abort();
605        }
606}
607
608// KERNEL ONLY
609bool force_yield( __Preemption_Reason reason ) libcfa_public {
610        __disable_interrupts_checked();
611                thread$ * thrd = kernelTLS().this_thread;
612                /* paranoid */ verify(thrd->state == Active);
613
614                // SKULLDUGGERY: It is possible that we are preempting this thread just before
615                // it was going to park itself. If that is the case and it is already using the
616                // intrusive fields then we can't use them to preempt the thread
617                // If that is the case, abandon the preemption.
618                bool preempted = false;
619                if(thrd->rdy_link.next == 0p) {
620                        preempted = true;
621                        thrd->preempted = reason;
622                        returnToKernel();
623                }
624        __enable_interrupts_checked( false );
625        return preempted;
626}
627
628//=============================================================================================
629// Kernel Idle Sleep
630//=============================================================================================
631// Wake a thread from the front if there are any
632static void __wake_one(cluster * this) {
633        eventfd_t val;
634
635        /* paranoid */ verify( ! __preemption_enabled() );
636        /* paranoid */ verify( ready_schedule_islocked() );
637
638        // Check if there is a sleeping processor
639        struct __fd_waitctx * fdp = __atomic_load_n(&this->procs.fdw, __ATOMIC_SEQ_CST);
640
641        // If no one is sleeping: we are done
642        if( fdp == 0p ) return;
643
644        int fd = 1;
645        if( __atomic_load_n(&fdp->sem, __ATOMIC_SEQ_CST) != 1 ) {
646                fd = __atomic_exchange_n(&fdp->sem, 1, __ATOMIC_RELAXED);
647        }
648
649        switch(fd) {
650                __attribute__((unused)) int ret;
651        case 0:
652                // If the processor isn't ready to sleep then the exchange will already wake it up
653                #if !defined(__CFA_NO_STATISTICS__)
654                        if( kernelTLS().this_stats ) { __tls_stats()->ready.sleep.early++;
655                        } else { __atomic_fetch_add(&this->stats->ready.sleep.early, 1, __ATOMIC_RELAXED); }
656                #endif
657                break;
658        case 1:
659                // If someone else already said they will wake them: we are done
660                #if !defined(__CFA_NO_STATISTICS__)
661                        if( kernelTLS().this_stats ) { __tls_stats()->ready.sleep.seen++;
662                        } else { __atomic_fetch_add(&this->stats->ready.sleep.seen, 1, __ATOMIC_RELAXED); }
663                #endif
664                break;
665        default:
666                // If the processor was ready to sleep, we need to wake it up with an actual write
667                val = 1;
668                ret = eventfd_write( fd, val );
669                /* paranoid */ verifyf( ret == 0, "Expected return to be 0, was %d\n", ret );
670
671                #if !defined(__CFA_NO_STATISTICS__)
672                        if( kernelTLS().this_stats ) { __tls_stats()->ready.sleep.wakes++;
673                        } else { __atomic_fetch_add(&this->stats->ready.sleep.wakes, 1, __ATOMIC_RELAXED); }
674                #endif
675                break;
676        }
677
678        /* paranoid */ verify( ready_schedule_islocked() );
679        /* paranoid */ verify( ! __preemption_enabled() );
680
681        return;
682}
683
684// Unconditionnaly wake a thread
685void __wake_proc(processor * this) {
686        /* paranoid */ verify( ! __preemption_enabled() );
687
688        __cfadbg_print_safe(runtime_core, "Kernel : waking Processor %p\n", this);
689
690        this->idle_wctx.sem = 1;
691
692        this->idle_wctx.wake__time = rdtscl();
693
694        eventfd_t val;
695        val = 1;
696        __attribute__((unused)) int ret = eventfd_write( this->idle_wctx.evfd, val );
697
698        /* paranoid */ verifyf( ret == 0, "Expected return to be 0, was %d\n", ret );
699        /* paranoid */ verify( ! __preemption_enabled() );
700}
701
702static void idle_sleep(processor * this) {
703        /* paranoid */ verify( this->idle_wctx.evfd != 1 );
704        /* paranoid */ verify( this->idle_wctx.evfd != 2 );
705
706        // Tell everyone we are ready to go do sleep
707        for() {
708                int expected = this->idle_wctx.sem;
709
710                // Someone already told us to wake-up! No time for a nap.
711                if(expected == 1) { return; }
712
713                // Try to mark that we are going to sleep
714                if(__atomic_compare_exchange_n(&this->idle_wctx.sem, &expected, this->idle_wctx.evfd, false,  __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST) ) {
715                        // Every one agreed, taking a nap
716                        break;
717                }
718        }
719
720
721        #if !defined(__CFA_NO_STATISTICS__)
722                if(this->print_halts) {
723                        __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 0\n", this->unique_id, rdtscl());
724                }
725        #endif
726
727        __cfadbg_print_safe(runtime_core, "Kernel : core %p waiting on eventfd %d\n", this, this->idle_fd);
728
729        {
730                eventfd_t val;
731                ssize_t ret = read( this->idle_wctx.evfd, &val, sizeof(val) );
732                if(ret < 0) {
733                        switch((int)errno) {
734                        case EAGAIN:
735                        #if EAGAIN != EWOULDBLOCK
736                                case EWOULDBLOCK:
737                        #endif
738                        case EINTR:
739                                // No need to do anything special here, just assume it's a legitimate wake-up
740                                break;
741                        default:
742                                abort( "KERNEL : internal error, read failure on idle eventfd, error(%d) %s.", (int)errno, strerror( (int)errno ) );
743                        }
744                }
745        }
746
747        #if !defined(__CFA_NO_STATISTICS__)
748                if(this->print_halts) {
749                        __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 1\n", this->unique_id, rdtscl());
750                }
751        #endif
752}
753
754static bool mark_idle(__cluster_proc_list & this, processor & proc) {
755        __STATS__(true, ready.sleep.halts++; )
756
757        proc.idle_wctx.sem = 0;
758
759        /* paranoid */ verify( ! __preemption_enabled() );
760        if(!try_lock( this )) return false;
761                this.idle++;
762                /* paranoid */ verify( this.idle <= this.total );
763                remove(proc);
764                insert_first(this.idles, proc);
765
766                // update the pointer to the head wait context, which should now point to this proc.
767                __atomic_store_n(&this.fdw, &proc.idle_wctx, __ATOMIC_SEQ_CST);
768        unlock( this );
769        /* paranoid */ verify( ! __preemption_enabled() );
770
771        return true;
772}
773
774static void mark_awake(__cluster_proc_list & this, processor & proc) {
775        /* paranoid */ verify( ! __preemption_enabled() );
776        lock( this );
777                this.idle--;
778                /* paranoid */ verify( this.idle >= 0 );
779                remove(proc);
780                insert_last(this.actives, proc);
781
782                {
783                        // update the pointer to the head wait context
784                        struct __fd_waitctx * wctx = 0;
785                        if(!this.idles`isEmpty) wctx = &this.idles`first.idle_wctx;
786                        __atomic_store_n(&this.fdw, wctx, __ATOMIC_SEQ_CST);
787                }
788
789        unlock( this );
790        /* paranoid */ verify( ! __preemption_enabled() );
791}
792
793//=============================================================================================
794// Unexpected Terminating logic
795//=============================================================================================
796void __kernel_abort_msg( char * abort_text, int abort_text_size ) {
797        thread$ * thrd = __cfaabi_tls.this_thread;
798
799        if(thrd) {
800                int len = snprintf( abort_text, abort_text_size, "Error occurred while executing thread %.256s (%p)", thrd->self_cor.name, thrd );
801                __cfaabi_bits_write( STDERR_FILENO, abort_text, len );
802
803                if ( &thrd->self_cor != thrd->curr_cor ) {
804                        len = snprintf( abort_text, abort_text_size, " in coroutine %.256s (%p).\n", thrd->curr_cor->name, thrd->curr_cor );
805                        __cfaabi_bits_write( STDERR_FILENO, abort_text, len );
806                }
807                else {
808                        __cfaabi_bits_write( STDERR_FILENO, ".\n", 2 );
809                }
810        }
811        else {
812                int len = snprintf( abort_text, abort_text_size, "Error occurred outside of any thread.\n" );
813                __cfaabi_bits_write( STDERR_FILENO, abort_text, len );
814        }
815}
816
817int __kernel_abort_lastframe( void ) __attribute__ ((__nothrow__)) {
818        return get_coroutine(__cfaabi_tls.this_thread) == get_coroutine(mainThread) ? 4 : 2;
819}
820
821static __spinlock_t kernel_debug_lock;
822
823extern "C" {
824        void __cfaabi_bits_acquire() {
825                lock( kernel_debug_lock __cfaabi_dbg_ctx2 );
826        }
827
828        void __cfaabi_bits_release() {
829                unlock( kernel_debug_lock );
830        }
831}
832
833//=============================================================================================
834// Kernel Utilities
835//=============================================================================================
836#if defined(CFA_HAVE_LINUX_IO_URING_H)
837#include "io/types.hfa"
838#endif
839
840//-----------------------------------------------------------------------------
841// Debug
842bool threading_enabled(void) __attribute__((const)) libcfa_public {
843        return true;
844}
845
846//-----------------------------------------------------------------------------
847// Statistics
848#if !defined(__CFA_NO_STATISTICS__)
849        void print_halts( processor & this ) libcfa_public {
850                this.print_halts = true;
851        }
852
853        static void crawl_list( cluster * cltr, dlist(processor) & list, unsigned count ) {
854                /* paranoid */ verify( cltr->stats );
855
856                processor * it = &list`first;
857                for(unsigned i = 0; i < count; i++) {
858                        /* paranoid */ verifyf( it, "Unexpected null iterator, at index %u of %u\n", i, count);
859                        /* paranoid */ verify( it->local_data->this_stats );
860                        // __print_stats( it->local_data->this_stats, cltr->print_stats, "Processor", it->name, (void*)it );
861                        __tally_stats( cltr->stats, it->local_data->this_stats );
862                        it = &(*it)`next;
863                }
864        }
865
866        static void crawl_cluster_stats( cluster & this ) {
867                // Stop the world, otherwise stats could get really messed-up
868                // this doesn't solve all problems but does solve many
869                // so it's probably good enough
870                disable_interrupts();
871                uint_fast32_t last_size = ready_mutate_lock();
872
873                        crawl_list(&this, this.procs.actives, this.procs.total - this.procs.idle);
874                        crawl_list(&this, this.procs.idles  , this.procs.idle );
875
876                // Unlock the RWlock
877                ready_mutate_unlock( last_size );
878                enable_interrupts();
879        }
880
881
882        void print_stats_now( cluster & this, int flags ) libcfa_public {
883                crawl_cluster_stats( this );
884                __print_stats( this.stats, flags, "Cluster", this.name, (void*)&this );
885        }
886#endif
887// Local Variables: //
888// mode: c //
889// tab-width: 4 //
890// End: //
Note: See TracBrowser for help on using the repository browser.