source: libcfa/src/concurrency/kernel/private.hfa @ 7a2057a

ADTast-experimental
Last change on this file since 7a2057a was 28372f7, checked in by Thierry Delisle <tdelisle@…>, 20 months ago

Changed to park/unpark permit/ticketing to make sure no one unparks a dead thread
  • Property mode set to 100644
File size: 14.2 KB
Line 
1//
2// Cforall Version 1.0.0 Copyright (C) 2016 University of Waterloo
3//
4// The contents of this file are covered under the licence agreement in the
5// file "LICENCE" distributed with Cforall.
6//
7// kernel/private.hfa --
8//
9// Author           : Thierry Delisle
10// Created On       : Mon Feb 13 12:27:26 2017
11// Last Modified By : Peter A. Buhr
12// Last Modified On : Wed Aug 12 08:21:33 2020
13// Update Count     : 9
14//
15
16#pragma once
17
18#if !defined(__cforall_thread__)
19        #error kernel/private.hfa should only be included in libcfathread source
20#endif
21
22#include <signal.h>
23
24#include "kernel.hfa"
25#include "thread.hfa"
26
27#include "alarm.hfa"
28#include "stats.hfa"
29
30extern "C" {
31#if   defined(CFA_HAVE_LINUX_LIBRSEQ)
32        #include <rseq/rseq.h>
33#elif defined(CFA_HAVE_LINUX_RSEQ_H)
34        #include <linux/rseq.h>
35#else
36        #ifndef _GNU_SOURCE
37        #error kernel/private requires gnu_source
38        #endif
39        #include <sched.h>
40#endif
41}
42
43// Defines whether or not we *want* to use io_uring_enter as the idle_sleep blocking call
44// #define CFA_WANT_IO_URING_IDLE
45
46// Defines whether or not we *can* use io_uring_enter as the idle_sleep blocking call
47#if defined(CFA_WANT_IO_URING_IDLE) && defined(CFA_HAVE_LINUX_IO_URING_H)
48        #if defined(CFA_HAVE_IORING_OP_READ) || (defined(CFA_HAVE_READV) && defined(CFA_HAVE_IORING_OP_READV))
49                #define CFA_WITH_IO_URING_IDLE
50        #endif
51#endif
52// #define READYQ_USE_LINEAR_AVG
53#define READYQ_USE_LOGDBL_AVG
54// #define READYQ_USE_LOGINT_AVG
55
56#if   defined(READYQ_USE_LINEAR_AVG)
57typedef unsigned long long __readyQ_avg_t;
58#elif defined(READYQ_USE_LOGDBL_AVG)
59typedef double __readyQ_avg_t;
60#elif defined(READYQ_USE_LOGDBL_AVG)
61typedef unsigned long long __readyQ_avg_t;
62#else
63#error must pick a scheme for averaging
64#endif
65
66extern "C" {
67        __attribute__((visibility("protected"))) int __cfaabi_pthread_create(pthread_t *_thread, const pthread_attr_t *attr, void *(*start_routine) (void *), void *arg);
68        __attribute__((visibility("protected"))) int __cfaabi_pthread_join(pthread_t _thread, void **retval);
69        __attribute__((visibility("protected"))) pthread_t __cfaabi_pthread_self(void);
70        __attribute__((visibility("protected"))) int __cfaabi_pthread_attr_init(pthread_attr_t *attr);
71        __attribute__((visibility("protected"))) int __cfaabi_pthread_attr_destroy(pthread_attr_t *attr);
72        __attribute__((visibility("protected"))) int __cfaabi_pthread_attr_setstack( pthread_attr_t *attr, void *stackaddr, size_t stacksize );
73        __attribute__((visibility("protected"))) int __cfaabi_pthread_attr_getstacksize( const pthread_attr_t *attr, size_t *stacksize );
74        __attribute__((visibility("protected"))) int __cfaabi_pthread_sigqueue(pthread_t _thread, int sig, const union sigval value);
75        __attribute__((visibility("protected"))) int __cfaabi_pthread_sigmask( int how, const sigset_t *set, sigset_t *oset);
76}
77
78//-----------------------------------------------------------------------------
79// Scheduler
80union __attribute__((aligned(64))) __timestamp_t {
81        struct {
82                volatile unsigned long long tv;
83                volatile __readyQ_avg_t ma;
84        } t;
85        char __padding[192];
86};
87
88extern "C" {
89        void disable_interrupts() OPTIONAL_THREAD;
90        void enable_interrupts( bool poll = true );
91}
92
93void schedule_thread$( thread$ *, unpark_hint hint ) __attribute__((nonnull (1)));
94
95extern bool __preemption_enabled();
96
97enum {
98        PREEMPT_NORMAL    = 0,
99        PREEMPT_TERMINATE = 1,
100        PREEMPT_IO = 2,
101};
102
103static inline void __disable_interrupts_checked() {
104        /* paranoid */ verify( __preemption_enabled() );
105        disable_interrupts();
106        /* paranoid */ verify( ! __preemption_enabled() );
107}
108
109static inline void __enable_interrupts_checked( bool poll = true ) {
110        /* paranoid */ verify( ! __preemption_enabled() );
111        enable_interrupts( poll );
112        /* paranoid */ verify( __preemption_enabled() );
113}
114
115//release/wake-up the following resources
116void __thread_finish( thread$ * thrd );
117
118//-----------------------------------------------------------------------------
119// Hardware
120
121#if   defined(CFA_HAVE_LINUX_LIBRSEQ)
122        // No data needed
123#elif defined(CFA_HAVE_LINUX_RSEQ_H)
124        extern "Cforall" {
125                extern __attribute__((aligned(64))) __thread volatile struct rseq __cfaabi_rseq;
126        }
127#else
128        // No data needed
129#endif
130
131static inline int __kernel_getcpu() {
132        /* paranoid */ verify( ! __preemption_enabled() );
133#if   defined(CFA_HAVE_LINUX_LIBRSEQ)
134        return rseq_current_cpu();
135#elif defined(CFA_HAVE_LINUX_RSEQ_H)
136        int r = __cfaabi_rseq.cpu_id;
137        /* paranoid */ verify( r >= 0 );
138        return r;
139#else
140        return sched_getcpu();
141#endif
142}
143
144//-----------------------------------------------------------------------------
145// Processor
146void main(processorCtx_t &);
147static inline coroutine$* get_coroutine(processorCtx_t & this) { return &this.self; }
148
149void * __create_pthread( pthread_t *, void * (*)(void *), void * );
150void __destroy_pthread( pthread_t pthread, void * stack, void ** retval );
151
152extern cluster * mainCluster;
153
154//-----------------------------------------------------------------------------
155// Threads
156extern "C" {
157      void __cfactx_invoke_thread(void (*main)(void *), void * this);
158}
159
160__cfaabi_dbg_debug_do(
161        extern void __cfaabi_dbg_thread_register  ( thread$ * thrd );
162        extern void __cfaabi_dbg_thread_unregister( thread$ * thrd );
163)
164
165#define TICKET_BLOCKED (-1) // thread is blocked
166#define TICKET_RUNNING ( 0) // thread is running
167#define TICKET_UNBLOCK ( 1) // thread should ignore next block
168#define TICKET_DEAD    (0xDEAD) // thread should never be unparked
169
170//-----------------------------------------------------------------------------
171// Utils
172void doregister( struct cluster * cltr, struct thread$ & thrd );
173void unregister( struct cluster * cltr, struct thread$ & thrd );
174
175//-----------------------------------------------------------------------------
176// I/O
177io_arbiter$ * create(void);
178void destroy(io_arbiter$ *);
179
180//=======================================================================
181// Cluster lock API
182//=======================================================================
183// Lock-Free registering/unregistering of threads
184// Register a processor to a given cluster and get its unique id in return
185unsigned register_proc_id( void );
186
187// Unregister a processor from a given cluster using its id, getting back the original pointer
188void unregister_proc_id( unsigned );
189
190//=======================================================================
191// Reader-writer lock implementation
192// Concurrent with doregister/unregister,
193//    i.e., threads can be added at any point during or between the entry/exit
194
195//-----------------------------------------------------------------------
196// simple spinlock underlying the RWLock
197// Blocking acquire
198static inline void __atomic_acquire(volatile bool * ll) {
199        /* paranoid */ verify( ! __preemption_enabled() );
200        /* paranoid */ verify(ll);
201
202        while( __builtin_expect(__atomic_exchange_n(ll, (bool)true, __ATOMIC_SEQ_CST), false) ) {
203                while(__atomic_load_n(ll, (int)__ATOMIC_RELAXED))
204                        Pause();
205        }
206        /* paranoid */ verify(*ll);
207        /* paranoid */ verify( ! __preemption_enabled() );
208}
209
210// Non-Blocking acquire
211static inline bool __atomic_try_acquire(volatile bool * ll) {
212        /* paranoid */ verify( ! __preemption_enabled() );
213        /* paranoid */ verify(ll);
214
215        return !__atomic_exchange_n(ll, (bool)true, __ATOMIC_SEQ_CST);
216}
217
218// Release
219static inline void __atomic_unlock(volatile bool * ll) {
220        /* paranoid */ verify( ! __preemption_enabled() );
221        /* paranoid */ verify(ll);
222        /* paranoid */ verify(*ll);
223        __atomic_store_n(ll, (bool)false, __ATOMIC_RELEASE);
224}
225
226//-----------------------------------------------------------------------
227// Reader-Writer lock protecting the ready-queues
228// while this lock is mostly generic some aspects
229// have been hard-coded to for the ready-queue for
230// simplicity and performance
231union __attribute__((aligned(64))) __scheduler_RWLock_t {
232        struct {
233                __attribute__((aligned(64))) char padding;
234
235                // total cachelines allocated
236                __attribute__((aligned(64))) unsigned int max;
237
238                // cachelines currently in use
239                volatile unsigned int alloc;
240
241                // cachelines ready to itereate over
242                // (!= to alloc when thread is in second half of doregister)
243                volatile unsigned int ready;
244
245                // writer lock
246                volatile bool write_lock;
247
248                // data pointer
249                volatile bool * volatile * data;
250        } lock;
251        char pad[192];
252};
253
254void  ?{}(__scheduler_RWLock_t & this);
255void ^?{}(__scheduler_RWLock_t & this);
256
257extern __scheduler_RWLock_t __scheduler_lock;
258
259//-----------------------------------------------------------------------
260// Reader side : acquire when using the ready queue to schedule but not
261//  creating/destroying queues
262static inline void ready_schedule_lock(void) with(__scheduler_lock.lock) {
263        /* paranoid */ verify( ! __preemption_enabled() );
264        /* paranoid */ verify( ! kernelTLS().in_sched_lock );
265        /* paranoid */ verify( data[kernelTLS().sched_id] == &kernelTLS().sched_lock );
266        /* paranoid */ verify( !kernelTLS().this_processor || kernelTLS().this_processor->unique_id == kernelTLS().sched_id );
267
268        // Step 1 : make sure no writer are in the middle of the critical section
269        while(__atomic_load_n(&write_lock, (int)__ATOMIC_RELAXED))
270                Pause();
271
272        // Fence needed because we don't want to start trying to acquire the lock
273        // before we read a false.
274        // Not needed on x86
275        // std::atomic_thread_fence(std::memory_order_seq_cst);
276
277        // Step 2 : acquire our local lock
278        __atomic_acquire( &kernelTLS().sched_lock );
279        /*paranoid*/ verify(kernelTLS().sched_lock);
280
281        #ifdef __CFA_WITH_VERIFY__
282                // Debug, check if this is owned for reading
283                kernelTLS().in_sched_lock = true;
284        #endif
285}
286
287static inline void ready_schedule_unlock(void) with(__scheduler_lock.lock) {
288        /* paranoid */ verify( ! __preemption_enabled() );
289        /* paranoid */ verify( data[kernelTLS().sched_id] == &kernelTLS().sched_lock );
290        /* paranoid */ verify( !kernelTLS().this_processor || kernelTLS().this_processor->unique_id == kernelTLS().sched_id );
291        /* paranoid */ verify( kernelTLS().sched_lock );
292        /* paranoid */ verify( kernelTLS().in_sched_lock );
293        #ifdef __CFA_WITH_VERIFY__
294                // Debug, check if this is owned for reading
295                kernelTLS().in_sched_lock = false;
296        #endif
297        __atomic_unlock(&kernelTLS().sched_lock);
298}
299
300#ifdef __CFA_WITH_VERIFY__
301        static inline bool ready_schedule_islocked(void) {
302                /* paranoid */ verify( ! __preemption_enabled() );
303                /* paranoid */ verify( (!kernelTLS().in_sched_lock) || kernelTLS().sched_lock );
304                return kernelTLS().sched_lock;
305        }
306
307        static inline bool ready_mutate_islocked() {
308                return __scheduler_lock.lock.write_lock;
309        }
310#endif
311
312//-----------------------------------------------------------------------
313// Writer side : acquire when changing the ready queue, e.g. adding more
314//  queues or removing them.
315uint_fast32_t ready_mutate_lock( void );
316
317void ready_mutate_unlock( uint_fast32_t /* value returned by lock */ );
318
319//-----------------------------------------------------------------------
320// Lock-Free registering/unregistering of threads
321// Register a processor to a given cluster and get its unique id in return
322// For convenience, also acquires the lock
323static inline [unsigned, uint_fast32_t] ready_mutate_register() {
324        unsigned id = register_proc_id();
325        uint_fast32_t last = ready_mutate_lock();
326        return [id, last];
327}
328
329// Unregister a processor from a given cluster using its id, getting back the original pointer
330// assumes the lock is acquired
331static inline void ready_mutate_unregister( unsigned id, uint_fast32_t last_s ) {
332        ready_mutate_unlock( last_s );
333        unregister_proc_id( id );
334}
335
336//-----------------------------------------------------------------------
337// Cluster idle lock/unlock
338static inline void lock(__cluster_proc_list & this) {
339        /* paranoid */ verify( ! __preemption_enabled() );
340
341        // Start by locking the global RWlock so that we know no-one is
342        // adding/removing processors while we mess with the idle lock
343        ready_schedule_lock();
344
345        lock( this.lock __cfaabi_dbg_ctx2 );
346
347        /* paranoid */ verify( ! __preemption_enabled() );
348}
349
350static inline bool try_lock(__cluster_proc_list & this) {
351        /* paranoid */ verify( ! __preemption_enabled() );
352
353        // Start by locking the global RWlock so that we know no-one is
354        // adding/removing processors while we mess with the idle lock
355        ready_schedule_lock();
356
357        if(try_lock( this.lock __cfaabi_dbg_ctx2 )) {
358                // success
359                /* paranoid */ verify( ! __preemption_enabled() );
360                return true;
361        }
362
363        // failed to lock
364        ready_schedule_unlock();
365
366        /* paranoid */ verify( ! __preemption_enabled() );
367        return false;
368}
369
370static inline void unlock(__cluster_proc_list & this) {
371        /* paranoid */ verify( ! __preemption_enabled() );
372
373        unlock(this.lock);
374
375        // Release the global lock, which we acquired when locking
376        ready_schedule_unlock();
377
378        /* paranoid */ verify( ! __preemption_enabled() );
379}
380
381//=======================================================================
382// Ready-Queue API
383//-----------------------------------------------------------------------
384// push thread onto a ready queue for a cluster
385// returns true if the list was previously empty, false otherwise
386__attribute__((hot)) void push(struct cluster * cltr, struct thread$ * thrd, unpark_hint hint);
387
388//-----------------------------------------------------------------------
389// pop thread from the local queues of a cluster
390// returns 0p if empty
391// May return 0p spuriously
392__attribute__((hot)) struct thread$ * pop_fast(struct cluster * cltr);
393
394//-----------------------------------------------------------------------
395// pop thread from any ready queue of a cluster
396// returns 0p if empty
397// May return 0p spuriously
398__attribute__((hot)) struct thread$ * pop_slow(struct cluster * cltr);
399
400//-----------------------------------------------------------------------
401// search all ready queues of a cluster for any thread
402// returns 0p if empty
403// guaranteed to find any threads added before this call
404__attribute__((hot)) struct thread$ * pop_search(struct cluster * cltr);
405
406//-----------------------------------------------------------------------
407// get preferred ready for new thread
408unsigned ready_queue_new_preferred();
409
410//-----------------------------------------------------------------------
411// Increase the width of the ready queue (number of lanes) by 4
412void ready_queue_grow  (struct cluster * cltr);
413
414//-----------------------------------------------------------------------
415// Decrease the width of the ready queue (number of lanes) by 4
416void ready_queue_shrink(struct cluster * cltr);
417
418//-----------------------------------------------------------------------
419// Decrease the width of the ready queue (number of lanes) by 4
420void ready_queue_close(struct cluster * cltr);
421
422// Local Variables: //
423// mode: c //
424// tab-width: 4 //
425// End: //
Note: See TracBrowser for help on using the repository browser.