source: libcfa/src/concurrency/kernel/private.hfa @ 7eac70e

ADTast-experimental
Last change on this file since 7eac70e was a757ba1, checked in by Thierry Delisle <tdelisle@…>, 2 years ago

Removed wait-with-io_uring feature which was slow, broken and ifdefed out.
  • Property mode set to 100644
File size: 13.8 KB
Line 
1//
2// Cforall Version 1.0.0 Copyright (C) 2016 University of Waterloo
3//
4// The contents of this file are covered under the licence agreement in the
5// file "LICENCE" distributed with Cforall.
6//
7// kernel/private.hfa --
8//
9// Author           : Thierry Delisle
10// Created On       : Mon Feb 13 12:27:26 2017
11// Last Modified By : Peter A. Buhr
12// Last Modified On : Wed Aug 12 08:21:33 2020
13// Update Count     : 9
14//
15
16#pragma once
17
18#if !defined(__cforall_thread__)
19        #error kernel/private.hfa should only be included in libcfathread source
20#endif
21
22#include <signal.h>
23
24#include "kernel.hfa"
25#include "thread.hfa"
26
27#include "alarm.hfa"
28#include "stats.hfa"
29
30extern "C" {
31#if   defined(CFA_HAVE_LINUX_LIBRSEQ)
32        #include <rseq/rseq.h>
33#elif defined(CFA_HAVE_LINUX_RSEQ_H)
34        #include <linux/rseq.h>
35#else
36        #ifndef _GNU_SOURCE
37        #error kernel/private requires gnu_source
38        #endif
39        #include <sched.h>
40#endif
41}
42
43// #define READYQ_USE_LINEAR_AVG
44#define READYQ_USE_LOGDBL_AVG
45// #define READYQ_USE_LOGINT_AVG
46
47#if   defined(READYQ_USE_LINEAR_AVG)
48typedef unsigned long long __readyQ_avg_t;
49#elif defined(READYQ_USE_LOGDBL_AVG)
50typedef double __readyQ_avg_t;
51#elif defined(READYQ_USE_LOGDBL_AVG)
52typedef unsigned long long __readyQ_avg_t;
53#else
54#error must pick a scheme for averaging
55#endif
56
57extern "C" {
58        __attribute__((visibility("protected"))) int __cfaabi_pthread_create(pthread_t *_thread, const pthread_attr_t *attr, void *(*start_routine) (void *), void *arg);
59        __attribute__((visibility("protected"))) int __cfaabi_pthread_join(pthread_t _thread, void **retval);
60        __attribute__((visibility("protected"))) pthread_t __cfaabi_pthread_self(void);
61        __attribute__((visibility("protected"))) int __cfaabi_pthread_attr_init(pthread_attr_t *attr);
62        __attribute__((visibility("protected"))) int __cfaabi_pthread_attr_destroy(pthread_attr_t *attr);
63        __attribute__((visibility("protected"))) int __cfaabi_pthread_attr_setstack( pthread_attr_t *attr, void *stackaddr, size_t stacksize );
64        __attribute__((visibility("protected"))) int __cfaabi_pthread_attr_getstacksize( const pthread_attr_t *attr, size_t *stacksize );
65        __attribute__((visibility("protected"))) int __cfaabi_pthread_sigqueue(pthread_t _thread, int sig, const union sigval value);
66        __attribute__((visibility("protected"))) int __cfaabi_pthread_sigmask( int how, const sigset_t *set, sigset_t *oset);
67}
68
69//-----------------------------------------------------------------------------
70// Scheduler
71union __attribute__((aligned(64))) __timestamp_t {
72        struct {
73                volatile unsigned long long tv;
74                volatile __readyQ_avg_t ma;
75        } t;
76        char __padding[192];
77};
78
79extern "C" {
80        void disable_interrupts() OPTIONAL_THREAD;
81        void enable_interrupts( bool poll = true );
82}
83
84void schedule_thread$( thread$ *, unpark_hint hint ) __attribute__((nonnull (1)));
85
86extern bool __preemption_enabled();
87
88enum {
89        PREEMPT_NORMAL    = 0,
90        PREEMPT_TERMINATE = 1,
91        PREEMPT_IO = 2,
92};
93
94static inline void __disable_interrupts_checked() {
95        /* paranoid */ verify( __preemption_enabled() );
96        disable_interrupts();
97        /* paranoid */ verify( ! __preemption_enabled() );
98}
99
100static inline void __enable_interrupts_checked( bool poll = true ) {
101        /* paranoid */ verify( ! __preemption_enabled() );
102        enable_interrupts( poll );
103        /* paranoid */ verify( __preemption_enabled() );
104}
105
106//release/wake-up the following resources
107void __thread_finish( thread$ * thrd );
108
109//-----------------------------------------------------------------------------
110// Hardware
111
112#if   defined(CFA_HAVE_LINUX_LIBRSEQ)
113        // No data needed
114#elif defined(CFA_HAVE_LINUX_RSEQ_H)
115        extern "Cforall" {
116                extern __attribute__((aligned(64))) __thread volatile struct rseq __cfaabi_rseq;
117        }
118#else
119        // No data needed
120#endif
121
122static inline int __kernel_getcpu() {
123        /* paranoid */ verify( ! __preemption_enabled() );
124#if   defined(CFA_HAVE_LINUX_LIBRSEQ)
125        return rseq_current_cpu();
126#elif defined(CFA_HAVE_LINUX_RSEQ_H)
127        int r = __cfaabi_rseq.cpu_id;
128        /* paranoid */ verify( r >= 0 );
129        return r;
130#else
131        return sched_getcpu();
132#endif
133}
134
135//-----------------------------------------------------------------------------
136// Processor
137void main(processorCtx_t &);
138static inline coroutine$* get_coroutine(processorCtx_t & this) { return &this.self; }
139
140void * __create_pthread( pthread_t *, void * (*)(void *), void * );
141void __destroy_pthread( pthread_t pthread, void * stack, void ** retval );
142
143extern cluster * mainCluster;
144
145//-----------------------------------------------------------------------------
146// Threads
147extern "C" {
148      void __cfactx_invoke_thread(void (*main)(void *), void * this);
149}
150
151__cfaabi_dbg_debug_do(
152        extern void __cfaabi_dbg_thread_register  ( thread$ * thrd );
153        extern void __cfaabi_dbg_thread_unregister( thread$ * thrd );
154)
155
156#define TICKET_BLOCKED (-1) // thread is blocked
157#define TICKET_RUNNING ( 0) // thread is running
158#define TICKET_UNBLOCK ( 1) // thread should ignore next block
159#define TICKET_DEAD    (0xDEAD) // thread should never be unparked
160
161//-----------------------------------------------------------------------------
162// Utils
163void doregister( struct cluster * cltr, struct thread$ & thrd );
164void unregister( struct cluster * cltr, struct thread$ & thrd );
165
166//-----------------------------------------------------------------------------
167// I/O
168io_arbiter$ * create(void);
169void destroy(io_arbiter$ *);
170
171//=======================================================================
172// Cluster lock API
173//=======================================================================
174// Lock-Free registering/unregistering of threads
175// Register a processor to a given cluster and get its unique id in return
176unsigned register_proc_id( void );
177
178// Unregister a processor from a given cluster using its id, getting back the original pointer
179void unregister_proc_id( unsigned );
180
181//=======================================================================
182// Reader-writer lock implementation
183// Concurrent with doregister/unregister,
184//    i.e., threads can be added at any point during or between the entry/exit
185
186//-----------------------------------------------------------------------
187// simple spinlock underlying the RWLock
188// Blocking acquire
189static inline void __atomic_acquire(volatile bool * ll) {
190        /* paranoid */ verify( ! __preemption_enabled() );
191        /* paranoid */ verify(ll);
192
193        while( __builtin_expect(__atomic_exchange_n(ll, (bool)true, __ATOMIC_SEQ_CST), false) ) {
194                while(__atomic_load_n(ll, (int)__ATOMIC_RELAXED))
195                        Pause();
196        }
197        /* paranoid */ verify(*ll);
198        /* paranoid */ verify( ! __preemption_enabled() );
199}
200
201// Non-Blocking acquire
202static inline bool __atomic_try_acquire(volatile bool * ll) {
203        /* paranoid */ verify( ! __preemption_enabled() );
204        /* paranoid */ verify(ll);
205
206        return !__atomic_exchange_n(ll, (bool)true, __ATOMIC_SEQ_CST);
207}
208
209// Release
210static inline void __atomic_unlock(volatile bool * ll) {
211        /* paranoid */ verify( ! __preemption_enabled() );
212        /* paranoid */ verify(ll);
213        /* paranoid */ verify(*ll);
214        __atomic_store_n(ll, (bool)false, __ATOMIC_RELEASE);
215}
216
217//-----------------------------------------------------------------------
218// Reader-Writer lock protecting the ready-queues
219// while this lock is mostly generic some aspects
220// have been hard-coded to for the ready-queue for
221// simplicity and performance
222union __attribute__((aligned(64))) __scheduler_RWLock_t {
223        struct {
224                __attribute__((aligned(64))) char padding;
225
226                // total cachelines allocated
227                __attribute__((aligned(64))) unsigned int max;
228
229                // cachelines currently in use
230                volatile unsigned int alloc;
231
232                // cachelines ready to itereate over
233                // (!= to alloc when thread is in second half of doregister)
234                volatile unsigned int ready;
235
236                // writer lock
237                volatile bool write_lock;
238
239                // data pointer
240                volatile bool * volatile * data;
241        } lock;
242        char pad[192];
243};
244
245void  ?{}(__scheduler_RWLock_t & this);
246void ^?{}(__scheduler_RWLock_t & this);
247
248extern __scheduler_RWLock_t __scheduler_lock;
249
250//-----------------------------------------------------------------------
251// Reader side : acquire when using the ready queue to schedule but not
252//  creating/destroying queues
253static inline void ready_schedule_lock(void) with(__scheduler_lock.lock) {
254        /* paranoid */ verify( ! __preemption_enabled() );
255        /* paranoid */ verify( ! kernelTLS().in_sched_lock );
256        /* paranoid */ verify( data[kernelTLS().sched_id] == &kernelTLS().sched_lock );
257        /* paranoid */ verify( !kernelTLS().this_processor || kernelTLS().this_processor->unique_id == kernelTLS().sched_id );
258
259        // Step 1 : make sure no writer are in the middle of the critical section
260        while(__atomic_load_n(&write_lock, (int)__ATOMIC_RELAXED))
261                Pause();
262
263        // Fence needed because we don't want to start trying to acquire the lock
264        // before we read a false.
265        // Not needed on x86
266        // std::atomic_thread_fence(std::memory_order_seq_cst);
267
268        // Step 2 : acquire our local lock
269        __atomic_acquire( &kernelTLS().sched_lock );
270        /*paranoid*/ verify(kernelTLS().sched_lock);
271
272        #ifdef __CFA_WITH_VERIFY__
273                // Debug, check if this is owned for reading
274                kernelTLS().in_sched_lock = true;
275        #endif
276}
277
278static inline void ready_schedule_unlock(void) with(__scheduler_lock.lock) {
279        /* paranoid */ verify( ! __preemption_enabled() );
280        /* paranoid */ verify( data[kernelTLS().sched_id] == &kernelTLS().sched_lock );
281        /* paranoid */ verify( !kernelTLS().this_processor || kernelTLS().this_processor->unique_id == kernelTLS().sched_id );
282        /* paranoid */ verify( kernelTLS().sched_lock );
283        /* paranoid */ verify( kernelTLS().in_sched_lock );
284        #ifdef __CFA_WITH_VERIFY__
285                // Debug, check if this is owned for reading
286                kernelTLS().in_sched_lock = false;
287        #endif
288        __atomic_unlock(&kernelTLS().sched_lock);
289}
290
291#ifdef __CFA_WITH_VERIFY__
292        static inline bool ready_schedule_islocked(void) {
293                /* paranoid */ verify( ! __preemption_enabled() );
294                /* paranoid */ verify( (!kernelTLS().in_sched_lock) || kernelTLS().sched_lock );
295                return kernelTLS().sched_lock;
296        }
297
298        static inline bool ready_mutate_islocked() {
299                return __scheduler_lock.lock.write_lock;
300        }
301#endif
302
303//-----------------------------------------------------------------------
304// Writer side : acquire when changing the ready queue, e.g. adding more
305//  queues or removing them.
306uint_fast32_t ready_mutate_lock( void );
307
308void ready_mutate_unlock( uint_fast32_t /* value returned by lock */ );
309
310//-----------------------------------------------------------------------
311// Lock-Free registering/unregistering of threads
312// Register a processor to a given cluster and get its unique id in return
313// For convenience, also acquires the lock
314static inline [unsigned, uint_fast32_t] ready_mutate_register() {
315        unsigned id = register_proc_id();
316        uint_fast32_t last = ready_mutate_lock();
317        return [id, last];
318}
319
320// Unregister a processor from a given cluster using its id, getting back the original pointer
321// assumes the lock is acquired
322static inline void ready_mutate_unregister( unsigned id, uint_fast32_t last_s ) {
323        ready_mutate_unlock( last_s );
324        unregister_proc_id( id );
325}
326
327//-----------------------------------------------------------------------
328// Cluster idle lock/unlock
329static inline void lock(__cluster_proc_list & this) {
330        /* paranoid */ verify( ! __preemption_enabled() );
331
332        // Start by locking the global RWlock so that we know no-one is
333        // adding/removing processors while we mess with the idle lock
334        ready_schedule_lock();
335
336        lock( this.lock __cfaabi_dbg_ctx2 );
337
338        /* paranoid */ verify( ! __preemption_enabled() );
339}
340
341static inline bool try_lock(__cluster_proc_list & this) {
342        /* paranoid */ verify( ! __preemption_enabled() );
343
344        // Start by locking the global RWlock so that we know no-one is
345        // adding/removing processors while we mess with the idle lock
346        ready_schedule_lock();
347
348        if(try_lock( this.lock __cfaabi_dbg_ctx2 )) {
349                // success
350                /* paranoid */ verify( ! __preemption_enabled() );
351                return true;
352        }
353
354        // failed to lock
355        ready_schedule_unlock();
356
357        /* paranoid */ verify( ! __preemption_enabled() );
358        return false;
359}
360
361static inline void unlock(__cluster_proc_list & this) {
362        /* paranoid */ verify( ! __preemption_enabled() );
363
364        unlock(this.lock);
365
366        // Release the global lock, which we acquired when locking
367        ready_schedule_unlock();
368
369        /* paranoid */ verify( ! __preemption_enabled() );
370}
371
372//=======================================================================
373// Ready-Queue API
374//-----------------------------------------------------------------------
375// push thread onto a ready queue for a cluster
376// returns true if the list was previously empty, false otherwise
377__attribute__((hot)) void push(struct cluster * cltr, struct thread$ * thrd, unpark_hint hint);
378
379//-----------------------------------------------------------------------
380// pop thread from the local queues of a cluster
381// returns 0p if empty
382// May return 0p spuriously
383__attribute__((hot)) struct thread$ * pop_fast(struct cluster * cltr);
384
385//-----------------------------------------------------------------------
386// pop thread from any ready queue of a cluster
387// returns 0p if empty
388// May return 0p spuriously
389__attribute__((hot)) struct thread$ * pop_slow(struct cluster * cltr);
390
391//-----------------------------------------------------------------------
392// search all ready queues of a cluster for any thread
393// returns 0p if empty
394// guaranteed to find any threads added before this call
395__attribute__((hot)) struct thread$ * pop_search(struct cluster * cltr);
396
397//-----------------------------------------------------------------------
398// get preferred ready for new thread
399unsigned ready_queue_new_preferred();
400
401//-----------------------------------------------------------------------
402// Increase the width of the ready queue (number of lanes) by 4
403void ready_queue_grow  (struct cluster * cltr);
404
405//-----------------------------------------------------------------------
406// Decrease the width of the ready queue (number of lanes) by 4
407void ready_queue_shrink(struct cluster * cltr);
408
409//-----------------------------------------------------------------------
410// Decrease the width of the ready queue (number of lanes) by 4
411void ready_queue_close(struct cluster * cltr);
412
413// Local Variables: //
414// mode: c //
415// tab-width: 4 //
416// End: //
Note: See TracBrowser for help on using the repository browser.