source: libcfa/src/concurrency/kernel/private.hfa @ 9cd5bd2

pthread-emulation
Last change on this file since 9cd5bd2 was 9cd5bd2, checked in by Thierry Delisle <tdelisle@…>, 4 months ago

Added an assembly to prevent null-checks from being optimized out.
Attempted to reduce the scope of needed headers.
Cleaned tabbing.

  • Property mode set to 100644
File size: 13.6 KB
Line 
1//
2// Cforall Version 1.0.0 Copyright (C) 2016 University of Waterloo
3//
4// The contents of this file are covered under the licence agreement in the
5// file "LICENCE" distributed with Cforall.
6//
7// kernel/private.hfa --
8//
9// Author           : Thierry Delisle
10// Created On       : Mon Feb 13 12:27:26 2017
11// Last Modified By : Peter A. Buhr
12// Last Modified On : Wed Aug 12 08:21:33 2020
13// Update Count     : 9
14//
15
16#pragma once
17
18#if !defined(__cforall_thread__)
19        #error kernel/private.hfa should only be included in libcfathread source
20#endif
21
22#include <signal.h>
23
24#include "kernel.hfa"
25#include "thread.hfa"
26
27#include "alarm.hfa"
28#include "stats.hfa"
29
30extern "C" {
31#if   defined(CFA_HAVE_LINUX_LIBRSEQ)
32        #include <rseq/rseq.h>
33#elif defined(CFA_HAVE_LINUX_RSEQ_H)
34        #include <linux/rseq.h>
35#else
36        #ifndef _GNU_SOURCE
37        #error kernel/private requires gnu_source
38        #endif
39        #include <sched.h>
40#endif
41}
42
43// Defines whether or not we *want* to use io_uring_enter as the idle_sleep blocking call
44// #define CFA_WANT_IO_URING_IDLE
45
46// Defines whether or not we *can* use io_uring_enter as the idle_sleep blocking call
47#if defined(CFA_WANT_IO_URING_IDLE) && defined(CFA_HAVE_LINUX_IO_URING_H)
48        #if defined(CFA_HAVE_IORING_OP_READ) || (defined(CFA_HAVE_READV) && defined(CFA_HAVE_IORING_OP_READV))
49                #define CFA_WITH_IO_URING_IDLE
50        #endif
51#endif
52
53extern "C" {
54        __attribute__((visibility("protected"))) int __cfaabi_pthread_create(pthread_t *_thread, const pthread_attr_t *attr, void *(*start_routine) (void *), void *arg);
55        __attribute__((visibility("protected"))) int __cfaabi_pthread_join(pthread_t _thread, void **retval);
56        __attribute__((visibility("protected"))) pthread_t __cfaabi_pthread_self(void);
57        __attribute__((visibility("protected"))) int __cfaabi_pthread_attr_init(pthread_attr_t *attr);
58        __attribute__((visibility("protected"))) int __cfaabi_pthread_attr_destroy(pthread_attr_t *attr);
59        __attribute__((visibility("protected"))) int __cfaabi_pthread_attr_setstack( pthread_attr_t *attr, void *stackaddr, size_t stacksize );
60        __attribute__((visibility("protected"))) int __cfaabi_pthread_attr_getstacksize( const pthread_attr_t *attr, size_t *stacksize );
61        __attribute__((visibility("protected"))) int __cfaabi_pthread_sigqueue(pthread_t _thread, int sig, const union sigval value);
62        __attribute__((visibility("protected"))) int __cfaabi_pthread_sigmask( int how, const sigset_t *set, sigset_t *oset);
63}
64
65//-----------------------------------------------------------------------------
66// Scheduler
67extern "C" {
68        void disable_interrupts() OPTIONAL_THREAD;
69        void enable_interrupts( bool poll = true );
70}
71
72void schedule_thread$( thread$ *, unpark_hint hint ) __attribute__((nonnull (1)));
73
74extern bool __preemption_enabled();
75
76enum {
77        PREEMPT_NORMAL    = 0,
78        PREEMPT_TERMINATE = 1,
79        PREEMPT_IO = 2,
80};
81
82static inline void __disable_interrupts_checked() {
83        /* paranoid */ verify( __preemption_enabled() );
84        disable_interrupts();
85        /* paranoid */ verify( ! __preemption_enabled() );
86}
87
88static inline void __enable_interrupts_checked( bool poll = true ) {
89        /* paranoid */ verify( ! __preemption_enabled() );
90        enable_interrupts( poll );
91        /* paranoid */ verify( __preemption_enabled() );
92}
93
94//release/wake-up the following resources
95void __thread_finish( thread$ * thrd );
96
97//-----------------------------------------------------------------------------
98// Hardware
99
100#if   defined(CFA_HAVE_LINUX_LIBRSEQ)
101        // No data needed
102#elif defined(CFA_HAVE_LINUX_RSEQ_H)
103        extern "Cforall" {
104                extern __attribute__((aligned(64))) __thread volatile struct rseq __cfaabi_rseq;
105        }
106#else
107        // No data needed
108#endif
109
110static inline int __kernel_getcpu() {
111        /* paranoid */ verify( ! __preemption_enabled() );
112#if   defined(CFA_HAVE_LINUX_LIBRSEQ)
113        return rseq_current_cpu();
114#elif defined(CFA_HAVE_LINUX_RSEQ_H)
115        int r = __cfaabi_rseq.cpu_id;
116        /* paranoid */ verify( r >= 0 );
117        return r;
118#else
119        return sched_getcpu();
120#endif
121}
122
123//-----------------------------------------------------------------------------
124// Processor
125void main(processorCtx_t &);
126static inline coroutine$* get_coroutine(processorCtx_t & this) { return &this.self; }
127
128void * __create_pthread( pthread_t *, void * (*)(void *), void * );
129void __destroy_pthread( pthread_t pthread, void * stack, void ** retval );
130
131extern cluster * mainCluster;
132
133//-----------------------------------------------------------------------------
134// Threads
135extern "C" {
136      void __cfactx_invoke_thread(void (*main)(void *), void * this);
137}
138
139__cfaabi_dbg_debug_do(
140        extern void __cfaabi_dbg_thread_register  ( thread$ * thrd );
141        extern void __cfaabi_dbg_thread_unregister( thread$ * thrd );
142)
143
144#define TICKET_BLOCKED (-1) // thread is blocked
145#define TICKET_RUNNING ( 0) // thread is running
146#define TICKET_UNBLOCK ( 1) // thread should ignore next block
147
148//-----------------------------------------------------------------------------
149// Utils
150void doregister( struct cluster * cltr, struct thread$ & thrd );
151void unregister( struct cluster * cltr, struct thread$ & thrd );
152
153//-----------------------------------------------------------------------------
154// I/O
155io_arbiter$ * create(void);
156void destroy(io_arbiter$ *);
157
158//=======================================================================
159// Cluster lock API
160//=======================================================================
161// Lock-Free registering/unregistering of threads
162// Register a processor to a given cluster and get its unique id in return
163unsigned register_proc_id( void );
164
165// Unregister a processor from a given cluster using its id, getting back the original pointer
166void unregister_proc_id( unsigned );
167
168//=======================================================================
169// Reader-writer lock implementation
170// Concurrent with doregister/unregister,
171//    i.e., threads can be added at any point during or between the entry/exit
172
173//-----------------------------------------------------------------------
174// simple spinlock underlying the RWLock
175// Blocking acquire
176static inline void __atomic_acquire(volatile bool * ll) {
177        /* paranoid */ verify( ! __preemption_enabled() );
178        /* paranoid */ verify(ll);
179
180        while( __builtin_expect(__atomic_exchange_n(ll, (bool)true, __ATOMIC_SEQ_CST), false) ) {
181                while(__atomic_load_n(ll, (int)__ATOMIC_RELAXED))
182                        Pause();
183        }
184        /* paranoid */ verify(*ll);
185        /* paranoid */ verify( ! __preemption_enabled() );
186}
187
188// Non-Blocking acquire
189static inline bool __atomic_try_acquire(volatile bool * ll) {
190        /* paranoid */ verify( ! __preemption_enabled() );
191        /* paranoid */ verify(ll);
192
193        return !__atomic_exchange_n(ll, (bool)true, __ATOMIC_SEQ_CST);
194}
195
196// Release
197static inline void __atomic_unlock(volatile bool * ll) {
198        /* paranoid */ verify( ! __preemption_enabled() );
199        /* paranoid */ verify(ll);
200        /* paranoid */ verify(*ll);
201        __atomic_store_n(ll, (bool)false, __ATOMIC_RELEASE);
202}
203
204//-----------------------------------------------------------------------
205// Reader-Writer lock protecting the ready-queues
206// while this lock is mostly generic some aspects
207// have been hard-coded to for the ready-queue for
208// simplicity and performance
209union __attribute__((aligned(64))) __scheduler_RWLock_t {
210        struct {
211                __attribute__((aligned(64))) char padding;
212
213                // total cachelines allocated
214                __attribute__((aligned(64))) unsigned int max;
215
216                // cachelines currently in use
217                volatile unsigned int alloc;
218
219                // cachelines ready to itereate over
220                // (!= to alloc when thread is in second half of doregister)
221                volatile unsigned int ready;
222
223                // writer lock
224                volatile bool write_lock;
225
226                // data pointer
227                volatile bool * volatile * data;
228        } lock;
229        char pad[192];
230};
231
232void  ?{}(__scheduler_RWLock_t & this);
233void ^?{}(__scheduler_RWLock_t & this);
234
235extern __scheduler_RWLock_t __scheduler_lock;
236
237//-----------------------------------------------------------------------
238// Reader side : acquire when using the ready queue to schedule but not
239//  creating/destroying queues
240static inline void ready_schedule_lock(void) with(__scheduler_lock.lock) {
241        /* paranoid */ verify( ! __preemption_enabled() );
242        /* paranoid */ verify( ! kernelTLS().in_sched_lock );
243        /* paranoid */ verify( data[kernelTLS().sched_id] == &kernelTLS().sched_lock );
244        /* paranoid */ verify( !kernelTLS().this_processor || kernelTLS().this_processor->unique_id == kernelTLS().sched_id );
245
246        // Step 1 : make sure no writer are in the middle of the critical section
247        while(__atomic_load_n(&write_lock, (int)__ATOMIC_RELAXED))
248                Pause();
249
250        // Fence needed because we don't want to start trying to acquire the lock
251        // before we read a false.
252        // Not needed on x86
253        // std::atomic_thread_fence(std::memory_order_seq_cst);
254
255        // Step 2 : acquire our local lock
256        __atomic_acquire( &kernelTLS().sched_lock );
257        /*paranoid*/ verify(kernelTLS().sched_lock);
258
259        #ifdef __CFA_WITH_VERIFY__
260                // Debug, check if this is owned for reading
261                kernelTLS().in_sched_lock = true;
262        #endif
263}
264
265static inline void ready_schedule_unlock(void) with(__scheduler_lock.lock) {
266        /* paranoid */ verify( ! __preemption_enabled() );
267        /* paranoid */ verify( data[kernelTLS().sched_id] == &kernelTLS().sched_lock );
268        /* paranoid */ verify( !kernelTLS().this_processor || kernelTLS().this_processor->unique_id == kernelTLS().sched_id );
269        /* paranoid */ verify( kernelTLS().sched_lock );
270        /* paranoid */ verify( kernelTLS().in_sched_lock );
271        #ifdef __CFA_WITH_VERIFY__
272                // Debug, check if this is owned for reading
273                kernelTLS().in_sched_lock = false;
274        #endif
275        __atomic_unlock(&kernelTLS().sched_lock);
276}
277
278#ifdef __CFA_WITH_VERIFY__
279        static inline bool ready_schedule_islocked(void) {
280                /* paranoid */ verify( ! __preemption_enabled() );
281                /* paranoid */ verify( (!kernelTLS().in_sched_lock) || kernelTLS().sched_lock );
282                return kernelTLS().sched_lock;
283        }
284
285        static inline bool ready_mutate_islocked() {
286                return __scheduler_lock.lock.write_lock;
287        }
288#endif
289
290//-----------------------------------------------------------------------
291// Writer side : acquire when changing the ready queue, e.g. adding more
292//  queues or removing them.
293uint_fast32_t ready_mutate_lock( void );
294
295void ready_mutate_unlock( uint_fast32_t /* value returned by lock */ );
296
297//-----------------------------------------------------------------------
298// Lock-Free registering/unregistering of threads
299// Register a processor to a given cluster and get its unique id in return
300// For convenience, also acquires the lock
301static inline [unsigned, uint_fast32_t] ready_mutate_register() {
302        unsigned id = register_proc_id();
303        uint_fast32_t last = ready_mutate_lock();
304        return [id, last];
305}
306
307// Unregister a processor from a given cluster using its id, getting back the original pointer
308// assumes the lock is acquired
309static inline void ready_mutate_unregister( unsigned id, uint_fast32_t last_s ) {
310        ready_mutate_unlock( last_s );
311        unregister_proc_id( id );
312}
313
314//-----------------------------------------------------------------------
315// Cluster idle lock/unlock
316static inline void lock(__cluster_proc_list & this) {
317        /* paranoid */ verify( ! __preemption_enabled() );
318
319        // Start by locking the global RWlock so that we know no-one is
320        // adding/removing processors while we mess with the idle lock
321        ready_schedule_lock();
322
323        lock( this.lock __cfaabi_dbg_ctx2 );
324
325        /* paranoid */ verify( ! __preemption_enabled() );
326}
327
328static inline bool try_lock(__cluster_proc_list & this) {
329        /* paranoid */ verify( ! __preemption_enabled() );
330
331        // Start by locking the global RWlock so that we know no-one is
332        // adding/removing processors while we mess with the idle lock
333        ready_schedule_lock();
334
335        if(try_lock( this.lock __cfaabi_dbg_ctx2 )) {
336                // success
337                /* paranoid */ verify( ! __preemption_enabled() );
338                return true;
339        }
340
341        // failed to lock
342        ready_schedule_unlock();
343
344        /* paranoid */ verify( ! __preemption_enabled() );
345        return false;
346}
347
348static inline void unlock(__cluster_proc_list & this) {
349        /* paranoid */ verify( ! __preemption_enabled() );
350
351        unlock(this.lock);
352
353        // Release the global lock, which we acquired when locking
354        ready_schedule_unlock();
355
356        /* paranoid */ verify( ! __preemption_enabled() );
357}
358
359//=======================================================================
360// Ready-Queue API
361//-----------------------------------------------------------------------
362// push thread onto a ready queue for a cluster
363// returns true if the list was previously empty, false otherwise
364__attribute__((hot)) void push(struct cluster * cltr, struct thread$ * thrd, unpark_hint hint);
365
366//-----------------------------------------------------------------------
367// pop thread from the local queues of a cluster
368// returns 0p if empty
369// May return 0p spuriously
370__attribute__((hot)) struct thread$ * pop_fast(struct cluster * cltr);
371
372//-----------------------------------------------------------------------
373// pop thread from any ready queue of a cluster
374// returns 0p if empty
375// May return 0p spuriously
376__attribute__((hot)) struct thread$ * pop_slow(struct cluster * cltr);
377
378//-----------------------------------------------------------------------
379// search all ready queues of a cluster for any thread
380// returns 0p if empty
381// guaranteed to find any threads added before this call
382__attribute__((hot)) struct thread$ * pop_search(struct cluster * cltr);
383
384//-----------------------------------------------------------------------
385// get preferred ready for new thread
386unsigned ready_queue_new_preferred();
387
388//-----------------------------------------------------------------------
389// Increase the width of the ready queue (number of lanes) by 4
390void ready_queue_grow  (struct cluster * cltr);
391
392//-----------------------------------------------------------------------
393// Decrease the width of the ready queue (number of lanes) by 4
394void ready_queue_shrink(struct cluster * cltr);
395
396//-----------------------------------------------------------------------
397// Decrease the width of the ready queue (number of lanes) by 4
398void ready_queue_close(struct cluster * cltr);
399
400// Local Variables: //
401// mode: c //
402// tab-width: 4 //
403// End: //
Note: See TracBrowser for help on using the repository browser.