source: libcfa/src/concurrency/kernel_private.hfa @ 46bbcaf

enumforall-pointer-decaypthread-emulationqualifiedEnum
Last change on this file since 46bbcaf was 46bbcaf, checked in by Thierry Delisle <tdelisle@…>, 8 months ago

Fix race condition in verify

  • Property mode set to 100644
File size: 11.8 KB
Line 
1//
2// Cforall Version 1.0.0 Copyright (C) 2016 University of Waterloo
3//
4// The contents of this file are covered under the licence agreement in the
5// file "LICENCE" distributed with Cforall.
6//
7// kernel_private.hfa --
8//
9// Author           : Thierry Delisle
10// Created On       : Mon Feb 13 12:27:26 2017
11// Last Modified By : Peter A. Buhr
12// Last Modified On : Wed Aug 12 08:21:33 2020
13// Update Count     : 9
14//
15
16#pragma once
17
18#if !defined(__cforall_thread__)
19        #error kernel_private.hfa should only be included in libcfathread source
20#endif
21
22#include "kernel.hfa"
23#include "thread.hfa"
24
25#include "alarm.hfa"
26#include "stats.hfa"
27
28extern "C" {
29#if   defined(CFA_HAVE_LINUX_LIBRSEQ)
30        #include <rseq/rseq.h>
31#elif defined(CFA_HAVE_LINUX_RSEQ_H)
32        #include <linux/rseq.h>
33#else
34        #ifndef _GNU_SOURCE
35        #error kernel_private requires gnu_source
36        #endif
37        #include <sched.h>
38#endif
39}
40
41// Defines whether or not we *want* to use io_uring_enter as the idle_sleep blocking call
42#define CFA_WANT_IO_URING_IDLE
43
44// Defines whether or not we *can* use io_uring_enter as the idle_sleep blocking call
45#if defined(CFA_WANT_IO_URING_IDLE) && defined(CFA_HAVE_LINUX_IO_URING_H)
46        #if defined(CFA_HAVE_IORING_OP_READ) || (defined(CFA_HAVE_READV) && defined(CFA_HAVE_IORING_OP_READV))
47                #define CFA_WITH_IO_URING_IDLE
48        #endif
49#endif
50
51//-----------------------------------------------------------------------------
52// Scheduler
53extern "C" {
54        void disable_interrupts() OPTIONAL_THREAD;
55        void enable_interrupts( bool poll = true );
56}
57
58void schedule_thread$( thread$ *, unpark_hint hint ) __attribute__((nonnull (1)));
59
60extern bool __preemption_enabled();
61
62static inline void __disable_interrupts_checked() {
63        /* paranoid */ verify( __preemption_enabled() );
64        disable_interrupts();
65        /* paranoid */ verify( ! __preemption_enabled() );
66}
67
68static inline void __enable_interrupts_checked( bool poll = true ) {
69        /* paranoid */ verify( ! __preemption_enabled() );
70        enable_interrupts( poll );
71        /* paranoid */ verify( __preemption_enabled() );
72}
73
74//release/wake-up the following resources
75void __thread_finish( thread$ * thrd );
76
77//-----------------------------------------------------------------------------
78// Hardware
79
80#if   defined(CFA_HAVE_LINUX_LIBRSEQ)
81        // No data needed
82#elif defined(CFA_HAVE_LINUX_RSEQ_H)
83        extern "Cforall" {
84                extern __attribute__((aligned(128))) thread_local volatile struct rseq __cfaabi_rseq;
85        }
86#else
87        // No data needed
88#endif
89
90static inline int __kernel_getcpu() {
91        /* paranoid */ verify( ! __preemption_enabled() );
92#if   defined(CFA_HAVE_LINUX_LIBRSEQ)
93        return rseq_current_cpu();
94#elif defined(CFA_HAVE_LINUX_RSEQ_H)
95        int r = __cfaabi_rseq.cpu_id;
96        /* paranoid */ verify( r >= 0 );
97        return r;
98#else
99        return sched_getcpu();
100#endif
101}
102
103//-----------------------------------------------------------------------------
104// Processor
105void main(processorCtx_t *);
106
107void * __create_pthread( pthread_t *, void * (*)(void *), void * );
108void __destroy_pthread( pthread_t pthread, void * stack, void ** retval );
109
110extern cluster * mainCluster;
111
112//-----------------------------------------------------------------------------
113// Threads
114extern "C" {
115      void __cfactx_invoke_thread(void (*main)(void *), void * this);
116}
117
118__cfaabi_dbg_debug_do(
119        extern void __cfaabi_dbg_thread_register  ( thread$ * thrd );
120        extern void __cfaabi_dbg_thread_unregister( thread$ * thrd );
121)
122
123#define TICKET_BLOCKED (-1) // thread is blocked
124#define TICKET_RUNNING ( 0) // thread is running
125#define TICKET_UNBLOCK ( 1) // thread should ignore next block
126
127//-----------------------------------------------------------------------------
128// Utils
129void doregister( struct cluster * cltr, struct thread$ & thrd );
130void unregister( struct cluster * cltr, struct thread$ & thrd );
131
132//-----------------------------------------------------------------------------
133// I/O
134$io_arbiter * create(void);
135void destroy($io_arbiter *);
136
137//=======================================================================
138// Cluster lock API
139//=======================================================================
140// Lock-Free registering/unregistering of threads
141// Register a processor to a given cluster and get its unique id in return
142unsigned register_proc_id( void );
143
144// Unregister a processor from a given cluster using its id, getting back the original pointer
145void unregister_proc_id( unsigned );
146
147//=======================================================================
148// Reader-writer lock implementation
149// Concurrent with doregister/unregister,
150//    i.e., threads can be added at any point during or between the entry/exit
151
152//-----------------------------------------------------------------------
153// simple spinlock underlying the RWLock
154// Blocking acquire
155static inline void __atomic_acquire(volatile bool * ll) {
156        while( __builtin_expect(__atomic_exchange_n(ll, (bool)true, __ATOMIC_SEQ_CST), false) ) {
157                while(__atomic_load_n(ll, (int)__ATOMIC_RELAXED))
158                        Pause();
159        }
160        /* paranoid */ verify(*ll);
161}
162
163// Non-Blocking acquire
164static inline bool __atomic_try_acquire(volatile bool * ll) {
165        return !__atomic_exchange_n(ll, (bool)true, __ATOMIC_SEQ_CST);
166}
167
168// Release
169static inline void __atomic_unlock(volatile bool * ll) {
170        /* paranoid */ verify(*ll);
171        __atomic_store_n(ll, (bool)false, __ATOMIC_RELEASE);
172}
173
174//-----------------------------------------------------------------------
175// Reader-Writer lock protecting the ready-queues
176// while this lock is mostly generic some aspects
177// have been hard-coded to for the ready-queue for
178// simplicity and performance
179struct __scheduler_RWLock_t {
180        // total cachelines allocated
181        unsigned int max;
182
183        // cachelines currently in use
184        volatile unsigned int alloc;
185
186        // cachelines ready to itereate over
187        // (!= to alloc when thread is in second half of doregister)
188        volatile unsigned int ready;
189
190        // writer lock
191        volatile bool write_lock;
192
193        // data pointer
194        volatile bool * volatile * data;
195};
196
197void  ?{}(__scheduler_RWLock_t & this);
198void ^?{}(__scheduler_RWLock_t & this);
199
200extern __scheduler_RWLock_t * __scheduler_lock;
201
202//-----------------------------------------------------------------------
203// Reader side : acquire when using the ready queue to schedule but not
204//  creating/destroying queues
205static inline void ready_schedule_lock(void) with(*__scheduler_lock) {
206        /* paranoid */ verify( ! __preemption_enabled() );
207        /* paranoid */ verify( ! kernelTLS().in_sched_lock );
208        /* paranoid */ verify( data[kernelTLS().sched_id] == &kernelTLS().sched_lock );
209        /* paranoid */ verify( !kernelTLS().this_processor || kernelTLS().this_processor->unique_id == kernelTLS().sched_id );
210
211        // Step 1 : make sure no writer are in the middle of the critical section
212        while(__atomic_load_n(&write_lock, (int)__ATOMIC_RELAXED))
213                Pause();
214
215        // Fence needed because we don't want to start trying to acquire the lock
216        // before we read a false.
217        // Not needed on x86
218        // std::atomic_thread_fence(std::memory_order_seq_cst);
219
220        // Step 2 : acquire our local lock
221        /*paranoid*/ verify(!kernelTLS().sched_lock);
222        __atomic_acquire( &kernelTLS().sched_lock );
223        /*paranoid*/ verify(kernelTLS().sched_lock);
224
225        #ifdef __CFA_WITH_VERIFY__
226                // Debug, check if this is owned for reading
227                kernelTLS().in_sched_lock = true;
228        #endif
229}
230
231static inline void ready_schedule_unlock(void) with(*__scheduler_lock) {
232        /* paranoid */ verify( ! __preemption_enabled() );
233        /* paranoid */ verify( data[kernelTLS().sched_id] == &kernelTLS().sched_lock );
234        /* paranoid */ verify( !kernelTLS().this_processor || kernelTLS().this_processor->unique_id == kernelTLS().sched_id );
235        /* paranoid */ verify( kernelTLS().sched_lock );
236        /* paranoid */ verify( kernelTLS().in_sched_lock );
237        #ifdef __CFA_WITH_VERIFY__
238                // Debug, check if this is owned for reading
239                kernelTLS().in_sched_lock = false;
240        #endif
241        __atomic_unlock(&kernelTLS().sched_lock);
242}
243
244#ifdef __CFA_WITH_VERIFY__
245        static inline bool ready_schedule_islocked(void) {
246                /* paranoid */ verify( ! __preemption_enabled() );
247                /* paranoid */ verify( (!kernelTLS().in_sched_lock) || kernelTLS().sched_lock );
248                return kernelTLS().sched_lock;
249        }
250
251        static inline bool ready_mutate_islocked() {
252                return __scheduler_lock->write_lock;
253        }
254#endif
255
256//-----------------------------------------------------------------------
257// Writer side : acquire when changing the ready queue, e.g. adding more
258//  queues or removing them.
259uint_fast32_t ready_mutate_lock( void );
260
261void ready_mutate_unlock( uint_fast32_t /* value returned by lock */ );
262
263//-----------------------------------------------------------------------
264// Lock-Free registering/unregistering of threads
265// Register a processor to a given cluster and get its unique id in return
266// For convenience, also acquires the lock
267static inline [unsigned, uint_fast32_t] ready_mutate_register() {
268        unsigned id = register_proc_id();
269        uint_fast32_t last = ready_mutate_lock();
270        return [id, last];
271}
272
273// Unregister a processor from a given cluster using its id, getting back the original pointer
274// assumes the lock is acquired
275static inline void ready_mutate_unregister( unsigned id, uint_fast32_t last_s ) {
276        ready_mutate_unlock( last_s );
277        unregister_proc_id( id );
278}
279
280//-----------------------------------------------------------------------
281// Cluster idle lock/unlock
282static inline void lock(__cluster_proc_list & this) {
283        /* paranoid */ verify( ! __preemption_enabled() );
284
285        // Start by locking the global RWlock so that we know no-one is
286        // adding/removing processors while we mess with the idle lock
287        ready_schedule_lock();
288
289        lock( this.lock __cfaabi_dbg_ctx2 );
290
291        /* paranoid */ verify( ! __preemption_enabled() );
292}
293
294static inline bool try_lock(__cluster_proc_list & this) {
295        /* paranoid */ verify( ! __preemption_enabled() );
296
297        // Start by locking the global RWlock so that we know no-one is
298        // adding/removing processors while we mess with the idle lock
299        ready_schedule_lock();
300
301        if(try_lock( this.lock __cfaabi_dbg_ctx2 )) {
302                // success
303                /* paranoid */ verify( ! __preemption_enabled() );
304                return true;
305        }
306
307        // failed to lock
308        ready_schedule_unlock();
309
310        /* paranoid */ verify( ! __preemption_enabled() );
311        return false;
312}
313
314static inline void unlock(__cluster_proc_list & this) {
315        /* paranoid */ verify( ! __preemption_enabled() );
316
317        unlock(this.lock);
318
319        // Release the global lock, which we acquired when locking
320        ready_schedule_unlock();
321
322        /* paranoid */ verify( ! __preemption_enabled() );
323}
324
325//=======================================================================
326// Ready-Queue API
327//-----------------------------------------------------------------------
328// push thread onto a ready queue for a cluster
329// returns true if the list was previously empty, false otherwise
330__attribute__((hot)) void push(struct cluster * cltr, struct thread$ * thrd, unpark_hint hint);
331
332//-----------------------------------------------------------------------
333// pop thread from the local queues of a cluster
334// returns 0p if empty
335// May return 0p spuriously
336__attribute__((hot)) struct thread$ * pop_fast(struct cluster * cltr);
337
338//-----------------------------------------------------------------------
339// pop thread from any ready queue of a cluster
340// returns 0p if empty
341// May return 0p spuriously
342__attribute__((hot)) struct thread$ * pop_slow(struct cluster * cltr);
343
344//-----------------------------------------------------------------------
345// search all ready queues of a cluster for any thread
346// returns 0p if empty
347// guaranteed to find any threads added before this call
348__attribute__((hot)) struct thread$ * pop_search(struct cluster * cltr);
349
350//-----------------------------------------------------------------------
351// get preferred ready for new thread
352unsigned ready_queue_new_preferred();
353
354//-----------------------------------------------------------------------
355// Increase the width of the ready queue (number of lanes) by 4
356void ready_queue_grow  (struct cluster * cltr);
357
358//-----------------------------------------------------------------------
359// Decrease the width of the ready queue (number of lanes) by 4
360void ready_queue_shrink(struct cluster * cltr);
361
362
363// Local Variables: //
364// mode: c //
365// tab-width: 4 //
366// End: //
Note: See TracBrowser for help on using the repository browser.