source: libcfa/src/concurrency/kernel_private.hfa @ 3489ea6

ADTast-experimentalenumforall-pointer-decayjacob/cs343-translationnew-ast-unique-exprpthread-emulationqualifiedEnum
Last change on this file since 3489ea6 was 3489ea6, checked in by Thierry Delisle <tdelisle@…>, 3 years ago

Started work on kernel_getcpu

  • Property mode set to 100644
File size: 10.7 KB
Line 
1//
2// Cforall Version 1.0.0 Copyright (C) 2016 University of Waterloo
3//
4// The contents of this file are covered under the licence agreement in the
5// file "LICENCE" distributed with Cforall.
6//
7// kernel_private.hfa --
8//
9// Author           : Thierry Delisle
10// Created On       : Mon Feb 13 12:27:26 2017
11// Last Modified By : Peter A. Buhr
12// Last Modified On : Wed Aug 12 08:21:33 2020
13// Update Count     : 9
14//
15
16#pragma once
17
18#if !defined(__cforall_thread__)
19        #error kernel_private.hfa should only be included in libcfathread source
20#endif
21
22#include "kernel.hfa"
23#include "thread.hfa"
24
25#include "alarm.hfa"
26#include "stats.hfa"
27
28extern "C" {
29#if   defined(CFA_HAVE_LINUX_LIBRSEQ)
30        #include <rseq/rseq.h>
31#elif defined(CFA_HAVE_LINUX_RSEQ_H)
32        #include <linux/rseq.h>
33#else
34        #ifndef _GNU_SOURCE
35        #error kernel_private requires gnu_source
36        #endif
37        #include <sched.h>
38#endif
39}
40
41//-----------------------------------------------------------------------------
42// Scheduler
43extern "C" {
44        void disable_interrupts() OPTIONAL_THREAD;
45        void enable_interrupts( bool poll = true );
46}
47
48void schedule_thread$( $thread * ) __attribute__((nonnull (1)));
49
50extern bool __preemption_enabled();
51
52//release/wake-up the following resources
53void __thread_finish( $thread * thrd );
54
55//-----------------------------------------------------------------------------
56// Hardware
57
58#if   defined(CFA_HAVE_LINUX_LIBRSEQ)
59        // No data needed
60#elif defined(CFA_HAVE_LINUX_RSEQ_H)
61        extern "Cforall" {
62                extern __attribute__((aligned(128))) thread_local volatile struct rseq __cfaabi_rseq;
63        }
64#else
65        // No data needed
66#endif
67
68static inline int __kernel_getcpu() {
69        /* paranoid */ verify( ! __preemption_enabled() );
70#if   defined(CFA_HAVE_LINUX_LIBRSEQ)
71        return rseq_current_cpu_raw();
72#elif defined(CFA_HAVE_LINUX_RSEQ_H)
73        return __cfaabi_rseq.cpu_id;
74#else
75        return sched_getcpu();
76#endif
77}
78
79//-----------------------------------------------------------------------------
80// Processor
81void main(processorCtx_t *);
82
83void * __create_pthread( pthread_t *, void * (*)(void *), void * );
84void __destroy_pthread( pthread_t pthread, void * stack, void ** retval );
85
86extern cluster * mainCluster;
87
88//-----------------------------------------------------------------------------
89// Threads
90extern "C" {
91      void __cfactx_invoke_thread(void (*main)(void *), void * this);
92}
93
94__cfaabi_dbg_debug_do(
95        extern void __cfaabi_dbg_thread_register  ( $thread * thrd );
96        extern void __cfaabi_dbg_thread_unregister( $thread * thrd );
97)
98
99#define TICKET_BLOCKED (-1) // thread is blocked
100#define TICKET_RUNNING ( 0) // thread is running
101#define TICKET_UNBLOCK ( 1) // thread should ignore next block
102
103//-----------------------------------------------------------------------------
104// Utils
105void doregister( struct cluster * cltr, struct $thread & thrd );
106void unregister( struct cluster * cltr, struct $thread & thrd );
107
108//-----------------------------------------------------------------------------
109// I/O
110$io_arbiter * create(void);
111void destroy($io_arbiter *);
112
113//=======================================================================
114// Cluster lock API
115//=======================================================================
116// Lock-Free registering/unregistering of threads
117// Register a processor to a given cluster and get its unique id in return
118unsigned register_proc_id( void );
119
120// Unregister a processor from a given cluster using its id, getting back the original pointer
121void unregister_proc_id( unsigned );
122
123//=======================================================================
124// Reader-writer lock implementation
125// Concurrent with doregister/unregister,
126//    i.e., threads can be added at any point during or between the entry/exit
127
128//-----------------------------------------------------------------------
129// simple spinlock underlying the RWLock
130// Blocking acquire
131static inline void __atomic_acquire(volatile bool * ll) {
132        while( __builtin_expect(__atomic_exchange_n(ll, (bool)true, __ATOMIC_SEQ_CST), false) ) {
133                while(__atomic_load_n(ll, (int)__ATOMIC_RELAXED))
134                        Pause();
135        }
136        /* paranoid */ verify(*ll);
137}
138
139// Non-Blocking acquire
140static inline bool __atomic_try_acquire(volatile bool * ll) {
141        return !__atomic_exchange_n(ll, (bool)true, __ATOMIC_SEQ_CST);
142}
143
144// Release
145static inline void __atomic_unlock(volatile bool * ll) {
146        /* paranoid */ verify(*ll);
147        __atomic_store_n(ll, (bool)false, __ATOMIC_RELEASE);
148}
149
150
151
152
153
154//-----------------------------------------------------------------------
155// Reader-Writer lock protecting the ready-queues
156// while this lock is mostly generic some aspects
157// have been hard-coded to for the ready-queue for
158// simplicity and performance
159struct __scheduler_RWLock_t {
160        // total cachelines allocated
161        unsigned int max;
162
163        // cachelines currently in use
164        volatile unsigned int alloc;
165
166        // cachelines ready to itereate over
167        // (!= to alloc when thread is in second half of doregister)
168        volatile unsigned int ready;
169
170        // writer lock
171        volatile bool write_lock;
172
173        // data pointer
174        volatile bool * volatile * data;
175};
176
177void  ?{}(__scheduler_RWLock_t & this);
178void ^?{}(__scheduler_RWLock_t & this);
179
180extern __scheduler_RWLock_t * __scheduler_lock;
181
182//-----------------------------------------------------------------------
183// Reader side : acquire when using the ready queue to schedule but not
184//  creating/destroying queues
185static inline void ready_schedule_lock(void) with(*__scheduler_lock) {
186        /* paranoid */ verify( ! __preemption_enabled() );
187        /* paranoid */ verify( ! kernelTLS().in_sched_lock );
188        /* paranoid */ verify( data[kernelTLS().sched_id] == &kernelTLS().sched_lock );
189        /* paranoid */ verify( !kernelTLS().this_processor || kernelTLS().this_processor->unique_id == kernelTLS().sched_id );
190
191        // Step 1 : make sure no writer are in the middle of the critical section
192        while(__atomic_load_n(&write_lock, (int)__ATOMIC_RELAXED))
193                Pause();
194
195        // Fence needed because we don't want to start trying to acquire the lock
196        // before we read a false.
197        // Not needed on x86
198        // std::atomic_thread_fence(std::memory_order_seq_cst);
199
200        // Step 2 : acquire our local lock
201        __atomic_acquire( &kernelTLS().sched_lock );
202        /*paranoid*/ verify(kernelTLS().sched_lock);
203
204        #ifdef __CFA_WITH_VERIFY__
205                // Debug, check if this is owned for reading
206                kernelTLS().in_sched_lock = true;
207        #endif
208}
209
210static inline void ready_schedule_unlock(void) with(*__scheduler_lock) {
211        /* paranoid */ verify( ! __preemption_enabled() );
212        /* paranoid */ verify( data[kernelTLS().sched_id] == &kernelTLS().sched_lock );
213        /* paranoid */ verify( !kernelTLS().this_processor || kernelTLS().this_processor->unique_id == kernelTLS().sched_id );
214        /* paranoid */ verify( kernelTLS().sched_lock );
215        /* paranoid */ verify( kernelTLS().in_sched_lock );
216        #ifdef __CFA_WITH_VERIFY__
217                // Debug, check if this is owned for reading
218                kernelTLS().in_sched_lock = false;
219        #endif
220        __atomic_unlock(&kernelTLS().sched_lock);
221}
222
223#ifdef __CFA_WITH_VERIFY__
224        static inline bool ready_schedule_islocked(void) {
225                /* paranoid */ verify( ! __preemption_enabled() );
226                /* paranoid */ verify( (!kernelTLS().in_sched_lock) || kernelTLS().sched_lock );
227                return kernelTLS().sched_lock;
228        }
229
230        static inline bool ready_mutate_islocked() {
231                return __scheduler_lock->write_lock;
232        }
233#endif
234
235//-----------------------------------------------------------------------
236// Writer side : acquire when changing the ready queue, e.g. adding more
237//  queues or removing them.
238uint_fast32_t ready_mutate_lock( void );
239
240void ready_mutate_unlock( uint_fast32_t /* value returned by lock */ );
241
242//-----------------------------------------------------------------------
243// Lock-Free registering/unregistering of threads
244// Register a processor to a given cluster and get its unique id in return
245// For convenience, also acquires the lock
246static inline [unsigned, uint_fast32_t] ready_mutate_register() {
247        unsigned id = register_proc_id();
248        uint_fast32_t last = ready_mutate_lock();
249        return [id, last];
250}
251
252// Unregister a processor from a given cluster using its id, getting back the original pointer
253// assumes the lock is acquired
254static inline void ready_mutate_unregister( unsigned id, uint_fast32_t last_s ) {
255        ready_mutate_unlock( last_s );
256        unregister_proc_id( id );
257}
258
259//-----------------------------------------------------------------------
260// Cluster idle lock/unlock
261static inline void lock(__cluster_proc_list & this) {
262        /* paranoid */ verify( ! __preemption_enabled() );
263
264        // Start by locking the global RWlock so that we know no-one is
265        // adding/removing processors while we mess with the idle lock
266        ready_schedule_lock();
267
268        // Simple counting lock, acquired, acquired by incrementing the counter
269        // to an odd number
270        for() {
271                uint64_t l = this.lock;
272                if(
273                        (0 == (l % 2))
274                        && __atomic_compare_exchange_n(&this.lock, &l, l + 1, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)
275                ) return;
276                Pause();
277        }
278
279        /* paranoid */ verify( ! __preemption_enabled() );
280}
281
282static inline void unlock(__cluster_proc_list & this) {
283        /* paranoid */ verify( ! __preemption_enabled() );
284
285        /* paranoid */ verify( 1 == (this.lock % 2) );
286        // Simple couting lock, release by incrementing to an even number
287        __atomic_fetch_add( &this.lock, 1, __ATOMIC_SEQ_CST );
288
289        // Release the global lock, which we acquired when locking
290        ready_schedule_unlock();
291
292        /* paranoid */ verify( ! __preemption_enabled() );
293}
294
295//=======================================================================
296// Ready-Queue API
297//-----------------------------------------------------------------------
298// push thread onto a ready queue for a cluster
299// returns true if the list was previously empty, false otherwise
300__attribute__((hot)) void push(struct cluster * cltr, struct $thread * thrd, bool local);
301
302//-----------------------------------------------------------------------
303// pop thread from the local queues of a cluster
304// returns 0p if empty
305// May return 0p spuriously
306__attribute__((hot)) struct $thread * pop_fast(struct cluster * cltr);
307
308//-----------------------------------------------------------------------
309// pop thread from any ready queue of a cluster
310// returns 0p if empty
311// May return 0p spuriously
312__attribute__((hot)) struct $thread * pop_slow(struct cluster * cltr);
313
314//-----------------------------------------------------------------------
315// search all ready queues of a cluster for any thread
316// returns 0p if empty
317// guaranteed to find any threads added before this call
318__attribute__((hot)) struct $thread * pop_search(struct cluster * cltr);
319
320//-----------------------------------------------------------------------
321// Increase the width of the ready queue (number of lanes) by 4
322void ready_queue_grow  (struct cluster * cltr);
323
324//-----------------------------------------------------------------------
325// Decrease the width of the ready queue (number of lanes) by 4
326void ready_queue_shrink(struct cluster * cltr);
327
328
329// Local Variables: //
330// mode: c //
331// tab-width: 4 //
332// End: //
Note: See TracBrowser for help on using the repository browser.