source: libcfa/src/concurrency/kernel_private.hfa @ 6a77224

ADTarm-ehast-experimentalenumforall-pointer-decayjacob/cs343-translationnew-ast-unique-exprpthread-emulationqualifiedEnum
Last change on this file since 6a77224 was 6a77224, checked in by Thierry Delisle <tdelisle@…>, 3 years ago

Changed park/unpark ticket to be -1,0,1 instead of 0,1,2 because it's more natural.
Also added defines in kernel_private.h to avoid these magic numbers

  • Property mode set to 100644
File size: 9.4 KB
Line 
1//
2// Cforall Version 1.0.0 Copyright (C) 2016 University of Waterloo
3//
4// The contents of this file are covered under the licence agreement in the
5// file "LICENCE" distributed with Cforall.
6//
7// kernel_private.hfa --
8//
9// Author           : Thierry Delisle
10// Created On       : Mon Feb 13 12:27:26 2017
11// Last Modified By : Peter A. Buhr
12// Last Modified On : Wed Aug 12 08:21:33 2020
13// Update Count     : 9
14//
15
16#pragma once
17
18#include "kernel.hfa"
19#include "thread.hfa"
20
21#include "alarm.hfa"
22#include "stats.hfa"
23
24//-----------------------------------------------------------------------------
25// Scheduler
26
27struct __attribute__((aligned(128))) __scheduler_lock_id_t;
28
29extern "C" {
30        void disable_interrupts() OPTIONAL_THREAD;
31        void enable_interrupts_noPoll();
32        void enable_interrupts( __cfaabi_dbg_ctx_param );
33}
34
35void __schedule_thread( struct __processor_id_t *, $thread * )
36#if defined(NDEBUG) || (!defined(__CFA_DEBUG__) && !defined(__CFA_VERIFY__))
37        __attribute__((nonnull (2)))
38#endif
39;
40
41//release/wake-up the following resources
42void __thread_finish( $thread * thrd );
43
44//-----------------------------------------------------------------------------
45// Processor
46void main(processorCtx_t *);
47
48void * __create_pthread( pthread_t *, void * (*)(void *), void * );
49
50
51
52extern cluster * mainCluster;
53
54//-----------------------------------------------------------------------------
55// Threads
56extern "C" {
57      void __cfactx_invoke_thread(void (*main)(void *), void * this);
58}
59
60__cfaabi_dbg_debug_do(
61        extern void __cfaabi_dbg_thread_register  ( $thread * thrd );
62        extern void __cfaabi_dbg_thread_unregister( $thread * thrd );
63)
64
65// KERNEL ONLY unpark with out disabling interrupts
66void __unpark( struct __processor_id_t *, $thread * thrd );
67
68#define TICKET_BLOCKED (-1) // thread is blocked
69#define TICKET_RUNNING ( 0) // thread is running
70#define TICKET_UNBLOCK ( 1) // thread should ignore next block
71
72static inline bool __post(single_sem & this, struct __processor_id_t * id) {
73        for() {
74                struct $thread * expected = this.ptr;
75                if(expected == 1p) return false;
76                if(expected == 0p) {
77                        if(__atomic_compare_exchange_n(&this.ptr, &expected, 1p, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) {
78                                return false;
79                        }
80                }
81                else {
82                        if(__atomic_compare_exchange_n(&this.ptr, &expected, 0p, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) {
83                                __unpark( id, expected );
84                                return true;
85                        }
86                }
87        }
88}
89
90//-----------------------------------------------------------------------------
91// Utils
92void doregister( struct cluster * cltr, struct $thread & thrd );
93void unregister( struct cluster * cltr, struct $thread & thrd );
94
95//-----------------------------------------------------------------------------
96// I/O
97void ^?{}(io_context & this, bool );
98
99//=======================================================================
100// Cluster lock API
101//=======================================================================
102// Cells use by the reader writer lock
103// while not generic it only relies on a opaque pointer
104struct __attribute__((aligned(128))) __scheduler_lock_id_t {
105        // Spin lock used as the underlying lock
106        volatile bool lock;
107
108        // Handle pointing to the proc owning this cell
109        // Used for allocating cells and debugging
110        __processor_id_t * volatile handle;
111
112        #ifdef __CFA_WITH_VERIFY__
113                // Debug, check if this is owned for reading
114                bool owned;
115        #endif
116};
117
118static_assert( sizeof(struct __scheduler_lock_id_t) <= __alignof(struct __scheduler_lock_id_t));
119
120// Lock-Free registering/unregistering of threads
121// Register a processor to a given cluster and get its unique id in return
122unsigned doregister( struct __processor_id_t * proc );
123
124// Unregister a processor from a given cluster using its id, getting back the original pointer
125void     unregister( struct __processor_id_t * proc );
126
127//-----------------------------------------------------------------------
128// Cluster idle lock/unlock
129static inline void lock(__cluster_idles & this) {
130        for() {
131                uint64_t l = this.lock;
132                if(
133                        (0 == (l % 2))
134                        && __atomic_compare_exchange_n(&this.lock, &l, l + 1, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)
135                ) return;
136                Pause();
137        }
138}
139
140static inline void unlock(__cluster_idles & this) {
141        /* paranoid */ verify( 1 == (this.lock % 2) );
142        __atomic_fetch_add( &this.lock, 1, __ATOMIC_SEQ_CST );
143}
144
145//=======================================================================
146// Reader-writer lock implementation
147// Concurrent with doregister/unregister,
148//    i.e., threads can be added at any point during or between the entry/exit
149
150//-----------------------------------------------------------------------
151// simple spinlock underlying the RWLock
152// Blocking acquire
153static inline void __atomic_acquire(volatile bool * ll) {
154        while( __builtin_expect(__atomic_exchange_n(ll, (bool)true, __ATOMIC_SEQ_CST), false) ) {
155                while(__atomic_load_n(ll, (int)__ATOMIC_RELAXED))
156                        Pause();
157        }
158        /* paranoid */ verify(*ll);
159}
160
161// Non-Blocking acquire
162static inline bool __atomic_try_acquire(volatile bool * ll) {
163        return !__atomic_exchange_n(ll, (bool)true, __ATOMIC_SEQ_CST);
164}
165
166// Release
167static inline void __atomic_unlock(volatile bool * ll) {
168        /* paranoid */ verify(*ll);
169        __atomic_store_n(ll, (bool)false, __ATOMIC_RELEASE);
170}
171
172//-----------------------------------------------------------------------
173// Reader-Writer lock protecting the ready-queues
174// while this lock is mostly generic some aspects
175// have been hard-coded to for the ready-queue for
176// simplicity and performance
177struct __scheduler_RWLock_t {
178        // total cachelines allocated
179        unsigned int max;
180
181        // cachelines currently in use
182        volatile unsigned int alloc;
183
184        // cachelines ready to itereate over
185        // (!= to alloc when thread is in second half of doregister)
186        volatile unsigned int ready;
187
188        // writer lock
189        volatile bool lock;
190
191        // data pointer
192        __scheduler_lock_id_t * data;
193};
194
195void  ?{}(__scheduler_RWLock_t & this);
196void ^?{}(__scheduler_RWLock_t & this);
197
198extern __scheduler_RWLock_t * __scheduler_lock;
199
200//-----------------------------------------------------------------------
201// Reader side : acquire when using the ready queue to schedule but not
202//  creating/destroying queues
203static inline void ready_schedule_lock( struct __processor_id_t * proc) with(*__scheduler_lock) {
204        unsigned iproc = proc->id;
205        /*paranoid*/ verify(data[iproc].handle == proc);
206        /*paranoid*/ verify(iproc < ready);
207
208        // Step 1 : make sure no writer are in the middle of the critical section
209        while(__atomic_load_n(&lock, (int)__ATOMIC_RELAXED))
210                Pause();
211
212        // Fence needed because we don't want to start trying to acquire the lock
213        // before we read a false.
214        // Not needed on x86
215        // std::atomic_thread_fence(std::memory_order_seq_cst);
216
217        // Step 2 : acquire our local lock
218        __atomic_acquire( &data[iproc].lock );
219        /*paranoid*/ verify(data[iproc].lock);
220
221        #ifdef __CFA_WITH_VERIFY__
222                // Debug, check if this is owned for reading
223                data[iproc].owned = true;
224        #endif
225}
226
227static inline void ready_schedule_unlock( struct __processor_id_t * proc) with(*__scheduler_lock) {
228        unsigned iproc = proc->id;
229        /*paranoid*/ verify(data[iproc].handle == proc);
230        /*paranoid*/ verify(iproc < ready);
231        /*paranoid*/ verify(data[iproc].lock);
232        /*paranoid*/ verify(data[iproc].owned);
233        #ifdef __CFA_WITH_VERIFY__
234                // Debug, check if this is owned for reading
235                data[iproc].owned = false;
236        #endif
237        __atomic_unlock(&data[iproc].lock);
238}
239
240#ifdef __CFA_WITH_VERIFY__
241        static inline bool ready_schedule_islocked( struct __processor_id_t * proc) {
242                return __scheduler_lock->data[proc->id].owned;
243        }
244
245        static inline bool ready_mutate_islocked() {
246                return __scheduler_lock->lock;
247        }
248#endif
249
250//-----------------------------------------------------------------------
251// Writer side : acquire when changing the ready queue, e.g. adding more
252//  queues or removing them.
253uint_fast32_t ready_mutate_lock( void );
254
255void ready_mutate_unlock( uint_fast32_t /* value returned by lock */ );
256
257//=======================================================================
258// Ready-Queue API
259//-----------------------------------------------------------------------
260// pop thread from the ready queue of a cluster
261// returns 0p if empty
262__attribute__((hot)) bool query(struct cluster * cltr);
263
264//-----------------------------------------------------------------------
265// push thread onto a ready queue for a cluster
266// returns true if the list was previously empty, false otherwise
267__attribute__((hot)) bool push(struct cluster * cltr, struct $thread * thrd);
268
269//-----------------------------------------------------------------------
270// pop thread from the ready queue of a cluster
271// returns 0p if empty
272// May return 0p spuriously
273__attribute__((hot)) struct $thread * pop(struct cluster * cltr);
274
275//-----------------------------------------------------------------------
276// pop thread from the ready queue of a cluster
277// returns 0p if empty
278// guaranteed to find any threads added before this call
279__attribute__((hot)) struct $thread * pop_slow(struct cluster * cltr);
280
281//-----------------------------------------------------------------------
282// remove thread from the ready queue of a cluster
283// returns bool if it wasn't found
284bool remove_head(struct cluster * cltr, struct $thread * thrd);
285
286//-----------------------------------------------------------------------
287// Increase the width of the ready queue (number of lanes) by 4
288void ready_queue_grow  (struct cluster * cltr, int target);
289
290//-----------------------------------------------------------------------
291// Decrease the width of the ready queue (number of lanes) by 4
292void ready_queue_shrink(struct cluster * cltr, int target);
293
294
295// Local Variables: //
296// mode: c //
297// tab-width: 4 //
298// End: //
Note: See TracBrowser for help on using the repository browser.