source: libcfa/src/concurrency/kernel_private.hfa @ fe63ae6

ADTarm-ehast-experimentalenumforall-pointer-decayjacob/cs343-translationnew-ast-unique-exprpthread-emulationqualifiedEnum
Last change on this file since fe63ae6 was bd0bdd37, checked in by Thierry Delisle <tdelisle@…>, 4 years ago

Fix how bias is handled in the ready queue to be more consistent with multiple clusters.
Does not handle churn of processors yet.

  • Property mode set to 100644
File size: 9.3 KB
Line 
1//
2// Cforall Version 1.0.0 Copyright (C) 2016 University of Waterloo
3//
4// The contents of this file are covered under the licence agreement in the
5// file "LICENCE" distributed with Cforall.
6//
7// kernel_private.hfa --
8//
9// Author           : Thierry Delisle
10// Created On       : Mon Feb 13 12:27:26 2017
11// Last Modified By : Peter A. Buhr
12// Last Modified On : Wed Aug 12 08:21:33 2020
13// Update Count     : 9
14//
15
16#pragma once
17
18#include "kernel.hfa"
19#include "thread.hfa"
20
21#include "alarm.hfa"
22#include "stats.hfa"
23
24//-----------------------------------------------------------------------------
25// Scheduler
26
27struct __attribute__((aligned(128))) __scheduler_lock_id_t;
28
29extern "C" {
30        void disable_interrupts() OPTIONAL_THREAD;
31        void enable_interrupts_noPoll();
32        void enable_interrupts( __cfaabi_dbg_ctx_param );
33}
34
35void __schedule_thread( $thread * )
36#if defined(NDEBUG) || (!defined(__CFA_DEBUG__) && !defined(__CFA_VERIFY__))
37        __attribute__((nonnull (1)))
38#endif
39;
40
41extern bool __preemption_enabled();
42
43//release/wake-up the following resources
44void __thread_finish( $thread * thrd );
45
46//-----------------------------------------------------------------------------
47// Processor
48void main(processorCtx_t *);
49
50void * __create_pthread( pthread_t *, void * (*)(void *), void * );
51void __destroy_pthread( pthread_t pthread, void * stack, void ** retval );
52
53
54
55extern cluster * mainCluster;
56
57//-----------------------------------------------------------------------------
58// Threads
59extern "C" {
60      void __cfactx_invoke_thread(void (*main)(void *), void * this);
61}
62
63__cfaabi_dbg_debug_do(
64        extern void __cfaabi_dbg_thread_register  ( $thread * thrd );
65        extern void __cfaabi_dbg_thread_unregister( $thread * thrd );
66)
67
68#define TICKET_BLOCKED (-1) // thread is blocked
69#define TICKET_RUNNING ( 0) // thread is running
70#define TICKET_UNBLOCK ( 1) // thread should ignore next block
71
72//-----------------------------------------------------------------------------
73// Utils
74void doregister( struct cluster * cltr, struct $thread & thrd );
75void unregister( struct cluster * cltr, struct $thread & thrd );
76
77//-----------------------------------------------------------------------------
78// I/O
79$io_arbiter * create(void);
80void destroy($io_arbiter *);
81
82//=======================================================================
83// Cluster lock API
84//=======================================================================
85// Cells use by the reader writer lock
86// while not generic it only relies on a opaque pointer
87struct __attribute__((aligned(128))) __scheduler_lock_id_t {
88        // Spin lock used as the underlying lock
89        volatile bool lock;
90
91        // Handle pointing to the proc owning this cell
92        // Used for allocating cells and debugging
93        __processor_id_t * volatile handle;
94
95        #ifdef __CFA_WITH_VERIFY__
96                // Debug, check if this is owned for reading
97                bool owned;
98        #endif
99};
100
101static_assert( sizeof(struct __scheduler_lock_id_t) <= __alignof(struct __scheduler_lock_id_t));
102
103// Lock-Free registering/unregistering of threads
104// Register a processor to a given cluster and get its unique id in return
105unsigned doregister( struct __processor_id_t * proc );
106
107// Unregister a processor from a given cluster using its id, getting back the original pointer
108void     unregister( struct __processor_id_t * proc );
109
110//-----------------------------------------------------------------------
111// Cluster idle lock/unlock
112static inline void lock(__cluster_idles & this) {
113        for() {
114                uint64_t l = this.lock;
115                if(
116                        (0 == (l % 2))
117                        && __atomic_compare_exchange_n(&this.lock, &l, l + 1, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)
118                ) return;
119                Pause();
120        }
121}
122
123static inline void unlock(__cluster_idles & this) {
124        /* paranoid */ verify( 1 == (this.lock % 2) );
125        __atomic_fetch_add( &this.lock, 1, __ATOMIC_SEQ_CST );
126}
127
128//=======================================================================
129// Reader-writer lock implementation
130// Concurrent with doregister/unregister,
131//    i.e., threads can be added at any point during or between the entry/exit
132
133//-----------------------------------------------------------------------
134// simple spinlock underlying the RWLock
135// Blocking acquire
136static inline void __atomic_acquire(volatile bool * ll) {
137        while( __builtin_expect(__atomic_exchange_n(ll, (bool)true, __ATOMIC_SEQ_CST), false) ) {
138                while(__atomic_load_n(ll, (int)__ATOMIC_RELAXED))
139                        Pause();
140        }
141        /* paranoid */ verify(*ll);
142}
143
144// Non-Blocking acquire
145static inline bool __atomic_try_acquire(volatile bool * ll) {
146        return !__atomic_exchange_n(ll, (bool)true, __ATOMIC_SEQ_CST);
147}
148
149// Release
150static inline void __atomic_unlock(volatile bool * ll) {
151        /* paranoid */ verify(*ll);
152        __atomic_store_n(ll, (bool)false, __ATOMIC_RELEASE);
153}
154
155//-----------------------------------------------------------------------
156// Reader-Writer lock protecting the ready-queues
157// while this lock is mostly generic some aspects
158// have been hard-coded to for the ready-queue for
159// simplicity and performance
160struct __scheduler_RWLock_t {
161        // total cachelines allocated
162        unsigned int max;
163
164        // cachelines currently in use
165        volatile unsigned int alloc;
166
167        // cachelines ready to itereate over
168        // (!= to alloc when thread is in second half of doregister)
169        volatile unsigned int ready;
170
171        // writer lock
172        volatile bool lock;
173
174        // data pointer
175        __scheduler_lock_id_t * data;
176};
177
178void  ?{}(__scheduler_RWLock_t & this);
179void ^?{}(__scheduler_RWLock_t & this);
180
181extern __scheduler_RWLock_t * __scheduler_lock;
182
183//-----------------------------------------------------------------------
184// Reader side : acquire when using the ready queue to schedule but not
185//  creating/destroying queues
186static inline void ready_schedule_lock(void) with(*__scheduler_lock) {
187        /* paranoid */ verify( ! __preemption_enabled() );
188        /* paranoid */ verify( kernelTLS().this_proc_id );
189
190        unsigned iproc = kernelTLS().this_proc_id->id;
191        /*paranoid*/ verify(data[iproc].handle == kernelTLS().this_proc_id);
192        /*paranoid*/ verify(iproc < ready);
193
194        // Step 1 : make sure no writer are in the middle of the critical section
195        while(__atomic_load_n(&lock, (int)__ATOMIC_RELAXED))
196                Pause();
197
198        // Fence needed because we don't want to start trying to acquire the lock
199        // before we read a false.
200        // Not needed on x86
201        // std::atomic_thread_fence(std::memory_order_seq_cst);
202
203        // Step 2 : acquire our local lock
204        __atomic_acquire( &data[iproc].lock );
205        /*paranoid*/ verify(data[iproc].lock);
206
207        #ifdef __CFA_WITH_VERIFY__
208                // Debug, check if this is owned for reading
209                data[iproc].owned = true;
210        #endif
211}
212
213static inline void ready_schedule_unlock(void) with(*__scheduler_lock) {
214        /* paranoid */ verify( ! __preemption_enabled() );
215        /* paranoid */ verify( kernelTLS().this_proc_id );
216
217        unsigned iproc = kernelTLS().this_proc_id->id;
218        /*paranoid*/ verify(data[iproc].handle == kernelTLS().this_proc_id);
219        /*paranoid*/ verify(iproc < ready);
220        /*paranoid*/ verify(data[iproc].lock);
221        /*paranoid*/ verify(data[iproc].owned);
222        #ifdef __CFA_WITH_VERIFY__
223                // Debug, check if this is owned for reading
224                data[iproc].owned = false;
225        #endif
226        __atomic_unlock(&data[iproc].lock);
227}
228
229#ifdef __CFA_WITH_VERIFY__
230        static inline bool ready_schedule_islocked(void) {
231                /* paranoid */ verify( ! __preemption_enabled() );
232                /*paranoid*/ verify( kernelTLS().this_proc_id );
233                __processor_id_t * proc = kernelTLS().this_proc_id;
234                return __scheduler_lock->data[proc->id].owned;
235        }
236
237        static inline bool ready_mutate_islocked() {
238                return __scheduler_lock->lock;
239        }
240#endif
241
242//-----------------------------------------------------------------------
243// Writer side : acquire when changing the ready queue, e.g. adding more
244//  queues or removing them.
245uint_fast32_t ready_mutate_lock( void );
246
247void ready_mutate_unlock( uint_fast32_t /* value returned by lock */ );
248
249//=======================================================================
250// Ready-Queue API
251//-----------------------------------------------------------------------
252// pop thread from the ready queue of a cluster
253// returns 0p if empty
254__attribute__((hot)) bool query(struct cluster * cltr);
255
256//-----------------------------------------------------------------------
257// push thread onto a ready queue for a cluster
258// returns true if the list was previously empty, false otherwise
259__attribute__((hot)) bool push(struct cluster * cltr, struct $thread * thrd);
260
261//-----------------------------------------------------------------------
262// pop thread from the ready queue of a cluster
263// returns 0p if empty
264// May return 0p spuriously
265__attribute__((hot)) struct $thread * pop(struct cluster * cltr);
266
267//-----------------------------------------------------------------------
268// pop thread from the ready queue of a cluster
269// returns 0p if empty
270// guaranteed to find any threads added before this call
271__attribute__((hot)) struct $thread * pop_slow(struct cluster * cltr);
272
273//-----------------------------------------------------------------------
274// remove thread from the ready queue of a cluster
275// returns bool if it wasn't found
276bool remove_head(struct cluster * cltr, struct $thread * thrd);
277
278//-----------------------------------------------------------------------
279// Increase the width of the ready queue (number of lanes) by 4
280unsigned ready_queue_grow  (struct cluster * cltr, int target);
281
282//-----------------------------------------------------------------------
283// Decrease the width of the ready queue (number of lanes) by 4
284void ready_queue_shrink(struct cluster * cltr, int target);
285
286
287// Local Variables: //
288// mode: c //
289// tab-width: 4 //
290// End: //
Note: See TracBrowser for help on using the repository browser.