source: libcfa/src/concurrency/kernel_private.hfa @ e660761

arm-ehenumforall-pointer-decayjacob/cs343-translationnew-astnew-ast-unique-exprpthread-emulationqualifiedEnum
Last change on this file since e660761 was e660761, checked in by Thierry Delisle <tdelisle@…>, 3 years ago

First attempt at reducing complation time by restructuring the code.
Notably, starting the runtime has been moved to kernel/startup.cfa

  • Property mode set to 100644
File size: 10.1 KB
Line 
1//
2// Cforall Version 1.0.0 Copyright (C) 2016 University of Waterloo
3//
4// The contents of this file are covered under the licence agreement in the
5// file "LICENCE" distributed with Cforall.
6//
7// kernel_private.hfa --
8//
9// Author           : Thierry Delisle
10// Created On       : Mon Feb 13 12:27:26 2017
11// Last Modified By : Peter A. Buhr
12// Last Modified On : Sat Nov 30 19:25:02 2019
13// Update Count     : 8
14//
15
16#pragma once
17
18#include "kernel.hfa"
19#include "thread.hfa"
20
21#include "alarm.hfa"
22#include "stats.hfa"
23
24#include "bits/random.hfa"
25
26
27//-----------------------------------------------------------------------------
28// Scheduler
29
30struct __attribute__((aligned(128))) __scheduler_lock_id_t;
31
32extern "C" {
33        void disable_interrupts() OPTIONAL_THREAD;
34        void enable_interrupts_noPoll();
35        void enable_interrupts( __cfaabi_dbg_ctx_param );
36}
37
38void __schedule_thread( struct __processor_id_t *, $thread * )
39#if defined(NDEBUG) || (!defined(__CFA_DEBUG__) && !defined(__CFA_VERIFY__))
40        __attribute__((nonnull (2)))
41#endif
42;
43
44//Block current thread and release/wake-up the following resources
45void __leave_thread() __attribute__((noreturn));
46
47//-----------------------------------------------------------------------------
48// Processor
49void main(processorCtx_t *);
50
51void * __create_pthread( pthread_t *, void * (*)(void *), void * );
52
53
54
55extern cluster * mainCluster;
56
57//-----------------------------------------------------------------------------
58// Threads
59extern "C" {
60      void __cfactx_invoke_thread(void (*main)(void *), void * this);
61}
62
63__cfaabi_dbg_debug_do(
64        extern void __cfaabi_dbg_thread_register  ( $thread * thrd );
65        extern void __cfaabi_dbg_thread_unregister( $thread * thrd );
66)
67
68// KERNEL ONLY unpark with out disabling interrupts
69void __unpark( struct __processor_id_t *, $thread * thrd __cfaabi_dbg_ctx_param2 );
70
71static inline bool __post(single_sem & this, struct __processor_id_t * id) {
72        for() {
73                struct $thread * expected = this.ptr;
74                if(expected == 1p) return false;
75                if(expected == 0p) {
76                        if(__atomic_compare_exchange_n(&this.ptr, &expected, 1p, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) {
77                                return false;
78                        }
79                }
80                else {
81                        if(__atomic_compare_exchange_n(&this.ptr, &expected, 0p, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) {
82                                __unpark( id, expected __cfaabi_dbg_ctx2 );
83                                return true;
84                        }
85                }
86        }
87}
88
89//-----------------------------------------------------------------------------
90// Utils
91static inline uint64_t __tls_rand() {
92        #if defined(__SIZEOF_INT128__)
93                return __lehmer64( kernelTLS.rand_seed );
94        #else
95                return __xorshift64( kernelTLS.rand_seed );
96        #endif
97}
98
99void doregister( struct cluster * cltr, struct $thread & thrd );
100void unregister( struct cluster * cltr, struct $thread & thrd );
101
102//-----------------------------------------------------------------------------
103// I/O
104void __kernel_io_startup     ();
105void __kernel_io_shutdown    ();
106
107static inline io_context * __get_io_context( void ) {
108        cluster * cltr = active_cluster();
109        /* paranoid */ verifyf( cltr, "No active cluster for io operation\n");
110        assertf( cltr->io.cnt > 0, "Cluster %p has no default io contexts and no context was specified\n", cltr );
111        /* paranoid */ verifyf( cltr->io.ctxs, "default io contexts for cluster %p are missing\n", cltr);
112        return &cltr->io.ctxs[ __tls_rand() % cltr->io.cnt ];
113}
114
115void ^?{}(io_context & this, bool );
116
117//=======================================================================
118// Cluster lock API
119//=======================================================================
120// Cells use by the reader writer lock
121// while not generic it only relies on a opaque pointer
122struct __attribute__((aligned(128))) __scheduler_lock_id_t {
123        // Spin lock used as the underlying lock
124        volatile bool lock;
125
126        // Handle pointing to the proc owning this cell
127        // Used for allocating cells and debugging
128        __processor_id_t * volatile handle;
129
130        #ifdef __CFA_WITH_VERIFY__
131                // Debug, check if this is owned for reading
132                bool owned;
133        #endif
134};
135
136static_assert( sizeof(struct __scheduler_lock_id_t) <= __alignof(struct __scheduler_lock_id_t));
137
138// Lock-Free registering/unregistering of threads
139// Register a processor to a given cluster and get its unique id in return
140unsigned doregister( struct __processor_id_t * proc );
141
142// Unregister a processor from a given cluster using its id, getting back the original pointer
143void     unregister( struct __processor_id_t * proc );
144
145//=======================================================================
146// Reader-writer lock implementation
147// Concurrent with doregister/unregister,
148//    i.e., threads can be added at any point during or between the entry/exit
149
150//-----------------------------------------------------------------------
151// simple spinlock underlying the RWLock
152// Blocking acquire
153static inline void __atomic_acquire(volatile bool * ll) {
154        while( __builtin_expect(__atomic_exchange_n(ll, (bool)true, __ATOMIC_SEQ_CST), false) ) {
155                while(__atomic_load_n(ll, (int)__ATOMIC_RELAXED))
156                        asm volatile("pause");
157        }
158        /* paranoid */ verify(*ll);
159}
160
161// Non-Blocking acquire
162static inline bool __atomic_try_acquire(volatile bool * ll) {
163        return !__atomic_exchange_n(ll, (bool)true, __ATOMIC_SEQ_CST);
164}
165
166// Release
167static inline void __atomic_unlock(volatile bool * ll) {
168        /* paranoid */ verify(*ll);
169        __atomic_store_n(ll, (bool)false, __ATOMIC_RELEASE);
170}
171
172//-----------------------------------------------------------------------
173// Reader-Writer lock protecting the ready-queues
174// while this lock is mostly generic some aspects
175// have been hard-coded to for the ready-queue for
176// simplicity and performance
177struct __scheduler_RWLock_t {
178        // total cachelines allocated
179        unsigned int max;
180
181        // cachelines currently in use
182        volatile unsigned int alloc;
183
184        // cachelines ready to itereate over
185        // (!= to alloc when thread is in second half of doregister)
186        volatile unsigned int ready;
187
188        // writer lock
189        volatile bool lock;
190
191        // data pointer
192        __scheduler_lock_id_t * data;
193};
194
195void  ?{}(__scheduler_RWLock_t & this);
196void ^?{}(__scheduler_RWLock_t & this);
197
198extern __scheduler_RWLock_t * __scheduler_lock;
199
200//-----------------------------------------------------------------------
201// Reader side : acquire when using the ready queue to schedule but not
202//  creating/destroying queues
203static inline void ready_schedule_lock( struct __processor_id_t * proc) with(*__scheduler_lock) {
204        unsigned iproc = proc->id;
205        /*paranoid*/ verify(data[iproc].handle == proc);
206        /*paranoid*/ verify(iproc < ready);
207
208        // Step 1 : make sure no writer are in the middle of the critical section
209        while(__atomic_load_n(&lock, (int)__ATOMIC_RELAXED))
210                asm volatile("pause");
211
212        // Fence needed because we don't want to start trying to acquire the lock
213        // before we read a false.
214        // Not needed on x86
215        // std::atomic_thread_fence(std::memory_order_seq_cst);
216
217        // Step 2 : acquire our local lock
218        __atomic_acquire( &data[iproc].lock );
219        /*paranoid*/ verify(data[iproc].lock);
220
221        #ifdef __CFA_WITH_VERIFY__
222                // Debug, check if this is owned for reading
223                data[iproc].owned = true;
224        #endif
225}
226
227static inline void ready_schedule_unlock( struct __processor_id_t * proc) with(*__scheduler_lock) {
228        unsigned iproc = proc->id;
229        /*paranoid*/ verify(data[iproc].handle == proc);
230        /*paranoid*/ verify(iproc < ready);
231        /*paranoid*/ verify(data[iproc].lock);
232        /*paranoid*/ verify(data[iproc].owned);
233        #ifdef __CFA_WITH_VERIFY__
234                // Debug, check if this is owned for reading
235                data[iproc].owned = false;
236        #endif
237        __atomic_unlock(&data[iproc].lock);
238}
239
240#ifdef __CFA_WITH_VERIFY__
241        static inline bool ready_schedule_islocked( struct __processor_id_t * proc) {
242                return __scheduler_lock->data[proc->id].owned;
243        }
244
245        static inline bool ready_mutate_islocked() {
246                return __scheduler_lock->lock;
247        }
248#endif
249
250//-----------------------------------------------------------------------
251// Writer side : acquire when changing the ready queue, e.g. adding more
252//  queues or removing them.
253uint_fast32_t ready_mutate_lock( void );
254
255void ready_mutate_unlock( uint_fast32_t /* value returned by lock */ );
256
257//=======================================================================
258// Ready-Queue API
259//-----------------------------------------------------------------------
260// pop thread from the ready queue of a cluster
261// returns 0p if empty
262__attribute__((hot)) bool query(struct cluster * cltr);
263
264//-----------------------------------------------------------------------
265// push thread onto a ready queue for a cluster
266// returns true if the list was previously empty, false otherwise
267__attribute__((hot)) bool push(struct cluster * cltr, struct $thread * thrd);
268
269//-----------------------------------------------------------------------
270// pop thread from the ready queue of a cluster
271// returns 0p if empty
272__attribute__((hot)) struct $thread * pop(struct cluster * cltr);
273
274//-----------------------------------------------------------------------
275// remove thread from the ready queue of a cluster
276// returns bool if it wasn't found
277bool remove_head(struct cluster * cltr, struct $thread * thrd);
278
279//-----------------------------------------------------------------------
280// Increase the width of the ready queue (number of lanes) by 4
281void ready_queue_grow  (struct cluster * cltr, int target);
282
283//-----------------------------------------------------------------------
284// Decrease the width of the ready queue (number of lanes) by 4
285void ready_queue_shrink(struct cluster * cltr, int target);
286
287//-----------------------------------------------------------------------
288// IO user data
289struct __io_user_data_t {
290        int32_t result;
291        $thread * thrd;
292};
293
294//-----------------------------------------------------------------------
295// Statics call at the end of each thread to register statistics
296#if !defined(__CFA_NO_STATISTICS__)
297        static inline struct __stats_t * __tls_stats() {
298                /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
299                /* paranoid */ verify( kernelTLS.this_stats );
300                return kernelTLS.this_stats;
301        }
302
303        #define __STATS__(in_kernel, ...) { \
304                if( !(in_kernel) ) disable_interrupts(); \
305                with( *__tls_stats() ) { \
306                        __VA_ARGS__ \
307                } \
308                if( !(in_kernel) ) enable_interrupts( __cfaabi_dbg_ctx ); \
309        }
310#else
311        #define __STATS__(in_kernel, ...)
312#endif
313
314// Local Variables: //
315// mode: c //
316// tab-width: 4 //
317// End: //
Note: See TracBrowser for help on using the repository browser.