source: libcfa/src/concurrency/kernel_private.hfa @ efc171d1

arm-ehenumforall-pointer-decayjacob/cs343-translationnew-astnew-ast-unique-exprpthread-emulationqualifiedEnum
Last change on this file since efc171d1 was efc171d1, checked in by Thierry Delisle <tdelisle@…>, 3 years ago

Fixed several concurrency warnings

  • Property mode set to 100644
File size: 9.7 KB
Line 
1//
2// Cforall Version 1.0.0 Copyright (C) 2016 University of Waterloo
3//
4// The contents of this file are covered under the licence agreement in the
5// file "LICENCE" distributed with Cforall.
6//
7// kernel_private.hfa --
8//
9// Author           : Thierry Delisle
10// Created On       : Mon Feb 13 12:27:26 2017
11// Last Modified By : Peter A. Buhr
12// Last Modified On : Sat Nov 30 19:25:02 2019
13// Update Count     : 8
14//
15
16#pragma once
17
18#include "kernel.hfa"
19#include "thread.hfa"
20
21#include "alarm.hfa"
22#include "stats.hfa"
23
24#include "bits/random.hfa"
25
26
27//-----------------------------------------------------------------------------
28// Scheduler
29
30struct __attribute__((aligned(128))) __scheduler_lock_id_t;
31
32extern "C" {
33        void disable_interrupts() OPTIONAL_THREAD;
34        void enable_interrupts_noPoll();
35        void enable_interrupts( __cfaabi_dbg_ctx_param );
36}
37
38void __schedule_thread( struct __processor_id_t *, $thread * )
39#if defined(NDEBUG) || (!defined(__CFA_DEBUG__) && !defined(__CFA_VERIFY__))
40        __attribute__((nonnull (2)))
41#endif
42;
43
44//Block current thread and release/wake-up the following resources
45void __leave_thread() __attribute__((noreturn));
46
47//-----------------------------------------------------------------------------
48// Processor
49void main(processorCtx_t *);
50
51void * __create_pthread( pthread_t *, void * (*)(void *), void * );
52
53
54
55struct event_kernel_t {
56        alarm_list_t alarms;
57        __spinlock_t lock;
58};
59
60extern event_kernel_t * event_kernel;
61
62struct __cfa_kernel_preemption_state_t {
63        bool enabled;
64        bool in_progress;
65        unsigned short disable_count;
66};
67
68extern volatile thread_local __cfa_kernel_preemption_state_t preemption_state __attribute__ ((tls_model ( "initial-exec" )));
69
70extern cluster * mainCluster;
71
72//-----------------------------------------------------------------------------
73// Threads
74extern "C" {
75      void __cfactx_invoke_thread(void (*main)(void *), void * this);
76}
77
78__cfaabi_dbg_debug_do(
79        extern void __cfaabi_dbg_thread_register  ( $thread * thrd );
80        extern void __cfaabi_dbg_thread_unregister( $thread * thrd );
81)
82
83// KERNEL ONLY unpark with out disabling interrupts
84void __unpark( struct __processor_id_t *, $thread * thrd __cfaabi_dbg_ctx_param2 );
85
86//-----------------------------------------------------------------------------
87// I/O
88void __kernel_io_startup     ( cluster &, unsigned, bool );
89void __kernel_io_finish_start( cluster & );
90void __kernel_io_prepare_stop( cluster & );
91void __kernel_io_shutdown    ( cluster &, bool );
92
93//-----------------------------------------------------------------------------
94// Utils
95#define KERNEL_STORAGE(T,X) __attribute((aligned(__alignof__(T)))) static char storage_##X[sizeof(T)]
96
97static inline uint64_t __tls_rand() {
98        #if defined(__SIZEOF_INT128__)
99                return __lehmer64( kernelTLS.rand_seed );
100        #else
101                return __xorshift64( kernelTLS.rand_seed );
102        #endif
103}
104
105
106void doregister( struct cluster & cltr );
107void unregister( struct cluster & cltr );
108
109void doregister( struct cluster * cltr, struct $thread & thrd );
110void unregister( struct cluster * cltr, struct $thread & thrd );
111
112//=======================================================================
113// Cluster lock API
114//=======================================================================
115// Cells use by the reader writer lock
116// while not generic it only relies on a opaque pointer
117struct __attribute__((aligned(128))) __scheduler_lock_id_t {
118        // Spin lock used as the underlying lock
119        volatile bool lock;
120
121        // Handle pointing to the proc owning this cell
122        // Used for allocating cells and debugging
123        __processor_id_t * volatile handle;
124
125        #ifdef __CFA_WITH_VERIFY__
126                // Debug, check if this is owned for reading
127                bool owned;
128        #endif
129};
130
131static_assert( sizeof(struct __scheduler_lock_id_t) <= __alignof(struct __scheduler_lock_id_t));
132
133// Lock-Free registering/unregistering of threads
134// Register a processor to a given cluster and get its unique id in return
135unsigned doregister( struct __processor_id_t * proc );
136
137// Unregister a processor from a given cluster using its id, getting back the original pointer
138void     unregister( struct __processor_id_t * proc );
139
140//=======================================================================
141// Reader-writer lock implementation
142// Concurrent with doregister/unregister,
143//    i.e., threads can be added at any point during or between the entry/exit
144
145//-----------------------------------------------------------------------
146// simple spinlock underlying the RWLock
147// Blocking acquire
148static inline void __atomic_acquire(volatile bool * ll) {
149        while( __builtin_expect(__atomic_exchange_n(ll, (bool)true, __ATOMIC_SEQ_CST), false) ) {
150                while(__atomic_load_n(ll, (int)__ATOMIC_RELAXED))
151                        asm volatile("pause");
152        }
153        /* paranoid */ verify(*ll);
154}
155
156// Non-Blocking acquire
157static inline bool __atomic_try_acquire(volatile bool * ll) {
158        return !__atomic_exchange_n(ll, (bool)true, __ATOMIC_SEQ_CST);
159}
160
161// Release
162static inline void __atomic_unlock(volatile bool * ll) {
163        /* paranoid */ verify(*ll);
164        __atomic_store_n(ll, (bool)false, __ATOMIC_RELEASE);
165}
166
167//-----------------------------------------------------------------------
168// Reader-Writer lock protecting the ready-queues
169// while this lock is mostly generic some aspects
170// have been hard-coded to for the ready-queue for
171// simplicity and performance
172struct __scheduler_RWLock_t {
173        // total cachelines allocated
174        unsigned int max;
175
176        // cachelines currently in use
177        volatile unsigned int alloc;
178
179        // cachelines ready to itereate over
180        // (!= to alloc when thread is in second half of doregister)
181        volatile unsigned int ready;
182
183        // writer lock
184        volatile bool lock;
185
186        // data pointer
187        __scheduler_lock_id_t * data;
188};
189
190void  ?{}(__scheduler_RWLock_t & this);
191void ^?{}(__scheduler_RWLock_t & this);
192
193extern __scheduler_RWLock_t * __scheduler_lock;
194
195//-----------------------------------------------------------------------
196// Reader side : acquire when using the ready queue to schedule but not
197//  creating/destroying queues
198static inline void ready_schedule_lock( struct __processor_id_t * proc) with(*__scheduler_lock) {
199        unsigned iproc = proc->id;
200        /*paranoid*/ verify(data[iproc].handle == proc);
201        /*paranoid*/ verify(iproc < ready);
202
203        // Step 1 : make sure no writer are in the middle of the critical section
204        while(__atomic_load_n(&lock, (int)__ATOMIC_RELAXED))
205                asm volatile("pause");
206
207        // Fence needed because we don't want to start trying to acquire the lock
208        // before we read a false.
209        // Not needed on x86
210        // std::atomic_thread_fence(std::memory_order_seq_cst);
211
212        // Step 2 : acquire our local lock
213        __atomic_acquire( &data[iproc].lock );
214        /*paranoid*/ verify(data[iproc].lock);
215
216        #ifdef __CFA_WITH_VERIFY__
217                // Debug, check if this is owned for reading
218                data[iproc].owned = true;
219        #endif
220}
221
222static inline void ready_schedule_unlock( struct __processor_id_t * proc) with(*__scheduler_lock) {
223        unsigned iproc = proc->id;
224        /*paranoid*/ verify(data[iproc].handle == proc);
225        /*paranoid*/ verify(iproc < ready);
226        /*paranoid*/ verify(data[iproc].lock);
227        /*paranoid*/ verify(data[iproc].owned);
228        #ifdef __CFA_WITH_VERIFY__
229                // Debug, check if this is owned for reading
230                data[iproc].owned = false;
231        #endif
232        __atomic_unlock(&data[iproc].lock);
233}
234
235#ifdef __CFA_WITH_VERIFY__
236        static inline bool ready_schedule_islocked( struct __processor_id_t * proc) {
237                return __scheduler_lock->data[proc->id].owned;
238        }
239
240        static inline bool ready_mutate_islocked() {
241                return __scheduler_lock->lock;
242        }
243#endif
244
245//-----------------------------------------------------------------------
246// Writer side : acquire when changing the ready queue, e.g. adding more
247//  queues or removing them.
248uint_fast32_t ready_mutate_lock( void );
249
250void ready_mutate_unlock( uint_fast32_t /* value returned by lock */ );
251
252//=======================================================================
253// Ready-Queue API
254//-----------------------------------------------------------------------
255// pop thread from the ready queue of a cluster
256// returns 0p if empty
257__attribute__((hot)) bool query(struct cluster * cltr);
258
259//-----------------------------------------------------------------------
260// push thread onto a ready queue for a cluster
261// returns true if the list was previously empty, false otherwise
262__attribute__((hot)) bool push(struct cluster * cltr, struct $thread * thrd);
263
264//-----------------------------------------------------------------------
265// pop thread from the ready queue of a cluster
266// returns 0p if empty
267__attribute__((hot)) struct $thread * pop(struct cluster * cltr);
268
269//-----------------------------------------------------------------------
270// remove thread from the ready queue of a cluster
271// returns bool if it wasn't found
272bool remove_head(struct cluster * cltr, struct $thread * thrd);
273
274//-----------------------------------------------------------------------
275// Increase the width of the ready queue (number of lanes) by 4
276void ready_queue_grow  (struct cluster * cltr);
277
278//-----------------------------------------------------------------------
279// Decrease the width of the ready queue (number of lanes) by 4
280void ready_queue_shrink(struct cluster * cltr);
281
282//-----------------------------------------------------------------------
283// IO user data
284struct __io_user_data_t {
285        int32_t result;
286        $thread * thrd;
287};
288
289//-----------------------------------------------------------------------
290// Statics call at the end of each thread to register statistics
291#if !defined(__CFA_NO_STATISTICS__)
292        static inline struct __stats_t * __tls_stats() {
293                /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );
294                /* paranoid */ verify( kernelTLS.this_stats );
295                return kernelTLS.this_stats;
296        }
297
298        #define __STATS__(in_kernel, ...) { \
299                if( !(in_kernel) ) disable_interrupts(); \
300                with( *__tls_stats() ) { \
301                        __VA_ARGS__ \
302                } \
303                if( !(in_kernel) ) enable_interrupts( __cfaabi_dbg_ctx ); \
304        }
305#else
306        #define __STATS__(in_kernel, ...)
307#endif
308
309// Local Variables: //
310// mode: c //
311// tab-width: 4 //
312// End: //
Note: See TracBrowser for help on using the repository browser.