source: libcfa/src/concurrency/kernel_private.hfa @ 3e2b9c9

arm-ehjacob/cs343-translationnew-astnew-ast-unique-expr
Last change on this file since 3e2b9c9 was 3e2b9c9, checked in by Thierry Delisle <tdelisle@…>, 16 months ago

More restructuring of translation units
Unclear if it improves compilation time.

  • Property mode set to 100644
File size: 8.6 KB
Line 
1//
2// Cforall Version 1.0.0 Copyright (C) 2016 University of Waterloo
3//
4// The contents of this file are covered under the licence agreement in the
5// file "LICENCE" distributed with Cforall.
6//
7// kernel_private.hfa --
8//
9// Author           : Thierry Delisle
10// Created On       : Mon Feb 13 12:27:26 2017
11// Last Modified By : Peter A. Buhr
12// Last Modified On : Sat Nov 30 19:25:02 2019
13// Update Count     : 8
14//
15
16#pragma once
17
18#include "kernel.hfa"
19#include "thread.hfa"
20
21#include "alarm.hfa"
22#include "stats.hfa"
23
24//-----------------------------------------------------------------------------
25// Scheduler
26
27struct __attribute__((aligned(128))) __scheduler_lock_id_t;
28
29extern "C" {
30        void disable_interrupts() OPTIONAL_THREAD;
31        void enable_interrupts_noPoll();
32        void enable_interrupts( __cfaabi_dbg_ctx_param );
33}
34
35void __schedule_thread( struct __processor_id_t *, $thread * )
36#if defined(NDEBUG) || (!defined(__CFA_DEBUG__) && !defined(__CFA_VERIFY__))
37        __attribute__((nonnull (2)))
38#endif
39;
40
41//Block current thread and release/wake-up the following resources
42void __leave_thread() __attribute__((noreturn));
43
44//-----------------------------------------------------------------------------
45// Processor
46void main(processorCtx_t *);
47
48void * __create_pthread( pthread_t *, void * (*)(void *), void * );
49
50
51
52extern cluster * mainCluster;
53
54//-----------------------------------------------------------------------------
55// Threads
56extern "C" {
57      void __cfactx_invoke_thread(void (*main)(void *), void * this);
58}
59
60__cfaabi_dbg_debug_do(
61        extern void __cfaabi_dbg_thread_register  ( $thread * thrd );
62        extern void __cfaabi_dbg_thread_unregister( $thread * thrd );
63)
64
65// KERNEL ONLY unpark with out disabling interrupts
66void __unpark( struct __processor_id_t *, $thread * thrd __cfaabi_dbg_ctx_param2 );
67
68static inline bool __post(single_sem & this, struct __processor_id_t * id) {
69        for() {
70                struct $thread * expected = this.ptr;
71                if(expected == 1p) return false;
72                if(expected == 0p) {
73                        if(__atomic_compare_exchange_n(&this.ptr, &expected, 1p, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) {
74                                return false;
75                        }
76                }
77                else {
78                        if(__atomic_compare_exchange_n(&this.ptr, &expected, 0p, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) {
79                                __unpark( id, expected __cfaabi_dbg_ctx2 );
80                                return true;
81                        }
82                }
83        }
84}
85
86//-----------------------------------------------------------------------------
87// Utils
88void doregister( struct cluster * cltr, struct $thread & thrd );
89void unregister( struct cluster * cltr, struct $thread & thrd );
90
91//-----------------------------------------------------------------------------
92// I/O
93void ^?{}(io_context & this, bool );
94
95//=======================================================================
96// Cluster lock API
97//=======================================================================
98// Cells use by the reader writer lock
99// while not generic it only relies on a opaque pointer
100struct __attribute__((aligned(128))) __scheduler_lock_id_t {
101        // Spin lock used as the underlying lock
102        volatile bool lock;
103
104        // Handle pointing to the proc owning this cell
105        // Used for allocating cells and debugging
106        __processor_id_t * volatile handle;
107
108        #ifdef __CFA_WITH_VERIFY__
109                // Debug, check if this is owned for reading
110                bool owned;
111        #endif
112};
113
114static_assert( sizeof(struct __scheduler_lock_id_t) <= __alignof(struct __scheduler_lock_id_t));
115
116// Lock-Free registering/unregistering of threads
117// Register a processor to a given cluster and get its unique id in return
118unsigned doregister( struct __processor_id_t * proc );
119
120// Unregister a processor from a given cluster using its id, getting back the original pointer
121void     unregister( struct __processor_id_t * proc );
122
123//=======================================================================
124// Reader-writer lock implementation
125// Concurrent with doregister/unregister,
126//    i.e., threads can be added at any point during or between the entry/exit
127
128//-----------------------------------------------------------------------
129// simple spinlock underlying the RWLock
130// Blocking acquire
131static inline void __atomic_acquire(volatile bool * ll) {
132        while( __builtin_expect(__atomic_exchange_n(ll, (bool)true, __ATOMIC_SEQ_CST), false) ) {
133                while(__atomic_load_n(ll, (int)__ATOMIC_RELAXED))
134                        asm volatile("pause");
135        }
136        /* paranoid */ verify(*ll);
137}
138
139// Non-Blocking acquire
140static inline bool __atomic_try_acquire(volatile bool * ll) {
141        return !__atomic_exchange_n(ll, (bool)true, __ATOMIC_SEQ_CST);
142}
143
144// Release
145static inline void __atomic_unlock(volatile bool * ll) {
146        /* paranoid */ verify(*ll);
147        __atomic_store_n(ll, (bool)false, __ATOMIC_RELEASE);
148}
149
150//-----------------------------------------------------------------------
151// Reader-Writer lock protecting the ready-queues
152// while this lock is mostly generic some aspects
153// have been hard-coded to for the ready-queue for
154// simplicity and performance
155struct __scheduler_RWLock_t {
156        // total cachelines allocated
157        unsigned int max;
158
159        // cachelines currently in use
160        volatile unsigned int alloc;
161
162        // cachelines ready to itereate over
163        // (!= to alloc when thread is in second half of doregister)
164        volatile unsigned int ready;
165
166        // writer lock
167        volatile bool lock;
168
169        // data pointer
170        __scheduler_lock_id_t * data;
171};
172
173void  ?{}(__scheduler_RWLock_t & this);
174void ^?{}(__scheduler_RWLock_t & this);
175
176extern __scheduler_RWLock_t * __scheduler_lock;
177
178//-----------------------------------------------------------------------
179// Reader side : acquire when using the ready queue to schedule but not
180//  creating/destroying queues
181static inline void ready_schedule_lock( struct __processor_id_t * proc) with(*__scheduler_lock) {
182        unsigned iproc = proc->id;
183        /*paranoid*/ verify(data[iproc].handle == proc);
184        /*paranoid*/ verify(iproc < ready);
185
186        // Step 1 : make sure no writer are in the middle of the critical section
187        while(__atomic_load_n(&lock, (int)__ATOMIC_RELAXED))
188                asm volatile("pause");
189
190        // Fence needed because we don't want to start trying to acquire the lock
191        // before we read a false.
192        // Not needed on x86
193        // std::atomic_thread_fence(std::memory_order_seq_cst);
194
195        // Step 2 : acquire our local lock
196        __atomic_acquire( &data[iproc].lock );
197        /*paranoid*/ verify(data[iproc].lock);
198
199        #ifdef __CFA_WITH_VERIFY__
200                // Debug, check if this is owned for reading
201                data[iproc].owned = true;
202        #endif
203}
204
205static inline void ready_schedule_unlock( struct __processor_id_t * proc) with(*__scheduler_lock) {
206        unsigned iproc = proc->id;
207        /*paranoid*/ verify(data[iproc].handle == proc);
208        /*paranoid*/ verify(iproc < ready);
209        /*paranoid*/ verify(data[iproc].lock);
210        /*paranoid*/ verify(data[iproc].owned);
211        #ifdef __CFA_WITH_VERIFY__
212                // Debug, check if this is owned for reading
213                data[iproc].owned = false;
214        #endif
215        __atomic_unlock(&data[iproc].lock);
216}
217
218#ifdef __CFA_WITH_VERIFY__
219        static inline bool ready_schedule_islocked( struct __processor_id_t * proc) {
220                return __scheduler_lock->data[proc->id].owned;
221        }
222
223        static inline bool ready_mutate_islocked() {
224                return __scheduler_lock->lock;
225        }
226#endif
227
228//-----------------------------------------------------------------------
229// Writer side : acquire when changing the ready queue, e.g. adding more
230//  queues or removing them.
231uint_fast32_t ready_mutate_lock( void );
232
233void ready_mutate_unlock( uint_fast32_t /* value returned by lock */ );
234
235//=======================================================================
236// Ready-Queue API
237//-----------------------------------------------------------------------
238// pop thread from the ready queue of a cluster
239// returns 0p if empty
240__attribute__((hot)) bool query(struct cluster * cltr);
241
242//-----------------------------------------------------------------------
243// push thread onto a ready queue for a cluster
244// returns true if the list was previously empty, false otherwise
245__attribute__((hot)) bool push(struct cluster * cltr, struct $thread * thrd);
246
247//-----------------------------------------------------------------------
248// pop thread from the ready queue of a cluster
249// returns 0p if empty
250__attribute__((hot)) struct $thread * pop(struct cluster * cltr);
251
252//-----------------------------------------------------------------------
253// remove thread from the ready queue of a cluster
254// returns bool if it wasn't found
255bool remove_head(struct cluster * cltr, struct $thread * thrd);
256
257//-----------------------------------------------------------------------
258// Increase the width of the ready queue (number of lanes) by 4
259void ready_queue_grow  (struct cluster * cltr, int target);
260
261//-----------------------------------------------------------------------
262// Decrease the width of the ready queue (number of lanes) by 4
263void ready_queue_shrink(struct cluster * cltr, int target);
264
265
266// Local Variables: //
267// mode: c //
268// tab-width: 4 //
269// End: //
Note: See TracBrowser for help on using the repository browser.