source: libcfa/src/concurrency/kernel_private.hfa @ 8fc652e0

arm-ehjacob/cs343-translationnew-ast-unique-expr
Last change on this file since 8fc652e0 was 8fc652e0, checked in by Thierry Delisle <tdelisle@…>, 15 months ago

Change usage of TLS to more strongly segregate in kernel and out of kernel usage.

  • Property mode set to 100644
File size: 9.2 KB
Line 
1//
2// Cforall Version 1.0.0 Copyright (C) 2016 University of Waterloo
3//
4// The contents of this file are covered under the licence agreement in the
5// file "LICENCE" distributed with Cforall.
6//
7// kernel_private.hfa --
8//
9// Author           : Thierry Delisle
10// Created On       : Mon Feb 13 12:27:26 2017
11// Last Modified By : Peter A. Buhr
12// Last Modified On : Wed Aug 12 08:21:33 2020
13// Update Count     : 9
14//
15
16#pragma once
17
18#include "kernel.hfa"
19#include "thread.hfa"
20
21#include "alarm.hfa"
22#include "stats.hfa"
23
24//-----------------------------------------------------------------------------
25// Scheduler
26
27struct __attribute__((aligned(128))) __scheduler_lock_id_t;
28
29extern "C" {
30        void disable_interrupts() OPTIONAL_THREAD;
31        void enable_interrupts_noPoll();
32        void enable_interrupts( __cfaabi_dbg_ctx_param );
33}
34
35void __schedule_thread( $thread * )
36#if defined(NDEBUG) || (!defined(__CFA_DEBUG__) && !defined(__CFA_VERIFY__))
37        __attribute__((nonnull (1)))
38#endif
39;
40
41extern bool __preemption_enabled();
42
43//release/wake-up the following resources
44void __thread_finish( $thread * thrd );
45
46//-----------------------------------------------------------------------------
47// Processor
48void main(processorCtx_t *);
49
50void * __create_pthread( pthread_t *, void * (*)(void *), void * );
51
52
53
54extern cluster * mainCluster;
55
56//-----------------------------------------------------------------------------
57// Threads
58extern "C" {
59      void __cfactx_invoke_thread(void (*main)(void *), void * this);
60}
61
62__cfaabi_dbg_debug_do(
63        extern void __cfaabi_dbg_thread_register  ( $thread * thrd );
64        extern void __cfaabi_dbg_thread_unregister( $thread * thrd );
65)
66
67#define TICKET_BLOCKED (-1) // thread is blocked
68#define TICKET_RUNNING ( 0) // thread is running
69#define TICKET_UNBLOCK ( 1) // thread should ignore next block
70
71//-----------------------------------------------------------------------------
72// Utils
73void doregister( struct cluster * cltr, struct $thread & thrd );
74void unregister( struct cluster * cltr, struct $thread & thrd );
75
76//-----------------------------------------------------------------------------
77// I/O
78void ^?{}(io_context & this, bool );
79
80//=======================================================================
81// Cluster lock API
82//=======================================================================
83// Cells use by the reader writer lock
84// while not generic it only relies on a opaque pointer
85struct __attribute__((aligned(128))) __scheduler_lock_id_t {
86        // Spin lock used as the underlying lock
87        volatile bool lock;
88
89        // Handle pointing to the proc owning this cell
90        // Used for allocating cells and debugging
91        __processor_id_t * volatile handle;
92
93        #ifdef __CFA_WITH_VERIFY__
94                // Debug, check if this is owned for reading
95                bool owned;
96        #endif
97};
98
99static_assert( sizeof(struct __scheduler_lock_id_t) <= __alignof(struct __scheduler_lock_id_t));
100
101// Lock-Free registering/unregistering of threads
102// Register a processor to a given cluster and get its unique id in return
103unsigned doregister( struct __processor_id_t * proc );
104
105// Unregister a processor from a given cluster using its id, getting back the original pointer
106void     unregister( struct __processor_id_t * proc );
107
108//-----------------------------------------------------------------------
109// Cluster idle lock/unlock
110static inline void lock(__cluster_idles & this) {
111        for() {
112                uint64_t l = this.lock;
113                if(
114                        (0 == (l % 2))
115                        && __atomic_compare_exchange_n(&this.lock, &l, l + 1, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)
116                ) return;
117                Pause();
118        }
119}
120
121static inline void unlock(__cluster_idles & this) {
122        /* paranoid */ verify( 1 == (this.lock % 2) );
123        __atomic_fetch_add( &this.lock, 1, __ATOMIC_SEQ_CST );
124}
125
126//=======================================================================
127// Reader-writer lock implementation
128// Concurrent with doregister/unregister,
129//    i.e., threads can be added at any point during or between the entry/exit
130
131//-----------------------------------------------------------------------
132// simple spinlock underlying the RWLock
133// Blocking acquire
134static inline void __atomic_acquire(volatile bool * ll) {
135        while( __builtin_expect(__atomic_exchange_n(ll, (bool)true, __ATOMIC_SEQ_CST), false) ) {
136                while(__atomic_load_n(ll, (int)__ATOMIC_RELAXED))
137                        Pause();
138        }
139        /* paranoid */ verify(*ll);
140}
141
142// Non-Blocking acquire
143static inline bool __atomic_try_acquire(volatile bool * ll) {
144        return !__atomic_exchange_n(ll, (bool)true, __ATOMIC_SEQ_CST);
145}
146
147// Release
148static inline void __atomic_unlock(volatile bool * ll) {
149        /* paranoid */ verify(*ll);
150        __atomic_store_n(ll, (bool)false, __ATOMIC_RELEASE);
151}
152
153//-----------------------------------------------------------------------
154// Reader-Writer lock protecting the ready-queues
155// while this lock is mostly generic some aspects
156// have been hard-coded to for the ready-queue for
157// simplicity and performance
158struct __scheduler_RWLock_t {
159        // total cachelines allocated
160        unsigned int max;
161
162        // cachelines currently in use
163        volatile unsigned int alloc;
164
165        // cachelines ready to itereate over
166        // (!= to alloc when thread is in second half of doregister)
167        volatile unsigned int ready;
168
169        // writer lock
170        volatile bool lock;
171
172        // data pointer
173        __scheduler_lock_id_t * data;
174};
175
176void  ?{}(__scheduler_RWLock_t & this);
177void ^?{}(__scheduler_RWLock_t & this);
178
179extern __scheduler_RWLock_t * __scheduler_lock;
180
181//-----------------------------------------------------------------------
182// Reader side : acquire when using the ready queue to schedule but not
183//  creating/destroying queues
184static inline void ready_schedule_lock(void) with(*__scheduler_lock) {
185        /* paranoid */ verify( ! __preemption_enabled() );
186        /* paranoid */ verify( kernelTLS().this_proc_id );
187
188        unsigned iproc = kernelTLS().this_proc_id->id;
189        /*paranoid*/ verify(data[iproc].handle == kernelTLS().this_proc_id);
190        /*paranoid*/ verify(iproc < ready);
191
192        // Step 1 : make sure no writer are in the middle of the critical section
193        while(__atomic_load_n(&lock, (int)__ATOMIC_RELAXED))
194                Pause();
195
196        // Fence needed because we don't want to start trying to acquire the lock
197        // before we read a false.
198        // Not needed on x86
199        // std::atomic_thread_fence(std::memory_order_seq_cst);
200
201        // Step 2 : acquire our local lock
202        __atomic_acquire( &data[iproc].lock );
203        /*paranoid*/ verify(data[iproc].lock);
204
205        #ifdef __CFA_WITH_VERIFY__
206                // Debug, check if this is owned for reading
207                data[iproc].owned = true;
208        #endif
209}
210
211static inline void ready_schedule_unlock(void) with(*__scheduler_lock) {
212        /* paranoid */ verify( ! __preemption_enabled() );
213        /* paranoid */ verify( kernelTLS().this_proc_id );
214
215        unsigned iproc = kernelTLS().this_proc_id->id;
216        /*paranoid*/ verify(data[iproc].handle == kernelTLS().this_proc_id);
217        /*paranoid*/ verify(iproc < ready);
218        /*paranoid*/ verify(data[iproc].lock);
219        /*paranoid*/ verify(data[iproc].owned);
220        #ifdef __CFA_WITH_VERIFY__
221                // Debug, check if this is owned for reading
222                data[iproc].owned = false;
223        #endif
224        __atomic_unlock(&data[iproc].lock);
225}
226
227#ifdef __CFA_WITH_VERIFY__
228        static inline bool ready_schedule_islocked(void) {
229                /* paranoid */ verify( ! __preemption_enabled() );
230                /*paranoid*/ verify( kernelTLS().this_proc_id );
231                __processor_id_t * proc = kernelTLS().this_proc_id;
232                return __scheduler_lock->data[proc->id].owned;
233        }
234
235        static inline bool ready_mutate_islocked() {
236                return __scheduler_lock->lock;
237        }
238#endif
239
240//-----------------------------------------------------------------------
241// Writer side : acquire when changing the ready queue, e.g. adding more
242//  queues or removing them.
243uint_fast32_t ready_mutate_lock( void );
244
245void ready_mutate_unlock( uint_fast32_t /* value returned by lock */ );
246
247//=======================================================================
248// Ready-Queue API
249//-----------------------------------------------------------------------
250// pop thread from the ready queue of a cluster
251// returns 0p if empty
252__attribute__((hot)) bool query(struct cluster * cltr);
253
254//-----------------------------------------------------------------------
255// push thread onto a ready queue for a cluster
256// returns true if the list was previously empty, false otherwise
257__attribute__((hot)) bool push(struct cluster * cltr, struct $thread * thrd);
258
259//-----------------------------------------------------------------------
260// pop thread from the ready queue of a cluster
261// returns 0p if empty
262// May return 0p spuriously
263__attribute__((hot)) struct $thread * pop(struct cluster * cltr);
264
265//-----------------------------------------------------------------------
266// pop thread from the ready queue of a cluster
267// returns 0p if empty
268// guaranteed to find any threads added before this call
269__attribute__((hot)) struct $thread * pop_slow(struct cluster * cltr);
270
271//-----------------------------------------------------------------------
272// remove thread from the ready queue of a cluster
273// returns bool if it wasn't found
274bool remove_head(struct cluster * cltr, struct $thread * thrd);
275
276//-----------------------------------------------------------------------
277// Increase the width of the ready queue (number of lanes) by 4
278void ready_queue_grow  (struct cluster * cltr, int target);
279
280//-----------------------------------------------------------------------
281// Decrease the width of the ready queue (number of lanes) by 4
282void ready_queue_shrink(struct cluster * cltr, int target);
283
284
285// Local Variables: //
286// mode: c //
287// tab-width: 4 //
288// End: //
Note: See TracBrowser for help on using the repository browser.