source: libcfa/src/concurrency/kernel_private.hfa@ b4b63e8

ADT arm-eh ast-experimental enum forall-pointer-decay jacob/cs343-translation new-ast-unique-expr pthread-emulation qualifiedEnum
Last change on this file since b4b63e8 was e235429, checked in by Thierry Delisle <tdelisle@…>, 5 years ago

Removed last parker/unparker information is it was not particularly useful

  • Property mode set to 100644
File size: 9.3 KB
RevLine 
[75f3522]1//
2// Cforall Version 1.0.0 Copyright (C) 2016 University of Waterloo
3//
4// The contents of this file are covered under the licence agreement in the
5// file "LICENCE" distributed with Cforall.
6//
[73abe95]7// kernel_private.hfa --
[75f3522]8//
9// Author : Thierry Delisle
10// Created On : Mon Feb 13 12:27:26 2017
[6b0b624]11// Last Modified By : Peter A. Buhr
[fd9b524]12// Last Modified On : Wed Aug 12 08:21:33 2020
13// Update Count : 9
[75f3522]14//
15
[6b0b624]16#pragma once
[75f3522]17
[58b6d1b]18#include "kernel.hfa"
19#include "thread.hfa"
[75f3522]20
[73abe95]21#include "alarm.hfa"
[8834751]22#include "stats.hfa"
[fa21ac9]23
[75f3522]24//-----------------------------------------------------------------------------
25// Scheduler
[1c273d0]26
[37ba662]27struct __attribute__((aligned(128))) __scheduler_lock_id_t;
[9b1dcc2]28
[1c273d0]29extern "C" {
[2026bb6]30 void disable_interrupts() OPTIONAL_THREAD;
[969b3fe]31 void enable_interrupts_noPoll();
[36982fc]32 void enable_interrupts( __cfaabi_dbg_ctx_param );
[1c273d0]33}
34
[efc171d1]35void __schedule_thread( struct __processor_id_t *, $thread * )
36#if defined(NDEBUG) || (!defined(__CFA_DEBUG__) && !defined(__CFA_VERIFY__))
37 __attribute__((nonnull (2)))
38#endif
39;
[75f3522]40
[e60e0dc]41//Block current thread and release/wake-up the following resources
[b0c7419]42void __leave_thread() __attribute__((noreturn));
[db6f06a]43
[75f3522]44//-----------------------------------------------------------------------------
45// Processor
46void main(processorCtx_t *);
[85b1deb]47
[8c50aed]48void * __create_pthread( pthread_t *, void * (*)(void *), void * );
[1805b1b]49
[85b1deb]50
[75f3522]51
[6502a2b]52extern cluster * mainCluster;
53
[75f3522]54//-----------------------------------------------------------------------------
55// Threads
56extern "C" {
[c7a900a]57 void __cfactx_invoke_thread(void (*main)(void *), void * this);
[75f3522]58}
59
[f7d6bb0]60__cfaabi_dbg_debug_do(
[ac2b598]61 extern void __cfaabi_dbg_thread_register ( $thread * thrd );
62 extern void __cfaabi_dbg_thread_unregister( $thread * thrd );
[f7d6bb0]63)
64
[2d8f7b0]65// KERNEL ONLY unpark with out disabling interrupts
[e235429]66void __unpark( struct __processor_id_t *, $thread * thrd );
[2d8f7b0]67
[f00b26d4]68static inline bool __post(single_sem & this, struct __processor_id_t * id) {
69 for() {
70 struct $thread * expected = this.ptr;
71 if(expected == 1p) return false;
72 if(expected == 0p) {
73 if(__atomic_compare_exchange_n(&this.ptr, &expected, 1p, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) {
74 return false;
75 }
76 }
77 else {
78 if(__atomic_compare_exchange_n(&this.ptr, &expected, 0p, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) {
[e235429]79 __unpark( id, expected );
[f00b26d4]80 return true;
81 }
82 }
83 }
84}
[92976d9]85
[969b3fe]86//-----------------------------------------------------------------------------
87// Utils
[ac2b598]88void doregister( struct cluster * cltr, struct $thread & thrd );
89void unregister( struct cluster * cltr, struct $thread & thrd );
[de94a60]90
[f00b26d4]91//-----------------------------------------------------------------------------
92// I/O
93void ^?{}(io_context & this, bool );
94
[7768b8d]95//=======================================================================
96// Cluster lock API
97//=======================================================================
[b388ee81]98// Cells use by the reader writer lock
99// while not generic it only relies on a opaque pointer
[37ba662]100struct __attribute__((aligned(128))) __scheduler_lock_id_t {
[64a7146]101 // Spin lock used as the underlying lock
[7768b8d]102 volatile bool lock;
[64a7146]103
104 // Handle pointing to the proc owning this cell
105 // Used for allocating cells and debugging
106 __processor_id_t * volatile handle;
107
108 #ifdef __CFA_WITH_VERIFY__
109 // Debug, check if this is owned for reading
110 bool owned;
111 #endif
[7768b8d]112};
113
[64a7146]114static_assert( sizeof(struct __scheduler_lock_id_t) <= __alignof(struct __scheduler_lock_id_t));
115
[7768b8d]116// Lock-Free registering/unregistering of threads
117// Register a processor to a given cluster and get its unique id in return
[9b1dcc2]118unsigned doregister( struct __processor_id_t * proc );
[7768b8d]119
120// Unregister a processor from a given cluster using its id, getting back the original pointer
[9b1dcc2]121void unregister( struct __processor_id_t * proc );
[7768b8d]122
[1eb239e4]123//-----------------------------------------------------------------------
124// Cluster idle lock/unlock
125static inline void lock(__cluster_idles & this) {
126 for() {
127 uint64_t l = this.lock;
128 if(
129 (0 == (l % 2))
130 && __atomic_compare_exchange_n(&this.lock, &l, l + 1, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)
131 ) return;
132 Pause();
133 }
134}
135
136static inline void unlock(__cluster_idles & this) {
137 /* paranoid */ verify( 1 == (this.lock % 2) );
138 __atomic_fetch_add( &this.lock, 1, __ATOMIC_SEQ_CST );
139}
140
[7768b8d]141//=======================================================================
142// Reader-writer lock implementation
143// Concurrent with doregister/unregister,
144// i.e., threads can be added at any point during or between the entry/exit
[dca5802]145
146//-----------------------------------------------------------------------
147// simple spinlock underlying the RWLock
148// Blocking acquire
[7768b8d]149static inline void __atomic_acquire(volatile bool * ll) {
150 while( __builtin_expect(__atomic_exchange_n(ll, (bool)true, __ATOMIC_SEQ_CST), false) ) {
151 while(__atomic_load_n(ll, (int)__ATOMIC_RELAXED))
[fd9b524]152 Pause();
[7768b8d]153 }
154 /* paranoid */ verify(*ll);
155}
156
[dca5802]157// Non-Blocking acquire
[7768b8d]158static inline bool __atomic_try_acquire(volatile bool * ll) {
[b798713]159 return !__atomic_exchange_n(ll, (bool)true, __ATOMIC_SEQ_CST);
[7768b8d]160}
161
[dca5802]162// Release
[7768b8d]163static inline void __atomic_unlock(volatile bool * ll) {
164 /* paranoid */ verify(*ll);
165 __atomic_store_n(ll, (bool)false, __ATOMIC_RELEASE);
166}
167
[b388ee81]168//-----------------------------------------------------------------------
169// Reader-Writer lock protecting the ready-queues
170// while this lock is mostly generic some aspects
171// have been hard-coded to for the ready-queue for
172// simplicity and performance
173struct __scheduler_RWLock_t {
174 // total cachelines allocated
175 unsigned int max;
176
177 // cachelines currently in use
178 volatile unsigned int alloc;
179
180 // cachelines ready to itereate over
181 // (!= to alloc when thread is in second half of doregister)
182 volatile unsigned int ready;
183
184 // writer lock
185 volatile bool lock;
186
187 // data pointer
[9b1dcc2]188 __scheduler_lock_id_t * data;
[b388ee81]189};
190
191void ?{}(__scheduler_RWLock_t & this);
192void ^?{}(__scheduler_RWLock_t & this);
193
194extern __scheduler_RWLock_t * __scheduler_lock;
195
[7768b8d]196//-----------------------------------------------------------------------
197// Reader side : acquire when using the ready queue to schedule but not
198// creating/destroying queues
[9b1dcc2]199static inline void ready_schedule_lock( struct __processor_id_t * proc) with(*__scheduler_lock) {
[7768b8d]200 unsigned iproc = proc->id;
201 /*paranoid*/ verify(data[iproc].handle == proc);
202 /*paranoid*/ verify(iproc < ready);
203
204 // Step 1 : make sure no writer are in the middle of the critical section
205 while(__atomic_load_n(&lock, (int)__ATOMIC_RELAXED))
[fd9b524]206 Pause();
[7768b8d]207
208 // Fence needed because we don't want to start trying to acquire the lock
209 // before we read a false.
210 // Not needed on x86
211 // std::atomic_thread_fence(std::memory_order_seq_cst);
212
213 // Step 2 : acquire our local lock
214 __atomic_acquire( &data[iproc].lock );
215 /*paranoid*/ verify(data[iproc].lock);
[64a7146]216
217 #ifdef __CFA_WITH_VERIFY__
218 // Debug, check if this is owned for reading
219 data[iproc].owned = true;
220 #endif
[7768b8d]221}
222
[9b1dcc2]223static inline void ready_schedule_unlock( struct __processor_id_t * proc) with(*__scheduler_lock) {
[7768b8d]224 unsigned iproc = proc->id;
225 /*paranoid*/ verify(data[iproc].handle == proc);
226 /*paranoid*/ verify(iproc < ready);
227 /*paranoid*/ verify(data[iproc].lock);
[64a7146]228 /*paranoid*/ verify(data[iproc].owned);
229 #ifdef __CFA_WITH_VERIFY__
230 // Debug, check if this is owned for reading
231 data[iproc].owned = false;
232 #endif
[dca5802]233 __atomic_unlock(&data[iproc].lock);
[7768b8d]234}
235
[64a7146]236#ifdef __CFA_WITH_VERIFY__
237 static inline bool ready_schedule_islocked( struct __processor_id_t * proc) {
238 return __scheduler_lock->data[proc->id].owned;
239 }
240
241 static inline bool ready_mutate_islocked() {
242 return __scheduler_lock->lock;
243 }
244#endif
245
[7768b8d]246//-----------------------------------------------------------------------
247// Writer side : acquire when changing the ready queue, e.g. adding more
248// queues or removing them.
[b388ee81]249uint_fast32_t ready_mutate_lock( void );
[7768b8d]250
[b388ee81]251void ready_mutate_unlock( uint_fast32_t /* value returned by lock */ );
[7768b8d]252
[b798713]253//=======================================================================
254// Ready-Queue API
[64a7146]255//-----------------------------------------------------------------------
256// pop thread from the ready queue of a cluster
257// returns 0p if empty
258__attribute__((hot)) bool query(struct cluster * cltr);
259
[dca5802]260//-----------------------------------------------------------------------
261// push thread onto a ready queue for a cluster
262// returns true if the list was previously empty, false otherwise
[504a7dc]263__attribute__((hot)) bool push(struct cluster * cltr, struct $thread * thrd);
[dca5802]264
265//-----------------------------------------------------------------------
266// pop thread from the ready queue of a cluster
267// returns 0p if empty
[1eb239e4]268// May return 0p spuriously
[504a7dc]269__attribute__((hot)) struct $thread * pop(struct cluster * cltr);
[dca5802]270
[1eb239e4]271//-----------------------------------------------------------------------
272// pop thread from the ready queue of a cluster
273// returns 0p if empty
274// guaranteed to find any threads added before this call
275__attribute__((hot)) struct $thread * pop_slow(struct cluster * cltr);
276
[13c5e19]277//-----------------------------------------------------------------------
278// remove thread from the ready queue of a cluster
279// returns bool if it wasn't found
280bool remove_head(struct cluster * cltr, struct $thread * thrd);
281
[dca5802]282//-----------------------------------------------------------------------
283// Increase the width of the ready queue (number of lanes) by 4
[320ec6fc]284void ready_queue_grow (struct cluster * cltr, int target);
[dca5802]285
286//-----------------------------------------------------------------------
287// Decrease the width of the ready queue (number of lanes) by 4
[320ec6fc]288void ready_queue_shrink(struct cluster * cltr, int target);
[b798713]289
[de94a60]290
[75f3522]291// Local Variables: //
292// mode: c //
293// tab-width: 4 //
[4aa2fb2]294// End: //
Note: See TracBrowser for help on using the repository browser.