source: libcfa/src/concurrency/kernel_private.hfa @ e873838

ADTarm-ehast-experimentalenumforall-pointer-decayjacob/cs343-translationnew-ast-unique-exprpthread-emulationqualifiedEnum
Last change on this file since e873838 was e873838, checked in by Thierry Delisle <tdelisle@…>, 3 years ago

Removed unpark and added support for unpark from the kernel (removing the distinction between the two

  • Property mode set to 100644
File size: 9.0 KB
RevLine 
[75f3522]1//
2// Cforall Version 1.0.0 Copyright (C) 2016 University of Waterloo
3//
4// The contents of this file are covered under the licence agreement in the
5// file "LICENCE" distributed with Cforall.
6//
[73abe95]7// kernel_private.hfa --
[75f3522]8//
9// Author           : Thierry Delisle
10// Created On       : Mon Feb 13 12:27:26 2017
[6b0b624]11// Last Modified By : Peter A. Buhr
[fd9b524]12// Last Modified On : Wed Aug 12 08:21:33 2020
13// Update Count     : 9
[75f3522]14//
15
[6b0b624]16#pragma once
[75f3522]17
[58b6d1b]18#include "kernel.hfa"
19#include "thread.hfa"
[75f3522]20
[73abe95]21#include "alarm.hfa"
[8834751]22#include "stats.hfa"
[fa21ac9]23
[75f3522]24//-----------------------------------------------------------------------------
25// Scheduler
[1c273d0]26
[37ba662]27struct __attribute__((aligned(128))) __scheduler_lock_id_t;
[9b1dcc2]28
[1c273d0]29extern "C" {
[2026bb6]30        void disable_interrupts() OPTIONAL_THREAD;
[969b3fe]31        void enable_interrupts_noPoll();
[36982fc]32        void enable_interrupts( __cfaabi_dbg_ctx_param );
[1c273d0]33}
34
[e873838]35void __schedule_thread( $thread * )
[efc171d1]36#if defined(NDEBUG) || (!defined(__CFA_DEBUG__) && !defined(__CFA_VERIFY__))
[e873838]37        __attribute__((nonnull (1)))
[efc171d1]38#endif
39;
[75f3522]40
[5afb49a]41//release/wake-up the following resources
42void __thread_finish( $thread * thrd );
[db6f06a]43
[75f3522]44//-----------------------------------------------------------------------------
45// Processor
46void main(processorCtx_t *);
[85b1deb]47
[8c50aed]48void * __create_pthread( pthread_t *, void * (*)(void *), void * );
[1805b1b]49
[85b1deb]50
[75f3522]51
[6502a2b]52extern cluster * mainCluster;
53
[75f3522]54//-----------------------------------------------------------------------------
55// Threads
56extern "C" {
[c7a900a]57      void __cfactx_invoke_thread(void (*main)(void *), void * this);
[75f3522]58}
59
[f7d6bb0]60__cfaabi_dbg_debug_do(
[ac2b598]61        extern void __cfaabi_dbg_thread_register  ( $thread * thrd );
62        extern void __cfaabi_dbg_thread_unregister( $thread * thrd );
[f7d6bb0]63)
64
[6a77224]65#define TICKET_BLOCKED (-1) // thread is blocked
66#define TICKET_RUNNING ( 0) // thread is running
67#define TICKET_UNBLOCK ( 1) // thread should ignore next block
68
[969b3fe]69//-----------------------------------------------------------------------------
70// Utils
[ac2b598]71void doregister( struct cluster * cltr, struct $thread & thrd );
72void unregister( struct cluster * cltr, struct $thread & thrd );
[de94a60]73
[f00b26d4]74//-----------------------------------------------------------------------------
75// I/O
76void ^?{}(io_context & this, bool );
77
[7768b8d]78//=======================================================================
79// Cluster lock API
80//=======================================================================
[b388ee81]81// Cells use by the reader writer lock
82// while not generic it only relies on a opaque pointer
[37ba662]83struct __attribute__((aligned(128))) __scheduler_lock_id_t {
[64a7146]84        // Spin lock used as the underlying lock
[7768b8d]85        volatile bool lock;
[64a7146]86
87        // Handle pointing to the proc owning this cell
88        // Used for allocating cells and debugging
89        __processor_id_t * volatile handle;
90
91        #ifdef __CFA_WITH_VERIFY__
92                // Debug, check if this is owned for reading
93                bool owned;
94        #endif
[7768b8d]95};
96
[64a7146]97static_assert( sizeof(struct __scheduler_lock_id_t) <= __alignof(struct __scheduler_lock_id_t));
98
[7768b8d]99// Lock-Free registering/unregistering of threads
100// Register a processor to a given cluster and get its unique id in return
[9b1dcc2]101unsigned doregister( struct __processor_id_t * proc );
[7768b8d]102
103// Unregister a processor from a given cluster using its id, getting back the original pointer
[9b1dcc2]104void     unregister( struct __processor_id_t * proc );
[7768b8d]105
[1eb239e4]106//-----------------------------------------------------------------------
107// Cluster idle lock/unlock
108static inline void lock(__cluster_idles & this) {
109        for() {
110                uint64_t l = this.lock;
111                if(
112                        (0 == (l % 2))
113                        && __atomic_compare_exchange_n(&this.lock, &l, l + 1, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)
114                ) return;
115                Pause();
116        }
117}
118
119static inline void unlock(__cluster_idles & this) {
120        /* paranoid */ verify( 1 == (this.lock % 2) );
121        __atomic_fetch_add( &this.lock, 1, __ATOMIC_SEQ_CST );
122}
123
[7768b8d]124//=======================================================================
125// Reader-writer lock implementation
126// Concurrent with doregister/unregister,
127//    i.e., threads can be added at any point during or between the entry/exit
[dca5802]128
129//-----------------------------------------------------------------------
130// simple spinlock underlying the RWLock
131// Blocking acquire
[7768b8d]132static inline void __atomic_acquire(volatile bool * ll) {
133        while( __builtin_expect(__atomic_exchange_n(ll, (bool)true, __ATOMIC_SEQ_CST), false) ) {
134                while(__atomic_load_n(ll, (int)__ATOMIC_RELAXED))
[fd9b524]135                        Pause();
[7768b8d]136        }
137        /* paranoid */ verify(*ll);
138}
139
[dca5802]140// Non-Blocking acquire
[7768b8d]141static inline bool __atomic_try_acquire(volatile bool * ll) {
[b798713]142        return !__atomic_exchange_n(ll, (bool)true, __ATOMIC_SEQ_CST);
[7768b8d]143}
144
[dca5802]145// Release
[7768b8d]146static inline void __atomic_unlock(volatile bool * ll) {
147        /* paranoid */ verify(*ll);
148        __atomic_store_n(ll, (bool)false, __ATOMIC_RELEASE);
149}
150
[b388ee81]151//-----------------------------------------------------------------------
152// Reader-Writer lock protecting the ready-queues
153// while this lock is mostly generic some aspects
154// have been hard-coded to for the ready-queue for
155// simplicity and performance
156struct __scheduler_RWLock_t {
157        // total cachelines allocated
158        unsigned int max;
159
160        // cachelines currently in use
161        volatile unsigned int alloc;
162
163        // cachelines ready to itereate over
164        // (!= to alloc when thread is in second half of doregister)
165        volatile unsigned int ready;
166
167        // writer lock
168        volatile bool lock;
169
170        // data pointer
[9b1dcc2]171        __scheduler_lock_id_t * data;
[b388ee81]172};
173
174void  ?{}(__scheduler_RWLock_t & this);
175void ^?{}(__scheduler_RWLock_t & this);
176
177extern __scheduler_RWLock_t * __scheduler_lock;
178
[7768b8d]179//-----------------------------------------------------------------------
180// Reader side : acquire when using the ready queue to schedule but not
181//  creating/destroying queues
[e873838]182static inline void ready_schedule_lock(void) with(*__scheduler_lock) {
183        /*paranoid*/ verify( kernelTLS.this_proc_id );
184
185        unsigned iproc = kernelTLS.this_proc_id->id;
186        /*paranoid*/ verify(data[iproc].handle == kernelTLS.this_proc_id);
[7768b8d]187        /*paranoid*/ verify(iproc < ready);
188
189        // Step 1 : make sure no writer are in the middle of the critical section
190        while(__atomic_load_n(&lock, (int)__ATOMIC_RELAXED))
[fd9b524]191                Pause();
[7768b8d]192
193        // Fence needed because we don't want to start trying to acquire the lock
194        // before we read a false.
195        // Not needed on x86
196        // std::atomic_thread_fence(std::memory_order_seq_cst);
197
198        // Step 2 : acquire our local lock
199        __atomic_acquire( &data[iproc].lock );
200        /*paranoid*/ verify(data[iproc].lock);
[64a7146]201
202        #ifdef __CFA_WITH_VERIFY__
203                // Debug, check if this is owned for reading
204                data[iproc].owned = true;
205        #endif
[7768b8d]206}
207
[e873838]208static inline void ready_schedule_unlock(void) with(*__scheduler_lock) {
209        /*paranoid*/ verify( kernelTLS.this_proc_id );
210
211        unsigned iproc = kernelTLS.this_proc_id->id;
212        /*paranoid*/ verify(data[iproc].handle == kernelTLS.this_proc_id);
[7768b8d]213        /*paranoid*/ verify(iproc < ready);
214        /*paranoid*/ verify(data[iproc].lock);
[64a7146]215        /*paranoid*/ verify(data[iproc].owned);
216        #ifdef __CFA_WITH_VERIFY__
217                // Debug, check if this is owned for reading
218                data[iproc].owned = false;
219        #endif
[dca5802]220        __atomic_unlock(&data[iproc].lock);
[7768b8d]221}
222
[64a7146]223#ifdef __CFA_WITH_VERIFY__
[e873838]224        static inline bool ready_schedule_islocked(void) {
225                /*paranoid*/ verify( kernelTLS.this_proc_id );
226                __processor_id_t * proc = kernelTLS.this_proc_id;
[64a7146]227                return __scheduler_lock->data[proc->id].owned;
228        }
229
230        static inline bool ready_mutate_islocked() {
231                return __scheduler_lock->lock;
232        }
233#endif
234
[7768b8d]235//-----------------------------------------------------------------------
236// Writer side : acquire when changing the ready queue, e.g. adding more
237//  queues or removing them.
[b388ee81]238uint_fast32_t ready_mutate_lock( void );
[7768b8d]239
[b388ee81]240void ready_mutate_unlock( uint_fast32_t /* value returned by lock */ );
[7768b8d]241
[b798713]242//=======================================================================
243// Ready-Queue API
[64a7146]244//-----------------------------------------------------------------------
245// pop thread from the ready queue of a cluster
246// returns 0p if empty
247__attribute__((hot)) bool query(struct cluster * cltr);
248
[dca5802]249//-----------------------------------------------------------------------
250// push thread onto a ready queue for a cluster
251// returns true if the list was previously empty, false otherwise
[504a7dc]252__attribute__((hot)) bool push(struct cluster * cltr, struct $thread * thrd);
[dca5802]253
254//-----------------------------------------------------------------------
255// pop thread from the ready queue of a cluster
256// returns 0p if empty
[1eb239e4]257// May return 0p spuriously
[504a7dc]258__attribute__((hot)) struct $thread * pop(struct cluster * cltr);
[dca5802]259
[1eb239e4]260//-----------------------------------------------------------------------
261// pop thread from the ready queue of a cluster
262// returns 0p if empty
263// guaranteed to find any threads added before this call
264__attribute__((hot)) struct $thread * pop_slow(struct cluster * cltr);
265
[13c5e19]266//-----------------------------------------------------------------------
267// remove thread from the ready queue of a cluster
268// returns bool if it wasn't found
269bool remove_head(struct cluster * cltr, struct $thread * thrd);
270
[dca5802]271//-----------------------------------------------------------------------
272// Increase the width of the ready queue (number of lanes) by 4
[320ec6fc]273void ready_queue_grow  (struct cluster * cltr, int target);
[dca5802]274
275//-----------------------------------------------------------------------
276// Decrease the width of the ready queue (number of lanes) by 4
[320ec6fc]277void ready_queue_shrink(struct cluster * cltr, int target);
[b798713]278
[de94a60]279
[75f3522]280// Local Variables: //
281// mode: c //
282// tab-width: 4 //
[4aa2fb2]283// End: //
Note: See TracBrowser for help on using the repository browser.