source: libcfa/src/concurrency/kernel_private.hfa@ 7770cc8

ADT ast-experimental enum pthread-emulation qualifiedEnum
Last change on this file since 7770cc8 was 059ad16, checked in by Thierry Delisle <tdelisle@…>, 4 years ago

Flush now supports blocking until at least one I/O op terminates.

  • Property mode set to 100644
File size: 11.0 KB
RevLine 
[75f3522]1//
2// Cforall Version 1.0.0 Copyright (C) 2016 University of Waterloo
3//
4// The contents of this file are covered under the licence agreement in the
5// file "LICENCE" distributed with Cforall.
6//
[73abe95]7// kernel_private.hfa --
[75f3522]8//
9// Author : Thierry Delisle
10// Created On : Mon Feb 13 12:27:26 2017
[6b0b624]11// Last Modified By : Peter A. Buhr
[fd9b524]12// Last Modified On : Wed Aug 12 08:21:33 2020
13// Update Count : 9
[75f3522]14//
15
[6b0b624]16#pragma once
[75f3522]17
[3489ea6]18#if !defined(__cforall_thread__)
19 #error kernel_private.hfa should only be included in libcfathread source
20#endif
21
[58b6d1b]22#include "kernel.hfa"
23#include "thread.hfa"
[75f3522]24
[73abe95]25#include "alarm.hfa"
[8834751]26#include "stats.hfa"
[fa21ac9]27
[3489ea6]28extern "C" {
29#if defined(CFA_HAVE_LINUX_LIBRSEQ)
30 #include <rseq/rseq.h>
31#elif defined(CFA_HAVE_LINUX_RSEQ_H)
[f558b5f]32 #include <linux/rseq.h>
[3489ea6]33#else
34 #ifndef _GNU_SOURCE
35 #error kernel_private requires gnu_source
36 #endif
37 #include <sched.h>
38#endif
39}
40
[059ad16]41// #define IO_URING_IDLE
42
[75f3522]43//-----------------------------------------------------------------------------
44// Scheduler
[1c273d0]45extern "C" {
[2026bb6]46 void disable_interrupts() OPTIONAL_THREAD;
[a3821fa]47 void enable_interrupts( bool poll = true );
[1c273d0]48}
49
[24e321c]50void schedule_thread$( thread$ *, unpark_hint hint ) __attribute__((nonnull (1)));
[75f3522]51
[8fc652e0]52extern bool __preemption_enabled();
53
[5afb49a]54//release/wake-up the following resources
[e84ab3d]55void __thread_finish( thread$ * thrd );
[db6f06a]56
[3489ea6]57//-----------------------------------------------------------------------------
58// Hardware
59
60#if defined(CFA_HAVE_LINUX_LIBRSEQ)
61 // No data needed
62#elif defined(CFA_HAVE_LINUX_RSEQ_H)
63 extern "Cforall" {
[f558b5f]64 extern __attribute__((aligned(128))) thread_local volatile struct rseq __cfaabi_rseq;
[3489ea6]65 }
66#else
67 // No data needed
68#endif
69
70static inline int __kernel_getcpu() {
71 /* paranoid */ verify( ! __preemption_enabled() );
72#if defined(CFA_HAVE_LINUX_LIBRSEQ)
[f558b5f]73 return rseq_current_cpu();
[3489ea6]74#elif defined(CFA_HAVE_LINUX_RSEQ_H)
[f558b5f]75 int r = __cfaabi_rseq.cpu_id;
76 /* paranoid */ verify( r >= 0 );
77 return r;
[3489ea6]78#else
79 return sched_getcpu();
80#endif
81}
82
[75f3522]83//-----------------------------------------------------------------------------
84// Processor
85void main(processorCtx_t *);
[85b1deb]86
[8c50aed]87void * __create_pthread( pthread_t *, void * (*)(void *), void * );
[bfcf6b9]88void __destroy_pthread( pthread_t pthread, void * stack, void ** retval );
[1805b1b]89
[6502a2b]90extern cluster * mainCluster;
91
[75f3522]92//-----------------------------------------------------------------------------
93// Threads
94extern "C" {
[c7a900a]95 void __cfactx_invoke_thread(void (*main)(void *), void * this);
[75f3522]96}
97
[f7d6bb0]98__cfaabi_dbg_debug_do(
[e84ab3d]99 extern void __cfaabi_dbg_thread_register ( thread$ * thrd );
100 extern void __cfaabi_dbg_thread_unregister( thread$ * thrd );
[f7d6bb0]101)
102
[6a77224]103#define TICKET_BLOCKED (-1) // thread is blocked
104#define TICKET_RUNNING ( 0) // thread is running
105#define TICKET_UNBLOCK ( 1) // thread should ignore next block
106
[969b3fe]107//-----------------------------------------------------------------------------
108// Utils
[e84ab3d]109void doregister( struct cluster * cltr, struct thread$ & thrd );
110void unregister( struct cluster * cltr, struct thread$ & thrd );
[de94a60]111
[f00b26d4]112//-----------------------------------------------------------------------------
113// I/O
[78da4ab]114$io_arbiter * create(void);
115void destroy($io_arbiter *);
[f00b26d4]116
[7768b8d]117//=======================================================================
118// Cluster lock API
119//=======================================================================
120// Lock-Free registering/unregistering of threads
121// Register a processor to a given cluster and get its unique id in return
[c993b15]122unsigned register_proc_id( void );
[7768b8d]123
124// Unregister a processor from a given cluster using its id, getting back the original pointer
[c993b15]125void unregister_proc_id( unsigned );
[7768b8d]126
127//=======================================================================
128// Reader-writer lock implementation
129// Concurrent with doregister/unregister,
130// i.e., threads can be added at any point during or between the entry/exit
[dca5802]131
132//-----------------------------------------------------------------------
133// simple spinlock underlying the RWLock
134// Blocking acquire
[7768b8d]135static inline void __atomic_acquire(volatile bool * ll) {
136 while( __builtin_expect(__atomic_exchange_n(ll, (bool)true, __ATOMIC_SEQ_CST), false) ) {
137 while(__atomic_load_n(ll, (int)__ATOMIC_RELAXED))
[fd9b524]138 Pause();
[7768b8d]139 }
140 /* paranoid */ verify(*ll);
141}
142
[dca5802]143// Non-Blocking acquire
[7768b8d]144static inline bool __atomic_try_acquire(volatile bool * ll) {
[b798713]145 return !__atomic_exchange_n(ll, (bool)true, __ATOMIC_SEQ_CST);
[7768b8d]146}
147
[dca5802]148// Release
[7768b8d]149static inline void __atomic_unlock(volatile bool * ll) {
150 /* paranoid */ verify(*ll);
151 __atomic_store_n(ll, (bool)false, __ATOMIC_RELEASE);
152}
153
[b388ee81]154//-----------------------------------------------------------------------
155// Reader-Writer lock protecting the ready-queues
156// while this lock is mostly generic some aspects
157// have been hard-coded to for the ready-queue for
158// simplicity and performance
159struct __scheduler_RWLock_t {
160 // total cachelines allocated
161 unsigned int max;
162
163 // cachelines currently in use
164 volatile unsigned int alloc;
165
166 // cachelines ready to itereate over
167 // (!= to alloc when thread is in second half of doregister)
168 volatile unsigned int ready;
169
170 // writer lock
[c993b15]171 volatile bool write_lock;
[b388ee81]172
173 // data pointer
[c993b15]174 volatile bool * volatile * data;
[b388ee81]175};
176
177void ?{}(__scheduler_RWLock_t & this);
178void ^?{}(__scheduler_RWLock_t & this);
179
180extern __scheduler_RWLock_t * __scheduler_lock;
181
[7768b8d]182//-----------------------------------------------------------------------
183// Reader side : acquire when using the ready queue to schedule but not
184// creating/destroying queues
[e873838]185static inline void ready_schedule_lock(void) with(*__scheduler_lock) {
[8fc652e0]186 /* paranoid */ verify( ! __preemption_enabled() );
[c993b15]187 /* paranoid */ verify( ! kernelTLS().in_sched_lock );
188 /* paranoid */ verify( data[kernelTLS().sched_id] == &kernelTLS().sched_lock );
189 /* paranoid */ verify( !kernelTLS().this_processor || kernelTLS().this_processor->unique_id == kernelTLS().sched_id );
[7768b8d]190
191 // Step 1 : make sure no writer are in the middle of the critical section
[c993b15]192 while(__atomic_load_n(&write_lock, (int)__ATOMIC_RELAXED))
[fd9b524]193 Pause();
[7768b8d]194
195 // Fence needed because we don't want to start trying to acquire the lock
196 // before we read a false.
197 // Not needed on x86
198 // std::atomic_thread_fence(std::memory_order_seq_cst);
199
200 // Step 2 : acquire our local lock
[c993b15]201 __atomic_acquire( &kernelTLS().sched_lock );
202 /*paranoid*/ verify(kernelTLS().sched_lock);
[64a7146]203
204 #ifdef __CFA_WITH_VERIFY__
205 // Debug, check if this is owned for reading
[c993b15]206 kernelTLS().in_sched_lock = true;
[64a7146]207 #endif
[7768b8d]208}
209
[e873838]210static inline void ready_schedule_unlock(void) with(*__scheduler_lock) {
[8fc652e0]211 /* paranoid */ verify( ! __preemption_enabled() );
[c993b15]212 /* paranoid */ verify( data[kernelTLS().sched_id] == &kernelTLS().sched_lock );
213 /* paranoid */ verify( !kernelTLS().this_processor || kernelTLS().this_processor->unique_id == kernelTLS().sched_id );
214 /* paranoid */ verify( kernelTLS().sched_lock );
215 /* paranoid */ verify( kernelTLS().in_sched_lock );
[64a7146]216 #ifdef __CFA_WITH_VERIFY__
217 // Debug, check if this is owned for reading
[c993b15]218 kernelTLS().in_sched_lock = false;
[64a7146]219 #endif
[c993b15]220 __atomic_unlock(&kernelTLS().sched_lock);
[7768b8d]221}
222
[64a7146]223#ifdef __CFA_WITH_VERIFY__
[e873838]224 static inline bool ready_schedule_islocked(void) {
[8fc652e0]225 /* paranoid */ verify( ! __preemption_enabled() );
[c993b15]226 /* paranoid */ verify( (!kernelTLS().in_sched_lock) || kernelTLS().sched_lock );
227 return kernelTLS().sched_lock;
[64a7146]228 }
229
230 static inline bool ready_mutate_islocked() {
[c993b15]231 return __scheduler_lock->write_lock;
[64a7146]232 }
233#endif
234
[7768b8d]235//-----------------------------------------------------------------------
236// Writer side : acquire when changing the ready queue, e.g. adding more
237// queues or removing them.
[b388ee81]238uint_fast32_t ready_mutate_lock( void );
[7768b8d]239
[b388ee81]240void ready_mutate_unlock( uint_fast32_t /* value returned by lock */ );
[7768b8d]241
[a33c113]242//-----------------------------------------------------------------------
243// Lock-Free registering/unregistering of threads
244// Register a processor to a given cluster and get its unique id in return
245// For convenience, also acquires the lock
[c993b15]246static inline [unsigned, uint_fast32_t] ready_mutate_register() {
247 unsigned id = register_proc_id();
248 uint_fast32_t last = ready_mutate_lock();
249 return [id, last];
[a33c113]250}
251
252// Unregister a processor from a given cluster using its id, getting back the original pointer
253// assumes the lock is acquired
[c993b15]254static inline void ready_mutate_unregister( unsigned id, uint_fast32_t last_s ) {
[a33c113]255 ready_mutate_unlock( last_s );
[c993b15]256 unregister_proc_id( id );
[a33c113]257}
258
[a7504db5]259//-----------------------------------------------------------------------
260// Cluster idle lock/unlock
[6a9b12b]261static inline void lock(__cluster_proc_list & this) {
[a7504db5]262 /* paranoid */ verify( ! __preemption_enabled() );
263
264 // Start by locking the global RWlock so that we know no-one is
265 // adding/removing processors while we mess with the idle lock
266 ready_schedule_lock();
267
[a633f6f]268 lock( this.lock __cfaabi_dbg_ctx2 );
[a7504db5]269
270 /* paranoid */ verify( ! __preemption_enabled() );
271}
272
[5f5a729]273static inline bool try_lock(__cluster_proc_list & this) {
274 /* paranoid */ verify( ! __preemption_enabled() );
275
276 // Start by locking the global RWlock so that we know no-one is
277 // adding/removing processors while we mess with the idle lock
278 ready_schedule_lock();
279
[a633f6f]280 if(try_lock( this.lock __cfaabi_dbg_ctx2 )) {
[5f5a729]281 // success
282 /* paranoid */ verify( ! __preemption_enabled() );
283 return true;
284 }
285
286 // failed to lock
287 ready_schedule_unlock();
288
289 /* paranoid */ verify( ! __preemption_enabled() );
290 return false;
291}
292
[6a9b12b]293static inline void unlock(__cluster_proc_list & this) {
[a7504db5]294 /* paranoid */ verify( ! __preemption_enabled() );
295
[a633f6f]296 unlock(this.lock);
[a7504db5]297
298 // Release the global lock, which we acquired when locking
299 ready_schedule_unlock();
300
301 /* paranoid */ verify( ! __preemption_enabled() );
302}
303
[b798713]304//=======================================================================
305// Ready-Queue API
[dca5802]306//-----------------------------------------------------------------------
307// push thread onto a ready queue for a cluster
308// returns true if the list was previously empty, false otherwise
[24e321c]309__attribute__((hot)) void push(struct cluster * cltr, struct thread$ * thrd, unpark_hint hint);
[dca5802]310
311//-----------------------------------------------------------------------
[fc59df78]312// pop thread from the local queues of a cluster
[dca5802]313// returns 0p if empty
[1eb239e4]314// May return 0p spuriously
[e84ab3d]315__attribute__((hot)) struct thread$ * pop_fast(struct cluster * cltr);
[dca5802]316
[1eb239e4]317//-----------------------------------------------------------------------
[fc59df78]318// pop thread from any ready queue of a cluster
[1eb239e4]319// returns 0p if empty
[fc59df78]320// May return 0p spuriously
[e84ab3d]321__attribute__((hot)) struct thread$ * pop_slow(struct cluster * cltr);
[1eb239e4]322
[fc59df78]323//-----------------------------------------------------------------------
324// search all ready queues of a cluster for any thread
325// returns 0p if empty
326// guaranteed to find any threads added before this call
[e84ab3d]327__attribute__((hot)) struct thread$ * pop_search(struct cluster * cltr);
[fc59df78]328
[24e321c]329//-----------------------------------------------------------------------
330// get preferred ready for new thread
331unsigned ready_queue_new_preferred();
332
[dca5802]333//-----------------------------------------------------------------------
334// Increase the width of the ready queue (number of lanes) by 4
[a017ee7]335void ready_queue_grow (struct cluster * cltr);
[dca5802]336
337//-----------------------------------------------------------------------
338// Decrease the width of the ready queue (number of lanes) by 4
[a017ee7]339void ready_queue_shrink(struct cluster * cltr);
[b798713]340
[de94a60]341
[75f3522]342// Local Variables: //
343// mode: c //
344// tab-width: 4 //
[4aa2fb2]345// End: //
Note: See TracBrowser for help on using the repository browser.