source: libcfa/src/concurrency/kernel/private.hfa@ edf247b

ADT ast-experimental enum pthread-emulation qualifiedEnum
Last change on this file since edf247b was 708ae38, checked in by Thierry Delisle <tdelisle@…>, 4 years ago

Some more cleanup and grow/shrink now readjusts io timestamps.
(They are still unused).
  • Property mode set to 100644
File size: 12.0 KB
Line 
1//
2// Cforall Version 1.0.0 Copyright (C) 2016 University of Waterloo
3//
4// The contents of this file are covered under the licence agreement in the
5// file "LICENCE" distributed with Cforall.
6//
7// kernel/private.hfa --
8//
9// Author : Thierry Delisle
10// Created On : Mon Feb 13 12:27:26 2017
11// Last Modified By : Peter A. Buhr
12// Last Modified On : Wed Aug 12 08:21:33 2020
13// Update Count : 9
14//
15
16#pragma once
17
18#if !defined(__cforall_thread__)
19 #error kernel/private.hfa should only be included in libcfathread source
20#endif
21
22#include "kernel.hfa"
23#include "thread.hfa"
24
25#include "alarm.hfa"
26#include "stats.hfa"
27
28extern "C" {
29#if defined(CFA_HAVE_LINUX_LIBRSEQ)
30 #include <rseq/rseq.h>
31#elif defined(CFA_HAVE_LINUX_RSEQ_H)
32 #include <linux/rseq.h>
33#else
34 #ifndef _GNU_SOURCE
35 #error kernel/private requires gnu_source
36 #endif
37 #include <sched.h>
38#endif
39}
40
41// Defines whether or not we *want* to use io_uring_enter as the idle_sleep blocking call
42#define CFA_WANT_IO_URING_IDLE
43
44// Defines whether or not we *can* use io_uring_enter as the idle_sleep blocking call
45#if defined(CFA_WANT_IO_URING_IDLE) && defined(CFA_HAVE_LINUX_IO_URING_H)
46 #if defined(CFA_HAVE_IORING_OP_READ) || (defined(CFA_HAVE_READV) && defined(CFA_HAVE_IORING_OP_READV))
47 #define CFA_WITH_IO_URING_IDLE
48 #endif
49#endif
50
51//-----------------------------------------------------------------------------
52// Scheduler
53extern "C" {
54 void disable_interrupts() OPTIONAL_THREAD;
55 void enable_interrupts( bool poll = true );
56}
57
58void schedule_thread$( thread$ *, unpark_hint hint ) __attribute__((nonnull (1)));
59
60extern bool __preemption_enabled();
61
62enum {
63 PREEMPT_NORMAL = 0,
64 PREEMPT_TERMINATE = 1,
65 PREEMPT_IO = 2,
66};
67
68static inline void __disable_interrupts_checked() {
69 /* paranoid */ verify( __preemption_enabled() );
70 disable_interrupts();
71 /* paranoid */ verify( ! __preemption_enabled() );
72}
73
74static inline void __enable_interrupts_checked( bool poll = true ) {
75 /* paranoid */ verify( ! __preemption_enabled() );
76 enable_interrupts( poll );
77 /* paranoid */ verify( __preemption_enabled() );
78}
79
80//release/wake-up the following resources
81void __thread_finish( thread$ * thrd );
82
83//-----------------------------------------------------------------------------
84// Hardware
85
86#if defined(CFA_HAVE_LINUX_LIBRSEQ)
87 // No data needed
88#elif defined(CFA_HAVE_LINUX_RSEQ_H)
89 extern "Cforall" {
90 extern __attribute__((aligned(128))) thread_local volatile struct rseq __cfaabi_rseq;
91 }
92#else
93 // No data needed
94#endif
95
96static inline int __kernel_getcpu() {
97 /* paranoid */ verify( ! __preemption_enabled() );
98#if defined(CFA_HAVE_LINUX_LIBRSEQ)
99 return rseq_current_cpu();
100#elif defined(CFA_HAVE_LINUX_RSEQ_H)
101 int r = __cfaabi_rseq.cpu_id;
102 /* paranoid */ verify( r >= 0 );
103 return r;
104#else
105 return sched_getcpu();
106#endif
107}
108
109//-----------------------------------------------------------------------------
110// Processor
111void main(processorCtx_t *);
112
113void * __create_pthread( pthread_t *, void * (*)(void *), void * );
114void __destroy_pthread( pthread_t pthread, void * stack, void ** retval );
115
116extern cluster * mainCluster;
117
118//-----------------------------------------------------------------------------
119// Threads
120extern "C" {
121 void __cfactx_invoke_thread(void (*main)(void *), void * this);
122}
123
124__cfaabi_dbg_debug_do(
125 extern void __cfaabi_dbg_thread_register ( thread$ * thrd );
126 extern void __cfaabi_dbg_thread_unregister( thread$ * thrd );
127)
128
129#define TICKET_BLOCKED (-1) // thread is blocked
130#define TICKET_RUNNING ( 0) // thread is running
131#define TICKET_UNBLOCK ( 1) // thread should ignore next block
132
133//-----------------------------------------------------------------------------
134// Utils
135void doregister( struct cluster * cltr, struct thread$ & thrd );
136void unregister( struct cluster * cltr, struct thread$ & thrd );
137
138//-----------------------------------------------------------------------------
139// I/O
140$io_arbiter * create(void);
141void destroy($io_arbiter *);
142
143//=======================================================================
144// Cluster lock API
145//=======================================================================
146// Lock-Free registering/unregistering of threads
147// Register a processor to a given cluster and get its unique id in return
148unsigned register_proc_id( void );
149
150// Unregister a processor from a given cluster using its id, getting back the original pointer
151void unregister_proc_id( unsigned );
152
153//=======================================================================
154// Reader-writer lock implementation
155// Concurrent with doregister/unregister,
156// i.e., threads can be added at any point during or between the entry/exit
157
158//-----------------------------------------------------------------------
159// simple spinlock underlying the RWLock
160// Blocking acquire
161static inline void __atomic_acquire(volatile bool * ll) {
162 while( __builtin_expect(__atomic_exchange_n(ll, (bool)true, __ATOMIC_SEQ_CST), false) ) {
163 while(__atomic_load_n(ll, (int)__ATOMIC_RELAXED))
164 Pause();
165 }
166 /* paranoid */ verify(*ll);
167}
168
169// Non-Blocking acquire
170static inline bool __atomic_try_acquire(volatile bool * ll) {
171 return !__atomic_exchange_n(ll, (bool)true, __ATOMIC_SEQ_CST);
172}
173
174// Release
175static inline void __atomic_unlock(volatile bool * ll) {
176 /* paranoid */ verify(*ll);
177 __atomic_store_n(ll, (bool)false, __ATOMIC_RELEASE);
178}
179
180//-----------------------------------------------------------------------
181// Reader-Writer lock protecting the ready-queues
182// while this lock is mostly generic some aspects
183// have been hard-coded to for the ready-queue for
184// simplicity and performance
185struct __scheduler_RWLock_t {
186 // total cachelines allocated
187 unsigned int max;
188
189 // cachelines currently in use
190 volatile unsigned int alloc;
191
192 // cachelines ready to itereate over
193 // (!= to alloc when thread is in second half of doregister)
194 volatile unsigned int ready;
195
196 // writer lock
197 volatile bool write_lock;
198
199 // data pointer
200 volatile bool * volatile * data;
201};
202
203void ?{}(__scheduler_RWLock_t & this);
204void ^?{}(__scheduler_RWLock_t & this);
205
206extern __scheduler_RWLock_t * __scheduler_lock;
207
208//-----------------------------------------------------------------------
209// Reader side : acquire when using the ready queue to schedule but not
210// creating/destroying queues
211static inline void ready_schedule_lock(void) with(*__scheduler_lock) {
212 /* paranoid */ verify( ! __preemption_enabled() );
213 /* paranoid */ verify( ! kernelTLS().in_sched_lock );
214 /* paranoid */ verify( data[kernelTLS().sched_id] == &kernelTLS().sched_lock );
215 /* paranoid */ verify( !kernelTLS().this_processor || kernelTLS().this_processor->unique_id == kernelTLS().sched_id );
216
217 // Step 1 : make sure no writer are in the middle of the critical section
218 while(__atomic_load_n(&write_lock, (int)__ATOMIC_RELAXED))
219 Pause();
220
221 // Fence needed because we don't want to start trying to acquire the lock
222 // before we read a false.
223 // Not needed on x86
224 // std::atomic_thread_fence(std::memory_order_seq_cst);
225
226 // Step 2 : acquire our local lock
227 __atomic_acquire( &kernelTLS().sched_lock );
228 /*paranoid*/ verify(kernelTLS().sched_lock);
229
230 #ifdef __CFA_WITH_VERIFY__
231 // Debug, check if this is owned for reading
232 kernelTLS().in_sched_lock = true;
233 #endif
234}
235
236static inline void ready_schedule_unlock(void) with(*__scheduler_lock) {
237 /* paranoid */ verify( ! __preemption_enabled() );
238 /* paranoid */ verify( data[kernelTLS().sched_id] == &kernelTLS().sched_lock );
239 /* paranoid */ verify( !kernelTLS().this_processor || kernelTLS().this_processor->unique_id == kernelTLS().sched_id );
240 /* paranoid */ verify( kernelTLS().sched_lock );
241 /* paranoid */ verify( kernelTLS().in_sched_lock );
242 #ifdef __CFA_WITH_VERIFY__
243 // Debug, check if this is owned for reading
244 kernelTLS().in_sched_lock = false;
245 #endif
246 __atomic_unlock(&kernelTLS().sched_lock);
247}
248
249#ifdef __CFA_WITH_VERIFY__
250 static inline bool ready_schedule_islocked(void) {
251 /* paranoid */ verify( ! __preemption_enabled() );
252 /* paranoid */ verify( (!kernelTLS().in_sched_lock) || kernelTLS().sched_lock );
253 return kernelTLS().sched_lock;
254 }
255
256 static inline bool ready_mutate_islocked() {
257 return __scheduler_lock->write_lock;
258 }
259#endif
260
261//-----------------------------------------------------------------------
262// Writer side : acquire when changing the ready queue, e.g. adding more
263// queues or removing them.
264uint_fast32_t ready_mutate_lock( void );
265
266void ready_mutate_unlock( uint_fast32_t /* value returned by lock */ );
267
268//-----------------------------------------------------------------------
269// Lock-Free registering/unregistering of threads
270// Register a processor to a given cluster and get its unique id in return
271// For convenience, also acquires the lock
272static inline [unsigned, uint_fast32_t] ready_mutate_register() {
273 unsigned id = register_proc_id();
274 uint_fast32_t last = ready_mutate_lock();
275 return [id, last];
276}
277
278// Unregister a processor from a given cluster using its id, getting back the original pointer
279// assumes the lock is acquired
280static inline void ready_mutate_unregister( unsigned id, uint_fast32_t last_s ) {
281 ready_mutate_unlock( last_s );
282 unregister_proc_id( id );
283}
284
285//-----------------------------------------------------------------------
286// Cluster idle lock/unlock
287static inline void lock(__cluster_proc_list & this) {
288 /* paranoid */ verify( ! __preemption_enabled() );
289
290 // Start by locking the global RWlock so that we know no-one is
291 // adding/removing processors while we mess with the idle lock
292 ready_schedule_lock();
293
294 lock( this.lock __cfaabi_dbg_ctx2 );
295
296 /* paranoid */ verify( ! __preemption_enabled() );
297}
298
299static inline bool try_lock(__cluster_proc_list & this) {
300 /* paranoid */ verify( ! __preemption_enabled() );
301
302 // Start by locking the global RWlock so that we know no-one is
303 // adding/removing processors while we mess with the idle lock
304 ready_schedule_lock();
305
306 if(try_lock( this.lock __cfaabi_dbg_ctx2 )) {
307 // success
308 /* paranoid */ verify( ! __preemption_enabled() );
309 return true;
310 }
311
312 // failed to lock
313 ready_schedule_unlock();
314
315 /* paranoid */ verify( ! __preemption_enabled() );
316 return false;
317}
318
319static inline void unlock(__cluster_proc_list & this) {
320 /* paranoid */ verify( ! __preemption_enabled() );
321
322 unlock(this.lock);
323
324 // Release the global lock, which we acquired when locking
325 ready_schedule_unlock();
326
327 /* paranoid */ verify( ! __preemption_enabled() );
328}
329
330//=======================================================================
331// Ready-Queue API
332//-----------------------------------------------------------------------
333// push thread onto a ready queue for a cluster
334// returns true if the list was previously empty, false otherwise
335__attribute__((hot)) void push(struct cluster * cltr, struct thread$ * thrd, unpark_hint hint);
336
337//-----------------------------------------------------------------------
338// pop thread from the local queues of a cluster
339// returns 0p if empty
340// May return 0p spuriously
341__attribute__((hot)) struct thread$ * pop_fast(struct cluster * cltr);
342
343//-----------------------------------------------------------------------
344// pop thread from any ready queue of a cluster
345// returns 0p if empty
346// May return 0p spuriously
347__attribute__((hot)) struct thread$ * pop_slow(struct cluster * cltr);
348
349//-----------------------------------------------------------------------
350// search all ready queues of a cluster for any thread
351// returns 0p if empty
352// guaranteed to find any threads added before this call
353__attribute__((hot)) struct thread$ * pop_search(struct cluster * cltr);
354
355//-----------------------------------------------------------------------
356// get preferred ready for new thread
357unsigned ready_queue_new_preferred();
358
359//-----------------------------------------------------------------------
360// Increase the width of the ready queue (number of lanes) by 4
361void ready_queue_grow (struct cluster * cltr);
362
363//-----------------------------------------------------------------------
364// Decrease the width of the ready queue (number of lanes) by 4
365void ready_queue_shrink(struct cluster * cltr);
366
367//-----------------------------------------------------------------------
368// Decrease the width of the ready queue (number of lanes) by 4
369void ready_queue_close(struct cluster * cltr);
370
371// Local Variables: //
372// mode: c //
373// tab-width: 4 //
374// End: //
Note: See TracBrowser for help on using the repository browser.