source: libcfa/src/concurrency/kernel.hfa @ d3ab183

ADTarm-ehast-experimentalenumforall-pointer-decayjacob/cs343-translationnew-astnew-ast-unique-exprpthread-emulationqualifiedEnum
Last change on this file since d3ab183 was f6660520, checked in by Thierry Delisle <tdelisle@…>, 4 years ago

Added new implementation of io_uring that uses user-thread

  • Property mode set to 100644
File size: 6.6 KB
Line 
1//
2// Cforall Version 1.0.0 Copyright (C) 2016 University of Waterloo
3//
4// The contents of this file are covered under the licence agreement in the
5// file "LICENCE" distributed with Cforall.
6//
7// kernel --
8//
9// Author           : Thierry Delisle
10// Created On       : Tue Jan 17 12:27:26 2017
11// Last Modified By : Peter A. Buhr
12// Last Modified On : Tue Feb  4 12:29:26 2020
13// Update Count     : 22
14//
15
16#pragma once
17
18#include <stdbool.h>
19#include <stdint.h>
20
21#include "invoke.h"
22#include "time_t.hfa"
23#include "coroutine.hfa"
24
25extern "C" {
26#include <pthread.h>
27#include <semaphore.h>
28}
29
30//-----------------------------------------------------------------------------
31// Locks
32struct semaphore {
33        __spinlock_t lock;
34        int count;
35        __queue_t($thread) waiting;
36};
37
38void  ?{}(semaphore & this, int count = 1);
39void ^?{}(semaphore & this);
40void   P (semaphore & this);
41bool   V (semaphore & this);
42bool   V (semaphore & this, unsigned count);
43
44
45//-----------------------------------------------------------------------------
46// Processor
47extern struct cluster * mainCluster;
48
49// Processor
50coroutine processorCtx_t {
51        struct processor * proc;
52};
53
54// Wrapper around kernel threads
55struct processor {
56        // Main state
57        // Coroutine ctx who does keeps the state of the processor
58        struct processorCtx_t runner;
59
60        // Cluster from which to get threads
61        struct cluster * cltr;
62
63        // Name of the processor
64        const char * name;
65
66        // Handle to pthreads
67        pthread_t kernel_thread;
68
69        // RunThread data
70        // Action to do after a thread is ran
71        $thread * destroyer;
72
73        // Preemption data
74        // Node which is added in the discrete event simulaiton
75        struct alarm_node_t * preemption_alarm;
76
77        // If true, a preemption was triggered in an unsafe region, the processor must preempt as soon as possible
78        bool pending_preemption;
79
80        // Idle lock (kernel semaphore)
81        __bin_sem_t idle;
82
83        // Termination
84        // Set to true to notify the processor should terminate
85        volatile bool do_terminate;
86
87        // Termination synchronisation (user semaphore)
88        semaphore terminated;
89
90        // pthread Stack
91        void * stack;
92
93        // Link lists fields
94        struct __dbg_node_proc {
95                struct processor * next;
96                struct processor * prev;
97        } node;
98
99#ifdef __CFA_DEBUG__
100        // Last function to enable preemption on this processor
101        const char * last_enable;
102#endif
103};
104
105void  ?{}(processor & this, const char name[], struct cluster & cltr);
106void ^?{}(processor & this);
107
108static inline void  ?{}(processor & this)                    { this{ "Anonymous Processor", *mainCluster}; }
109static inline void  ?{}(processor & this, struct cluster & cltr)    { this{ "Anonymous Processor", cltr}; }
110static inline void  ?{}(processor & this, const char name[]) { this{name, *mainCluster }; }
111
112static inline [processor *&, processor *& ] __get( processor & this ) __attribute__((const)) { return this.node.[next, prev]; }
113
114//-----------------------------------------------------------------------------
115// I/O
116#if defined(HAVE_LINUX_IO_URING_H)
117struct io_uring_sq {
118        // Head and tail of the ring (associated with array)
119        volatile uint32_t * head;
120        volatile uint32_t * tail;
121
122        // The actual kernel ring which uses head/tail
123        // indexes into the sqes arrays
124        uint32_t * array;
125
126        // number of entries and mask to go with it
127        const uint32_t * num;
128        const uint32_t * mask;
129
130        // Submission flags (Not sure what for)
131        uint32_t * flags;
132
133        // number of sqes not submitted (whatever that means)
134        uint32_t * dropped;
135
136        // Like head/tail but not seen by the kernel
137        volatile uint32_t alloc;
138        volatile uint32_t ready;
139
140        __spinlock_t lock;
141
142        // A buffer of sqes (not the actual ring)
143        struct io_uring_sqe * sqes;
144
145        // The location and size of the mmaped area
146        void * ring_ptr;
147        size_t ring_sz;
148
149        // Statistics
150        #if !defined(__CFA_NO_STATISTICS__)
151                struct {
152                        struct {
153                                unsigned long long int val;
154                                unsigned long long int cnt;
155                        } submit_avg;
156                } stats;
157        #endif
158};
159
160struct io_uring_cq {
161        // Head and tail of the ring
162        volatile uint32_t * head;
163        volatile uint32_t * tail;
164
165        // number of entries and mask to go with it
166        const uint32_t * mask;
167        const uint32_t * num;
168
169        // number of cqes not submitted (whatever that means)
170        uint32_t * overflow;
171
172        // the kernel ring
173        struct io_uring_cqe * cqes;
174
175        // The location and size of the mmaped area
176        void * ring_ptr;
177        size_t ring_sz;
178
179        // Statistics
180        #if !defined(__CFA_NO_STATISTICS__)
181                struct {
182                        struct {
183                                unsigned long long int val;
184                                unsigned long long int cnt;
185                        } completed_avg;
186                } stats;
187        #endif
188};
189
190#if defined(__CFA_IO_POLLING_USER__)
191        struct __io_poller_fast {
192                struct io_ring * ring;
193                $thread thrd;
194        };
195#endif
196
197struct io_ring {
198        struct io_uring_sq submit_q;
199        struct io_uring_cq completion_q;
200        uint32_t flags;
201        int fd;
202        semaphore submit;
203        volatile bool done;
204        struct {
205                struct {
206                        void * stack;
207                        pthread_t kthrd;
208                } slow;
209                #if defined(__CFA_IO_POLLING_USER__)
210                        __io_poller_fast fast;
211                        __bin_sem_t sem;
212                #endif
213        } poller;
214};
215#endif
216
217//-----------------------------------------------------------------------------
218// Cluster
219struct cluster {
220        // Ready queue locks
221        __spinlock_t ready_queue_lock;
222
223        // Ready queue for threads
224        __queue_t($thread) ready_queue;
225
226        // Name of the cluster
227        const char * name;
228
229        // Preemption rate on this cluster
230        Duration preemption_rate;
231
232        // List of processors
233        __spinlock_t idle_lock;
234        __dllist_t(struct processor) procs;
235        __dllist_t(struct processor) idles;
236        unsigned int nprocessors;
237
238        // List of threads
239        __spinlock_t thread_list_lock;
240        __dllist_t(struct $thread) threads;
241        unsigned int nthreads;
242
243        // Link lists fields
244        struct __dbg_node_cltr {
245                cluster * next;
246                cluster * prev;
247        } node;
248
249        #if defined(HAVE_LINUX_IO_URING_H)
250                struct io_ring io;
251        #endif
252
253        #if !defined(__CFA_NO_STATISTICS__)
254                bool print_stats;
255        #endif
256};
257extern Duration default_preemption();
258
259void ?{} (cluster & this, const char name[], Duration preemption_rate);
260void ^?{}(cluster & this);
261
262static inline void ?{} (cluster & this)                           { this{"Anonymous Cluster", default_preemption()}; }
263static inline void ?{} (cluster & this, Duration preemption_rate) { this{"Anonymous Cluster", preemption_rate}; }
264static inline void ?{} (cluster & this, const char name[])        { this{name, default_preemption()}; }
265
266static inline [cluster *&, cluster *& ] __get( cluster & this ) __attribute__((const)) { return this.node.[next, prev]; }
267
268static inline struct processor * active_processor() { return TL_GET( this_processor ); } // UNSAFE
269static inline struct cluster   * active_cluster  () { return TL_GET( this_processor )->cltr; }
270
271#if !defined(__CFA_NO_STATISTICS__)
272        static inline void print_stats_at_exit( cluster & this ) {
273                this.print_stats = true;
274        }
275#endif
276
277// Local Variables: //
278// mode: c //
279// tab-width: 4 //
280// End: //
Note: See TracBrowser for help on using the repository browser.