source: libcfa/src/concurrency/io/setup.cfa@ 9082d7e8

ADT ast-experimental
Last change on this file since 9082d7e8 was f5f2768, checked in by Peter A. Buhr <pabuhr@…>, 3 years ago

make _GNU_SOURCE default, change IO to use SOCKADDR_ARG and CONST_SOCKADDR_ARG, move sys/socket.h to first include because of anonymous naming problem
  • Property mode set to 100644
File size: 11.7 KB
RevLine 
[3e2b9c9]1//
2// Cforall Version 1.0.0 Copyright (C) 2020 University of Waterloo
3//
4// The contents of this file are covered under the licence agreement in the
5// file "LICENCE" distributed with Cforall.
6//
7// io/setup.cfa --
8//
9// Author : Thierry Delisle
10// Created On : Fri Jul 31 16:25:51 2020
11// Last Modified By :
12// Last Modified On :
13// Update Count :
14//
15
16#define __cforall_thread__
17
[80444bb]18#if defined(__CFA_DEBUG__)
19 // #define __CFA_DEBUG_PRINT_IO__
20 // #define __CFA_DEBUG_PRINT_IO_CORE__
21#endif
22
[3e2b9c9]23#include "io/types.hfa"
[c44d652]24#include "kernel.hfa"
[3e2b9c9]25
26#if !defined(CFA_HAVE_LINUX_IO_URING_H)
[108345a]27 void ?{}(io_context_params & this) libcfa_public {}
[f277633e]28
[8bee858]29 void ?{}(io_context$ & this, struct cluster & cl) {}
30 void ^?{}(io_context$ & this) {}
[3e2b9c9]31
[f815c46]32 void __cfa_io_start( processor * proc ) {}
[18f7858]33 bool __cfa_io_flush( processor * proc ) { return false; }
[7425720]34 bool __cfa_io_drain( processor * proc ) __attribute__((nonnull (1))) { return false; }
[f815c46]35 void __cfa_io_stop ( processor * proc ) {}
36
[8bee858]37 io_arbiter$ * create(void) { return 0p; }
38 void destroy(io_arbiter$ *) {}
[b7664a03]39
[3e2b9c9]40#else
[bfb9bf5]41#pragma GCC diagnostic push
42#pragma GCC diagnostic ignored "-Waddress-of-packed-member"
[3e2b9c9]43 #include <errno.h>
44 #include <stdint.h>
45 #include <string.h>
46 #include <signal.h>
47 #include <unistd.h>
48
49 extern "C" {
50 #include <pthread.h>
51 #include <sys/epoll.h>
[426f60c]52 #include <sys/eventfd.h>
[3e2b9c9]53 #include <sys/mman.h>
54 #include <sys/syscall.h>
55
56 #include <linux/io_uring.h>
57 }
58
59 #include "bitmanip.hfa"
[b0d0285]60 #include "fstream.hfa"
[708ae38]61 #include "kernel/private.hfa"
[78a580d]62 #include "limits.hfa"
[3e2b9c9]63 #include "thread.hfa"
[bfb9bf5]64#pragma GCC diagnostic pop
[3e2b9c9]65
[108345a]66 void ?{}(io_context_params & this) libcfa_public {
[3e2b9c9]67 this.num_entries = 256;
68 }
69
70 static void * __io_poller_slow( void * arg );
71
72 // Weirdly, some systems that do support io_uring don't actually define these
73 #ifdef __alpha__
74 /*
75 * alpha is the only exception, all other architectures
76 * have common numbers for new system calls.
77 */
78 #ifndef __NR_io_uring_setup
79 #define __NR_io_uring_setup 535
80 #endif
81 #ifndef __NR_io_uring_enter
82 #define __NR_io_uring_enter 536
83 #endif
84 #ifndef __NR_io_uring_register
85 #define __NR_io_uring_register 537
86 #endif
87 #else /* !__alpha__ */
88 #ifndef __NR_io_uring_setup
89 #define __NR_io_uring_setup 425
90 #endif
91 #ifndef __NR_io_uring_enter
92 #define __NR_io_uring_enter 426
93 #endif
94 #ifndef __NR_io_uring_register
95 #define __NR_io_uring_register 427
96 #endif
97 #endif
98
99//=============================================================================================
[dddb3dd0]100// I/O Context Constrution/Destruction
[3e2b9c9]101//=============================================================================================
102
103
104
[8bee858]105 static void __io_uring_setup ( io_context$ & this, const io_context_params & params_in, int procfd );
106 static void __io_uring_teardown( io_context$ & this );
107 static void __epoll_register(io_context$ & ctx);
108 static void __epoll_unregister(io_context$ & ctx);
109 void __ioarbiter_register( io_arbiter$ & mutex, io_context$ & ctx );
110 void __ioarbiter_unregister( io_arbiter$ & mutex, io_context$ & ctx );
[3e2b9c9]111
[8bee858]112 void ?{}(io_context$ & this, processor * proc, struct cluster & cl) {
[dddb3dd0]113 /* paranoid */ verify( cl.io.arbiter );
114 this.proc = proc;
115 this.arbiter = cl.io.arbiter;
[78da4ab]116 this.ext_sq.empty = true;
[11054eb]117 (this.ext_sq.queue){};
[22226e4]118 __io_uring_setup( this, cl.io.params, proc->idle_wctx.evfd );
[78da4ab]119 __cfadbg_print_safe(io_core, "Kernel I/O : Created ring for io_context %u (%p)\n", this.fd, &this);
[3e2b9c9]120 }
121
[8bee858]122 void ^?{}(io_context$ & this) {
[78da4ab]123 __cfadbg_print_safe(io_core, "Kernel I/O : tearing down io_context %u\n", this.fd);
[3e2b9c9]124
[78da4ab]125 __io_uring_teardown( this );
126 __cfadbg_print_safe(io_core, "Kernel I/O : Destroyed ring for io_context %u\n", this.fd);
127 }
[3e2b9c9]128
[8bee858]129 static void __io_uring_setup( io_context$ & this, const io_context_params & params_in, int procfd ) {
[3e2b9c9]130 // Step 1 : call to setup
131 struct io_uring_params params;
132 memset(&params, 0, sizeof(params));
[78da4ab]133 // if( params_in.poll_submit ) params.flags |= IORING_SETUP_SQPOLL;
134 // if( params_in.poll_complete ) params.flags |= IORING_SETUP_IOPOLL;
[3e2b9c9]135
[4998155]136 __u32 nentries = params_in.num_entries != 0 ? params_in.num_entries : 256;
[63fe427c]137 if( !is_pow2(nentries) ) {
[bf0263c]138 abort("ERROR: I/O setup 'num_entries' must be a power of 2, was %u\n", nentries);
[63fe427c]139 }
[3e2b9c9]140
141 int fd = syscall(__NR_io_uring_setup, nentries, &params );
142 if(fd < 0) {
143 abort("KERNEL ERROR: IO_URING SETUP - %s\n", strerror(errno));
144 }
145
146 // Step 2 : mmap result
[78da4ab]147 struct __sub_ring_t & sq = this.sq;
148 struct __cmp_ring_t & cq = this.cq;
[3e2b9c9]149
150 // calculate the right ring size
151 sq.ring_sz = params.sq_off.array + (params.sq_entries * sizeof(unsigned) );
152 cq.ring_sz = params.cq_off.cqes + (params.cq_entries * sizeof(struct io_uring_cqe));
153
154 // Requires features
155 #if defined(IORING_FEAT_SINGLE_MMAP)
156 // adjust the size according to the parameters
157 if ((params.features & IORING_FEAT_SINGLE_MMAP) != 0) {
158 cq.ring_sz = sq.ring_sz = max(cq.ring_sz, sq.ring_sz);
159 }
160 #endif
161
162 // mmap the Submit Queue into existence
163 sq.ring_ptr = mmap(0, sq.ring_sz, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd, IORING_OFF_SQ_RING);
164 if (sq.ring_ptr == (void*)MAP_FAILED) {
165 abort("KERNEL ERROR: IO_URING MMAP1 - %s\n", strerror(errno));
166 }
167
168 // Requires features
169 #if defined(IORING_FEAT_SINGLE_MMAP)
170 // mmap the Completion Queue into existence (may or may not be needed)
171 if ((params.features & IORING_FEAT_SINGLE_MMAP) != 0) {
172 cq.ring_ptr = sq.ring_ptr;
173 }
174 else
175 #endif
176 {
177 // We need multiple call to MMAP
178 cq.ring_ptr = mmap(0, cq.ring_sz, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd, IORING_OFF_CQ_RING);
179 if (cq.ring_ptr == (void*)MAP_FAILED) {
180 munmap(sq.ring_ptr, sq.ring_sz);
181 abort("KERNEL ERROR: IO_URING MMAP2 - %s\n", strerror(errno));
182 }
183 }
184
185 // mmap the submit queue entries
186 size_t size = params.sq_entries * sizeof(struct io_uring_sqe);
187 sq.sqes = (struct io_uring_sqe *)mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd, IORING_OFF_SQES);
188 if (sq.sqes == (struct io_uring_sqe *)MAP_FAILED) {
189 munmap(sq.ring_ptr, sq.ring_sz);
190 if (cq.ring_ptr != sq.ring_ptr) munmap(cq.ring_ptr, cq.ring_sz);
191 abort("KERNEL ERROR: IO_URING MMAP3 - %s\n", strerror(errno));
192 }
193
[426f60c]194 // Step 3 : Initialize the data structure
[3e2b9c9]195 // Get the pointers from the kernel to fill the structure
196 // submit queue
[78da4ab]197 sq.kring.head = (volatile __u32 *)(((intptr_t)sq.ring_ptr) + params.sq_off.head);
198 sq.kring.tail = (volatile __u32 *)(((intptr_t)sq.ring_ptr) + params.sq_off.tail);
199 sq.kring.array = ( __u32 *)(((intptr_t)sq.ring_ptr) + params.sq_off.array);
200 sq.mask = ( const __u32 *)(((intptr_t)sq.ring_ptr) + params.sq_off.ring_mask);
201 sq.num = ( const __u32 *)(((intptr_t)sq.ring_ptr) + params.sq_off.ring_entries);
202 sq.flags = ( __u32 *)(((intptr_t)sq.ring_ptr) + params.sq_off.flags);
203 sq.dropped = ( __u32 *)(((intptr_t)sq.ring_ptr) + params.sq_off.dropped);
204
205 sq.kring.released = 0;
206
207 sq.free_ring.head = 0;
208 sq.free_ring.tail = *sq.num;
209 sq.free_ring.array = alloc( *sq.num, 128`align );
210 for(i; (__u32)*sq.num) {
211 sq.free_ring.array[i] = i;
[3e2b9c9]212 }
213
[78da4ab]214 sq.to_submit = 0;
[3e2b9c9]215
216 // completion queue
[26544f9]217 cq.try_lock = false;
[78a580d]218 cq.id = MAX;
219 cq.ts = rdtscl();
[4998155]220 cq.head = (volatile __u32 *)(((intptr_t)cq.ring_ptr) + params.cq_off.head);
221 cq.tail = (volatile __u32 *)(((intptr_t)cq.ring_ptr) + params.cq_off.tail);
222 cq.mask = ( const __u32 *)(((intptr_t)cq.ring_ptr) + params.cq_off.ring_mask);
223 cq.num = ( const __u32 *)(((intptr_t)cq.ring_ptr) + params.cq_off.ring_entries);
224 cq.overflow = ( __u32 *)(((intptr_t)cq.ring_ptr) + params.cq_off.overflow);
225 cq.cqes = (struct io_uring_cqe *)(((intptr_t)cq.ring_ptr) + params.cq_off.cqes);
[3e2b9c9]226
[d3605f8]227 #if !defined(CFA_WITH_IO_URING_IDLE)
[19cb0cb]228 {
[7ef162b2]229 // Step 4 : eventfd
230 __cfadbg_print_safe(io_core, "Kernel I/O : registering %d for completion with ring %d\n", procfd, fd);
[bb58825]231
[7ef162b2]232 int ret = syscall( __NR_io_uring_register, fd, IORING_REGISTER_EVENTFD, &procfd, 1);
233 if (ret < 0) {
234 abort("KERNEL ERROR: IO_URING EVENTFD REGISTER - %s\n", strerror(errno));
235 }
[426f60c]236
[7ef162b2]237 __cfadbg_print_safe(io_core, "Kernel I/O : registered %d for completion with ring %d\n", procfd, fd);
[19cb0cb]238 }
[7ef162b2]239 #endif
[dddb3dd0]240
[19cb0cb]241 // TODO: implement a proper version of this.
242 // I have not found a better maximum that works in general but users should be able to configure it
243 // the same way they configure other I/O options
[7ce8873]244 // #if defined(CFA_HAVE_IORING_REGISTER_IOWQ_MAX_WORKERS)
[19cb0cb]245 // {
[7ce8873]246 // // Step 5 : max worker count
247 // __cfadbg_print_safe(io_core, "Kernel I/O : lmiting max workers for ring %d\n", fd);
248
249 // unsigned int maxes[2];
250 // maxes[0] = 64; // max number of bounded workers (Regular files / block)
251 // maxes[1] = 64; // max number of unbounded workers (IOSQE_ASYNC)
252 // int ret = syscall( __NR_io_uring_register, fd, IORING_REGISTER_IOWQ_MAX_WORKERS, maxes, 2);
253 // if (ret < 0) {
254 // abort("KERNEL ERROR: IO_URING MAX WORKER REGISTER - %s\n", strerror(errno));
255 // }
256
257 // __cfadbg_print_safe(io_core, "Kernel I/O : lmited max workers for ring %d\n", fd);
[19cb0cb]258 // }
[7ce8873]259 // #endif
260
[3e2b9c9]261 // some paranoid checks
262 /* paranoid */ verifyf( (*cq.mask) == ((*cq.num) - 1ul32), "IO_URING Expected mask to be %u (%u entries), was %u", (*cq.num) - 1ul32, *cq.num, *cq.mask );
263 /* paranoid */ verifyf( (*cq.num) >= nentries, "IO_URING Expected %u entries, got %u", nentries, *cq.num );
264 /* paranoid */ verifyf( (*cq.head) == 0, "IO_URING Expected head to be 0, got %u", *cq.head );
265 /* paranoid */ verifyf( (*cq.tail) == 0, "IO_URING Expected tail to be 0, got %u", *cq.tail );
266
267 /* paranoid */ verifyf( (*sq.mask) == ((*sq.num) - 1ul32), "IO_URING Expected mask to be %u (%u entries), was %u", (*sq.num) - 1ul32, *sq.num, *sq.mask );
268 /* paranoid */ verifyf( (*sq.num) >= nentries, "IO_URING Expected %u entries, got %u", nentries, *sq.num );
[78da4ab]269 /* paranoid */ verifyf( (*sq.kring.head) == 0, "IO_URING Expected head to be 0, got %u", *sq.kring.head );
270 /* paranoid */ verifyf( (*sq.kring.tail) == 0, "IO_URING Expected tail to be 0, got %u", *sq.kring.tail );
[3e2b9c9]271
272 // Update the global ring info
[78da4ab]273 this.ring_flags = 0;
[3e2b9c9]274 this.fd = fd;
275 }
276
[8bee858]277 static void __io_uring_teardown( io_context$ & this ) {
[3e2b9c9]278 // Shutdown the io rings
[78da4ab]279 struct __sub_ring_t & sq = this.sq;
280 struct __cmp_ring_t & cq = this.cq;
[b0d0285]281 {
282 __u32 fhead = sq.free_ring.head;
283 __u32 ftail = sq.free_ring.tail;
284
285 __u32 total = *sq.num;
286 __u32 avail = ftail - fhead;
287
288 if(avail != total) abort | "Processor (" | (void*)this.proc | ") tearing down ring with" | (total - avail) | "entries allocated but not submitted, out of" | total;
289 }
[3e2b9c9]290
291 // unmap the submit queue entries
292 munmap(sq.sqes, (*sq.num) * sizeof(struct io_uring_sqe));
293
294 // unmap the Submit Queue ring
295 munmap(sq.ring_ptr, sq.ring_sz);
296
297 // unmap the Completion Queue ring, if it is different
298 if (cq.ring_ptr != sq.ring_ptr) {
299 munmap(cq.ring_ptr, cq.ring_sz);
300 }
301
302 // close the file descriptor
303 close(this.fd);
304
[78da4ab]305 free( this.sq.free_ring.array ); // Maybe null, doesn't matter
[3e2b9c9]306 }
307
[dddb3dd0]308 void __cfa_io_start( processor * proc ) {
309 proc->io.ctx = alloc();
310 (*proc->io.ctx){proc, *proc->cltr};
311 }
312 void __cfa_io_stop ( processor * proc ) {
313 ^(*proc->io.ctx){};
314 free(proc->io.ctx);
315 }
316
[78da4ab]317
[3e2b9c9]318//=============================================================================================
319// I/O Context Misc Setup
320//=============================================================================================
[8bee858]321 void ?{}( io_arbiter$ & this ) {
[11054eb]322 this.pending.empty = true;
[78da4ab]323 }
[3e2b9c9]324
[8bee858]325 void ^?{}( io_arbiter$ & mutex this ) {}
[3e2b9c9]326
[8bee858]327 io_arbiter$ * create(void) {
[78da4ab]328 return new();
[3e2b9c9]329 }
[8bee858]330 void destroy(io_arbiter$ * arbiter) {
[78da4ab]331 delete(arbiter);
332 }
333
334//=============================================================================================
335// I/O Context Misc Setup
336//=============================================================================================
337
[c44d652]338#endif
Note: See TracBrowser for help on using the repository browser.