source: libcfa/src/concurrency/io.cfa@ 44aad8f

ADT arm-eh ast-experimental enum forall-pointer-decay jacob/cs343-translation new-ast new-ast-unique-expr pthread-emulation qualifiedEnum
Last change on this file since 44aad8f was 038be32, checked in by Thierry Delisle <tdelisle@…>, 6 years ago

added defines and bool for whether or not to print statistics
  • Property mode set to 100644
File size: 25.9 KB
Line 
1//
2// Cforall Version 1.0.0 Copyright (C) 2020 University of Waterloo
3//
4// The contents of this file are covered under the licence agreement in the
5// file "LICENCE" distributed with Cforall.
6//
7// io.cfa --
8//
9// Author : Thierry Delisle
10// Created On : Thu Apr 23 17:31:00 2020
11// Last Modified By :
12// Last Modified On :
13// Update Count :
14//
15
16#include "kernel.hfa"
17
18#if !defined(HAVE_LINUX_IO_URING_H)
19 void __kernel_io_startup( cluster & this ) {
20 // Nothing to do without io_uring
21 }
22
23 void __kernel_io_shutdown( cluster & this ) {
24 // Nothing to do without io_uring
25 }
26
27#else
28 extern "C" {
29 #define _GNU_SOURCE /* See feature_test_macros(7) */
30 #include <errno.h>
31 #include <stdint.h>
32 #include <string.h>
33 #include <unistd.h>
34 #include <sys/mman.h>
35 #include <sys/syscall.h>
36
37 #include <linux/io_uring.h>
38 }
39
40 #include "bits/signal.hfa"
41 #include "kernel_private.hfa"
42 #include "thread.hfa"
43
44 uint32_t entries_per_cluster() {
45 return 256;
46 }
47
48 static void * __io_poller( void * arg );
49
50 // Weirdly, some systems that do support io_uring don't actually define these
51 #ifdef __alpha__
52 /*
53 * alpha is the only exception, all other architectures
54 * have common numbers for new system calls.
55 */
56 # ifndef __NR_io_uring_setup
57 # define __NR_io_uring_setup 535
58 # endif
59 # ifndef __NR_io_uring_enter
60 # define __NR_io_uring_enter 536
61 # endif
62 # ifndef __NR_io_uring_register
63 # define __NR_io_uring_register 537
64 # endif
65 #else /* !__alpha__ */
66 # ifndef __NR_io_uring_setup
67 # define __NR_io_uring_setup 425
68 # endif
69 # ifndef __NR_io_uring_enter
70 # define __NR_io_uring_enter 426
71 # endif
72 # ifndef __NR_io_uring_register
73 # define __NR_io_uring_register 427
74 # endif
75 #endif
76
77
78//=============================================================================================
79// I/O Startup / Shutdown logic
80//=============================================================================================
81 void __kernel_io_startup( cluster & this ) {
82 // Step 1 : call to setup
83 struct io_uring_params params;
84 memset(&params, 0, sizeof(params));
85
86 uint32_t nentries = entries_per_cluster();
87
88 int fd = syscall(__NR_io_uring_setup, nentries, &params );
89 if(fd < 0) {
90 abort("KERNEL ERROR: IO_URING SETUP - %s\n", strerror(errno));
91 }
92
93 // Step 2 : mmap result
94 memset(&this.io, 0, sizeof(struct io_ring));
95 struct io_uring_sq & sq = this.io.submit_q;
96 struct io_uring_cq & cq = this.io.completion_q;
97
98 // calculate the right ring size
99 sq.ring_sz = params.sq_off.array + (params.sq_entries * sizeof(unsigned) );
100 cq.ring_sz = params.cq_off.cqes + (params.cq_entries * sizeof(struct io_uring_cqe));
101
102 // Requires features
103 #if defined(IORING_FEAT_SINGLE_MMAP)
104 // adjust the size according to the parameters
105 if ((params.features & IORING_FEAT_SINGLE_MMAP) != 0) {
106 cq->ring_sz = sq->ring_sz = max(cq->ring_sz, sq->ring_sz);
107 }
108 #endif
109
110 // mmap the Submit Queue into existence
111 sq.ring_ptr = mmap(0, sq.ring_sz, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd, IORING_OFF_SQ_RING);
112 if (sq.ring_ptr == (void*)MAP_FAILED) {
113 abort("KERNEL ERROR: IO_URING MMAP1 - %s\n", strerror(errno));
114 }
115
116 // Requires features
117 #if defined(IORING_FEAT_SINGLE_MMAP)
118 // mmap the Completion Queue into existence (may or may not be needed)
119 if ((params.features & IORING_FEAT_SINGLE_MMAP) != 0) {
120 cq->ring_ptr = sq->ring_ptr;
121 }
122 else
123 #endif
124 {
125 // We need multiple call to MMAP
126 cq.ring_ptr = mmap(0, cq.ring_sz, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd, IORING_OFF_CQ_RING);
127 if (cq.ring_ptr == (void*)MAP_FAILED) {
128 munmap(sq.ring_ptr, sq.ring_sz);
129 abort("KERNEL ERROR: IO_URING MMAP2 - %s\n", strerror(errno));
130 }
131 }
132
133 // mmap the submit queue entries
134 size_t size = params.sq_entries * sizeof(struct io_uring_sqe);
135 sq.sqes = (struct io_uring_sqe *)mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd, IORING_OFF_SQES);
136 if (sq.sqes == (struct io_uring_sqe *)MAP_FAILED) {
137 munmap(sq.ring_ptr, sq.ring_sz);
138 if (cq.ring_ptr != sq.ring_ptr) munmap(cq.ring_ptr, cq.ring_sz);
139 abort("KERNEL ERROR: IO_URING MMAP3 - %s\n", strerror(errno));
140 }
141
142 // Get the pointers from the kernel to fill the structure
143 // submit queue
144 sq.head = (volatile uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.head);
145 sq.tail = (volatile uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.tail);
146 sq.mask = ( const uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.ring_mask);
147 sq.num = ( const uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.ring_entries);
148 sq.flags = ( uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.flags);
149 sq.dropped = ( uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.dropped);
150 sq.array = ( uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.array);
151 sq.alloc = *sq.tail;
152
153 // completion queue
154 cq.head = (volatile uint32_t *)(((intptr_t)cq.ring_ptr) + params.cq_off.head);
155 cq.tail = (volatile uint32_t *)(((intptr_t)cq.ring_ptr) + params.cq_off.tail);
156 cq.mask = ( const uint32_t *)(((intptr_t)cq.ring_ptr) + params.cq_off.ring_mask);
157 cq.num = ( const uint32_t *)(((intptr_t)cq.ring_ptr) + params.cq_off.ring_entries);
158 cq.overflow = ( uint32_t *)(((intptr_t)cq.ring_ptr) + params.cq_off.overflow);
159 cq.cqes = (struct io_uring_cqe *)(((intptr_t)cq.ring_ptr) + params.cq_off.cqes);
160
161 // some paranoid checks
162 /* paranoid */ verifyf( (*cq.mask) == ((*cq.num) - 1ul32), "IO_URING Expected mask to be %u (%u entries), was %u", (*cq.num) - 1ul32, *cq.num, *cq.mask );
163 /* paranoid */ verifyf( (*cq.num) >= nentries, "IO_URING Expected %u entries, got %u", nentries, *cq.num );
164 /* paranoid */ verifyf( (*cq.head) == 0, "IO_URING Expected head to be 0, got %u", *cq.head );
165 /* paranoid */ verifyf( (*cq.tail) == 0, "IO_URING Expected tail to be 0, got %u", *cq.tail );
166
167 /* paranoid */ verifyf( (*sq.mask) == ((*sq.num) - 1ul32), "IO_URING Expected mask to be %u (%u entries), was %u", (*sq.num) - 1ul32, *sq.num, *sq.mask );
168 /* paranoid */ verifyf( (*sq.num) >= nentries, "IO_URING Expected %u entries, got %u", nentries, *sq.num );
169 /* paranoid */ verifyf( (*sq.head) == 0, "IO_URING Expected head to be 0, got %u", *sq.head );
170 /* paranoid */ verifyf( (*sq.tail) == 0, "IO_URING Expected tail to be 0, got %u", *sq.tail );
171
172 // Update the global ring info
173 this.io.flags = params.flags;
174 this.io.fd = fd;
175 this.io.done = false;
176 (this.io.submit){ min(*sq.num, *cq.num) };
177
178 // Initialize statistics
179 #if !defined(__CFA_NO_STATISTICS__)
180 this.io.submit_q.stats.submit_avg.val = 0;
181 this.io.submit_q.stats.submit_avg.cnt = 0;
182 this.io.completion_q.stats.completed_avg.val = 0;
183 this.io.completion_q.stats.completed_avg.cnt = 0;
184 #endif
185
186 // Create the poller thread
187 this.io.stack = __create_pthread( &this.io.poller, __io_poller, &this );
188 }
189
190 void __kernel_io_shutdown( cluster & this ) {
191 // Stop the IO Poller
192 // Notify the poller thread of the shutdown
193 __atomic_store_n(&this.io.done, true, __ATOMIC_SEQ_CST);
194 sigval val = { 1 };
195 pthread_sigqueue( this.io.poller, SIGUSR1, val );
196
197 // Wait for the poller thread to finish
198 pthread_join( this.io.poller, 0p );
199 free( this.io.stack );
200
201 // print statistics
202 #if !defined(__CFA_NO_STATISTICS__)
203 if(this.print_stats) {
204 __cfaabi_bits_print_safe( STDERR_FILENO,
205 "----- I/O uRing Stats -----\n"
206 "- total submit calls : %llu\n"
207 "- avg submit : %lf\n"
208 "- total wait calls : %llu\n"
209 "- avg completion/wait : %lf\n",
210 this.io.submit_q.stats.submit_avg.cnt,
211 ((double)this.io.submit_q.stats.submit_avg.val) / this.io.submit_q.stats.submit_avg.cnt,
212 this.io.completion_q.stats.completed_avg.cnt,
213 ((double)this.io.completion_q.stats.completed_avg.val) / this.io.completion_q.stats.completed_avg.cnt
214 );
215 }
216 #endif
217
218 // Shutdown the io rings
219 struct io_uring_sq & sq = this.io.submit_q;
220 struct io_uring_cq & cq = this.io.completion_q;
221
222 // unmap the submit queue entries
223 munmap(sq.sqes, (*sq.num) * sizeof(struct io_uring_sqe));
224
225 // unmap the Submit Queue ring
226 munmap(sq.ring_ptr, sq.ring_sz);
227
228 // unmap the Completion Queue ring, if it is different
229 if (cq.ring_ptr != sq.ring_ptr) {
230 munmap(cq.ring_ptr, cq.ring_sz);
231 }
232
233 // close the file descriptor
234 close(this.io.fd);
235 }
236
237//=============================================================================================
238// I/O Polling
239//=============================================================================================
240 struct io_user_data {
241 int32_t result;
242 $thread * thrd;
243 };
244
245 // Process a single completion message from the io_uring
246 // This is NOT thread-safe
247 int __drain_io( struct io_ring & ring, sigset_t & mask, int waitcnt ) {
248 int ret = syscall( __NR_io_uring_enter, ring.fd, 0, waitcnt, IORING_ENTER_GETEVENTS, &mask, _NSIG / 8);
249 if( ret < 0 ) {
250 switch((int)errno) {
251 case EAGAIN:
252 case EINTR:
253 return -EAGAIN;
254 default:
255 abort( "KERNEL ERROR: IO_URING WAIT - %s\n", strerror(errno) );
256 }
257 }
258
259 // Drain the queue
260 unsigned head = *ring.completion_q.head;
261 unsigned tail = __atomic_load_n(ring.completion_q.tail, __ATOMIC_ACQUIRE);
262
263 // Nothing was new return 0
264 if (head == tail) {
265 #if !defined(__CFA_NO_STATISTICS__)
266 ring.completion_q.stats.completed_avg.cnt += 1;
267 #endif
268 return 0;
269 }
270
271 uint32_t count = tail - head;
272 for(i; count) {
273 unsigned idx = (head + i) & (*ring.completion_q.mask);
274 struct io_uring_cqe & cqe = ring.completion_q.cqes[idx];
275
276 /* paranoid */ verify(&cqe);
277
278 struct io_user_data * data = (struct io_user_data *)cqe.user_data;
279 // __cfaabi_bits_print_safe( STDERR_FILENO, "Performed reading io cqe %p, result %d for %p\n", data, cqe.res, data->thrd );
280
281 data->result = cqe.res;
282 __unpark( data->thrd __cfaabi_dbg_ctx2 );
283 }
284
285 // Allow new submissions to happen
286 V(ring.submit, count);
287
288 // Mark to the kernel that the cqe has been seen
289 // Ensure that the kernel only sees the new value of the head index after the CQEs have been read.
290 __atomic_fetch_add( ring.completion_q.head, count, __ATOMIC_RELAXED );
291
292 // Update statistics
293 #if !defined(__CFA_NO_STATISTICS__)
294 ring.completion_q.stats.completed_avg.val += count;
295 ring.completion_q.stats.completed_avg.cnt += 1;
296 #endif
297
298 return count;
299 }
300
301 static void * __io_poller( void * arg ) {
302 cluster * cltr = (cluster *)arg;
303 struct io_ring & ring = cltr->io;
304
305 sigset_t mask;
306 sigfillset(&mask);
307 if ( pthread_sigmask( SIG_BLOCK, &mask, 0p ) == -1 ) {
308 abort( "KERNEL ERROR: IO_URING - pthread_sigmask" );
309 }
310
311 sigdelset( &mask, SIGUSR1 );
312
313 verify( (*ring.submit_q.head) == (*ring.submit_q.tail) );
314 verify( (*ring.completion_q.head) == (*ring.completion_q.tail) );
315
316 while(!__atomic_load_n(&ring.done, __ATOMIC_SEQ_CST)) {
317 __drain_io( ring, mask, 1 );
318 }
319
320 return 0p;
321 }
322
323//=============================================================================================
324// I/O Submissions
325//=============================================================================================
326
327// Submition steps :
328// 1 - We need to make sure we don't overflow any of the buffer, P(ring.submit) to make sure
329// entries are available. The semaphore make sure that there is no more operations in
330// progress then the number of entries in the buffer. This probably limits concurrency
331// more than necessary since submitted but not completed operations don't need any
332// entries in user space. However, I don't know what happens if we overflow the buffers
333// because too many requests completed at once. This is a safe approach in all cases.
334// Furthermore, with hundreds of entries, this may be okay.
335//
336// 2 - Allocate a queue entry. The ring already has memory for all entries but only the ones
337// listed in sq.array are visible by the kernel. For those not listed, the kernel does not
338// offer any assurance that an entry is not being filled by multiple flags. Therefore, we
339// need to write an allocator that allows allocating concurrently.
340//
341// 3 - Actually fill the submit entry, this is the only simple and straightforward step.
342//
343// 4 - Append the entry index to the array and adjust the tail accordingly. This operation
344// needs to arrive to two concensus at the same time:
345// A - The order in which entries are listed in the array: no two threads must pick the
346// same index for their entries
347// B - When can the tail be update for the kernel. EVERY entries in the array between
348// head and tail must be fully filled and shouldn't ever be touched again.
349//
350
351 static inline [* struct io_uring_sqe, uint32_t] __submit_alloc( struct io_ring & ring ) {
352 // Wait for a spot to be available
353 P(ring.submit);
354
355 // Allocate the sqe
356 uint32_t idx = __atomic_fetch_add(&ring.submit_q.alloc, 1ul32, __ATOMIC_SEQ_CST);
357
358 // Validate that we didn't overflow anything
359 // Check that nothing overflowed
360 /* paranoid */ verify( true );
361
362 // Check that it goes head -> tail -> alloc and never head -> alloc -> tail
363 /* paranoid */ verify( true );
364
365 // Return the sqe
366 return [&ring.submit_q.sqes[ idx & (*ring.submit_q.mask)], idx];
367 }
368
369 static inline void __submit( struct io_ring & ring, uint32_t idx ) {
370 // get mutual exclusion
371 lock(ring.submit_q.lock __cfaabi_dbg_ctx2);
372
373 // Append to the list of ready entries
374 uint32_t * tail = ring.submit_q.tail;
375 const uint32_t mask = *ring.submit_q.mask;
376
377 ring.submit_q.array[ (*tail) & mask ] = idx & mask;
378 __atomic_fetch_add(tail, 1ul32, __ATOMIC_SEQ_CST);
379
380 // Submit however, many entries need to be submitted
381 int ret = syscall( __NR_io_uring_enter, ring.fd, 1, 0, 0, 0p, 0);
382 // __cfaabi_bits_print_safe( STDERR_FILENO, "Performed io_submit, returned %d\n", ret );
383 if( ret < 0 ) {
384 switch((int)errno) {
385 default:
386 abort( "KERNEL ERROR: IO_URING SUBMIT - %s\n", strerror(errno) );
387 }
388 }
389
390 // update statistics
391 #if !defined(__CFA_NO_STATISTICS__)
392 ring.submit_q.stats.submit_avg.val += 1;
393 ring.submit_q.stats.submit_avg.cnt += 1;
394 #endif
395
396 unlock(ring.submit_q.lock);
397 // Make sure that idx was submitted
398 // Be careful to not get false positive if we cycled the entire list or that someone else submitted for us
399 }
400
401 static inline void ?{}(struct io_uring_sqe & this, uint8_t opcode, int fd) {
402 this.opcode = opcode;
403 #if !defined(IOSQE_ASYNC)
404 this.flags = 0;
405 #else
406 this.flags = IOSQE_ASYNC;
407 #endif
408 this.ioprio = 0;
409 this.fd = fd;
410 this.off = 0;
411 this.addr = 0;
412 this.len = 0;
413 this.rw_flags = 0;
414 this.__pad2[0] = this.__pad2[1] = this.__pad2[2] = 0;
415 }
416
417 static inline void ?{}(struct io_uring_sqe & this, uint8_t opcode, int fd, void * addr, uint32_t len, uint64_t off ) {
418 (this){ opcode, fd };
419 this.off = off;
420 this.addr = (uint64_t)addr;
421 this.len = len;
422 }
423#endif
424
425//=============================================================================================
426// I/O Interface
427//=============================================================================================
428#if defined(HAVE_LINUX_IO_URING_H)
429 #define __submit_prelude \
430 struct io_ring & ring = active_cluster()->io; \
431 struct io_uring_sqe * sqe; \
432 uint32_t idx; \
433 [sqe, idx] = __submit_alloc( ring );
434
435 #define __submit_wait \
436 io_user_data data = { 0, active_thread() }; \
437 /*__cfaabi_bits_print_safe( STDERR_FILENO, "Preparing user data %p for %p\n", &data, data.thrd );*/ \
438 sqe->user_data = (uint64_t)&data; \
439 __submit( ring, idx ); \
440 park( __cfaabi_dbg_ctx ); \
441 return data.result;
442#endif
443
444// Some forward declarations
445extern "C" {
446 #include <sys/types.h>
447 struct iovec;
448 extern ssize_t preadv2 (int fd, const struct iovec *iov, int iovcnt, off_t offset, int flags);
449 extern ssize_t pwritev2(int fd, const struct iovec *iov, int iovcnt, off_t offset, int flags);
450
451 extern int fsync(int fd);
452 extern int sync_file_range(int fd, int64_t offset, int64_t nbytes, unsigned int flags);
453
454 struct msghdr;
455 struct sockaddr;
456 extern ssize_t sendmsg(int sockfd, const struct msghdr *msg, int flags);
457 extern ssize_t recvmsg(int sockfd, struct msghdr *msg, int flags);
458 extern ssize_t send(int sockfd, const void *buf, size_t len, int flags);
459 extern ssize_t recv(int sockfd, void *buf, size_t len, int flags);
460 extern int accept4(int sockfd, struct sockaddr *addr, socklen_t *addrlen, int flags);
461 extern int connect(int sockfd, const struct sockaddr *addr, socklen_t addrlen);
462
463 extern int fallocate(int fd, int mode, uint64_t offset, uint64_t len);
464 extern int posix_fadvise(int fd, uint64_t offset, uint64_t len, int advice);
465 extern int madvise(void *addr, size_t length, int advice);
466
467 extern int openat(int dirfd, const char *pathname, int flags, mode_t mode);
468 extern int close(int fd);
469
470 struct statx;
471 extern int statx(int dirfd, const char *pathname, int flags, unsigned int mask, struct statx *statxbuf);
472
473 extern ssize_t read (int fd, void *buf, size_t count);
474}
475
476//-----------------------------------------------------------------------------
477// Asynchronous operations
478ssize_t cfa_preadv2(int fd, const struct iovec *iov, int iovcnt, off_t offset, int flags) {
479 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_READV)
480 return preadv2(fd, iov, iovcnt, offset, flags);
481 #else
482 __submit_prelude
483
484 (*sqe){ IORING_OP_READV, fd, iov, iovcnt, offset };
485
486 __submit_wait
487 #endif
488}
489
490ssize_t cfa_pwritev2(int fd, const struct iovec *iov, int iovcnt, off_t offset, int flags) {
491 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_WRITEV)
492 return pwritev2(fd, iov, iovcnt, offset, flags);
493 #else
494 __submit_prelude
495
496 (*sqe){ IORING_OP_WRITEV, fd, iov, iovcnt, offset };
497
498 __submit_wait
499 #endif
500}
501
502int cfa_fsync(int fd) {
503 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_FSYNC)
504 return fsync(fd);
505 #else
506 __submit_prelude
507
508 (*sqe){ IORING_OP_FSYNC, fd };
509
510 __submit_wait
511 #endif
512}
513
514int cfa_sync_file_range(int fd, int64_t offset, int64_t nbytes, unsigned int flags) {
515 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_SYNC_FILE_RANGE)
516 return sync_file_range(fd, offset, nbytes, flags);
517 #else
518 __submit_prelude
519
520 (*sqe){ IORING_OP_SYNC_FILE_RANGE, fd };
521 sqe->off = offset;
522 sqe->len = nbytes;
523 sqe->sync_range_flags = flags;
524
525 __submit_wait
526 #endif
527}
528
529
530ssize_t cfa_sendmsg(int sockfd, const struct msghdr *msg, int flags) {
531 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_SENDMSG)
532 return recv(sockfd, msg, flags);
533 #else
534 __submit_prelude
535
536 (*sqe){ IORING_OP_SENDMSG, sockfd, msg, 1, 0 };
537 sqe->msg_flags = flags;
538
539 __submit_wait
540 #endif
541}
542
543ssize_t cfa_recvmsg(int sockfd, struct msghdr *msg, int flags) {
544 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_RECVMSG)
545 return recv(sockfd, msg, flags);
546 #else
547 __submit_prelude
548
549 (*sqe){ IORING_OP_RECVMSG, sockfd, msg, 1, 0 };
550 sqe->msg_flags = flags;
551
552 __submit_wait
553 #endif
554}
555
556ssize_t cfa_send(int sockfd, const void *buf, size_t len, int flags) {
557 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_SEND)
558 return send( sockfd, buf, len, flags );
559 #else
560 __submit_prelude
561
562 (*sqe){ IORING_OP_SEND, sockfd };
563 sqe->addr = (uint64_t)buf;
564 sqe->len = len;
565 sqe->msg_flags = flags;
566
567 __submit_wait
568 #endif
569}
570
571ssize_t cfa_recv(int sockfd, void *buf, size_t len, int flags) {
572 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_RECV)
573 return recv( sockfd, buf, len, flags );
574 #else
575 __submit_prelude
576
577 (*sqe){ IORING_OP_RECV, sockfd };
578 sqe->addr = (uint64_t)buf;
579 sqe->len = len;
580 sqe->msg_flags = flags;
581
582 __submit_wait
583 #endif
584}
585
586int cfa_accept4(int sockfd, struct sockaddr *addr, socklen_t *addrlen, int flags) {
587 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_ACCEPT)
588 return accept4( sockfd, addr, addrlen, flags );
589 #else
590 __submit_prelude
591
592 (*sqe){ IORING_OP_ACCEPT, sockfd };
593 sqe->addr = addr;
594 sqe->addr2 = addrlen;
595 sqe->accept_flags = flags;
596
597 __submit_wait
598 #endif
599}
600
601int cfa_connect(int sockfd, const struct sockaddr *addr, socklen_t addrlen) {
602 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_CONNECT)
603 return connect( sockfd, addr, addrlen );
604 #else
605 __submit_prelude
606
607 (*sqe){ IORING_OP_CONNECT, sockfd };
608 sqe->addr = (uint64_t)addr;
609 sqe->off = addrlen;
610
611 __submit_wait
612 #endif
613}
614
615int cfa_fallocate(int fd, int mode, uint64_t offset, uint64_t len) {
616 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_FALLOCATE)
617 return fallocate( fd, mode, offset, len );
618 #else
619 __submit_prelude
620
621 (*sqe){ IORING_OP_FALLOCATE, fd };
622 sqe->off = offset;
623 sqe->len = length;
624 sqe->mode = mode;
625
626 __submit_wait
627 #endif
628}
629
630int cfa_fadvise(int fd, uint64_t offset, uint64_t len, int advice) {
631 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_FADVISE)
632 return posix_fadvise( fd, offset, len, advice );
633 #else
634 __submit_prelude
635
636 (*sqe){ IORING_OP_FADVISE, fd };
637 sqe->off = (uint64_t)offset;
638 sqe->len = length;
639 sqe->fadvise_advice = advice;
640
641 __submit_wait
642 #endif
643}
644
645int cfa_madvise(void *addr, size_t length, int advice) {
646 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_MADVISE)
647 return madvise( addr, length, advice );
648 #else
649 __submit_prelude
650
651 (*sqe){ IORING_OP_MADVISE, 0 };
652 sqe->addr = (uint64_t)addr;
653 sqe->len = length;
654 sqe->fadvise_advice = advice;
655
656 __submit_wait
657 #endif
658}
659
660int cfa_openat(int dirfd, const char *pathname, int flags, mode_t mode) {
661 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_OPENAT)
662 return openat( dirfd, pathname, flags, mode );
663 #else
664 __submit_prelude
665
666 (*sqe){ IORING_OP_OPENAT, dirfd };
667 sqe->addr = (uint64_t)pathname;
668 sqe->open_flags = flags;
669 sqe->mode = mode;
670
671 __submit_wait
672 #endif
673}
674
675int cfa_close(int fd) {
676 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_CLOSE)
677 return close( fd );
678 #else
679 __submit_prelude
680
681 (*sqe){ IORING_OP_CLOSE, fd };
682
683 __submit_wait
684 #endif
685}
686
687int cfa_statx(int dirfd, const char *pathname, int flags, unsigned int mask, struct statx *statxbuf) {
688 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_STATX)
689 //return statx( dirfd, pathname, flags, mask, statxbuf );
690 return syscall( __NR_io_uring_setup, dirfd, pathname, flags, mask, statxbuf );
691 #else
692 __submit_prelude
693
694 (*sqe){ IORING_OP_STATX, dirfd };
695 sqe->addr = (uint64_t)pathname;
696 sqe->statx_flags = flags;
697 sqe->len = mask;
698 sqe->off = (uint64_t)statxbuf;
699
700 __submit_wait
701 #endif
702}
703
704
705ssize_t cfa_read(int fd, void *buf, size_t count) {
706 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_READ)
707 return read( fd, buf, count );
708 #else
709 __submit_prelude
710
711 (*sqe){ IORING_OP_READ, fd, buf, count, 0 };
712
713 __submit_wait
714 #endif
715}
716
717ssize_t cfa_write(int fd, void *buf, size_t count) {
718 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_WRITE)
719 return read( fd, buf, count );
720 #else
721 __submit_prelude
722
723 (*sqe){ IORING_OP_WRITE, fd, buf, count, 0 };
724
725 __submit_wait
726 #endif
727}
728
729//-----------------------------------------------------------------------------
730// Check if a function is asynchronous
731
732// Macro magic to reduce the size of the following switch case
733#define IS_DEFINED_APPLY(f, ...) f(__VA_ARGS__)
734#define IS_DEFINED_SECOND(first, second, ...) second
735#define IS_DEFINED_TEST(expansion) _CFA_IO_FEATURE_##expansion
736#define IS_DEFINED(macro) IS_DEFINED_APPLY( IS_DEFINED_SECOND,IS_DEFINED_TEST(macro) false, true)
737
738bool has_user_level_blocking( fptr_t func ) {
739 #if defined(HAVE_LINUX_IO_URING_H)
740 if( /*func == (fptr_t)preadv2 || */
741 func == (fptr_t)cfa_preadv2 )
742 #define _CFA_IO_FEATURE_IORING_OP_READV ,
743 return IS_DEFINED(IORING_OP_READV);
744
745 if( /*func == (fptr_t)pwritev2 || */
746 func == (fptr_t)cfa_pwritev2 )
747 #define _CFA_IO_FEATURE_IORING_OP_WRITEV ,
748 return IS_DEFINED(IORING_OP_WRITEV);
749
750 if( /*func == (fptr_t)fsync || */
751 func == (fptr_t)cfa_fsync )
752 #define _CFA_IO_FEATURE_IORING_OP_FSYNC ,
753 return IS_DEFINED(IORING_OP_FSYNC);
754
755 if( /*func == (fptr_t)ync_file_range || */
756 func == (fptr_t)cfa_sync_file_range )
757 #define _CFA_IO_FEATURE_IORING_OP_SYNC_FILE_RANGE ,
758 return IS_DEFINED(IORING_OP_SYNC_FILE_RANGE);
759
760 if( /*func == (fptr_t)sendmsg || */
761 func == (fptr_t)cfa_sendmsg )
762 #define _CFA_IO_FEATURE_IORING_OP_SENDMSG ,
763 return IS_DEFINED(IORING_OP_SENDMSG);
764
765 if( /*func == (fptr_t)recvmsg || */
766 func == (fptr_t)cfa_recvmsg )
767 #define _CFA_IO_FEATURE_IORING_OP_RECVMSG ,
768 return IS_DEFINED(IORING_OP_RECVMSG);
769
770 if( /*func == (fptr_t)send || */
771 func == (fptr_t)cfa_send )
772 #define _CFA_IO_FEATURE_IORING_OP_SEND ,
773 return IS_DEFINED(IORING_OP_SEND);
774
775 if( /*func == (fptr_t)recv || */
776 func == (fptr_t)cfa_recv )
777 #define _CFA_IO_FEATURE_IORING_OP_RECV ,
778 return IS_DEFINED(IORING_OP_RECV);
779
780 if( /*func == (fptr_t)accept4 || */
781 func == (fptr_t)cfa_accept4 )
782 #define _CFA_IO_FEATURE_IORING_OP_ACCEPT ,
783 return IS_DEFINED(IORING_OP_ACCEPT);
784
785 if( /*func == (fptr_t)connect || */
786 func == (fptr_t)cfa_connect )
787 #define _CFA_IO_FEATURE_IORING_OP_CONNECT ,
788 return IS_DEFINED(IORING_OP_CONNECT);
789
790 if( /*func == (fptr_t)fallocate || */
791 func == (fptr_t)cfa_fallocate )
792 #define _CFA_IO_FEATURE_IORING_OP_FALLOCATE ,
793 return IS_DEFINED(IORING_OP_FALLOCATE);
794
795 if( /*func == (fptr_t)posix_fadvise || */
796 func == (fptr_t)cfa_fadvise )
797 #define _CFA_IO_FEATURE_IORING_OP_FADVISE ,
798 return IS_DEFINED(IORING_OP_FADVISE);
799
800 if( /*func == (fptr_t)madvise || */
801 func == (fptr_t)cfa_madvise )
802 #define _CFA_IO_FEATURE_IORING_OP_MADVISE ,
803 return IS_DEFINED(IORING_OP_MADVISE);
804
805 if( /*func == (fptr_t)openat || */
806 func == (fptr_t)cfa_openat )
807 #define _CFA_IO_FEATURE_IORING_OP_OPENAT ,
808 return IS_DEFINED(IORING_OP_OPENAT);
809
810 if( /*func == (fptr_t)close || */
811 func == (fptr_t)cfa_close )
812 #define _CFA_IO_FEATURE_IORING_OP_CLOSE ,
813 return IS_DEFINED(IORING_OP_CLOSE);
814
815 if( /*func == (fptr_t)statx || */
816 func == (fptr_t)cfa_statx )
817 #define _CFA_IO_FEATURE_IORING_OP_STATX ,
818 return IS_DEFINED(IORING_OP_STATX);
819
820 if( /*func == (fptr_t)read || */
821 func == (fptr_t)cfa_read )
822 #define _CFA_IO_FEATURE_IORING_OP_READ ,
823 return IS_DEFINED(IORING_OP_READ);
824
825 if( /*func == (fptr_t)write || */
826 func == (fptr_t)cfa_write )
827 #define _CFA_IO_FEATURE_IORING_OP_WRITE ,
828 return IS_DEFINED(IORING_OP_WRITE);
829 #endif
830
831 return false;
832}
Note: See TracBrowser for help on using the repository browser.