source: libcfa/src/concurrency/io.cfa@ 6136ecc

ADT arm-eh ast-experimental enum forall-pointer-decay jacob/cs343-translation new-ast new-ast-unique-expr pthread-emulation qualifiedEnum
Last change on this file since 6136ecc was 6136ecc, checked in by Thierry Delisle <tdelisle@…>, 6 years ago

Apparently, "standard headers" have way more diversity then I expected.
  • Property mode set to 100644
File size: 28.8 KB
RevLine 
[ecf6b46]1//
2// Cforall Version 1.0.0 Copyright (C) 2020 University of Waterloo
3//
4// The contents of this file are covered under the licence agreement in the
5// file "LICENCE" distributed with Cforall.
6//
7// io.cfa --
8//
9// Author : Thierry Delisle
10// Created On : Thu Apr 23 17:31:00 2020
11// Last Modified By :
12// Last Modified On :
13// Update Count :
14//
15
[4069faad]16// #define __CFA_DEBUG_PRINT_IO__
[0a805f2]17// #define __CFA_DEBUG_PRINT_IO_CORE__
[4069faad]18
[92976d9]19#include "kernel.hfa"
20
21#if !defined(HAVE_LINUX_IO_URING_H)
[f6660520]22 void __kernel_io_startup( cluster & ) {
[92976d9]23 // Nothing to do without io_uring
24 }
25
[f6660520]26 void __kernel_io_start_thrd( cluster & ) {
27 // Nothing to do without io_uring
28 }
29
30 void __kernel_io_stop_thrd ( cluster & ) {
31 // Nothing to do without io_uring
32 }
33
34 void __kernel_io_shutdown( cluster & ) {
[92976d9]35 // Nothing to do without io_uring
36 }
37
38#else
39 extern "C" {
40 #define _GNU_SOURCE /* See feature_test_macros(7) */
41 #include <errno.h>
42 #include <stdint.h>
43 #include <string.h>
44 #include <unistd.h>
45 #include <sys/mman.h>
46 #include <sys/syscall.h>
47
48 #include <linux/io_uring.h>
49 }
50
51 #include "bits/signal.hfa"
52 #include "kernel_private.hfa"
53 #include "thread.hfa"
54
55 uint32_t entries_per_cluster() {
56 return 256;
57 }
58
[f6660520]59 static void * __io_poller_slow( void * arg );
60
61 // Weirdly, some systems that do support io_uring don't actually define these
62 #ifdef __alpha__
63 /*
64 * alpha is the only exception, all other architectures
65 * have common numbers for new system calls.
66 */
67 #ifndef __NR_io_uring_setup
68 #define __NR_io_uring_setup 535
69 #endif
70 #ifndef __NR_io_uring_enter
71 #define __NR_io_uring_enter 536
72 #endif
73 #ifndef __NR_io_uring_register
74 #define __NR_io_uring_register 537
75 #endif
76 #else /* !__alpha__ */
77 #ifndef __NR_io_uring_setup
78 #define __NR_io_uring_setup 425
79 #endif
80 #ifndef __NR_io_uring_enter
81 #define __NR_io_uring_enter 426
82 #endif
83 #ifndef __NR_io_uring_register
84 #define __NR_io_uring_register 427
85 #endif
86 #endif
87
88 #if defined(__CFA_IO_POLLING_USER__)
89 void ?{}( __io_poller_fast & this, struct cluster & cltr ) {
90 this.ring = &cltr.io;
[93f7c001]91 (this.thrd){ "Fast I/O Poller", cltr };
[f6660520]92 }
93 void ^?{}( __io_poller_fast & mutex this );
94 void main( __io_poller_fast & this );
95 static inline $thread * get_thread( __io_poller_fast & this ) { return &this.thrd; }
96 void ^?{}( __io_poller_fast & mutex this ) {}
97 #endif
[185efe6]98
[92976d9]99//=============================================================================================
100// I/O Startup / Shutdown logic
101//=============================================================================================
[f6660520]102 void __kernel_io_startup( cluster & this, bool main_cluster ) {
[92976d9]103 // Step 1 : call to setup
104 struct io_uring_params params;
105 memset(&params, 0, sizeof(params));
106
[2d8f7b0]107 uint32_t nentries = entries_per_cluster();
108
109 int fd = syscall(__NR_io_uring_setup, nentries, &params );
[92976d9]110 if(fd < 0) {
111 abort("KERNEL ERROR: IO_URING SETUP - %s\n", strerror(errno));
112 }
113
114 // Step 2 : mmap result
115 memset(&this.io, 0, sizeof(struct io_ring));
[2d8f7b0]116 struct io_uring_sq & sq = this.io.submit_q;
117 struct io_uring_cq & cq = this.io.completion_q;
[92976d9]118
119 // calculate the right ring size
[2d8f7b0]120 sq.ring_sz = params.sq_off.array + (params.sq_entries * sizeof(unsigned) );
121 cq.ring_sz = params.cq_off.cqes + (params.cq_entries * sizeof(struct io_uring_cqe));
[92976d9]122
123 // Requires features
[d384787]124 #if defined(IORING_FEAT_SINGLE_MMAP)
125 // adjust the size according to the parameters
126 if ((params.features & IORING_FEAT_SINGLE_MMAP) != 0) {
127 cq->ring_sz = sq->ring_sz = max(cq->ring_sz, sq->ring_sz);
128 }
129 #endif
[92976d9]130
131 // mmap the Submit Queue into existence
[2d8f7b0]132 sq.ring_ptr = mmap(0, sq.ring_sz, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd, IORING_OFF_SQ_RING);
133 if (sq.ring_ptr == (void*)MAP_FAILED) {
[92976d9]134 abort("KERNEL ERROR: IO_URING MMAP1 - %s\n", strerror(errno));
135 }
136
137 // Requires features
[d384787]138 #if defined(IORING_FEAT_SINGLE_MMAP)
139 // mmap the Completion Queue into existence (may or may not be needed)
140 if ((params.features & IORING_FEAT_SINGLE_MMAP) != 0) {
141 cq->ring_ptr = sq->ring_ptr;
142 }
143 else
144 #endif
145 {
[92976d9]146 // We need multiple call to MMAP
[2d8f7b0]147 cq.ring_ptr = mmap(0, cq.ring_sz, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd, IORING_OFF_CQ_RING);
148 if (cq.ring_ptr == (void*)MAP_FAILED) {
149 munmap(sq.ring_ptr, sq.ring_sz);
[92976d9]150 abort("KERNEL ERROR: IO_URING MMAP2 - %s\n", strerror(errno));
151 }
[d384787]152 }
[92976d9]153
154 // mmap the submit queue entries
155 size_t size = params.sq_entries * sizeof(struct io_uring_sqe);
[2d8f7b0]156 sq.sqes = (struct io_uring_sqe *)mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd, IORING_OFF_SQES);
157 if (sq.sqes == (struct io_uring_sqe *)MAP_FAILED) {
158 munmap(sq.ring_ptr, sq.ring_sz);
159 if (cq.ring_ptr != sq.ring_ptr) munmap(cq.ring_ptr, cq.ring_sz);
[92976d9]160 abort("KERNEL ERROR: IO_URING MMAP3 - %s\n", strerror(errno));
161 }
162
163 // Get the pointers from the kernel to fill the structure
164 // submit queue
[2d8f7b0]165 sq.head = (volatile uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.head);
166 sq.tail = (volatile uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.tail);
167 sq.mask = ( const uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.ring_mask);
168 sq.num = ( const uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.ring_entries);
169 sq.flags = ( uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.flags);
170 sq.dropped = ( uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.dropped);
171 sq.array = ( uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.array);
172 sq.alloc = *sq.tail;
[c59a346]173 sq.ready = *sq.tail;
[92976d9]174
175 // completion queue
[2d8f7b0]176 cq.head = (volatile uint32_t *)(((intptr_t)cq.ring_ptr) + params.cq_off.head);
177 cq.tail = (volatile uint32_t *)(((intptr_t)cq.ring_ptr) + params.cq_off.tail);
178 cq.mask = ( const uint32_t *)(((intptr_t)cq.ring_ptr) + params.cq_off.ring_mask);
179 cq.num = ( const uint32_t *)(((intptr_t)cq.ring_ptr) + params.cq_off.ring_entries);
180 cq.overflow = ( uint32_t *)(((intptr_t)cq.ring_ptr) + params.cq_off.overflow);
181 cq.cqes = (struct io_uring_cqe *)(((intptr_t)cq.ring_ptr) + params.cq_off.cqes);
182
183 // some paranoid checks
184 /* paranoid */ verifyf( (*cq.mask) == ((*cq.num) - 1ul32), "IO_URING Expected mask to be %u (%u entries), was %u", (*cq.num) - 1ul32, *cq.num, *cq.mask );
185 /* paranoid */ verifyf( (*cq.num) >= nentries, "IO_URING Expected %u entries, got %u", nentries, *cq.num );
186 /* paranoid */ verifyf( (*cq.head) == 0, "IO_URING Expected head to be 0, got %u", *cq.head );
187 /* paranoid */ verifyf( (*cq.tail) == 0, "IO_URING Expected tail to be 0, got %u", *cq.tail );
188
189 /* paranoid */ verifyf( (*sq.mask) == ((*sq.num) - 1ul32), "IO_URING Expected mask to be %u (%u entries), was %u", (*sq.num) - 1ul32, *sq.num, *sq.mask );
190 /* paranoid */ verifyf( (*sq.num) >= nentries, "IO_URING Expected %u entries, got %u", nentries, *sq.num );
191 /* paranoid */ verifyf( (*sq.head) == 0, "IO_URING Expected head to be 0, got %u", *sq.head );
192 /* paranoid */ verifyf( (*sq.tail) == 0, "IO_URING Expected tail to be 0, got %u", *sq.tail );
[92976d9]193
194 // Update the global ring info
195 this.io.flags = params.flags;
196 this.io.fd = fd;
197 this.io.done = false;
[2d8f7b0]198 (this.io.submit){ min(*sq.num, *cq.num) };
[92976d9]199
[d384787]200 // Initialize statistics
[038be32]201 #if !defined(__CFA_NO_STATISTICS__)
202 this.io.submit_q.stats.submit_avg.val = 0;
203 this.io.submit_q.stats.submit_avg.cnt = 0;
204 this.io.completion_q.stats.completed_avg.val = 0;
205 this.io.completion_q.stats.completed_avg.cnt = 0;
206 #endif
[d384787]207
[f6660520]208 if(!main_cluster) {
209 __kernel_io_finish_start( this );
210 }
211 }
212
213 void __kernel_io_finish_start( cluster & this ) {
214 #if defined(__CFA_IO_POLLING_USER__)
[0a805f2]215 __cfadbg_print_safe(io_core, "Kernel I/O : Creating fast poller for cluter %p\n", &this);
[93f7c001]216 (this.io.poller.fast){ this };
[f6660520]217 __thrd_start( this.io.poller.fast, main );
218 #endif
219
[92976d9]220 // Create the poller thread
[0a805f2]221 __cfadbg_print_safe(io_core, "Kernel I/O : Creating slow poller for cluter %p\n", &this);
[f6660520]222 this.io.poller.slow.stack = __create_pthread( &this.io.poller.slow.kthrd, __io_poller_slow, &this );
[92976d9]223 }
224
[f6660520]225 void __kernel_io_prepare_stop( cluster & this ) {
[0a805f2]226 __cfadbg_print_safe(io_core, "Kernel I/O : Stopping pollers for cluster\n", &this);
[92976d9]227 // Notify the poller thread of the shutdown
228 __atomic_store_n(&this.io.done, true, __ATOMIC_SEQ_CST);
[f6660520]229
230 // Stop the IO Poller
[92976d9]231 sigval val = { 1 };
[f6660520]232 pthread_sigqueue( this.io.poller.slow.kthrd, SIGUSR1, val );
233 #if defined(__CFA_IO_POLLING_USER__)
234 post( this.io.poller.sem );
235 #endif
[92976d9]236
237 // Wait for the poller thread to finish
[f6660520]238 pthread_join( this.io.poller.slow.kthrd, 0p );
239 free( this.io.poller.slow.stack );
240
[0a805f2]241 __cfadbg_print_safe(io_core, "Kernel I/O : Slow poller stopped for cluster\n", &this);
[4069faad]242
[f6660520]243 #if defined(__CFA_IO_POLLING_USER__)
244 // unpark the fast io_poller
245 unpark( &this.io.poller.fast.thrd __cfaabi_dbg_ctx2 );
246
247 ^(this.io.poller.fast){};
[4069faad]248
[0a805f2]249 __cfadbg_print_safe(io_core, "Kernel I/O : Fast poller stopped for cluster\n", &this);
[f6660520]250 #endif
251 }
252
253 void __kernel_io_shutdown( cluster & this, bool main_cluster ) {
254 if(!main_cluster) {
255 __kernel_io_prepare_stop( this );
256 }
[92976d9]257
[d384787]258 // print statistics
[038be32]259 #if !defined(__CFA_NO_STATISTICS__)
260 if(this.print_stats) {
261 __cfaabi_bits_print_safe( STDERR_FILENO,
262 "----- I/O uRing Stats -----\n"
263 "- total submit calls : %llu\n"
264 "- avg submit : %lf\n"
265 "- total wait calls : %llu\n"
266 "- avg completion/wait : %lf\n",
267 this.io.submit_q.stats.submit_avg.cnt,
268 ((double)this.io.submit_q.stats.submit_avg.val) / this.io.submit_q.stats.submit_avg.cnt,
269 this.io.completion_q.stats.completed_avg.cnt,
270 ((double)this.io.completion_q.stats.completed_avg.val) / this.io.completion_q.stats.completed_avg.cnt
271 );
272 }
273 #endif
[d384787]274
[92976d9]275 // Shutdown the io rings
276 struct io_uring_sq & sq = this.io.submit_q;
277 struct io_uring_cq & cq = this.io.completion_q;
278
279 // unmap the submit queue entries
[2d8f7b0]280 munmap(sq.sqes, (*sq.num) * sizeof(struct io_uring_sqe));
[92976d9]281
282 // unmap the Submit Queue ring
283 munmap(sq.ring_ptr, sq.ring_sz);
284
285 // unmap the Completion Queue ring, if it is different
286 if (cq.ring_ptr != sq.ring_ptr) {
287 munmap(cq.ring_ptr, cq.ring_sz);
288 }
289
290 // close the file descriptor
291 close(this.io.fd);
292 }
293
294//=============================================================================================
295// I/O Polling
296//=============================================================================================
297 struct io_user_data {
298 int32_t result;
299 $thread * thrd;
300 };
301
302 // Process a single completion message from the io_uring
303 // This is NOT thread-safe
[f6660520]304 static int __drain_io( struct io_ring & ring, sigset_t * mask, int waitcnt, bool in_kernel ) {
305 int ret = syscall( __NR_io_uring_enter, ring.fd, 0, waitcnt, IORING_ENTER_GETEVENTS, mask, _NSIG / 8);
[d384787]306 if( ret < 0 ) {
307 switch((int)errno) {
308 case EAGAIN:
309 case EINTR:
310 return -EAGAIN;
311 default:
312 abort( "KERNEL ERROR: IO_URING WAIT - %s\n", strerror(errno) );
313 }
314 }
315
316 // Drain the queue
[92976d9]317 unsigned head = *ring.completion_q.head;
318 unsigned tail = __atomic_load_n(ring.completion_q.tail, __ATOMIC_ACQUIRE);
319
[d384787]320 // Nothing was new return 0
321 if (head == tail) {
[038be32]322 #if !defined(__CFA_NO_STATISTICS__)
323 ring.completion_q.stats.completed_avg.cnt += 1;
324 #endif
[d384787]325 return 0;
326 }
[92976d9]327
[d384787]328 uint32_t count = tail - head;
329 for(i; count) {
330 unsigned idx = (head + i) & (*ring.completion_q.mask);
331 struct io_uring_cqe & cqe = ring.completion_q.cqes[idx];
[92976d9]332
[d384787]333 /* paranoid */ verify(&cqe);
[92976d9]334
[d384787]335 struct io_user_data * data = (struct io_user_data *)cqe.user_data;
[4069faad]336 __cfadbg_print_safe( io, "Kernel I/O : Performed reading io cqe %p, result %d for %p\n", data, cqe.res, data->thrd );
[2d8f7b0]337
[d384787]338 data->result = cqe.res;
[f6660520]339 if(!in_kernel) { unpark( data->thrd __cfaabi_dbg_ctx2 ); }
340 else { __unpark( data->thrd __cfaabi_dbg_ctx2 ); }
[d384787]341 }
[2d8f7b0]342
343 // Allow new submissions to happen
[d384787]344 V(ring.submit, count);
[92976d9]345
346 // Mark to the kernel that the cqe has been seen
347 // Ensure that the kernel only sees the new value of the head index after the CQEs have been read.
[d384787]348 __atomic_fetch_add( ring.completion_q.head, count, __ATOMIC_RELAXED );
[92976d9]349
[038be32]350 // Update statistics
351 #if !defined(__CFA_NO_STATISTICS__)
352 ring.completion_q.stats.completed_avg.val += count;
353 ring.completion_q.stats.completed_avg.cnt += 1;
354 #endif
[d384787]355
356 return count;
[92976d9]357 }
358
[f6660520]359 static void * __io_poller_slow( void * arg ) {
[92976d9]360 cluster * cltr = (cluster *)arg;
361 struct io_ring & ring = cltr->io;
362
363 sigset_t mask;
364 sigfillset(&mask);
365 if ( pthread_sigmask( SIG_BLOCK, &mask, 0p ) == -1 ) {
366 abort( "KERNEL ERROR: IO_URING - pthread_sigmask" );
367 }
368
369 sigdelset( &mask, SIGUSR1 );
370
371 verify( (*ring.submit_q.head) == (*ring.submit_q.tail) );
372 verify( (*ring.completion_q.head) == (*ring.completion_q.tail) );
373
[d384787]374 while(!__atomic_load_n(&ring.done, __ATOMIC_SEQ_CST)) {
[f6660520]375 #if defined(__CFA_IO_POLLING_USER__)
376
377 // In the user-thread approach drain and if anything was drained,
378 // batton pass to the user-thread
379 int count = __drain_io( ring, &mask, 1, true );
380 if(count > 0) {
[0a805f2]381 __cfadbg_print_safe(io_core, "Kernel I/O : Moving to ring %p to fast poller\n", &ring);
[f6660520]382 __unpark( &ring.poller.fast.thrd __cfaabi_dbg_ctx2 );
383 wait( ring.poller.sem );
384 }
385
386 #else
387
388 //In the naive approach, just poll the io completion queue directly
389 __drain_io( ring, &mask, 1, true );
390
391 #endif
[92976d9]392 }
393
394 return 0p;
395 }
396
[f6660520]397 #if defined(__CFA_IO_POLLING_USER__)
398 void main( __io_poller_fast & this ) {
399 // Start parked
400 park( __cfaabi_dbg_ctx );
401
402 // Then loop until we need to start
403 while(!__atomic_load_n(&this.ring->done, __ATOMIC_SEQ_CST)) {
404 // Drain the io
405 if(0 > __drain_io( *this.ring, 0p, 0, false )) {
406 // If we got something, just yield and check again
407 yield();
408 }
409 else {
410 // We didn't get anything baton pass to the slow poller
[0a805f2]411 __cfadbg_print_safe(io_core, "Kernel I/O : Moving to ring %p to slow poller\n", &this.ring);
[f6660520]412 post( this.ring->poller.sem );
413 park( __cfaabi_dbg_ctx );
414 }
415 }
416 }
417 #endif
418
[92976d9]419//=============================================================================================
420// I/O Submissions
421//=============================================================================================
422
[2d8f7b0]423// Submition steps :
424// 1 - We need to make sure we don't overflow any of the buffer, P(ring.submit) to make sure
425// entries are available. The semaphore make sure that there is no more operations in
426// progress then the number of entries in the buffer. This probably limits concurrency
427// more than necessary since submitted but not completed operations don't need any
428// entries in user space. However, I don't know what happens if we overflow the buffers
429// because too many requests completed at once. This is a safe approach in all cases.
430// Furthermore, with hundreds of entries, this may be okay.
431//
432// 2 - Allocate a queue entry. The ring already has memory for all entries but only the ones
433// listed in sq.array are visible by the kernel. For those not listed, the kernel does not
434// offer any assurance that an entry is not being filled by multiple flags. Therefore, we
435// need to write an allocator that allows allocating concurrently.
436//
437// 3 - Actually fill the submit entry, this is the only simple and straightforward step.
438//
439// 4 - Append the entry index to the array and adjust the tail accordingly. This operation
440// needs to arrive to two concensus at the same time:
441// A - The order in which entries are listed in the array: no two threads must pick the
442// same index for their entries
443// B - When can the tail be update for the kernel. EVERY entries in the array between
444// head and tail must be fully filled and shouldn't ever be touched again.
445//
446
[2489d31]447 static inline [* struct io_uring_sqe, uint32_t] __submit_alloc( struct io_ring & ring ) {
448 // Wait for a spot to be available
449 P(ring.submit);
450
451 // Allocate the sqe
452 uint32_t idx = __atomic_fetch_add(&ring.submit_q.alloc, 1ul32, __ATOMIC_SEQ_CST);
453
454 // Validate that we didn't overflow anything
455 // Check that nothing overflowed
456 /* paranoid */ verify( true );
457
458 // Check that it goes head -> tail -> alloc and never head -> alloc -> tail
459 /* paranoid */ verify( true );
460
461 // Return the sqe
462 return [&ring.submit_q.sqes[ idx & (*ring.submit_q.mask)], idx];
463 }
464
465 static inline void __submit( struct io_ring & ring, uint32_t idx ) {
466 // get mutual exclusion
467 lock(ring.submit_q.lock __cfaabi_dbg_ctx2);
468
469 // Append to the list of ready entries
470 uint32_t * tail = ring.submit_q.tail;
471 const uint32_t mask = *ring.submit_q.mask;
472
473 ring.submit_q.array[ (*tail) & mask ] = idx & mask;
474 __atomic_fetch_add(tail, 1ul32, __ATOMIC_SEQ_CST);
475
476 // Submit however, many entries need to be submitted
477 int ret = syscall( __NR_io_uring_enter, ring.fd, 1, 0, 0, 0p, 0);
478 if( ret < 0 ) {
479 switch((int)errno) {
480 default:
481 abort( "KERNEL ERROR: IO_URING SUBMIT - %s\n", strerror(errno) );
482 }
[2d8f7b0]483 }
[2489d31]484
[038be32]485 // update statistics
486 #if !defined(__CFA_NO_STATISTICS__)
487 ring.submit_q.stats.submit_avg.val += 1;
488 ring.submit_q.stats.submit_avg.cnt += 1;
489 #endif
[d384787]490
[2489d31]491 unlock(ring.submit_q.lock);
492 // Make sure that idx was submitted
493 // Be careful to not get false positive if we cycled the entire list or that someone else submitted for us
[4069faad]494 __cfadbg_print_safe( io, "Kernel I/O : Performed io_submit for %p, returned %d\n", active_thread(), ret );
[2489d31]495 }
496
497 static inline void ?{}(struct io_uring_sqe & this, uint8_t opcode, int fd) {
498 this.opcode = opcode;
499 #if !defined(IOSQE_ASYNC)
500 this.flags = 0;
501 #else
502 this.flags = IOSQE_ASYNC;
503 #endif
504 this.ioprio = 0;
505 this.fd = fd;
506 this.off = 0;
507 this.addr = 0;
508 this.len = 0;
509 this.rw_flags = 0;
510 this.__pad2[0] = this.__pad2[1] = this.__pad2[2] = 0;
[2d8f7b0]511 }
512
[2489d31]513 static inline void ?{}(struct io_uring_sqe & this, uint8_t opcode, int fd, void * addr, uint32_t len, uint64_t off ) {
514 (this){ opcode, fd };
515 this.off = off;
516 this.addr = (uint64_t)addr;
517 this.len = len;
518 }
[f6660520]519
[92976d9]520
521//=============================================================================================
522// I/O Interface
523//=============================================================================================
[f6660520]524
[2d8f7b0]525 #define __submit_prelude \
526 struct io_ring & ring = active_cluster()->io; \
527 struct io_uring_sqe * sqe; \
528 uint32_t idx; \
529 [sqe, idx] = __submit_alloc( ring );
530
531 #define __submit_wait \
532 io_user_data data = { 0, active_thread() }; \
[185efe6]533 /*__cfaabi_bits_print_safe( STDERR_FILENO, "Preparing user data %p for %p\n", &data, data.thrd );*/ \
[2d8f7b0]534 sqe->user_data = (uint64_t)&data; \
535 __submit( ring, idx ); \
536 park( __cfaabi_dbg_ctx ); \
537 return data.result;
[ecf6b46]538#endif
[2d8f7b0]539
[0ea6c5a]540// Some forward declarations
541extern "C" {
542 #include <sys/types.h>
[93f7c001]543 #include <sys/socket.h>
[6136ecc]544 #include <sys/syscall.h>
[0ea6c5a]545 struct iovec;
546 extern ssize_t preadv2 (int fd, const struct iovec *iov, int iovcnt, off_t offset, int flags);
547 extern ssize_t pwritev2(int fd, const struct iovec *iov, int iovcnt, off_t offset, int flags);
548
549 extern int fsync(int fd);
550 extern int sync_file_range(int fd, int64_t offset, int64_t nbytes, unsigned int flags);
551
552 struct msghdr;
553 struct sockaddr;
554 extern ssize_t sendmsg(int sockfd, const struct msghdr *msg, int flags);
555 extern ssize_t recvmsg(int sockfd, struct msghdr *msg, int flags);
556 extern ssize_t send(int sockfd, const void *buf, size_t len, int flags);
557 extern ssize_t recv(int sockfd, void *buf, size_t len, int flags);
558 extern int accept4(int sockfd, struct sockaddr *addr, socklen_t *addrlen, int flags);
559 extern int connect(int sockfd, const struct sockaddr *addr, socklen_t addrlen);
560
561 extern int fallocate(int fd, int mode, uint64_t offset, uint64_t len);
562 extern int posix_fadvise(int fd, uint64_t offset, uint64_t len, int advice);
563 extern int madvise(void *addr, size_t length, int advice);
564
565 extern int openat(int dirfd, const char *pathname, int flags, mode_t mode);
566 extern int close(int fd);
567
568 struct statx;
569 extern int statx(int dirfd, const char *pathname, int flags, unsigned int mask, struct statx *statxbuf);
570
571 extern ssize_t read (int fd, void *buf, size_t count);
572}
573
[2d8f7b0]574//-----------------------------------------------------------------------------
575// Asynchronous operations
[ecf6b46]576ssize_t cfa_preadv2(int fd, const struct iovec *iov, int iovcnt, off_t offset, int flags) {
577 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_READV)
578 return preadv2(fd, iov, iovcnt, offset, flags);
579 #else
580 __submit_prelude
581
582 (*sqe){ IORING_OP_READV, fd, iov, iovcnt, offset };
583
584 __submit_wait
585 #endif
586}
587
588ssize_t cfa_pwritev2(int fd, const struct iovec *iov, int iovcnt, off_t offset, int flags) {
589 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_WRITEV)
590 return pwritev2(fd, iov, iovcnt, offset, flags);
591 #else
592 __submit_prelude
593
594 (*sqe){ IORING_OP_WRITEV, fd, iov, iovcnt, offset };
595
596 __submit_wait
597 #endif
598}
599
600int cfa_fsync(int fd) {
601 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_FSYNC)
602 return fsync(fd);
603 #else
604 __submit_prelude
605
606 (*sqe){ IORING_OP_FSYNC, fd };
607
608 __submit_wait
609 #endif
610}
611
612int cfa_sync_file_range(int fd, int64_t offset, int64_t nbytes, unsigned int flags) {
613 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_SYNC_FILE_RANGE)
614 return sync_file_range(fd, offset, nbytes, flags);
615 #else
616 __submit_prelude
617
618 (*sqe){ IORING_OP_SYNC_FILE_RANGE, fd };
619 sqe->off = offset;
620 sqe->len = nbytes;
621 sqe->sync_range_flags = flags;
622
623 __submit_wait
624 #endif
625}
626
627
628ssize_t cfa_sendmsg(int sockfd, const struct msghdr *msg, int flags) {
629 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_SENDMSG)
630 return recv(sockfd, msg, flags);
631 #else
632 __submit_prelude
633
634 (*sqe){ IORING_OP_SENDMSG, sockfd, msg, 1, 0 };
635 sqe->msg_flags = flags;
636
637 __submit_wait
638 #endif
639}
640
641ssize_t cfa_recvmsg(int sockfd, struct msghdr *msg, int flags) {
642 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_RECVMSG)
643 return recv(sockfd, msg, flags);
644 #else
645 __submit_prelude
646
647 (*sqe){ IORING_OP_RECVMSG, sockfd, msg, 1, 0 };
648 sqe->msg_flags = flags;
649
650 __submit_wait
651 #endif
652}
653
654ssize_t cfa_send(int sockfd, const void *buf, size_t len, int flags) {
655 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_SEND)
656 return send( sockfd, buf, len, flags );
657 #else
658 __submit_prelude
659
660 (*sqe){ IORING_OP_SEND, sockfd };
661 sqe->addr = (uint64_t)buf;
662 sqe->len = len;
663 sqe->msg_flags = flags;
664
665 __submit_wait
666 #endif
667}
668
669ssize_t cfa_recv(int sockfd, void *buf, size_t len, int flags) {
670 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_RECV)
671 return recv( sockfd, buf, len, flags );
672 #else
673 __submit_prelude
674
675 (*sqe){ IORING_OP_RECV, sockfd };
676 sqe->addr = (uint64_t)buf;
677 sqe->len = len;
678 sqe->msg_flags = flags;
679
680 __submit_wait
681 #endif
682}
683
684int cfa_accept4(int sockfd, struct sockaddr *addr, socklen_t *addrlen, int flags) {
685 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_ACCEPT)
[0ea6c5a]686 return accept4( sockfd, addr, addrlen, flags );
[ecf6b46]687 #else
688 __submit_prelude
689
690 (*sqe){ IORING_OP_ACCEPT, sockfd };
691 sqe->addr = addr;
692 sqe->addr2 = addrlen;
693 sqe->accept_flags = flags;
694
695 __submit_wait
696 #endif
697}
698
699int cfa_connect(int sockfd, const struct sockaddr *addr, socklen_t addrlen) {
700 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_CONNECT)
[0ea6c5a]701 return connect( sockfd, addr, addrlen );
[ecf6b46]702 #else
703 __submit_prelude
704
705 (*sqe){ IORING_OP_CONNECT, sockfd };
706 sqe->addr = (uint64_t)addr;
707 sqe->off = addrlen;
708
709 __submit_wait
710 #endif
711}
712
713int cfa_fallocate(int fd, int mode, uint64_t offset, uint64_t len) {
714 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_FALLOCATE)
715 return fallocate( fd, mode, offset, len );
716 #else
717 __submit_prelude
718
719 (*sqe){ IORING_OP_FALLOCATE, fd };
720 sqe->off = offset;
721 sqe->len = length;
722 sqe->mode = mode;
723
724 __submit_wait
725 #endif
726}
727
728int cfa_fadvise(int fd, uint64_t offset, uint64_t len, int advice) {
729 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_FADVISE)
730 return posix_fadvise( fd, offset, len, advice );
731 #else
732 __submit_prelude
733
734 (*sqe){ IORING_OP_FADVISE, fd };
735 sqe->off = (uint64_t)offset;
736 sqe->len = length;
737 sqe->fadvise_advice = advice;
738
739 __submit_wait
740 #endif
741}
742
743int cfa_madvise(void *addr, size_t length, int advice) {
744 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_MADVISE)
745 return madvise( addr, length, advice );
746 #else
747 __submit_prelude
748
749 (*sqe){ IORING_OP_MADVISE, 0 };
750 sqe->addr = (uint64_t)addr;
751 sqe->len = length;
752 sqe->fadvise_advice = advice;
753
754 __submit_wait
755 #endif
756}
757
758int cfa_openat(int dirfd, const char *pathname, int flags, mode_t mode) {
759 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_OPENAT)
760 return openat( dirfd, pathname, flags, mode );
761 #else
762 __submit_prelude
763
764 (*sqe){ IORING_OP_OPENAT, dirfd };
765 sqe->addr = (uint64_t)pathname;
766 sqe->open_flags = flags;
767 sqe->mode = mode;
768
769 __submit_wait
770 #endif
771}
772
773int cfa_close(int fd) {
774 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_CLOSE)
775 return close( fd );
776 #else
777 __submit_prelude
778
779 (*sqe){ IORING_OP_CLOSE, fd };
780
781 __submit_wait
782 #endif
783}
784
785int cfa_statx(int dirfd, const char *pathname, int flags, unsigned int mask, struct statx *statxbuf) {
786 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_STATX)
787 //return statx( dirfd, pathname, flags, mask, statxbuf );
788 return syscall( __NR_io_uring_setup, dirfd, pathname, flags, mask, statxbuf );
789 #else
790 __submit_prelude
791
792 (*sqe){ IORING_OP_STATX, dirfd };
793 sqe->addr = (uint64_t)pathname;
794 sqe->statx_flags = flags;
795 sqe->len = mask;
796 sqe->off = (uint64_t)statxbuf;
797
798 __submit_wait
799 #endif
800}
801
802
803ssize_t cfa_read(int fd, void *buf, size_t count) {
804 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_READ)
805 return read( fd, buf, count );
806 #else
807 __submit_prelude
808
809 (*sqe){ IORING_OP_READ, fd, buf, count, 0 };
810
811 __submit_wait
812 #endif
813}
814
815ssize_t cfa_write(int fd, void *buf, size_t count) {
816 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_WRITE)
817 return read( fd, buf, count );
818 #else
819 __submit_prelude
820
821 (*sqe){ IORING_OP_WRITE, fd, buf, count, 0 };
822
823 __submit_wait
824 #endif
825}
[2d8f7b0]826
827//-----------------------------------------------------------------------------
828// Check if a function is asynchronous
829
830// Macro magic to reduce the size of the following switch case
[ecf6b46]831#define IS_DEFINED_APPLY(f, ...) f(__VA_ARGS__)
832#define IS_DEFINED_SECOND(first, second, ...) second
833#define IS_DEFINED_TEST(expansion) _CFA_IO_FEATURE_##expansion
834#define IS_DEFINED(macro) IS_DEFINED_APPLY( IS_DEFINED_SECOND,IS_DEFINED_TEST(macro) false, true)
[2d8f7b0]835
[ecf6b46]836bool has_user_level_blocking( fptr_t func ) {
837 #if defined(HAVE_LINUX_IO_URING_H)
[2d8f7b0]838 if( /*func == (fptr_t)preadv2 || */
[2489d31]839 func == (fptr_t)cfa_preadv2 )
[2d8f7b0]840 #define _CFA_IO_FEATURE_IORING_OP_READV ,
841 return IS_DEFINED(IORING_OP_READV);
842
843 if( /*func == (fptr_t)pwritev2 || */
[ecf6b46]844 func == (fptr_t)cfa_pwritev2 )
[2d8f7b0]845 #define _CFA_IO_FEATURE_IORING_OP_WRITEV ,
846 return IS_DEFINED(IORING_OP_WRITEV);
847
848 if( /*func == (fptr_t)fsync || */
[ecf6b46]849 func == (fptr_t)cfa_fsync )
[2d8f7b0]850 #define _CFA_IO_FEATURE_IORING_OP_FSYNC ,
851 return IS_DEFINED(IORING_OP_FSYNC);
852
853 if( /*func == (fptr_t)ync_file_range || */
[ecf6b46]854 func == (fptr_t)cfa_sync_file_range )
[2d8f7b0]855 #define _CFA_IO_FEATURE_IORING_OP_SYNC_FILE_RANGE ,
856 return IS_DEFINED(IORING_OP_SYNC_FILE_RANGE);
857
858 if( /*func == (fptr_t)sendmsg || */
[ecf6b46]859 func == (fptr_t)cfa_sendmsg )
[2d8f7b0]860 #define _CFA_IO_FEATURE_IORING_OP_SENDMSG ,
861 return IS_DEFINED(IORING_OP_SENDMSG);
862
863 if( /*func == (fptr_t)recvmsg || */
[ecf6b46]864 func == (fptr_t)cfa_recvmsg )
[2d8f7b0]865 #define _CFA_IO_FEATURE_IORING_OP_RECVMSG ,
866 return IS_DEFINED(IORING_OP_RECVMSG);
867
868 if( /*func == (fptr_t)send || */
[2489d31]869 func == (fptr_t)cfa_send )
[2d8f7b0]870 #define _CFA_IO_FEATURE_IORING_OP_SEND ,
871 return IS_DEFINED(IORING_OP_SEND);
872
873 if( /*func == (fptr_t)recv || */
[2489d31]874 func == (fptr_t)cfa_recv )
[2d8f7b0]875 #define _CFA_IO_FEATURE_IORING_OP_RECV ,
876 return IS_DEFINED(IORING_OP_RECV);
877
878 if( /*func == (fptr_t)accept4 || */
[2489d31]879 func == (fptr_t)cfa_accept4 )
[2d8f7b0]880 #define _CFA_IO_FEATURE_IORING_OP_ACCEPT ,
881 return IS_DEFINED(IORING_OP_ACCEPT);
882
883 if( /*func == (fptr_t)connect || */
[2489d31]884 func == (fptr_t)cfa_connect )
[2d8f7b0]885 #define _CFA_IO_FEATURE_IORING_OP_CONNECT ,
886 return IS_DEFINED(IORING_OP_CONNECT);
887
888 if( /*func == (fptr_t)fallocate || */
[2489d31]889 func == (fptr_t)cfa_fallocate )
[2d8f7b0]890 #define _CFA_IO_FEATURE_IORING_OP_FALLOCATE ,
891 return IS_DEFINED(IORING_OP_FALLOCATE);
892
[0ea6c5a]893 if( /*func == (fptr_t)posix_fadvise || */
[2489d31]894 func == (fptr_t)cfa_fadvise )
[2d8f7b0]895 #define _CFA_IO_FEATURE_IORING_OP_FADVISE ,
896 return IS_DEFINED(IORING_OP_FADVISE);
897
898 if( /*func == (fptr_t)madvise || */
[2489d31]899 func == (fptr_t)cfa_madvise )
[2d8f7b0]900 #define _CFA_IO_FEATURE_IORING_OP_MADVISE ,
901 return IS_DEFINED(IORING_OP_MADVISE);
902
903 if( /*func == (fptr_t)openat || */
[2489d31]904 func == (fptr_t)cfa_openat )
[2d8f7b0]905 #define _CFA_IO_FEATURE_IORING_OP_OPENAT ,
906 return IS_DEFINED(IORING_OP_OPENAT);
907
908 if( /*func == (fptr_t)close || */
[2489d31]909 func == (fptr_t)cfa_close )
[2d8f7b0]910 #define _CFA_IO_FEATURE_IORING_OP_CLOSE ,
911 return IS_DEFINED(IORING_OP_CLOSE);
912
913 if( /*func == (fptr_t)statx || */
[2489d31]914 func == (fptr_t)cfa_statx )
[2d8f7b0]915 #define _CFA_IO_FEATURE_IORING_OP_STATX ,
916 return IS_DEFINED(IORING_OP_STATX);
917
918 if( /*func == (fptr_t)read || */
[ecf6b46]919 func == (fptr_t)cfa_read )
[2d8f7b0]920 #define _CFA_IO_FEATURE_IORING_OP_READ ,
921 return IS_DEFINED(IORING_OP_READ);
922
923 if( /*func == (fptr_t)write || */
[ecf6b46]924 func == (fptr_t)cfa_write )
[2d8f7b0]925 #define _CFA_IO_FEATURE_IORING_OP_WRITE ,
926 return IS_DEFINED(IORING_OP_WRITE);
[ecf6b46]927 #endif
[2d8f7b0]928
[ecf6b46]929 return false;
930}
Note: See TracBrowser for help on using the repository browser.