Context Navigation

io.cfa@ c59a346

Visit:

ADT arm-eh ast-experimental enum forall-pointer-decay jacob/cs343-translation new-ast new-ast-unique-expr pthread-emulation qualifiedEnum

Last change on this file since c59a346 was c59a346, checked in by Thierry Delisle <tdelisle@…>, 6 years ago
Minor fixes to I/O
Property mode set to `100644`
File size: 26.0 KB

Line
1	//
2	// Cforall Version 1.0.0 Copyright (C) 2020 University of Waterloo
3	//
4	// The contents of this file are covered under the licence agreement in the
5	// file "LICENCE" distributed with Cforall.
6	//
7	// io.cfa --
8	//
9	// Author : Thierry Delisle
10	// Created On : Thu Apr 23 17:31:00 2020
11	// Last Modified By :
12	// Last Modified On :
13	// Update Count :
14	//
15
16	#include "kernel.hfa"
17
18	#if !defined(HAVE_LINUX_IO_URING_H)
19	void __kernel_io_startup( cluster & this ) {
20	// Nothing to do without io_uring
21	}
22
23	void __kernel_io_shutdown( cluster & this ) {
24	// Nothing to do without io_uring
25	}
26
27	#else
28	extern "C" {
29	#define _GNU_SOURCE /* See feature_test_macros(7) */
30	#include <errno.h>
31	#include <stdint.h>
32	#include <string.h>
33	#include <unistd.h>
34	#include <sys/mman.h>
35	#include <sys/syscall.h>
36
37	#include <linux/io_uring.h>
38	}
39
40	#include "bits/signal.hfa"
41	#include "kernel_private.hfa"
42	#include "thread.hfa"
43
44	uint32_t entries_per_cluster() {
45	return 256;
46	}
47
48	static void * __io_poller( void * arg );
49
50	// Weirdly, some systems that do support io_uring don't actually define these
51	#ifdef __alpha__
52	/*
53	* alpha is the only exception, all other architectures
54	* have common numbers for new system calls.
55	*/
56	# ifndef __NR_io_uring_setup
57	# define __NR_io_uring_setup 535
58	# endif
59	# ifndef __NR_io_uring_enter
60	# define __NR_io_uring_enter 536
61	# endif
62	# ifndef __NR_io_uring_register
63	# define __NR_io_uring_register 537
64	# endif
65	#else /* !__alpha__ */
66	# ifndef __NR_io_uring_setup
67	# define __NR_io_uring_setup 425
68	# endif
69	# ifndef __NR_io_uring_enter
70	# define __NR_io_uring_enter 426
71	# endif
72	# ifndef __NR_io_uring_register
73	# define __NR_io_uring_register 427
74	# endif
75	#endif
76
77	//=============================================================================================
78	// I/O Startup / Shutdown logic
79	//=============================================================================================
80	void __kernel_io_startup( cluster & this ) {
81	// Step 1 : call to setup
82	struct io_uring_params params;
83	memset(&params, 0, sizeof(params));
84
85	uint32_t nentries = entries_per_cluster();
86
87	int fd = syscall(__NR_io_uring_setup, nentries, &params );
88	if(fd < 0) {
89	abort("KERNEL ERROR: IO_URING SETUP - %s\n", strerror(errno));
90	}
91
92	// Step 2 : mmap result
93	memset(&this.io, 0, sizeof(struct io_ring));
94	struct io_uring_sq & sq = this.io.submit_q;
95	struct io_uring_cq & cq = this.io.completion_q;
96
97	// calculate the right ring size
98	sq.ring_sz = params.sq_off.array + (params.sq_entries * sizeof(unsigned) );
99	cq.ring_sz = params.cq_off.cqes + (params.cq_entries * sizeof(struct io_uring_cqe));
100
101	// Requires features
102	#if defined(IORING_FEAT_SINGLE_MMAP)
103	// adjust the size according to the parameters
104	if ((params.features & IORING_FEAT_SINGLE_MMAP) != 0) {
105	cq->ring_sz = sq->ring_sz = max(cq->ring_sz, sq->ring_sz);
106	}
107	#endif
108
109	// mmap the Submit Queue into existence
110	sq.ring_ptr = mmap(0, sq.ring_sz, PROT_READ \| PROT_WRITE, MAP_SHARED \| MAP_POPULATE, fd, IORING_OFF_SQ_RING);
111	if (sq.ring_ptr == (void*)MAP_FAILED) {
112	abort("KERNEL ERROR: IO_URING MMAP1 - %s\n", strerror(errno));
113	}
114
115	// Requires features
116	#if defined(IORING_FEAT_SINGLE_MMAP)
117	// mmap the Completion Queue into existence (may or may not be needed)
118	if ((params.features & IORING_FEAT_SINGLE_MMAP) != 0) {
119	cq->ring_ptr = sq->ring_ptr;
120	}
121	else
122	#endif
123	{
124	// We need multiple call to MMAP
125	cq.ring_ptr = mmap(0, cq.ring_sz, PROT_READ \| PROT_WRITE, MAP_SHARED \| MAP_POPULATE, fd, IORING_OFF_CQ_RING);
126	if (cq.ring_ptr == (void*)MAP_FAILED) {
127	munmap(sq.ring_ptr, sq.ring_sz);
128	abort("KERNEL ERROR: IO_URING MMAP2 - %s\n", strerror(errno));
129	}
130	}
131
132	// mmap the submit queue entries
133	size_t size = params.sq_entries * sizeof(struct io_uring_sqe);
134	sq.sqes = (struct io_uring_sqe *)mmap(0, size, PROT_READ \| PROT_WRITE, MAP_SHARED \| MAP_POPULATE, fd, IORING_OFF_SQES);
135	if (sq.sqes == (struct io_uring_sqe *)MAP_FAILED) {
136	munmap(sq.ring_ptr, sq.ring_sz);
137	if (cq.ring_ptr != sq.ring_ptr) munmap(cq.ring_ptr, cq.ring_sz);
138	abort("KERNEL ERROR: IO_URING MMAP3 - %s\n", strerror(errno));
139	}
140
141	// Get the pointers from the kernel to fill the structure
142	// submit queue
143	sq.head = (volatile uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.head);
144	sq.tail = (volatile uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.tail);
145	sq.mask = ( const uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.ring_mask);
146	sq.num = ( const uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.ring_entries);
147	sq.flags = ( uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.flags);
148	sq.dropped = ( uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.dropped);
149	sq.array = ( uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.array);
150	sq.alloc = *sq.tail;
151	sq.ready = *sq.tail;
152
153	// completion queue
154	cq.head = (volatile uint32_t *)(((intptr_t)cq.ring_ptr) + params.cq_off.head);
155	cq.tail = (volatile uint32_t *)(((intptr_t)cq.ring_ptr) + params.cq_off.tail);
156	cq.mask = ( const uint32_t *)(((intptr_t)cq.ring_ptr) + params.cq_off.ring_mask);
157	cq.num = ( const uint32_t *)(((intptr_t)cq.ring_ptr) + params.cq_off.ring_entries);
158	cq.overflow = ( uint32_t *)(((intptr_t)cq.ring_ptr) + params.cq_off.overflow);
159	cq.cqes = (struct io_uring_cqe *)(((intptr_t)cq.ring_ptr) + params.cq_off.cqes);
160
161	// some paranoid checks
162	/* paranoid / verifyf( (cq.mask) == ((cq.num) - 1ul32), "IO_URING Expected mask to be %u (%u entries), was %u", (cq.num) - 1ul32, cq.num, cq.mask );
163	/* paranoid / verifyf( (cq.num) >= nentries, "IO_URING Expected %u entries, got %u", nentries, *cq.num );
164	/* paranoid / verifyf( (cq.head) == 0, "IO_URING Expected head to be 0, got %u", *cq.head );
165	/* paranoid / verifyf( (cq.tail) == 0, "IO_URING Expected tail to be 0, got %u", *cq.tail );
166
167	/* paranoid / verifyf( (sq.mask) == ((sq.num) - 1ul32), "IO_URING Expected mask to be %u (%u entries), was %u", (sq.num) - 1ul32, sq.num, sq.mask );
168	/* paranoid / verifyf( (sq.num) >= nentries, "IO_URING Expected %u entries, got %u", nentries, *sq.num );
169	/* paranoid / verifyf( (sq.head) == 0, "IO_URING Expected head to be 0, got %u", *sq.head );
170	/* paranoid / verifyf( (sq.tail) == 0, "IO_URING Expected tail to be 0, got %u", *sq.tail );
171
172	// Update the global ring info
173	this.io.flags = params.flags;
174	this.io.fd = fd;
175	this.io.done = false;
176	(this.io.submit){ min(sq.num, cq.num) };
177
178	// Initialize statistics
179	#if !defined(__CFA_NO_STATISTICS__)
180	this.io.submit_q.stats.submit_avg.val = 0;
181	this.io.submit_q.stats.submit_avg.cnt = 0;
182	this.io.completion_q.stats.completed_avg.val = 0;
183	this.io.completion_q.stats.completed_avg.cnt = 0;
184	#endif
185
186	// Create the poller thread
187	this.io.stack = __create_pthread( &this.io.poller, __io_poller, &this );
188	}
189
190	void __kernel_io_shutdown( cluster & this ) {
191	// Stop the IO Poller
192	#if __CFA_IO_POLLING__ == __CFA_IO_POLLING_NAIVE__
193	// Notify the poller thread of the shutdown
194	__atomic_store_n(&this.io.done, true, __ATOMIC_SEQ_CST);
195	sigval val = { 1 };
196	pthread_sigqueue( this.io.poller, SIGUSR1, val );
197
198	// Wait for the poller thread to finish
199	pthread_join( this.io.poller, 0p );
200	free( this.io.stack );
201
202	// print statistics
203	#if !defined(__CFA_NO_STATISTICS__)
204	if(this.print_stats) {
205	__cfaabi_bits_print_safe( STDERR_FILENO,
206	"----- I/O uRing Stats -----\n"
207	"- total submit calls : %llu\n"
208	"- avg submit : %lf\n"
209	"- total wait calls : %llu\n"
210	"- avg completion/wait : %lf\n",
211	this.io.submit_q.stats.submit_avg.cnt,
212	((double)this.io.submit_q.stats.submit_avg.val) / this.io.submit_q.stats.submit_avg.cnt,
213	this.io.completion_q.stats.completed_avg.cnt,
214	((double)this.io.completion_q.stats.completed_avg.val) / this.io.completion_q.stats.completed_avg.cnt
215	);
216	}
217	#endif
218
219	// Shutdown the io rings
220	struct io_uring_sq & sq = this.io.submit_q;
221	struct io_uring_cq & cq = this.io.completion_q;
222
223	// unmap the submit queue entries
224	munmap(sq.sqes, (sq.num) sizeof(struct io_uring_sqe));
225
226	// unmap the Submit Queue ring
227	munmap(sq.ring_ptr, sq.ring_sz);
228
229	// unmap the Completion Queue ring, if it is different
230	if (cq.ring_ptr != sq.ring_ptr) {
231	munmap(cq.ring_ptr, cq.ring_sz);
232	}
233
234	// close the file descriptor
235	close(this.io.fd);
236	}
237
238	//=============================================================================================
239	// I/O Polling
240	//=============================================================================================
241	struct io_user_data {
242	int32_t result;
243	$thread * thrd;
244	};
245
246	// Process a single completion message from the io_uring
247	// This is NOT thread-safe
248	static int __drain_io( struct io_ring & ring, sigset_t & mask, int waitcnt ) {
249	int ret = syscall( __NR_io_uring_enter, ring.fd, 0, waitcnt, IORING_ENTER_GETEVENTS, &mask, _NSIG / 8);
250	if( ret < 0 ) {
251	switch((int)errno) {
252	case EAGAIN:
253	case EINTR:
254	return -EAGAIN;
255	default:
256	abort( "KERNEL ERROR: IO_URING WAIT - %s\n", strerror(errno) );
257	}
258	}
259
260	// Drain the queue
261	unsigned head = *ring.completion_q.head;
262	unsigned tail = __atomic_load_n(ring.completion_q.tail, __ATOMIC_ACQUIRE);
263
264	// Nothing was new return 0
265	if (head == tail) {
266	#if !defined(__CFA_NO_STATISTICS__)
267	ring.completion_q.stats.completed_avg.cnt += 1;
268	#endif
269	return 0;
270	}
271
272	uint32_t count = tail - head;
273	for(i; count) {
274	unsigned idx = (head + i) & (*ring.completion_q.mask);
275	struct io_uring_cqe & cqe = ring.completion_q.cqes[idx];
276
277	/* paranoid */ verify(&cqe);
278
279	struct io_user_data * data = (struct io_user_data *)cqe.user_data;
280	// __cfaabi_bits_print_safe( STDERR_FILENO, "Performed reading io cqe %p, result %d for %p\n", data, cqe.res, data->thrd );
281
282	data->result = cqe.res;
283	__unpark( data->thrd __cfaabi_dbg_ctx2 );
284	}
285
286	// Allow new submissions to happen
287	V(ring.submit, count);
288
289	// Mark to the kernel that the cqe has been seen
290	// Ensure that the kernel only sees the new value of the head index after the CQEs have been read.
291	__atomic_fetch_add( ring.completion_q.head, count, __ATOMIC_RELAXED );
292
293	// Update statistics
294	#if !defined(__CFA_NO_STATISTICS__)
295	ring.completion_q.stats.completed_avg.val += count;
296	ring.completion_q.stats.completed_avg.cnt += 1;
297	#endif
298
299	return count;
300	}
301
302	static void * __io_poller( void * arg ) {
303	cluster * cltr = (cluster *)arg;
304	struct io_ring & ring = cltr->io;
305
306	sigset_t mask;
307	sigfillset(&mask);
308	if ( pthread_sigmask( SIG_BLOCK, &mask, 0p ) == -1 ) {
309	abort( "KERNEL ERROR: IO_URING - pthread_sigmask" );
310	}
311
312	sigdelset( &mask, SIGUSR1 );
313
314	verify( (ring.submit_q.head) == (ring.submit_q.tail) );
315	verify( (ring.completion_q.head) == (ring.completion_q.tail) );
316
317	while(!__atomic_load_n(&ring.done, __ATOMIC_SEQ_CST)) {
318	__drain_io( ring, mask, 1 );
319	}
320
321	return 0p;
322	}
323
324	//=============================================================================================
325	// I/O Submissions
326	//=============================================================================================
327
328	// Submition steps :
329	// 1 - We need to make sure we don't overflow any of the buffer, P(ring.submit) to make sure
330	// entries are available. The semaphore make sure that there is no more operations in
331	// progress then the number of entries in the buffer. This probably limits concurrency
332	// more than necessary since submitted but not completed operations don't need any
333	// entries in user space. However, I don't know what happens if we overflow the buffers
334	// because too many requests completed at once. This is a safe approach in all cases.
335	// Furthermore, with hundreds of entries, this may be okay.
336	//
337	// 2 - Allocate a queue entry. The ring already has memory for all entries but only the ones
338	// listed in sq.array are visible by the kernel. For those not listed, the kernel does not
339	// offer any assurance that an entry is not being filled by multiple flags. Therefore, we
340	// need to write an allocator that allows allocating concurrently.
341	//
342	// 3 - Actually fill the submit entry, this is the only simple and straightforward step.
343	//
344	// 4 - Append the entry index to the array and adjust the tail accordingly. This operation
345	// needs to arrive to two concensus at the same time:
346	// A - The order in which entries are listed in the array: no two threads must pick the
347	// same index for their entries
348	// B - When can the tail be update for the kernel. EVERY entries in the array between
349	// head and tail must be fully filled and shouldn't ever be touched again.
350	//
351
352	static inline [* struct io_uring_sqe, uint32_t] __submit_alloc( struct io_ring & ring ) {
353	// Wait for a spot to be available
354	P(ring.submit);
355
356	// Allocate the sqe
357	uint32_t idx = __atomic_fetch_add(&ring.submit_q.alloc, 1ul32, __ATOMIC_SEQ_CST);
358
359	// Validate that we didn't overflow anything
360	// Check that nothing overflowed
361	/* paranoid */ verify( true );
362
363	// Check that it goes head -> tail -> alloc and never head -> alloc -> tail
364	/* paranoid */ verify( true );
365
366	// Return the sqe
367	return [&ring.submit_q.sqes[ idx & (*ring.submit_q.mask)], idx];
368	}
369
370	static inline void __submit( struct io_ring & ring, uint32_t idx ) {
371	// get mutual exclusion
372	lock(ring.submit_q.lock __cfaabi_dbg_ctx2);
373
374	// Append to the list of ready entries
375	uint32_t * tail = ring.submit_q.tail;
376	const uint32_t mask = *ring.submit_q.mask;
377
378	ring.submit_q.array[ (*tail) & mask ] = idx & mask;
379	__atomic_fetch_add(tail, 1ul32, __ATOMIC_SEQ_CST);
380
381	// Submit however, many entries need to be submitted
382	int ret = syscall( __NR_io_uring_enter, ring.fd, 1, 0, 0, 0p, 0);
383	// __cfaabi_bits_print_safe( STDERR_FILENO, "Performed io_submit, returned %d\n", ret );
384	if( ret < 0 ) {
385	switch((int)errno) {
386	default:
387	abort( "KERNEL ERROR: IO_URING SUBMIT - %s\n", strerror(errno) );
388	}
389	}
390
391	// update statistics
392	#if !defined(__CFA_NO_STATISTICS__)
393	ring.submit_q.stats.submit_avg.val += 1;
394	ring.submit_q.stats.submit_avg.cnt += 1;
395	#endif
396
397	unlock(ring.submit_q.lock);
398	// Make sure that idx was submitted
399	// Be careful to not get false positive if we cycled the entire list or that someone else submitted for us
400	}
401
402	static inline void ?{}(struct io_uring_sqe & this, uint8_t opcode, int fd) {
403	this.opcode = opcode;
404	#if !defined(IOSQE_ASYNC)
405	this.flags = 0;
406	#else
407	this.flags = IOSQE_ASYNC;
408	#endif
409	this.ioprio = 0;
410	this.fd = fd;
411	this.off = 0;
412	this.addr = 0;
413	this.len = 0;
414	this.rw_flags = 0;
415	this.__pad2[0] = this.__pad2[1] = this.__pad2[2] = 0;
416	}
417
418	static inline void ?{}(struct io_uring_sqe & this, uint8_t opcode, int fd, void * addr, uint32_t len, uint64_t off ) {
419	(this){ opcode, fd };
420	this.off = off;
421	this.addr = (uint64_t)addr;
422	this.len = len;
423	}
424	#endif
425
426	//=============================================================================================
427	// I/O Interface
428	//=============================================================================================
429	#if defined(HAVE_LINUX_IO_URING_H)
430	#define __submit_prelude \
431	struct io_ring & ring = active_cluster()->io; \
432	struct io_uring_sqe * sqe; \
433	uint32_t idx; \
434	[sqe, idx] = __submit_alloc( ring );
435
436	#define __submit_wait \
437	io_user_data data = { 0, active_thread() }; \
438	/__cfaabi_bits_print_safe( STDERR_FILENO, "Preparing user data %p for %p\n", &data, data.thrd );/ \
439	sqe->user_data = (uint64_t)&data; \
440	__submit( ring, idx ); \
441	park( __cfaabi_dbg_ctx ); \
442	return data.result;
443	#endif
444
445	// Some forward declarations
446	extern "C" {
447	#include <sys/types.h>
448	struct iovec;
449	extern ssize_t preadv2 (int fd, const struct iovec *iov, int iovcnt, off_t offset, int flags);
450	extern ssize_t pwritev2(int fd, const struct iovec *iov, int iovcnt, off_t offset, int flags);
451
452	extern int fsync(int fd);
453	extern int sync_file_range(int fd, int64_t offset, int64_t nbytes, unsigned int flags);
454
455	struct msghdr;
456	struct sockaddr;
457	extern ssize_t sendmsg(int sockfd, const struct msghdr *msg, int flags);
458	extern ssize_t recvmsg(int sockfd, struct msghdr *msg, int flags);
459	extern ssize_t send(int sockfd, const void *buf, size_t len, int flags);
460	extern ssize_t recv(int sockfd, void *buf, size_t len, int flags);
461	extern int accept4(int sockfd, struct sockaddr addr, socklen_t addrlen, int flags);
462	extern int connect(int sockfd, const struct sockaddr *addr, socklen_t addrlen);
463
464	extern int fallocate(int fd, int mode, uint64_t offset, uint64_t len);
465	extern int posix_fadvise(int fd, uint64_t offset, uint64_t len, int advice);
466	extern int madvise(void *addr, size_t length, int advice);
467
468	extern int openat(int dirfd, const char *pathname, int flags, mode_t mode);
469	extern int close(int fd);
470
471	struct statx;
472	extern int statx(int dirfd, const char pathname, int flags, unsigned int mask, struct statx statxbuf);
473
474	extern ssize_t read (int fd, void *buf, size_t count);
475	}
476
477	//-----------------------------------------------------------------------------
478	// Asynchronous operations
479	ssize_t cfa_preadv2(int fd, const struct iovec *iov, int iovcnt, off_t offset, int flags) {
480	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_READV)
481	return preadv2(fd, iov, iovcnt, offset, flags);
482	#else
483	__submit_prelude
484
485	(*sqe){ IORING_OP_READV, fd, iov, iovcnt, offset };
486
487	__submit_wait
488	#endif
489	}
490
491	ssize_t cfa_pwritev2(int fd, const struct iovec *iov, int iovcnt, off_t offset, int flags) {
492	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_WRITEV)
493	return pwritev2(fd, iov, iovcnt, offset, flags);
494	#else
495	__submit_prelude
496
497	(*sqe){ IORING_OP_WRITEV, fd, iov, iovcnt, offset };
498
499	__submit_wait
500	#endif
501	}
502
503	int cfa_fsync(int fd) {
504	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_FSYNC)
505	return fsync(fd);
506	#else
507	__submit_prelude
508
509	(*sqe){ IORING_OP_FSYNC, fd };
510
511	__submit_wait
512	#endif
513	}
514
515	int cfa_sync_file_range(int fd, int64_t offset, int64_t nbytes, unsigned int flags) {
516	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_SYNC_FILE_RANGE)
517	return sync_file_range(fd, offset, nbytes, flags);
518	#else
519	__submit_prelude
520
521	(*sqe){ IORING_OP_SYNC_FILE_RANGE, fd };
522	sqe->off = offset;
523	sqe->len = nbytes;
524	sqe->sync_range_flags = flags;
525
526	__submit_wait
527	#endif
528	}
529
530
531	ssize_t cfa_sendmsg(int sockfd, const struct msghdr *msg, int flags) {
532	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_SENDMSG)
533	return recv(sockfd, msg, flags);
534	#else
535	__submit_prelude
536
537	(*sqe){ IORING_OP_SENDMSG, sockfd, msg, 1, 0 };
538	sqe->msg_flags = flags;
539
540	__submit_wait
541	#endif
542	}
543
544	ssize_t cfa_recvmsg(int sockfd, struct msghdr *msg, int flags) {
545	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_RECVMSG)
546	return recv(sockfd, msg, flags);
547	#else
548	__submit_prelude
549
550	(*sqe){ IORING_OP_RECVMSG, sockfd, msg, 1, 0 };
551	sqe->msg_flags = flags;
552
553	__submit_wait
554	#endif
555	}
556
557	ssize_t cfa_send(int sockfd, const void *buf, size_t len, int flags) {
558	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_SEND)
559	return send( sockfd, buf, len, flags );
560	#else
561	__submit_prelude
562
563	(*sqe){ IORING_OP_SEND, sockfd };
564	sqe->addr = (uint64_t)buf;
565	sqe->len = len;
566	sqe->msg_flags = flags;
567
568	__submit_wait
569	#endif
570	}
571
572	ssize_t cfa_recv(int sockfd, void *buf, size_t len, int flags) {
573	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_RECV)
574	return recv( sockfd, buf, len, flags );
575	#else
576	__submit_prelude
577
578	(*sqe){ IORING_OP_RECV, sockfd };
579	sqe->addr = (uint64_t)buf;
580	sqe->len = len;
581	sqe->msg_flags = flags;
582
583	__submit_wait
584	#endif
585	}
586
587	int cfa_accept4(int sockfd, struct sockaddr addr, socklen_t addrlen, int flags) {
588	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_ACCEPT)
589	return accept4( sockfd, addr, addrlen, flags );
590	#else
591	__submit_prelude
592
593	(*sqe){ IORING_OP_ACCEPT, sockfd };
594	sqe->addr = addr;
595	sqe->addr2 = addrlen;
596	sqe->accept_flags = flags;
597
598	__submit_wait
599	#endif
600	}
601
602	int cfa_connect(int sockfd, const struct sockaddr *addr, socklen_t addrlen) {
603	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_CONNECT)
604	return connect( sockfd, addr, addrlen );
605	#else
606	__submit_prelude
607
608	(*sqe){ IORING_OP_CONNECT, sockfd };
609	sqe->addr = (uint64_t)addr;
610	sqe->off = addrlen;
611
612	__submit_wait
613	#endif
614	}
615
616	int cfa_fallocate(int fd, int mode, uint64_t offset, uint64_t len) {
617	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_FALLOCATE)
618	return fallocate( fd, mode, offset, len );
619	#else
620	__submit_prelude
621
622	(*sqe){ IORING_OP_FALLOCATE, fd };
623	sqe->off = offset;
624	sqe->len = length;
625	sqe->mode = mode;
626
627	__submit_wait
628	#endif
629	}
630
631	int cfa_fadvise(int fd, uint64_t offset, uint64_t len, int advice) {
632	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_FADVISE)
633	return posix_fadvise( fd, offset, len, advice );
634	#else
635	__submit_prelude
636
637	(*sqe){ IORING_OP_FADVISE, fd };
638	sqe->off = (uint64_t)offset;
639	sqe->len = length;
640	sqe->fadvise_advice = advice;
641
642	__submit_wait
643	#endif
644	}
645
646	int cfa_madvise(void *addr, size_t length, int advice) {
647	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_MADVISE)
648	return madvise( addr, length, advice );
649	#else
650	__submit_prelude
651
652	(*sqe){ IORING_OP_MADVISE, 0 };
653	sqe->addr = (uint64_t)addr;
654	sqe->len = length;
655	sqe->fadvise_advice = advice;
656
657	__submit_wait
658	#endif
659	}
660
661	int cfa_openat(int dirfd, const char *pathname, int flags, mode_t mode) {
662	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_OPENAT)
663	return openat( dirfd, pathname, flags, mode );
664	#else
665	__submit_prelude
666
667	(*sqe){ IORING_OP_OPENAT, dirfd };
668	sqe->addr = (uint64_t)pathname;
669	sqe->open_flags = flags;
670	sqe->mode = mode;
671
672	__submit_wait
673	#endif
674	}
675
676	int cfa_close(int fd) {
677	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_CLOSE)
678	return close( fd );
679	#else
680	__submit_prelude
681
682	(*sqe){ IORING_OP_CLOSE, fd };
683
684	__submit_wait
685	#endif
686	}
687
688	int cfa_statx(int dirfd, const char pathname, int flags, unsigned int mask, struct statx statxbuf) {
689	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_STATX)
690	//return statx( dirfd, pathname, flags, mask, statxbuf );
691	return syscall( __NR_io_uring_setup, dirfd, pathname, flags, mask, statxbuf );
692	#else
693	__submit_prelude
694
695	(*sqe){ IORING_OP_STATX, dirfd };
696	sqe->addr = (uint64_t)pathname;
697	sqe->statx_flags = flags;
698	sqe->len = mask;
699	sqe->off = (uint64_t)statxbuf;
700
701	__submit_wait
702	#endif
703	}
704
705
706	ssize_t cfa_read(int fd, void *buf, size_t count) {
707	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_READ)
708	return read( fd, buf, count );
709	#else
710	__submit_prelude
711
712	(*sqe){ IORING_OP_READ, fd, buf, count, 0 };
713
714	__submit_wait
715	#endif
716	}
717
718	ssize_t cfa_write(int fd, void *buf, size_t count) {
719	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_WRITE)
720	return read( fd, buf, count );
721	#else
722	__submit_prelude
723
724	(*sqe){ IORING_OP_WRITE, fd, buf, count, 0 };
725
726	__submit_wait
727	#endif
728	}
729
730	//-----------------------------------------------------------------------------
731	// Check if a function is asynchronous
732
733	// Macro magic to reduce the size of the following switch case
734	#define IS_DEFINED_APPLY(f, ...) f(__VA_ARGS__)
735	#define IS_DEFINED_SECOND(first, second, ...) second
736	#define IS_DEFINED_TEST(expansion) _CFA_IO_FEATURE_##expansion
737	#define IS_DEFINED(macro) IS_DEFINED_APPLY( IS_DEFINED_SECOND,IS_DEFINED_TEST(macro) false, true)
738
739	bool has_user_level_blocking( fptr_t func ) {
740	#if defined(HAVE_LINUX_IO_URING_H)
741	if( /func == (fptr_t)preadv2 \|\| /
742	func == (fptr_t)cfa_preadv2 )
743	#define _CFA_IO_FEATURE_IORING_OP_READV ,
744	return IS_DEFINED(IORING_OP_READV);
745
746	if( /func == (fptr_t)pwritev2 \|\| /
747	func == (fptr_t)cfa_pwritev2 )
748	#define _CFA_IO_FEATURE_IORING_OP_WRITEV ,
749	return IS_DEFINED(IORING_OP_WRITEV);
750
751	if( /func == (fptr_t)fsync \|\| /
752	func == (fptr_t)cfa_fsync )
753	#define _CFA_IO_FEATURE_IORING_OP_FSYNC ,
754	return IS_DEFINED(IORING_OP_FSYNC);
755
756	if( /func == (fptr_t)ync_file_range \|\| /
757	func == (fptr_t)cfa_sync_file_range )
758	#define _CFA_IO_FEATURE_IORING_OP_SYNC_FILE_RANGE ,
759	return IS_DEFINED(IORING_OP_SYNC_FILE_RANGE);
760
761	if( /func == (fptr_t)sendmsg \|\| /
762	func == (fptr_t)cfa_sendmsg )
763	#define _CFA_IO_FEATURE_IORING_OP_SENDMSG ,
764	return IS_DEFINED(IORING_OP_SENDMSG);
765
766	if( /func == (fptr_t)recvmsg \|\| /
767	func == (fptr_t)cfa_recvmsg )
768	#define _CFA_IO_FEATURE_IORING_OP_RECVMSG ,
769	return IS_DEFINED(IORING_OP_RECVMSG);
770
771	if( /func == (fptr_t)send \|\| /
772	func == (fptr_t)cfa_send )
773	#define _CFA_IO_FEATURE_IORING_OP_SEND ,
774	return IS_DEFINED(IORING_OP_SEND);
775
776	if( /func == (fptr_t)recv \|\| /
777	func == (fptr_t)cfa_recv )
778	#define _CFA_IO_FEATURE_IORING_OP_RECV ,
779	return IS_DEFINED(IORING_OP_RECV);
780
781	if( /func == (fptr_t)accept4 \|\| /
782	func == (fptr_t)cfa_accept4 )
783	#define _CFA_IO_FEATURE_IORING_OP_ACCEPT ,
784	return IS_DEFINED(IORING_OP_ACCEPT);
785
786	if( /func == (fptr_t)connect \|\| /
787	func == (fptr_t)cfa_connect )
788	#define _CFA_IO_FEATURE_IORING_OP_CONNECT ,
789	return IS_DEFINED(IORING_OP_CONNECT);
790
791	if( /func == (fptr_t)fallocate \|\| /
792	func == (fptr_t)cfa_fallocate )
793	#define _CFA_IO_FEATURE_IORING_OP_FALLOCATE ,
794	return IS_DEFINED(IORING_OP_FALLOCATE);
795
796	if( /func == (fptr_t)posix_fadvise \|\| /
797	func == (fptr_t)cfa_fadvise )
798	#define _CFA_IO_FEATURE_IORING_OP_FADVISE ,
799	return IS_DEFINED(IORING_OP_FADVISE);
800
801	if( /func == (fptr_t)madvise \|\| /
802	func == (fptr_t)cfa_madvise )
803	#define _CFA_IO_FEATURE_IORING_OP_MADVISE ,
804	return IS_DEFINED(IORING_OP_MADVISE);
805
806	if( /func == (fptr_t)openat \|\| /
807	func == (fptr_t)cfa_openat )
808	#define _CFA_IO_FEATURE_IORING_OP_OPENAT ,
809	return IS_DEFINED(IORING_OP_OPENAT);
810
811	if( /func == (fptr_t)close \|\| /
812	func == (fptr_t)cfa_close )
813	#define _CFA_IO_FEATURE_IORING_OP_CLOSE ,
814	return IS_DEFINED(IORING_OP_CLOSE);
815
816	if( /func == (fptr_t)statx \|\| /
817	func == (fptr_t)cfa_statx )
818	#define _CFA_IO_FEATURE_IORING_OP_STATX ,
819	return IS_DEFINED(IORING_OP_STATX);
820
821	if( /func == (fptr_t)read \|\| /
822	func == (fptr_t)cfa_read )
823	#define _CFA_IO_FEATURE_IORING_OP_READ ,
824	return IS_DEFINED(IORING_OP_READ);
825
826	if( /func == (fptr_t)write \|\| /
827	func == (fptr_t)cfa_write )
828	#define _CFA_IO_FEATURE_IORING_OP_WRITE ,
829	return IS_DEFINED(IORING_OP_WRITE);
830	#endif
831
832	return false;
833	}

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

Original Format