Context Navigation

io.cfa @ d384787

ADTarm-ehast-experimentalenumforall-pointer-decayjacob/cs343-translationnew-astnew-ast-unique-exprpthread-emulationqualifiedEnum

Last change on this file since d384787 was d384787, checked in by Thierry Delisle <tdelisle@…>, 5 years ago
Added basic statistics to io.cfa
Property mode set to `100644`
File size: 24.7 KB

Line
1	//
2	// Cforall Version 1.0.0 Copyright (C) 2020 University of Waterloo
3	//
4	// The contents of this file are covered under the licence agreement in the
5	// file "LICENCE" distributed with Cforall.
6	//
7	// io.cfa --
8	//
9	// Author : Thierry Delisle
10	// Created On : Thu Apr 23 17:31:00 2020
11	// Last Modified By :
12	// Last Modified On :
13	// Update Count :
14	//
15
16	#include "kernel.hfa"
17
18	#if !defined(HAVE_LINUX_IO_URING_H)
19	void __kernel_io_startup( cluster & this ) {
20	// Nothing to do without io_uring
21	}
22
23	void __kernel_io_shutdown( cluster & this ) {
24	// Nothing to do without io_uring
25	}
26
27	#else
28	extern "C" {
29	#define _GNU_SOURCE /* See feature_test_macros(7) */
30	#include <errno.h>
31	#include <stdint.h>
32	#include <string.h>
33	#include <unistd.h>
34	#include <sys/mman.h>
35	#include <sys/syscall.h>
36
37	#include <linux/io_uring.h>
38	}
39
40	#include "bits/signal.hfa"
41	#include "kernel_private.hfa"
42	#include "thread.hfa"
43
44	uint32_t entries_per_cluster() {
45	return 256;
46	}
47
48	static void * __io_poller( void * arg );
49
50	// Weirdly, some systems that do support io_uring don't actually define these
51	#ifdef __alpha__
52	/*
53	* alpha is the only exception, all other architectures
54	* have common numbers for new system calls.
55	*/
56	# ifndef __NR_io_uring_setup
57	# define __NR_io_uring_setup 535
58	# endif
59	# ifndef __NR_io_uring_enter
60	# define __NR_io_uring_enter 536
61	# endif
62	# ifndef __NR_io_uring_register
63	# define __NR_io_uring_register 537
64	# endif
65	#else /* !__alpha__ */
66	# ifndef __NR_io_uring_setup
67	# define __NR_io_uring_setup 425
68	# endif
69	# ifndef __NR_io_uring_enter
70	# define __NR_io_uring_enter 426
71	# endif
72	# ifndef __NR_io_uring_register
73	# define __NR_io_uring_register 427
74	# endif
75	#endif
76
77
78	//=============================================================================================
79	// I/O Startup / Shutdown logic
80	//=============================================================================================
81	void __kernel_io_startup( cluster & this ) {
82	// Step 1 : call to setup
83	struct io_uring_params params;
84	memset(&params, 0, sizeof(params));
85
86	uint32_t nentries = entries_per_cluster();
87
88	int fd = syscall(__NR_io_uring_setup, nentries, &params );
89	if(fd < 0) {
90	abort("KERNEL ERROR: IO_URING SETUP - %s\n", strerror(errno));
91	}
92
93	// Step 2 : mmap result
94	memset(&this.io, 0, sizeof(struct io_ring));
95	struct io_uring_sq & sq = this.io.submit_q;
96	struct io_uring_cq & cq = this.io.completion_q;
97
98	// calculate the right ring size
99	sq.ring_sz = params.sq_off.array + (params.sq_entries * sizeof(unsigned) );
100	cq.ring_sz = params.cq_off.cqes + (params.cq_entries * sizeof(struct io_uring_cqe));
101
102	// Requires features
103	#if defined(IORING_FEAT_SINGLE_MMAP)
104	// adjust the size according to the parameters
105	if ((params.features & IORING_FEAT_SINGLE_MMAP) != 0) {
106	cq->ring_sz = sq->ring_sz = max(cq->ring_sz, sq->ring_sz);
107	}
108	#endif
109
110	// mmap the Submit Queue into existence
111	sq.ring_ptr = mmap(0, sq.ring_sz, PROT_READ \| PROT_WRITE, MAP_SHARED \| MAP_POPULATE, fd, IORING_OFF_SQ_RING);
112	if (sq.ring_ptr == (void*)MAP_FAILED) {
113	abort("KERNEL ERROR: IO_URING MMAP1 - %s\n", strerror(errno));
114	}
115
116	// Requires features
117	#if defined(IORING_FEAT_SINGLE_MMAP)
118	// mmap the Completion Queue into existence (may or may not be needed)
119	if ((params.features & IORING_FEAT_SINGLE_MMAP) != 0) {
120	cq->ring_ptr = sq->ring_ptr;
121	}
122	else
123	#endif
124	{
125	// We need multiple call to MMAP
126	cq.ring_ptr = mmap(0, cq.ring_sz, PROT_READ \| PROT_WRITE, MAP_SHARED \| MAP_POPULATE, fd, IORING_OFF_CQ_RING);
127	if (cq.ring_ptr == (void*)MAP_FAILED) {
128	munmap(sq.ring_ptr, sq.ring_sz);
129	abort("KERNEL ERROR: IO_URING MMAP2 - %s\n", strerror(errno));
130	}
131	}
132
133	// mmap the submit queue entries
134	size_t size = params.sq_entries * sizeof(struct io_uring_sqe);
135	sq.sqes = (struct io_uring_sqe *)mmap(0, size, PROT_READ \| PROT_WRITE, MAP_SHARED \| MAP_POPULATE, fd, IORING_OFF_SQES);
136	if (sq.sqes == (struct io_uring_sqe *)MAP_FAILED) {
137	munmap(sq.ring_ptr, sq.ring_sz);
138	if (cq.ring_ptr != sq.ring_ptr) munmap(cq.ring_ptr, cq.ring_sz);
139	abort("KERNEL ERROR: IO_URING MMAP3 - %s\n", strerror(errno));
140	}
141
142	// Get the pointers from the kernel to fill the structure
143	// submit queue
144	sq.head = (volatile uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.head);
145	sq.tail = (volatile uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.tail);
146	sq.mask = ( const uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.ring_mask);
147	sq.num = ( const uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.ring_entries);
148	sq.flags = ( uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.flags);
149	sq.dropped = ( uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.dropped);
150	sq.array = ( uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.array);
151	sq.alloc = *sq.tail;
152
153	// completion queue
154	cq.head = (volatile uint32_t *)(((intptr_t)cq.ring_ptr) + params.cq_off.head);
155	cq.tail = (volatile uint32_t *)(((intptr_t)cq.ring_ptr) + params.cq_off.tail);
156	cq.mask = ( const uint32_t *)(((intptr_t)cq.ring_ptr) + params.cq_off.ring_mask);
157	cq.num = ( const uint32_t *)(((intptr_t)cq.ring_ptr) + params.cq_off.ring_entries);
158	cq.overflow = ( uint32_t *)(((intptr_t)cq.ring_ptr) + params.cq_off.overflow);
159	cq.cqes = (struct io_uring_cqe *)(((intptr_t)cq.ring_ptr) + params.cq_off.cqes);
160
161	// some paranoid checks
162	/* paranoid / verifyf( (cq.mask) == ((cq.num) - 1ul32), "IO_URING Expected mask to be %u (%u entries), was %u", (cq.num) - 1ul32, cq.num, cq.mask );
163	/* paranoid / verifyf( (cq.num) >= nentries, "IO_URING Expected %u entries, got %u", nentries, *cq.num );
164	/* paranoid / verifyf( (cq.head) == 0, "IO_URING Expected head to be 0, got %u", *cq.head );
165	/* paranoid / verifyf( (cq.tail) == 0, "IO_URING Expected tail to be 0, got %u", *cq.tail );
166
167	/* paranoid / verifyf( (sq.mask) == ((sq.num) - 1ul32), "IO_URING Expected mask to be %u (%u entries), was %u", (sq.num) - 1ul32, sq.num, sq.mask );
168	/* paranoid / verifyf( (sq.num) >= nentries, "IO_URING Expected %u entries, got %u", nentries, *sq.num );
169	/* paranoid / verifyf( (sq.head) == 0, "IO_URING Expected head to be 0, got %u", *sq.head );
170	/* paranoid / verifyf( (sq.tail) == 0, "IO_URING Expected tail to be 0, got %u", *sq.tail );
171
172	// Update the global ring info
173	this.io.flags = params.flags;
174	this.io.fd = fd;
175	this.io.done = false;
176	(this.io.submit){ min(sq.num, cq.num) };
177
178	// Initialize statistics
179	this.io.submit_q.stats.submit_avg.val = 0;
180	this.io.submit_q.stats.submit_avg.cnt = 0;
181	this.io.completion_q.stats.completed_avg.val = 0;
182	this.io.completion_q.stats.completed_avg.cnt = 0;
183
184	// Create the poller thread
185	this.io.stack = __create_pthread( &this.io.poller, __io_poller, &this );
186	}
187
188	void __kernel_io_shutdown( cluster & this ) {
189	// Stop the IO Poller
190	// Notify the poller thread of the shutdown
191	__atomic_store_n(&this.io.done, true, __ATOMIC_SEQ_CST);
192	sigval val = { 1 };
193	pthread_sigqueue( this.io.poller, SIGUSR1, val );
194
195	// Wait for the poller thread to finish
196	pthread_join( this.io.poller, 0p );
197	free( this.io.stack );
198
199	// print statistics
200	__cfaabi_bits_print_safe( STDERR_FILENO,
201	"----- I/O uRing Stats -----\n"
202	"- total submit calls : %llu\n"
203	"- avg submit : %lf\n"
204	"- total wait calls : %llu\n"
205	"- avg completion/wait : %lf\n",
206	this.io.submit_q.stats.submit_avg.cnt,
207	((double)this.io.submit_q.stats.submit_avg.val) / this.io.submit_q.stats.submit_avg.cnt,
208	this.io.completion_q.stats.completed_avg.cnt,
209	((double)this.io.completion_q.stats.completed_avg.val) / this.io.completion_q.stats.completed_avg.cnt
210	);
211
212	// Shutdown the io rings
213	struct io_uring_sq & sq = this.io.submit_q;
214	struct io_uring_cq & cq = this.io.completion_q;
215
216	// unmap the submit queue entries
217	munmap(sq.sqes, (sq.num) sizeof(struct io_uring_sqe));
218
219	// unmap the Submit Queue ring
220	munmap(sq.ring_ptr, sq.ring_sz);
221
222	// unmap the Completion Queue ring, if it is different
223	if (cq.ring_ptr != sq.ring_ptr) {
224	munmap(cq.ring_ptr, cq.ring_sz);
225	}
226
227	// close the file descriptor
228	close(this.io.fd);
229	}
230
231	//=============================================================================================
232	// I/O Polling
233	//=============================================================================================
234	struct io_user_data {
235	int32_t result;
236	$thread * thrd;
237	};
238
239	// Process a single completion message from the io_uring
240	// This is NOT thread-safe
241	int __drain_io( struct io_ring & ring, sigset_t & mask, int waitcnt ) {
242	int ret = syscall( __NR_io_uring_enter, ring.fd, 0, waitcnt, IORING_ENTER_GETEVENTS, &mask, _NSIG / 8);
243	if( ret < 0 ) {
244	switch((int)errno) {
245	case EAGAIN:
246	case EINTR:
247	return -EAGAIN;
248	default:
249	abort( "KERNEL ERROR: IO_URING WAIT - %s\n", strerror(errno) );
250	}
251	}
252
253	// Drain the queue
254	unsigned head = *ring.completion_q.head;
255	unsigned tail = __atomic_load_n(ring.completion_q.tail, __ATOMIC_ACQUIRE);
256
257	// Nothing was new return 0
258	if (head == tail) {
259	ring.completion_q.stats.completed_avg.cnt += 1;
260	return 0;
261	}
262
263	uint32_t count = tail - head;
264	for(i; count) {
265	unsigned idx = (head + i) & (*ring.completion_q.mask);
266	struct io_uring_cqe & cqe = ring.completion_q.cqes[idx];
267
268	/* paranoid */ verify(&cqe);
269
270	struct io_user_data * data = (struct io_user_data *)cqe.user_data;
271	// __cfaabi_bits_print_safe( STDERR_FILENO, "Performed reading io cqe %p, result %d for %p\n", data, cqe.res, data->thrd );
272
273	data->result = cqe.res;
274	__unpark( data->thrd __cfaabi_dbg_ctx2 );
275	}
276
277	// Allow new submissions to happen
278	V(ring.submit, count);
279
280	// Mark to the kernel that the cqe has been seen
281	// Ensure that the kernel only sees the new value of the head index after the CQEs have been read.
282	__atomic_fetch_add( ring.completion_q.head, count, __ATOMIC_RELAXED );
283
284	ring.completion_q.stats.completed_avg.val += count;
285	ring.completion_q.stats.completed_avg.cnt += 1;
286
287	return count;
288	}
289
290	static void * __io_poller( void * arg ) {
291	cluster * cltr = (cluster *)arg;
292	struct io_ring & ring = cltr->io;
293
294	sigset_t mask;
295	sigfillset(&mask);
296	if ( pthread_sigmask( SIG_BLOCK, &mask, 0p ) == -1 ) {
297	abort( "KERNEL ERROR: IO_URING - pthread_sigmask" );
298	}
299
300	sigdelset( &mask, SIGUSR1 );
301
302	verify( (ring.submit_q.head) == (ring.submit_q.tail) );
303	verify( (ring.completion_q.head) == (ring.completion_q.tail) );
304
305	while(!__atomic_load_n(&ring.done, __ATOMIC_SEQ_CST)) {
306	__drain_io( ring, mask, 1 );
307	}
308
309	return 0p;
310	}
311
312	//=============================================================================================
313	// I/O Submissions
314	//=============================================================================================
315
316	// Submition steps :
317	// 1 - We need to make sure we don't overflow any of the buffer, P(ring.submit) to make sure
318	// entries are available. The semaphore make sure that there is no more operations in
319	// progress then the number of entries in the buffer. This probably limits concurrency
320	// more than necessary since submitted but not completed operations don't need any
321	// entries in user space. However, I don't know what happens if we overflow the buffers
322	// because too many requests completed at once. This is a safe approach in all cases.
323	// Furthermore, with hundreds of entries, this may be okay.
324	//
325	// 2 - Allocate a queue entry. The ring already has memory for all entries but only the ones
326	// listed in sq.array are visible by the kernel. For those not listed, the kernel does not
327	// offer any assurance that an entry is not being filled by multiple flags. Therefore, we
328	// need to write an allocator that allows allocating concurrently.
329	//
330	// 3 - Actually fill the submit entry, this is the only simple and straightforward step.
331	//
332	// 4 - Append the entry index to the array and adjust the tail accordingly. This operation
333	// needs to arrive to two concensus at the same time:
334	// A - The order in which entries are listed in the array: no two threads must pick the
335	// same index for their entries
336	// B - When can the tail be update for the kernel. EVERY entries in the array between
337	// head and tail must be fully filled and shouldn't ever be touched again.
338	//
339
340	static inline [* struct io_uring_sqe, uint32_t] __submit_alloc( struct io_ring & ring ) {
341	// Wait for a spot to be available
342	P(ring.submit);
343
344	// Allocate the sqe
345	uint32_t idx = __atomic_fetch_add(&ring.submit_q.alloc, 1ul32, __ATOMIC_SEQ_CST);
346
347	// Validate that we didn't overflow anything
348	// Check that nothing overflowed
349	/* paranoid */ verify( true );
350
351	// Check that it goes head -> tail -> alloc and never head -> alloc -> tail
352	/* paranoid */ verify( true );
353
354	// Return the sqe
355	return [&ring.submit_q.sqes[ idx & (*ring.submit_q.mask)], idx];
356	}
357
358	static inline void __submit( struct io_ring & ring, uint32_t idx ) {
359	// get mutual exclusion
360	lock(ring.submit_q.lock __cfaabi_dbg_ctx2);
361
362	// Append to the list of ready entries
363	uint32_t * tail = ring.submit_q.tail;
364	const uint32_t mask = *ring.submit_q.mask;
365
366	ring.submit_q.array[ (*tail) & mask ] = idx & mask;
367	__atomic_fetch_add(tail, 1ul32, __ATOMIC_SEQ_CST);
368
369	// Submit however, many entries need to be submitted
370	int ret = syscall( __NR_io_uring_enter, ring.fd, 1, 0, 0, 0p, 0);
371	// __cfaabi_bits_print_safe( STDERR_FILENO, "Performed io_submit, returned %d\n", ret );
372	if( ret < 0 ) {
373	switch((int)errno) {
374	default:
375	abort( "KERNEL ERROR: IO_URING SUBMIT - %s\n", strerror(errno) );
376	}
377	}
378
379	ring.submit_q.stats.submit_avg.val += 1;
380	ring.submit_q.stats.submit_avg.cnt += 1;
381
382	unlock(ring.submit_q.lock);
383	// Make sure that idx was submitted
384	// Be careful to not get false positive if we cycled the entire list or that someone else submitted for us
385	}
386
387	static inline void ?{}(struct io_uring_sqe & this, uint8_t opcode, int fd) {
388	this.opcode = opcode;
389	#if !defined(IOSQE_ASYNC)
390	this.flags = 0;
391	#else
392	this.flags = IOSQE_ASYNC;
393	#endif
394	this.ioprio = 0;
395	this.fd = fd;
396	this.off = 0;
397	this.addr = 0;
398	this.len = 0;
399	this.rw_flags = 0;
400	this.__pad2[0] = this.__pad2[1] = this.__pad2[2] = 0;
401	}
402
403	static inline void ?{}(struct io_uring_sqe & this, uint8_t opcode, int fd, void * addr, uint32_t len, uint64_t off ) {
404	(this){ opcode, fd };
405	this.off = off;
406	this.addr = (uint64_t)addr;
407	this.len = len;
408	}
409	#endif
410
411	//=============================================================================================
412	// I/O Interface
413	//=============================================================================================
414	extern "C" {
415	#define __USE_GNU
416	#define _GNU_SOURCE
417	#include <fcntl.h>
418	#include <sys/uio.h>
419	#include <sys/socket.h>
420	#include <sys/stat.h>
421	}
422
423	#if defined(HAVE_LINUX_IO_URING_H)
424	#define __submit_prelude \
425	struct io_ring & ring = active_cluster()->io; \
426	struct io_uring_sqe * sqe; \
427	uint32_t idx; \
428	[sqe, idx] = __submit_alloc( ring );
429
430	#define __submit_wait \
431	io_user_data data = { 0, active_thread() }; \
432	/__cfaabi_bits_print_safe( STDERR_FILENO, "Preparing user data %p for %p\n", &data, data.thrd );/ \
433	sqe->user_data = (uint64_t)&data; \
434	__submit( ring, idx ); \
435	park( __cfaabi_dbg_ctx ); \
436	return data.result;
437	#endif
438
439	//-----------------------------------------------------------------------------
440	// Asynchronous operations
441	ssize_t cfa_preadv2(int fd, const struct iovec *iov, int iovcnt, off_t offset, int flags) {
442	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_READV)
443	return preadv2(fd, iov, iovcnt, offset, flags);
444	#else
445	__submit_prelude
446
447	(*sqe){ IORING_OP_READV, fd, iov, iovcnt, offset };
448
449	__submit_wait
450	#endif
451	}
452
453	ssize_t cfa_pwritev2(int fd, const struct iovec *iov, int iovcnt, off_t offset, int flags) {
454	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_WRITEV)
455	return pwritev2(fd, iov, iovcnt, offset, flags);
456	#else
457	__submit_prelude
458
459	(*sqe){ IORING_OP_WRITEV, fd, iov, iovcnt, offset };
460
461	__submit_wait
462	#endif
463	}
464
465	int cfa_fsync(int fd) {
466	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_FSYNC)
467	return fsync(fd);
468	#else
469	__submit_prelude
470
471	(*sqe){ IORING_OP_FSYNC, fd };
472
473	__submit_wait
474	#endif
475	}
476
477	int cfa_sync_file_range(int fd, int64_t offset, int64_t nbytes, unsigned int flags) {
478	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_SYNC_FILE_RANGE)
479	return sync_file_range(fd, offset, nbytes, flags);
480	#else
481	__submit_prelude
482
483	(*sqe){ IORING_OP_SYNC_FILE_RANGE, fd };
484	sqe->off = offset;
485	sqe->len = nbytes;
486	sqe->sync_range_flags = flags;
487
488	__submit_wait
489	#endif
490	}
491
492
493	ssize_t cfa_sendmsg(int sockfd, const struct msghdr *msg, int flags) {
494	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_SENDMSG)
495	return recv(sockfd, msg, flags);
496	#else
497	__submit_prelude
498
499	(*sqe){ IORING_OP_SENDMSG, sockfd, msg, 1, 0 };
500	sqe->msg_flags = flags;
501
502	__submit_wait
503	#endif
504	}
505
506	ssize_t cfa_recvmsg(int sockfd, struct msghdr *msg, int flags) {
507	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_RECVMSG)
508	return recv(sockfd, msg, flags);
509	#else
510	__submit_prelude
511
512	(*sqe){ IORING_OP_RECVMSG, sockfd, msg, 1, 0 };
513	sqe->msg_flags = flags;
514
515	__submit_wait
516	#endif
517	}
518
519	ssize_t cfa_send(int sockfd, const void *buf, size_t len, int flags) {
520	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_SEND)
521	return send( sockfd, buf, len, flags );
522	#else
523	__submit_prelude
524
525	(*sqe){ IORING_OP_SEND, sockfd };
526	sqe->addr = (uint64_t)buf;
527	sqe->len = len;
528	sqe->msg_flags = flags;
529
530	__submit_wait
531	#endif
532	}
533
534	ssize_t cfa_recv(int sockfd, void *buf, size_t len, int flags) {
535	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_RECV)
536	return recv( sockfd, buf, len, flags );
537	#else
538	__submit_prelude
539
540	(*sqe){ IORING_OP_RECV, sockfd };
541	sqe->addr = (uint64_t)buf;
542	sqe->len = len;
543	sqe->msg_flags = flags;
544
545	__submit_wait
546	#endif
547	}
548
549	int cfa_accept4(int sockfd, struct sockaddr addr, socklen_t addrlen, int flags) {
550	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_ACCEPT)
551	#pragma GCC diagnostic push
552	#pragma GCC diagnostic ignored "-Wattributes"
553	__SOCKADDR_ARG _addr;
554	_addr.__sockaddr__ = addr;
555	return accept4( sockfd, _addr, addrlen, flags );
556	#pragma GCC diagnostic pop
557	#else
558	__submit_prelude
559
560	(*sqe){ IORING_OP_ACCEPT, sockfd };
561	sqe->addr = addr;
562	sqe->addr2 = addrlen;
563	sqe->accept_flags = flags;
564
565	__submit_wait
566	#endif
567	}
568
569	int cfa_connect(int sockfd, const struct sockaddr *addr, socklen_t addrlen) {
570	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_CONNECT)
571	#pragma GCC diagnostic push
572	#pragma GCC diagnostic ignored "-Wattributes"
573	__CONST_SOCKADDR_ARG _addr;
574	_addr.__sockaddr__ = addr;
575	return connect( sockfd, _addr, addrlen );
576	#pragma GCC diagnostic pop
577	#else
578	__submit_prelude
579
580	(*sqe){ IORING_OP_CONNECT, sockfd };
581	sqe->addr = (uint64_t)addr;
582	sqe->off = addrlen;
583
584	__submit_wait
585	#endif
586	}
587
588	int cfa_fallocate(int fd, int mode, uint64_t offset, uint64_t len) {
589	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_FALLOCATE)
590	return fallocate( fd, mode, offset, len );
591	#else
592	__submit_prelude
593
594	(*sqe){ IORING_OP_FALLOCATE, fd };
595	sqe->off = offset;
596	sqe->len = length;
597	sqe->mode = mode;
598
599	__submit_wait
600	#endif
601	}
602
603	int cfa_fadvise(int fd, uint64_t offset, uint64_t len, int advice) {
604	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_FADVISE)
605	return posix_fadvise( fd, offset, len, advice );
606	#else
607	__submit_prelude
608
609	(*sqe){ IORING_OP_FADVISE, fd };
610	sqe->off = (uint64_t)offset;
611	sqe->len = length;
612	sqe->fadvise_advice = advice;
613
614	__submit_wait
615	#endif
616	}
617
618	int cfa_madvise(void *addr, size_t length, int advice) {
619	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_MADVISE)
620	return madvise( addr, length, advice );
621	#else
622	__submit_prelude
623
624	(*sqe){ IORING_OP_MADVISE, 0 };
625	sqe->addr = (uint64_t)addr;
626	sqe->len = length;
627	sqe->fadvise_advice = advice;
628
629	__submit_wait
630	#endif
631	}
632
633	int cfa_openat(int dirfd, const char *pathname, int flags, mode_t mode) {
634	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_OPENAT)
635	return openat( dirfd, pathname, flags, mode );
636	#else
637	__submit_prelude
638
639	(*sqe){ IORING_OP_OPENAT, dirfd };
640	sqe->addr = (uint64_t)pathname;
641	sqe->open_flags = flags;
642	sqe->mode = mode;
643
644	__submit_wait
645	#endif
646	}
647
648	int cfa_close(int fd) {
649	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_CLOSE)
650	return close( fd );
651	#else
652	__submit_prelude
653
654	(*sqe){ IORING_OP_CLOSE, fd };
655
656	__submit_wait
657	#endif
658	}
659
660	int cfa_statx(int dirfd, const char pathname, int flags, unsigned int mask, struct statx statxbuf) {
661	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_STATX)
662	//return statx( dirfd, pathname, flags, mask, statxbuf );
663	return syscall( __NR_io_uring_setup, dirfd, pathname, flags, mask, statxbuf );
664	#else
665	__submit_prelude
666
667	(*sqe){ IORING_OP_STATX, dirfd };
668	sqe->addr = (uint64_t)pathname;
669	sqe->statx_flags = flags;
670	sqe->len = mask;
671	sqe->off = (uint64_t)statxbuf;
672
673	__submit_wait
674	#endif
675	}
676
677
678	ssize_t cfa_read(int fd, void *buf, size_t count) {
679	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_READ)
680	return read( fd, buf, count );
681	#else
682	__submit_prelude
683
684	(*sqe){ IORING_OP_READ, fd, buf, count, 0 };
685
686	__submit_wait
687	#endif
688	}
689
690	ssize_t cfa_write(int fd, void *buf, size_t count) {
691	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_WRITE)
692	return read( fd, buf, count );
693	#else
694	__submit_prelude
695
696	(*sqe){ IORING_OP_WRITE, fd, buf, count, 0 };
697
698	__submit_wait
699	#endif
700	}
701
702	//-----------------------------------------------------------------------------
703	// Check if a function is asynchronous
704
705	// Macro magic to reduce the size of the following switch case
706	#define IS_DEFINED_APPLY(f, ...) f(__VA_ARGS__)
707	#define IS_DEFINED_SECOND(first, second, ...) second
708	#define IS_DEFINED_TEST(expansion) _CFA_IO_FEATURE_##expansion
709	#define IS_DEFINED(macro) IS_DEFINED_APPLY( IS_DEFINED_SECOND,IS_DEFINED_TEST(macro) false, true)
710
711	bool has_user_level_blocking( fptr_t func ) {
712	#if defined(HAVE_LINUX_IO_URING_H)
713	if( /func == (fptr_t)preadv2 \|\| /
714	func == (fptr_t)cfa_preadv2 )
715	#define _CFA_IO_FEATURE_IORING_OP_READV ,
716	return IS_DEFINED(IORING_OP_READV);
717
718	if( /func == (fptr_t)pwritev2 \|\| /
719	func == (fptr_t)cfa_pwritev2 )
720	#define _CFA_IO_FEATURE_IORING_OP_WRITEV ,
721	return IS_DEFINED(IORING_OP_WRITEV);
722
723	if( /func == (fptr_t)fsync \|\| /
724	func == (fptr_t)cfa_fsync )
725	#define _CFA_IO_FEATURE_IORING_OP_FSYNC ,
726	return IS_DEFINED(IORING_OP_FSYNC);
727
728	if( /func == (fptr_t)ync_file_range \|\| /
729	func == (fptr_t)cfa_sync_file_range )
730	#define _CFA_IO_FEATURE_IORING_OP_SYNC_FILE_RANGE ,
731	return IS_DEFINED(IORING_OP_SYNC_FILE_RANGE);
732
733	if( /func == (fptr_t)sendmsg \|\| /
734	func == (fptr_t)cfa_sendmsg )
735	#define _CFA_IO_FEATURE_IORING_OP_SENDMSG ,
736	return IS_DEFINED(IORING_OP_SENDMSG);
737
738	if( /func == (fptr_t)recvmsg \|\| /
739	func == (fptr_t)cfa_recvmsg )
740	#define _CFA_IO_FEATURE_IORING_OP_RECVMSG ,
741	return IS_DEFINED(IORING_OP_RECVMSG);
742
743	if( /func == (fptr_t)send \|\| /
744	func == (fptr_t)cfa_send )
745	#define _CFA_IO_FEATURE_IORING_OP_SEND ,
746	return IS_DEFINED(IORING_OP_SEND);
747
748	if( /func == (fptr_t)recv \|\| /
749	func == (fptr_t)cfa_recv )
750	#define _CFA_IO_FEATURE_IORING_OP_RECV ,
751	return IS_DEFINED(IORING_OP_RECV);
752
753	if( /func == (fptr_t)accept4 \|\| /
754	func == (fptr_t)cfa_accept4 )
755	#define _CFA_IO_FEATURE_IORING_OP_ACCEPT ,
756	return IS_DEFINED(IORING_OP_ACCEPT);
757
758	if( /func == (fptr_t)connect \|\| /
759	func == (fptr_t)cfa_connect )
760	#define _CFA_IO_FEATURE_IORING_OP_CONNECT ,
761	return IS_DEFINED(IORING_OP_CONNECT);
762
763	if( /func == (fptr_t)fallocate \|\| /
764	func == (fptr_t)cfa_fallocate )
765	#define _CFA_IO_FEATURE_IORING_OP_FALLOCATE ,
766	return IS_DEFINED(IORING_OP_FALLOCATE);
767
768	if( /func == (fptr_t)fadvise \|\| /
769	func == (fptr_t)cfa_fadvise )
770	#define _CFA_IO_FEATURE_IORING_OP_FADVISE ,
771	return IS_DEFINED(IORING_OP_FADVISE);
772
773	if( /func == (fptr_t)madvise \|\| /
774	func == (fptr_t)cfa_madvise )
775	#define _CFA_IO_FEATURE_IORING_OP_MADVISE ,
776	return IS_DEFINED(IORING_OP_MADVISE);
777
778	if( /func == (fptr_t)openat \|\| /
779	func == (fptr_t)cfa_openat )
780	#define _CFA_IO_FEATURE_IORING_OP_OPENAT ,
781	return IS_DEFINED(IORING_OP_OPENAT);
782
783	if( /func == (fptr_t)close \|\| /
784	func == (fptr_t)cfa_close )
785	#define _CFA_IO_FEATURE_IORING_OP_CLOSE ,
786	return IS_DEFINED(IORING_OP_CLOSE);
787
788	if( /func == (fptr_t)statx \|\| /
789	func == (fptr_t)cfa_statx )
790	#define _CFA_IO_FEATURE_IORING_OP_STATX ,
791	return IS_DEFINED(IORING_OP_STATX);
792
793	if( /func == (fptr_t)read \|\| /
794	func == (fptr_t)cfa_read )
795	#define _CFA_IO_FEATURE_IORING_OP_READ ,
796	return IS_DEFINED(IORING_OP_READ);
797
798	if( /func == (fptr_t)write \|\| /
799	func == (fptr_t)cfa_write )
800	#define _CFA_IO_FEATURE_IORING_OP_WRITE ,
801	return IS_DEFINED(IORING_OP_WRITE);
802	#endif
803
804	return false;
805	}

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

Original Format