Context Navigation

io.cfa@ d384787

Visit:

ADT arm-eh ast-experimental enum forall-pointer-decay jacob/cs343-translation new-ast new-ast-unique-expr pthread-emulation qualifiedEnum

Last change on this file since d384787 was d384787, checked in by Thierry Delisle <tdelisle@…>, 6 years ago
Added basic statistics to io.cfa
Property mode set to `100644`
File size: 24.7 KB

Rev	Line
[ecf6b46]	1	//
	2	// Cforall Version 1.0.0 Copyright (C) 2020 University of Waterloo
	3	//
	4	// The contents of this file are covered under the licence agreement in the
	5	// file "LICENCE" distributed with Cforall.
	6	//
	7	// io.cfa --
	8	//
	9	// Author : Thierry Delisle
	10	// Created On : Thu Apr 23 17:31:00 2020
	11	// Last Modified By :
	12	// Last Modified On :
	13	// Update Count :
	14	//
	15
[92976d9]	16	#include "kernel.hfa"
	17
	18	#if !defined(HAVE_LINUX_IO_URING_H)
	19	void __kernel_io_startup( cluster & this ) {
	20	// Nothing to do without io_uring
	21	}
	22
	23	void __kernel_io_shutdown( cluster & this ) {
	24	// Nothing to do without io_uring
	25	}
	26
	27	#else
	28	extern "C" {
	29	#define _GNU_SOURCE /* See feature_test_macros(7) */
	30	#include <errno.h>
	31	#include <stdint.h>
	32	#include <string.h>
	33	#include <unistd.h>
	34	#include <sys/mman.h>
	35	#include <sys/syscall.h>
	36
	37	#include <linux/io_uring.h>
	38	}
	39
	40	#include "bits/signal.hfa"
	41	#include "kernel_private.hfa"
	42	#include "thread.hfa"
	43
	44	uint32_t entries_per_cluster() {
	45	return 256;
	46	}
	47
	48	static void * __io_poller( void * arg );
	49
[185efe6]	50	// Weirdly, some systems that do support io_uring don't actually define these
	51	#ifdef __alpha__
	52	/*
	53	* alpha is the only exception, all other architectures
	54	* have common numbers for new system calls.
	55	*/
	56	# ifndef __NR_io_uring_setup
	57	# define __NR_io_uring_setup 535
	58	# endif
	59	# ifndef __NR_io_uring_enter
	60	# define __NR_io_uring_enter 536
	61	# endif
	62	# ifndef __NR_io_uring_register
	63	# define __NR_io_uring_register 537
	64	# endif
	65	#else /* !__alpha__ */
	66	# ifndef __NR_io_uring_setup
	67	# define __NR_io_uring_setup 425
	68	# endif
	69	# ifndef __NR_io_uring_enter
	70	# define __NR_io_uring_enter 426
	71	# endif
	72	# ifndef __NR_io_uring_register
	73	# define __NR_io_uring_register 427
	74	# endif
	75	#endif
	76
	77
[92976d9]	78	//=============================================================================================
	79	// I/O Startup / Shutdown logic
	80	//=============================================================================================
	81	void __kernel_io_startup( cluster & this ) {
	82	// Step 1 : call to setup
	83	struct io_uring_params params;
	84	memset(&params, 0, sizeof(params));
	85
[2d8f7b0]	86	uint32_t nentries = entries_per_cluster();
	87
	88	int fd = syscall(__NR_io_uring_setup, nentries, &params );
[92976d9]	89	if(fd < 0) {
	90	abort("KERNEL ERROR: IO_URING SETUP - %s\n", strerror(errno));
	91	}
	92
	93	// Step 2 : mmap result
	94	memset(&this.io, 0, sizeof(struct io_ring));
[2d8f7b0]	95	struct io_uring_sq & sq = this.io.submit_q;
	96	struct io_uring_cq & cq = this.io.completion_q;
[92976d9]	97
	98	// calculate the right ring size
[2d8f7b0]	99	sq.ring_sz = params.sq_off.array + (params.sq_entries * sizeof(unsigned) );
	100	cq.ring_sz = params.cq_off.cqes + (params.cq_entries * sizeof(struct io_uring_cqe));
[92976d9]	101
	102	// Requires features
[d384787]	103	#if defined(IORING_FEAT_SINGLE_MMAP)
	104	// adjust the size according to the parameters
	105	if ((params.features & IORING_FEAT_SINGLE_MMAP) != 0) {
	106	cq->ring_sz = sq->ring_sz = max(cq->ring_sz, sq->ring_sz);
	107	}
	108	#endif
[92976d9]	109
	110	// mmap the Submit Queue into existence
[2d8f7b0]	111	sq.ring_ptr = mmap(0, sq.ring_sz, PROT_READ \| PROT_WRITE, MAP_SHARED \| MAP_POPULATE, fd, IORING_OFF_SQ_RING);
	112	if (sq.ring_ptr == (void*)MAP_FAILED) {
[92976d9]	113	abort("KERNEL ERROR: IO_URING MMAP1 - %s\n", strerror(errno));
	114	}
	115
	116	// Requires features
[d384787]	117	#if defined(IORING_FEAT_SINGLE_MMAP)
	118	// mmap the Completion Queue into existence (may or may not be needed)
	119	if ((params.features & IORING_FEAT_SINGLE_MMAP) != 0) {
	120	cq->ring_ptr = sq->ring_ptr;
	121	}
	122	else
	123	#endif
	124	{
[92976d9]	125	// We need multiple call to MMAP
[2d8f7b0]	126	cq.ring_ptr = mmap(0, cq.ring_sz, PROT_READ \| PROT_WRITE, MAP_SHARED \| MAP_POPULATE, fd, IORING_OFF_CQ_RING);
	127	if (cq.ring_ptr == (void*)MAP_FAILED) {
	128	munmap(sq.ring_ptr, sq.ring_sz);
[92976d9]	129	abort("KERNEL ERROR: IO_URING MMAP2 - %s\n", strerror(errno));
	130	}
[d384787]	131	}
[92976d9]	132
	133	// mmap the submit queue entries
	134	size_t size = params.sq_entries * sizeof(struct io_uring_sqe);
[2d8f7b0]	135	sq.sqes = (struct io_uring_sqe *)mmap(0, size, PROT_READ \| PROT_WRITE, MAP_SHARED \| MAP_POPULATE, fd, IORING_OFF_SQES);
	136	if (sq.sqes == (struct io_uring_sqe *)MAP_FAILED) {
	137	munmap(sq.ring_ptr, sq.ring_sz);
	138	if (cq.ring_ptr != sq.ring_ptr) munmap(cq.ring_ptr, cq.ring_sz);
[92976d9]	139	abort("KERNEL ERROR: IO_URING MMAP3 - %s\n", strerror(errno));
	140	}
	141
	142	// Get the pointers from the kernel to fill the structure
	143	// submit queue
[2d8f7b0]	144	sq.head = (volatile uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.head);
	145	sq.tail = (volatile uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.tail);
	146	sq.mask = ( const uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.ring_mask);
	147	sq.num = ( const uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.ring_entries);
	148	sq.flags = ( uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.flags);
	149	sq.dropped = ( uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.dropped);
	150	sq.array = ( uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.array);
	151	sq.alloc = *sq.tail;
[92976d9]	152
	153	// completion queue
[2d8f7b0]	154	cq.head = (volatile uint32_t *)(((intptr_t)cq.ring_ptr) + params.cq_off.head);
	155	cq.tail = (volatile uint32_t *)(((intptr_t)cq.ring_ptr) + params.cq_off.tail);
	156	cq.mask = ( const uint32_t *)(((intptr_t)cq.ring_ptr) + params.cq_off.ring_mask);
	157	cq.num = ( const uint32_t *)(((intptr_t)cq.ring_ptr) + params.cq_off.ring_entries);
	158	cq.overflow = ( uint32_t *)(((intptr_t)cq.ring_ptr) + params.cq_off.overflow);
	159	cq.cqes = (struct io_uring_cqe *)(((intptr_t)cq.ring_ptr) + params.cq_off.cqes);
	160
	161	// some paranoid checks
	162	/* paranoid / verifyf( (cq.mask) == ((cq.num) - 1ul32), "IO_URING Expected mask to be %u (%u entries), was %u", (cq.num) - 1ul32, cq.num, cq.mask );
	163	/* paranoid / verifyf( (cq.num) >= nentries, "IO_URING Expected %u entries, got %u", nentries, *cq.num );
	164	/* paranoid / verifyf( (cq.head) == 0, "IO_URING Expected head to be 0, got %u", *cq.head );
	165	/* paranoid / verifyf( (cq.tail) == 0, "IO_URING Expected tail to be 0, got %u", *cq.tail );
	166
	167	/* paranoid / verifyf( (sq.mask) == ((sq.num) - 1ul32), "IO_URING Expected mask to be %u (%u entries), was %u", (sq.num) - 1ul32, sq.num, sq.mask );
	168	/* paranoid / verifyf( (sq.num) >= nentries, "IO_URING Expected %u entries, got %u", nentries, *sq.num );
	169	/* paranoid / verifyf( (sq.head) == 0, "IO_URING Expected head to be 0, got %u", *sq.head );
	170	/* paranoid / verifyf( (sq.tail) == 0, "IO_URING Expected tail to be 0, got %u", *sq.tail );
[92976d9]	171
	172	// Update the global ring info
	173	this.io.flags = params.flags;
	174	this.io.fd = fd;
	175	this.io.done = false;
[2d8f7b0]	176	(this.io.submit){ min(sq.num, cq.num) };
[92976d9]	177
[d384787]	178	// Initialize statistics
	179	this.io.submit_q.stats.submit_avg.val = 0;
	180	this.io.submit_q.stats.submit_avg.cnt = 0;
	181	this.io.completion_q.stats.completed_avg.val = 0;
	182	this.io.completion_q.stats.completed_avg.cnt = 0;
	183
[92976d9]	184	// Create the poller thread
	185	this.io.stack = __create_pthread( &this.io.poller, __io_poller, &this );
	186	}
	187
	188	void __kernel_io_shutdown( cluster & this ) {
	189	// Stop the IO Poller
	190	// Notify the poller thread of the shutdown
	191	__atomic_store_n(&this.io.done, true, __ATOMIC_SEQ_CST);
	192	sigval val = { 1 };
	193	pthread_sigqueue( this.io.poller, SIGUSR1, val );
	194
	195	// Wait for the poller thread to finish
	196	pthread_join( this.io.poller, 0p );
	197	free( this.io.stack );
	198
[d384787]	199	// print statistics
	200	__cfaabi_bits_print_safe( STDERR_FILENO,
	201	"----- I/O uRing Stats -----\n"
	202	"- total submit calls : %llu\n"
	203	"- avg submit : %lf\n"
	204	"- total wait calls : %llu\n"
	205	"- avg completion/wait : %lf\n",
	206	this.io.submit_q.stats.submit_avg.cnt,
	207	((double)this.io.submit_q.stats.submit_avg.val) / this.io.submit_q.stats.submit_avg.cnt,
	208	this.io.completion_q.stats.completed_avg.cnt,
	209	((double)this.io.completion_q.stats.completed_avg.val) / this.io.completion_q.stats.completed_avg.cnt
	210	);
	211
[92976d9]	212	// Shutdown the io rings
	213	struct io_uring_sq & sq = this.io.submit_q;
	214	struct io_uring_cq & cq = this.io.completion_q;
	215
	216	// unmap the submit queue entries
[2d8f7b0]	217	munmap(sq.sqes, (sq.num) sizeof(struct io_uring_sqe));
[92976d9]	218
	219	// unmap the Submit Queue ring
	220	munmap(sq.ring_ptr, sq.ring_sz);
	221
	222	// unmap the Completion Queue ring, if it is different
	223	if (cq.ring_ptr != sq.ring_ptr) {
	224	munmap(cq.ring_ptr, cq.ring_sz);
	225	}
	226
	227	// close the file descriptor
	228	close(this.io.fd);
	229	}
	230
	231	//=============================================================================================
	232	// I/O Polling
	233	//=============================================================================================
	234	struct io_user_data {
	235	int32_t result;
	236	$thread * thrd;
	237	};
	238
	239	// Process a single completion message from the io_uring
	240	// This is NOT thread-safe
[d384787]	241	int __drain_io( struct io_ring & ring, sigset_t & mask, int waitcnt ) {
	242	int ret = syscall( __NR_io_uring_enter, ring.fd, 0, waitcnt, IORING_ENTER_GETEVENTS, &mask, _NSIG / 8);
	243	if( ret < 0 ) {
	244	switch((int)errno) {
	245	case EAGAIN:
	246	case EINTR:
	247	return -EAGAIN;
	248	default:
	249	abort( "KERNEL ERROR: IO_URING WAIT - %s\n", strerror(errno) );
	250	}
	251	}
	252
	253	// Drain the queue
[92976d9]	254	unsigned head = *ring.completion_q.head;
	255	unsigned tail = __atomic_load_n(ring.completion_q.tail, __ATOMIC_ACQUIRE);
	256
[d384787]	257	// Nothing was new return 0
	258	if (head == tail) {
	259	ring.completion_q.stats.completed_avg.cnt += 1;
	260	return 0;
	261	}
[92976d9]	262
[d384787]	263	uint32_t count = tail - head;
	264	for(i; count) {
	265	unsigned idx = (head + i) & (*ring.completion_q.mask);
	266	struct io_uring_cqe & cqe = ring.completion_q.cqes[idx];
[92976d9]	267
[d384787]	268	/* paranoid */ verify(&cqe);
[92976d9]	269
[d384787]	270	struct io_user_data * data = (struct io_user_data *)cqe.user_data;
	271	// __cfaabi_bits_print_safe( STDERR_FILENO, "Performed reading io cqe %p, result %d for %p\n", data, cqe.res, data->thrd );
[2d8f7b0]	272
[d384787]	273	data->result = cqe.res;
	274	__unpark( data->thrd __cfaabi_dbg_ctx2 );
	275	}
[2d8f7b0]	276
	277	// Allow new submissions to happen
[d384787]	278	V(ring.submit, count);
[92976d9]	279
	280	// Mark to the kernel that the cqe has been seen
	281	// Ensure that the kernel only sees the new value of the head index after the CQEs have been read.
[d384787]	282	__atomic_fetch_add( ring.completion_q.head, count, __ATOMIC_RELAXED );
[92976d9]	283
[d384787]	284	ring.completion_q.stats.completed_avg.val += count;
	285	ring.completion_q.stats.completed_avg.cnt += 1;
	286
	287	return count;
[92976d9]	288	}
	289
	290	static void * __io_poller( void * arg ) {
	291	cluster * cltr = (cluster *)arg;
	292	struct io_ring & ring = cltr->io;
	293
	294	sigset_t mask;
	295	sigfillset(&mask);
	296	if ( pthread_sigmask( SIG_BLOCK, &mask, 0p ) == -1 ) {
	297	abort( "KERNEL ERROR: IO_URING - pthread_sigmask" );
	298	}
	299
	300	sigdelset( &mask, SIGUSR1 );
	301
	302	verify( (ring.submit_q.head) == (ring.submit_q.tail) );
	303	verify( (ring.completion_q.head) == (ring.completion_q.tail) );
	304
[d384787]	305	while(!__atomic_load_n(&ring.done, __ATOMIC_SEQ_CST)) {
	306	__drain_io( ring, mask, 1 );
[92976d9]	307	}
	308
	309	return 0p;
	310	}
	311
	312	//=============================================================================================
	313	// I/O Submissions
	314	//=============================================================================================
	315
[2d8f7b0]	316	// Submition steps :
	317	// 1 - We need to make sure we don't overflow any of the buffer, P(ring.submit) to make sure
	318	// entries are available. The semaphore make sure that there is no more operations in
	319	// progress then the number of entries in the buffer. This probably limits concurrency
	320	// more than necessary since submitted but not completed operations don't need any
	321	// entries in user space. However, I don't know what happens if we overflow the buffers
	322	// because too many requests completed at once. This is a safe approach in all cases.
	323	// Furthermore, with hundreds of entries, this may be okay.
	324	//
	325	// 2 - Allocate a queue entry. The ring already has memory for all entries but only the ones
	326	// listed in sq.array are visible by the kernel. For those not listed, the kernel does not
	327	// offer any assurance that an entry is not being filled by multiple flags. Therefore, we
	328	// need to write an allocator that allows allocating concurrently.
	329	//
	330	// 3 - Actually fill the submit entry, this is the only simple and straightforward step.
	331	//
	332	// 4 - Append the entry index to the array and adjust the tail accordingly. This operation
	333	// needs to arrive to two concensus at the same time:
	334	// A - The order in which entries are listed in the array: no two threads must pick the
	335	// same index for their entries
	336	// B - When can the tail be update for the kernel. EVERY entries in the array between
	337	// head and tail must be fully filled and shouldn't ever be touched again.
	338	//
	339
[2489d31]	340	static inline [* struct io_uring_sqe, uint32_t] __submit_alloc( struct io_ring & ring ) {
	341	// Wait for a spot to be available
	342	P(ring.submit);
	343
	344	// Allocate the sqe
	345	uint32_t idx = __atomic_fetch_add(&ring.submit_q.alloc, 1ul32, __ATOMIC_SEQ_CST);
	346
	347	// Validate that we didn't overflow anything
	348	// Check that nothing overflowed
	349	/* paranoid */ verify( true );
	350
	351	// Check that it goes head -> tail -> alloc and never head -> alloc -> tail
	352	/* paranoid */ verify( true );
	353
	354	// Return the sqe
	355	return [&ring.submit_q.sqes[ idx & (*ring.submit_q.mask)], idx];
	356	}
	357
	358	static inline void __submit( struct io_ring & ring, uint32_t idx ) {
	359	// get mutual exclusion
	360	lock(ring.submit_q.lock __cfaabi_dbg_ctx2);
	361
	362	// Append to the list of ready entries
	363	uint32_t * tail = ring.submit_q.tail;
	364	const uint32_t mask = *ring.submit_q.mask;
	365
	366	ring.submit_q.array[ (*tail) & mask ] = idx & mask;
	367	__atomic_fetch_add(tail, 1ul32, __ATOMIC_SEQ_CST);
	368
	369	// Submit however, many entries need to be submitted
	370	int ret = syscall( __NR_io_uring_enter, ring.fd, 1, 0, 0, 0p, 0);
	371	// __cfaabi_bits_print_safe( STDERR_FILENO, "Performed io_submit, returned %d\n", ret );
	372	if( ret < 0 ) {
	373	switch((int)errno) {
	374	default:
	375	abort( "KERNEL ERROR: IO_URING SUBMIT - %s\n", strerror(errno) );
	376	}
[2d8f7b0]	377	}
[2489d31]	378
[d384787]	379	ring.submit_q.stats.submit_avg.val += 1;
	380	ring.submit_q.stats.submit_avg.cnt += 1;
	381
[2489d31]	382	unlock(ring.submit_q.lock);
	383	// Make sure that idx was submitted
	384	// Be careful to not get false positive if we cycled the entire list or that someone else submitted for us
	385	}
	386
	387	static inline void ?{}(struct io_uring_sqe & this, uint8_t opcode, int fd) {
	388	this.opcode = opcode;
	389	#if !defined(IOSQE_ASYNC)
	390	this.flags = 0;
	391	#else
	392	this.flags = IOSQE_ASYNC;
	393	#endif
	394	this.ioprio = 0;
	395	this.fd = fd;
	396	this.off = 0;
	397	this.addr = 0;
	398	this.len = 0;
	399	this.rw_flags = 0;
	400	this.__pad2[0] = this.__pad2[1] = this.__pad2[2] = 0;
[2d8f7b0]	401	}
	402
[2489d31]	403	static inline void ?{}(struct io_uring_sqe & this, uint8_t opcode, int fd, void * addr, uint32_t len, uint64_t off ) {
	404	(this){ opcode, fd };
	405	this.off = off;
	406	this.addr = (uint64_t)addr;
	407	this.len = len;
	408	}
[ecf6b46]	409	#endif
[92976d9]	410
	411	//=============================================================================================
	412	// I/O Interface
	413	//=============================================================================================
[ecf6b46]	414	extern "C" {
	415	#define __USE_GNU
	416	#define _GNU_SOURCE
	417	#include <fcntl.h>
	418	#include <sys/uio.h>
	419	#include <sys/socket.h>
	420	#include <sys/stat.h>
	421	}
	422
	423	#if defined(HAVE_LINUX_IO_URING_H)
[2d8f7b0]	424	#define __submit_prelude \
	425	struct io_ring & ring = active_cluster()->io; \
	426	struct io_uring_sqe * sqe; \
	427	uint32_t idx; \
	428	[sqe, idx] = __submit_alloc( ring );
	429
	430	#define __submit_wait \
	431	io_user_data data = { 0, active_thread() }; \
[185efe6]	432	/__cfaabi_bits_print_safe( STDERR_FILENO, "Preparing user data %p for %p\n", &data, data.thrd );/ \
[2d8f7b0]	433	sqe->user_data = (uint64_t)&data; \
	434	__submit( ring, idx ); \
	435	park( __cfaabi_dbg_ctx ); \
	436	return data.result;
[ecf6b46]	437	#endif
[2d8f7b0]	438
	439	//-----------------------------------------------------------------------------
	440	// Asynchronous operations
[ecf6b46]	441	ssize_t cfa_preadv2(int fd, const struct iovec *iov, int iovcnt, off_t offset, int flags) {
	442	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_READV)
	443	return preadv2(fd, iov, iovcnt, offset, flags);
	444	#else
	445	__submit_prelude
	446
	447	(*sqe){ IORING_OP_READV, fd, iov, iovcnt, offset };
	448
	449	__submit_wait
	450	#endif
	451	}
	452
	453	ssize_t cfa_pwritev2(int fd, const struct iovec *iov, int iovcnt, off_t offset, int flags) {
	454	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_WRITEV)
	455	return pwritev2(fd, iov, iovcnt, offset, flags);
	456	#else
	457	__submit_prelude
	458
	459	(*sqe){ IORING_OP_WRITEV, fd, iov, iovcnt, offset };
	460
	461	__submit_wait
	462	#endif
	463	}
	464
	465	int cfa_fsync(int fd) {
	466	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_FSYNC)
	467	return fsync(fd);
	468	#else
	469	__submit_prelude
	470
	471	(*sqe){ IORING_OP_FSYNC, fd };
	472
	473	__submit_wait
	474	#endif
	475	}
	476
	477	int cfa_sync_file_range(int fd, int64_t offset, int64_t nbytes, unsigned int flags) {
	478	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_SYNC_FILE_RANGE)
	479	return sync_file_range(fd, offset, nbytes, flags);
	480	#else
	481	__submit_prelude
	482
	483	(*sqe){ IORING_OP_SYNC_FILE_RANGE, fd };
	484	sqe->off = offset;
	485	sqe->len = nbytes;
	486	sqe->sync_range_flags = flags;
	487
	488	__submit_wait
	489	#endif
	490	}
	491
	492
	493	ssize_t cfa_sendmsg(int sockfd, const struct msghdr *msg, int flags) {
	494	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_SENDMSG)
	495	return recv(sockfd, msg, flags);
	496	#else
	497	__submit_prelude
	498
	499	(*sqe){ IORING_OP_SENDMSG, sockfd, msg, 1, 0 };
	500	sqe->msg_flags = flags;
	501
	502	__submit_wait
	503	#endif
	504	}
	505
	506	ssize_t cfa_recvmsg(int sockfd, struct msghdr *msg, int flags) {
	507	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_RECVMSG)
	508	return recv(sockfd, msg, flags);
	509	#else
	510	__submit_prelude
	511
	512	(*sqe){ IORING_OP_RECVMSG, sockfd, msg, 1, 0 };
	513	sqe->msg_flags = flags;
	514
	515	__submit_wait
	516	#endif
	517	}
	518
	519	ssize_t cfa_send(int sockfd, const void *buf, size_t len, int flags) {
	520	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_SEND)
	521	return send( sockfd, buf, len, flags );
	522	#else
	523	__submit_prelude
	524
	525	(*sqe){ IORING_OP_SEND, sockfd };
	526	sqe->addr = (uint64_t)buf;
	527	sqe->len = len;
	528	sqe->msg_flags = flags;
	529
	530	__submit_wait
	531	#endif
	532	}
	533
	534	ssize_t cfa_recv(int sockfd, void *buf, size_t len, int flags) {
	535	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_RECV)
	536	return recv( sockfd, buf, len, flags );
	537	#else
	538	__submit_prelude
	539
	540	(*sqe){ IORING_OP_RECV, sockfd };
	541	sqe->addr = (uint64_t)buf;
	542	sqe->len = len;
	543	sqe->msg_flags = flags;
	544
	545	__submit_wait
	546	#endif
	547	}
	548
	549	int cfa_accept4(int sockfd, struct sockaddr addr, socklen_t addrlen, int flags) {
	550	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_ACCEPT)
[d384787]	551	#pragma GCC diagnostic push
	552	#pragma GCC diagnostic ignored "-Wattributes"
	553	__SOCKADDR_ARG _addr;
	554	_addr.__sockaddr__ = addr;
	555	return accept4( sockfd, _addr, addrlen, flags );
	556	#pragma GCC diagnostic pop
[ecf6b46]	557	#else
	558	__submit_prelude
	559
	560	(*sqe){ IORING_OP_ACCEPT, sockfd };
	561	sqe->addr = addr;
	562	sqe->addr2 = addrlen;
	563	sqe->accept_flags = flags;
	564
	565	__submit_wait
	566	#endif
	567	}
	568
	569	int cfa_connect(int sockfd, const struct sockaddr *addr, socklen_t addrlen) {
	570	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_CONNECT)
[d384787]	571	#pragma GCC diagnostic push
	572	#pragma GCC diagnostic ignored "-Wattributes"
	573	__CONST_SOCKADDR_ARG _addr;
	574	_addr.__sockaddr__ = addr;
	575	return connect( sockfd, _addr, addrlen );
	576	#pragma GCC diagnostic pop
[ecf6b46]	577	#else
	578	__submit_prelude
	579
	580	(*sqe){ IORING_OP_CONNECT, sockfd };
	581	sqe->addr = (uint64_t)addr;
	582	sqe->off = addrlen;
	583
	584	__submit_wait
	585	#endif
	586	}
	587
	588	int cfa_fallocate(int fd, int mode, uint64_t offset, uint64_t len) {
	589	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_FALLOCATE)
	590	return fallocate( fd, mode, offset, len );
	591	#else
	592	__submit_prelude
	593
	594	(*sqe){ IORING_OP_FALLOCATE, fd };
	595	sqe->off = offset;
	596	sqe->len = length;
	597	sqe->mode = mode;
	598
	599	__submit_wait
	600	#endif
	601	}
	602
	603	int cfa_fadvise(int fd, uint64_t offset, uint64_t len, int advice) {
	604	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_FADVISE)
	605	return posix_fadvise( fd, offset, len, advice );
	606	#else
	607	__submit_prelude
	608
	609	(*sqe){ IORING_OP_FADVISE, fd };
	610	sqe->off = (uint64_t)offset;
	611	sqe->len = length;
	612	sqe->fadvise_advice = advice;
	613
	614	__submit_wait
	615	#endif
	616	}
	617
	618	int cfa_madvise(void *addr, size_t length, int advice) {
	619	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_MADVISE)
	620	return madvise( addr, length, advice );
	621	#else
	622	__submit_prelude
	623
	624	(*sqe){ IORING_OP_MADVISE, 0 };
	625	sqe->addr = (uint64_t)addr;
	626	sqe->len = length;
	627	sqe->fadvise_advice = advice;
	628
	629	__submit_wait
	630	#endif
	631	}
	632
	633	int cfa_openat(int dirfd, const char *pathname, int flags, mode_t mode) {
	634	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_OPENAT)
	635	return openat( dirfd, pathname, flags, mode );
	636	#else
	637	__submit_prelude
	638
	639	(*sqe){ IORING_OP_OPENAT, dirfd };
	640	sqe->addr = (uint64_t)pathname;
	641	sqe->open_flags = flags;
	642	sqe->mode = mode;
	643
	644	__submit_wait
	645	#endif
	646	}
	647
	648	int cfa_close(int fd) {
	649	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_CLOSE)
	650	return close( fd );
	651	#else
	652	__submit_prelude
	653
	654	(*sqe){ IORING_OP_CLOSE, fd };
	655
	656	__submit_wait
	657	#endif
	658	}
	659
	660	int cfa_statx(int dirfd, const char pathname, int flags, unsigned int mask, struct statx statxbuf) {
	661	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_STATX)
	662	//return statx( dirfd, pathname, flags, mask, statxbuf );
	663	return syscall( __NR_io_uring_setup, dirfd, pathname, flags, mask, statxbuf );
	664	#else
	665	__submit_prelude
	666
	667	(*sqe){ IORING_OP_STATX, dirfd };
	668	sqe->addr = (uint64_t)pathname;
	669	sqe->statx_flags = flags;
	670	sqe->len = mask;
	671	sqe->off = (uint64_t)statxbuf;
	672
	673	__submit_wait
	674	#endif
	675	}
	676
	677
	678	ssize_t cfa_read(int fd, void *buf, size_t count) {
	679	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_READ)
	680	return read( fd, buf, count );
	681	#else
	682	__submit_prelude
	683
	684	(*sqe){ IORING_OP_READ, fd, buf, count, 0 };
	685
	686	__submit_wait
	687	#endif
	688	}
	689
	690	ssize_t cfa_write(int fd, void *buf, size_t count) {
	691	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_WRITE)
	692	return read( fd, buf, count );
	693	#else
	694	__submit_prelude
	695
	696	(*sqe){ IORING_OP_WRITE, fd, buf, count, 0 };
	697
	698	__submit_wait
	699	#endif
	700	}
[2d8f7b0]	701
	702	//-----------------------------------------------------------------------------
	703	// Check if a function is asynchronous
	704
	705	// Macro magic to reduce the size of the following switch case
[ecf6b46]	706	#define IS_DEFINED_APPLY(f, ...) f(__VA_ARGS__)
	707	#define IS_DEFINED_SECOND(first, second, ...) second
	708	#define IS_DEFINED_TEST(expansion) _CFA_IO_FEATURE_##expansion
	709	#define IS_DEFINED(macro) IS_DEFINED_APPLY( IS_DEFINED_SECOND,IS_DEFINED_TEST(macro) false, true)
[2d8f7b0]	710
[ecf6b46]	711	bool has_user_level_blocking( fptr_t func ) {
	712	#if defined(HAVE_LINUX_IO_URING_H)
[2d8f7b0]	713	if( /func == (fptr_t)preadv2 \|\| /
[2489d31]	714	func == (fptr_t)cfa_preadv2 )
[2d8f7b0]	715	#define _CFA_IO_FEATURE_IORING_OP_READV ,
	716	return IS_DEFINED(IORING_OP_READV);
	717
	718	if( /func == (fptr_t)pwritev2 \|\| /
[ecf6b46]	719	func == (fptr_t)cfa_pwritev2 )
[2d8f7b0]	720	#define _CFA_IO_FEATURE_IORING_OP_WRITEV ,
	721	return IS_DEFINED(IORING_OP_WRITEV);
	722
	723	if( /func == (fptr_t)fsync \|\| /
[ecf6b46]	724	func == (fptr_t)cfa_fsync )
[2d8f7b0]	725	#define _CFA_IO_FEATURE_IORING_OP_FSYNC ,
	726	return IS_DEFINED(IORING_OP_FSYNC);
	727
	728	if( /func == (fptr_t)ync_file_range \|\| /
[ecf6b46]	729	func == (fptr_t)cfa_sync_file_range )
[2d8f7b0]	730	#define _CFA_IO_FEATURE_IORING_OP_SYNC_FILE_RANGE ,
	731	return IS_DEFINED(IORING_OP_SYNC_FILE_RANGE);
	732
	733	if( /func == (fptr_t)sendmsg \|\| /
[ecf6b46]	734	func == (fptr_t)cfa_sendmsg )
[2d8f7b0]	735	#define _CFA_IO_FEATURE_IORING_OP_SENDMSG ,
	736	return IS_DEFINED(IORING_OP_SENDMSG);
	737
	738	if( /func == (fptr_t)recvmsg \|\| /
[ecf6b46]	739	func == (fptr_t)cfa_recvmsg )
[2d8f7b0]	740	#define _CFA_IO_FEATURE_IORING_OP_RECVMSG ,
	741	return IS_DEFINED(IORING_OP_RECVMSG);
	742
	743	if( /func == (fptr_t)send \|\| /
[2489d31]	744	func == (fptr_t)cfa_send )
[2d8f7b0]	745	#define _CFA_IO_FEATURE_IORING_OP_SEND ,
	746	return IS_DEFINED(IORING_OP_SEND);
	747
	748	if( /func == (fptr_t)recv \|\| /
[2489d31]	749	func == (fptr_t)cfa_recv )
[2d8f7b0]	750	#define _CFA_IO_FEATURE_IORING_OP_RECV ,
	751	return IS_DEFINED(IORING_OP_RECV);
	752
	753	if( /func == (fptr_t)accept4 \|\| /
[2489d31]	754	func == (fptr_t)cfa_accept4 )
[2d8f7b0]	755	#define _CFA_IO_FEATURE_IORING_OP_ACCEPT ,
	756	return IS_DEFINED(IORING_OP_ACCEPT);
	757
	758	if( /func == (fptr_t)connect \|\| /
[2489d31]	759	func == (fptr_t)cfa_connect )
[2d8f7b0]	760	#define _CFA_IO_FEATURE_IORING_OP_CONNECT ,
	761	return IS_DEFINED(IORING_OP_CONNECT);
	762
	763	if( /func == (fptr_t)fallocate \|\| /
[2489d31]	764	func == (fptr_t)cfa_fallocate )
[2d8f7b0]	765	#define _CFA_IO_FEATURE_IORING_OP_FALLOCATE ,
	766	return IS_DEFINED(IORING_OP_FALLOCATE);
	767
	768	if( /func == (fptr_t)fadvise \|\| /
[2489d31]	769	func == (fptr_t)cfa_fadvise )
[2d8f7b0]	770	#define _CFA_IO_FEATURE_IORING_OP_FADVISE ,
	771	return IS_DEFINED(IORING_OP_FADVISE);
	772
	773	if( /func == (fptr_t)madvise \|\| /
[2489d31]	774	func == (fptr_t)cfa_madvise )
[2d8f7b0]	775	#define _CFA_IO_FEATURE_IORING_OP_MADVISE ,
	776	return IS_DEFINED(IORING_OP_MADVISE);
	777
	778	if( /func == (fptr_t)openat \|\| /
[2489d31]	779	func == (fptr_t)cfa_openat )
[2d8f7b0]	780	#define _CFA_IO_FEATURE_IORING_OP_OPENAT ,
	781	return IS_DEFINED(IORING_OP_OPENAT);
	782
	783	if( /func == (fptr_t)close \|\| /
[2489d31]	784	func == (fptr_t)cfa_close )
[2d8f7b0]	785	#define _CFA_IO_FEATURE_IORING_OP_CLOSE ,
	786	return IS_DEFINED(IORING_OP_CLOSE);
	787
	788	if( /func == (fptr_t)statx \|\| /
[2489d31]	789	func == (fptr_t)cfa_statx )
[2d8f7b0]	790	#define _CFA_IO_FEATURE_IORING_OP_STATX ,
	791	return IS_DEFINED(IORING_OP_STATX);
	792
	793	if( /func == (fptr_t)read \|\| /
[ecf6b46]	794	func == (fptr_t)cfa_read )
[2d8f7b0]	795	#define _CFA_IO_FEATURE_IORING_OP_READ ,
	796	return IS_DEFINED(IORING_OP_READ);
	797
	798	if( /func == (fptr_t)write \|\| /
[ecf6b46]	799	func == (fptr_t)cfa_write )
[2d8f7b0]	800	#define _CFA_IO_FEATURE_IORING_OP_WRITE ,
	801	return IS_DEFINED(IORING_OP_WRITE);
[ecf6b46]	802	#endif
[2d8f7b0]	803
[ecf6b46]	804	return false;
	805	}

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

Original Format