#include "kernel.hfa"

#if !defined(HAVE_LINUX_IO_URING_H)
	void __kernel_io_startup( cluster & this ) {
		// Nothing to do without io_uring
	}

	void __kernel_io_shutdown( cluster & this ) {
		// Nothing to do without io_uring
	}

	bool is_async( void (*)() ) {
		return false;
	}

#else
	extern "C" {
		#define _GNU_SOURCE         /* See feature_test_macros(7) */
		#include <errno.h>
		#include <stdint.h>
		#include <string.h>
		#include <unistd.h>
		#include <sys/mman.h>
		#include <sys/syscall.h>

		#include <linux/io_uring.h>
	}

	#include "bits/signal.hfa"
	#include "kernel_private.hfa"
	#include "thread.hfa"

	uint32_t entries_per_cluster() {
		return 256;
	}

	static void * __io_poller( void * arg );

       // Weirdly, some systems that do support io_uring don't actually define these
       #ifdef __alpha__
       /*
       * alpha is the only exception, all other architectures
       * have common numbers for new system calls.
       */
       # ifndef __NR_io_uring_setup
       #  define __NR_io_uring_setup           535
       # endif
       # ifndef __NR_io_uring_enter
       #  define __NR_io_uring_enter           536
       # endif
       # ifndef __NR_io_uring_register
       #  define __NR_io_uring_register        537
       # endif
       #else /* !__alpha__ */
       # ifndef __NR_io_uring_setup
       #  define __NR_io_uring_setup           425
       # endif
       # ifndef __NR_io_uring_enter
       #  define __NR_io_uring_enter           426
       # endif
       # ifndef __NR_io_uring_register
       #  define __NR_io_uring_register        427
       # endif
       #endif


//=============================================================================================
// I/O Startup / Shutdown logic
//=============================================================================================
	void __kernel_io_startup( cluster & this ) {
		// Step 1 : call to setup
		struct io_uring_params params;
		memset(&params, 0, sizeof(params));

		uint32_t nentries = entries_per_cluster();

		int fd = syscall(__NR_io_uring_setup, nentries, &params );
		if(fd < 0) {
			abort("KERNEL ERROR: IO_URING SETUP - %s\n", strerror(errno));
		}

		// Step 2 : mmap result
		memset(&this.io, 0, sizeof(struct io_ring));
		struct io_uring_sq & sq = this.io.submit_q;
		struct io_uring_cq & cq = this.io.completion_q;

		// calculate the right ring size
		sq.ring_sz = params.sq_off.array + (params.sq_entries * sizeof(unsigned)           );
		cq.ring_sz = params.cq_off.cqes  + (params.cq_entries * sizeof(struct io_uring_cqe));

		// Requires features
		// // adjust the size according to the parameters
		// if ((params.features & IORING_FEAT_SINGLE_MMAP) != 0) {
		// 	cq->ring_sz = sq->ring_sz = max(cq->ring_sz, sq->ring_sz);
		// }

		// mmap the Submit Queue into existence
		sq.ring_ptr = mmap(0, sq.ring_sz, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd, IORING_OFF_SQ_RING);
		if (sq.ring_ptr == (void*)MAP_FAILED) {
			abort("KERNEL ERROR: IO_URING MMAP1 - %s\n", strerror(errno));
		}

		// mmap the Completion Queue into existence (may or may not be needed)
		// Requires features
		// if ((params.features & IORING_FEAT_SINGLE_MMAP) != 0) {
		// 	cq->ring_ptr = sq->ring_ptr;
		// }
		// else {
			// We need multiple call to MMAP
			cq.ring_ptr = mmap(0, cq.ring_sz, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd, IORING_OFF_CQ_RING);
			if (cq.ring_ptr == (void*)MAP_FAILED) {
				munmap(sq.ring_ptr, sq.ring_sz);
				abort("KERNEL ERROR: IO_URING MMAP2 - %s\n", strerror(errno));
			}
		// }

		// mmap the submit queue entries
		size_t size = params.sq_entries * sizeof(struct io_uring_sqe);
		sq.sqes = (struct io_uring_sqe *)mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd, IORING_OFF_SQES);
		if (sq.sqes == (struct io_uring_sqe *)MAP_FAILED) {
			munmap(sq.ring_ptr, sq.ring_sz);
			if (cq.ring_ptr != sq.ring_ptr) munmap(cq.ring_ptr, cq.ring_sz);
			abort("KERNEL ERROR: IO_URING MMAP3 - %s\n", strerror(errno));
		}

		// Get the pointers from the kernel to fill the structure
		// submit queue
		sq.head    = (volatile uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.head);
		sq.tail    = (volatile uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.tail);
		sq.mask    = (   const uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.ring_mask);
		sq.num     = (   const uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.ring_entries);
		sq.flags   = (         uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.flags);
		sq.dropped = (         uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.dropped);
		sq.array   = (         uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.array);
		sq.alloc = *sq.tail;

		// completion queue
		cq.head     = (volatile uint32_t *)(((intptr_t)cq.ring_ptr) + params.cq_off.head);
		cq.tail     = (volatile uint32_t *)(((intptr_t)cq.ring_ptr) + params.cq_off.tail);
		cq.mask     = (   const uint32_t *)(((intptr_t)cq.ring_ptr) + params.cq_off.ring_mask);
		cq.num      = (   const uint32_t *)(((intptr_t)cq.ring_ptr) + params.cq_off.ring_entries);
		cq.overflow = (         uint32_t *)(((intptr_t)cq.ring_ptr) + params.cq_off.overflow);
		cq.cqes   = (struct io_uring_cqe *)(((intptr_t)cq.ring_ptr) + params.cq_off.cqes);

		// some paranoid checks
		/* paranoid */ verifyf( (*cq.mask) == ((*cq.num) - 1ul32), "IO_URING Expected mask to be %u (%u entries), was %u", (*cq.num) - 1ul32, *cq.num, *cq.mask  );
		/* paranoid */ verifyf( (*cq.num)  >= nentries, "IO_URING Expected %u entries, got %u", nentries, *cq.num );
		/* paranoid */ verifyf( (*cq.head) == 0, "IO_URING Expected head to be 0, got %u", *cq.head );
		/* paranoid */ verifyf( (*cq.tail) == 0, "IO_URING Expected tail to be 0, got %u", *cq.tail );

		/* paranoid */ verifyf( (*sq.mask) == ((*sq.num) - 1ul32), "IO_URING Expected mask to be %u (%u entries), was %u", (*sq.num) - 1ul32, *sq.num, *sq.mask );
		/* paranoid */ verifyf( (*sq.num) >= nentries, "IO_URING Expected %u entries, got %u", nentries, *sq.num );
		/* paranoid */ verifyf( (*sq.head) == 0, "IO_URING Expected head to be 0, got %u", *sq.head );
		/* paranoid */ verifyf( (*sq.tail) == 0, "IO_URING Expected tail to be 0, got %u", *sq.tail );

		// Update the global ring info
		this.io.flags = params.flags;
		this.io.fd    = fd;
		this.io.done  = false;
		(this.io.submit){ min(*sq.num, *cq.num) };

		// Create the poller thread
		this.io.stack = __create_pthread( &this.io.poller, __io_poller, &this );
	}

	void __kernel_io_shutdown( cluster & this ) {
		// Stop the IO Poller
		// Notify the poller thread of the shutdown
		__atomic_store_n(&this.io.done, true, __ATOMIC_SEQ_CST);
		sigval val = { 1 };
		pthread_sigqueue( this.io.poller, SIGUSR1, val );

		// Wait for the poller thread to finish
		pthread_join( this.io.poller, 0p );
		free( this.io.stack );

		// Shutdown the io rings
		struct io_uring_sq & sq = this.io.submit_q;
		struct io_uring_cq & cq = this.io.completion_q;

		// unmap the submit queue entries
		munmap(sq.sqes, (*sq.num) * sizeof(struct io_uring_sqe));

		// unmap the Submit Queue ring
		munmap(sq.ring_ptr, sq.ring_sz);

		// unmap the Completion Queue ring, if it is different
		if (cq.ring_ptr != sq.ring_ptr) {
			munmap(cq.ring_ptr, cq.ring_sz);
		}

		// close the file descriptor
		close(this.io.fd);
	}

//=============================================================================================
// I/O Polling
//=============================================================================================
	struct io_user_data {
		int32_t result;
		$thread * thrd;
	};

	// Process a single completion message from the io_uring
	// This is NOT thread-safe
	static bool __io_process(struct io_ring & ring) {
		unsigned head = *ring.completion_q.head;
		unsigned tail = __atomic_load_n(ring.completion_q.tail, __ATOMIC_ACQUIRE);

		if (head == tail) return false;

		unsigned idx = head & (*ring.completion_q.mask);
		struct io_uring_cqe & cqe = ring.completion_q.cqes[idx];

		/* paranoid */ verify(&cqe);

		struct io_user_data * data = (struct io_user_data *)cqe.user_data;
		// __cfaabi_bits_print_safe( STDERR_FILENO, "Performed reading io cqe %p, result %d for %p\n", data, cqe.res, data->thrd );

		data->result = cqe.res;
		__unpark( data->thrd __cfaabi_dbg_ctx2 );

		// Allow new submissions to happen
		V(ring.submit);

		// Mark to the kernel that the cqe has been seen
		// Ensure that the kernel only sees the new value of the head index after the CQEs have been read.
		__atomic_fetch_add( ring.completion_q.head, 1, __ATOMIC_RELAXED );

		return true;
	}

	static void * __io_poller( void * arg ) {
		cluster * cltr = (cluster *)arg;
		struct io_ring & ring = cltr->io;

		sigset_t mask;
		sigfillset(&mask);
		if ( pthread_sigmask( SIG_BLOCK, &mask, 0p ) == -1 ) {
			abort( "KERNEL ERROR: IO_URING - pthread_sigmask" );
		}

		sigdelset( &mask, SIGUSR1 );

		verify( (*ring.submit_q.head) == (*ring.submit_q.tail) );
		verify( (*ring.completion_q.head) == (*ring.completion_q.tail) );

		LOOP: while(!__atomic_load_n(&ring.done, __ATOMIC_SEQ_CST)) {
			int ret = syscall( __NR_io_uring_enter, ring.fd, 0, 1, IORING_ENTER_GETEVENTS, &mask, _NSIG / 8);
			if( ret < 0 ) {
				switch((int)errno) {
				case EAGAIN:
				case EINTR:
					continue LOOP;
				default:
					abort( "KERNEL ERROR: IO_URING WAIT - %s\n", strerror(errno) );
				}
			}

			// Drain the queue
			while(__io_process(ring)) {}
		}

		return 0p;
	}

//=============================================================================================
// I/O Submissions
//=============================================================================================

// Submition steps :
// 1 - We need to make sure we don't overflow any of the buffer, P(ring.submit) to make sure
//     entries are available. The semaphore make sure that there is no more operations in
//     progress then the number of entries in the buffer. This probably limits concurrency
//     more than necessary since submitted but not completed operations don't need any
//     entries in user space. However, I don't know what happens if we overflow the buffers
//     because too many requests completed at once. This is a safe approach in all cases.
//     Furthermore, with hundreds of entries, this may be okay.
//
// 2 - Allocate a queue entry. The ring already has memory for all entries but only the ones
//     listed in sq.array are visible by the kernel. For those not listed, the kernel does not
//     offer any assurance that an entry is not being filled by multiple flags. Therefore, we
//     need to write an allocator that allows allocating concurrently.
//
// 3 - Actually fill the submit entry, this is the only simple and straightforward step.
//
// 4 - Append the entry index to the array and adjust the tail accordingly. This operation
//     needs to arrive to two concensus at the same time:
//     A - The order in which entries are listed in the array: no two threads must pick the
//         same index for their entries
//     B - When can the tail be update for the kernel. EVERY entries in the array between
//         head and tail must be fully filled and shouldn't ever be touched again.
//

static inline [* struct io_uring_sqe, uint32_t] __submit_alloc( struct io_ring & ring ) {
	// Wait for a spot to be available
	P(ring.submit);

	// Allocate the sqe
	uint32_t idx = __atomic_fetch_add(&ring.submit_q.alloc, 1ul32, __ATOMIC_SEQ_CST);

	// Validate that we didn't overflow anything
	// Check that nothing overflowed
	/* paranoid */ verify( true );

	// Check that it goes head -> tail -> alloc and never head -> alloc -> tail
	/* paranoid */ verify( true );

	// Return the sqe
	return [&ring.submit_q.sqes[ idx & (*ring.submit_q.mask)], idx];
}

static inline void __submit( struct io_ring & ring, uint32_t idx ) {
	// get mutual exclusion
	lock(ring.submit_q.lock __cfaabi_dbg_ctx2);

	// Append to the list of ready entries
	uint32_t * tail = ring.submit_q.tail;
	const uint32_t mask = *ring.submit_q.mask;

	ring.submit_q.array[ (*tail) & mask ] = idx & mask;
	__atomic_fetch_add(tail, 1ul32, __ATOMIC_SEQ_CST);

	// Submit however, many entries need to be submitted
	int ret = syscall( __NR_io_uring_enter, ring.fd, 1, 0, 0, 0p, 0);
	// __cfaabi_bits_print_safe( STDERR_FILENO, "Performed io_submit, returned %d\n", ret );
	if( ret < 0 ) {
		switch((int)errno) {
		default:
			abort( "KERNEL ERROR: IO_URING SUBMIT - %s\n", strerror(errno) );
		}
	}

	unlock(ring.submit_q.lock);
	// Make sure that idx was submitted
	// Be careful to not get false positive if we cycled the entire list or that someone else submitted for us
}

static inline void ?{}(struct io_uring_sqe & this, uint8_t opcode, int fd) {
	this.opcode = opcode;
	#if !defined(IOSQE_ASYNC)
		this.flags = 0;
	#else
		this.flags = IOSQE_ASYNC;
	#endif
	this.ioprio = 0;
	this.fd = fd;
	this.off = 0;
	this.addr = 0;
	this.len = 0;
	this.rw_flags = 0;
	this.__pad2[0] = this.__pad2[1] = this.__pad2[2] = 0;
}

static inline void ?{}(struct io_uring_sqe & this, uint8_t opcode, int fd, void * addr, uint32_t len, uint64_t off ) {
	(this){ opcode, fd };
	this.off = off;
	this.addr = (uint64_t)addr;
	this.len = len;
}

//=============================================================================================
// I/O Interface
//=============================================================================================
	extern "C" {
		#define __USE_GNU
		#define _GNU_SOURCE
		#include <fcntl.h>
		#include <sys/uio.h>
		#include <sys/socket.h>
		#include <sys/stat.h>
	}

	#define __submit_prelude \
		struct io_ring & ring = active_cluster()->io; \
		struct io_uring_sqe * sqe; \
		uint32_t idx; \
		[sqe, idx] = __submit_alloc( ring );

	#define __submit_wait \
		io_user_data data = { 0, active_thread() }; \
		/*__cfaabi_bits_print_safe( STDERR_FILENO, "Preparing user data %p for %p\n", &data, data.thrd );*/ \
		sqe->user_data = (uint64_t)&data; \
		__submit( ring, idx ); \
		park( __cfaabi_dbg_ctx ); \
		return data.result;

//-----------------------------------------------------------------------------
// Asynchronous operations
	ssize_t async_preadv2(int fd, const struct iovec *iov, int iovcnt, off_t offset, int flags) {
		#if !defined(IORING_OP_READV)
			return preadv2(fd, iov, iovcnt, offset, flags);
		#else
			__submit_prelude

			(*sqe){ IORING_OP_READV, fd, iov, iovcnt, offset };

			__submit_wait
		#endif
	}

	ssize_t async_pwritev2(int fd, const struct iovec *iov, int iovcnt, off_t offset, int flags) {
		#if !defined(IORING_OP_WRITEV)
			return pwritev2(fd, iov, iovcnt, offset, flags);
		#else
			__submit_prelude

			(*sqe){ IORING_OP_WRITEV, fd, iov, iovcnt, offset };

			__submit_wait
		#endif
	}

	int async_fsync(int fd) {
		#if !defined(IORING_OP_FSYNC)
			return fsync(fd);
		#else
			__submit_prelude

			(*sqe){ IORING_OP_FSYNC, fd };

			__submit_wait
		#endif
	}

	int async_sync_file_range(int fd, int64_t offset, int64_t nbytes, unsigned int flags) {
		#if !defined(IORING_OP_SYNC_FILE_RANGE)
			return sync_file_range(fd, offset, nbytes, flags);
		#else
			__submit_prelude

			(*sqe){ IORING_OP_SYNC_FILE_RANGE, fd };
			sqe->off = offset;
			sqe->len = nbytes;
			sqe->sync_range_flags = flags;

			__submit_wait
		#endif
	}


	ssize_t async_sendmsg(int sockfd, const struct msghdr *msg, int flags) {
		#if !defined(IORING_OP_SENDMSG)
			return recv(sockfd, msg, flags);
		#else
			__submit_prelude

			(*sqe){ IORING_OP_SENDMSG, sockfd, msg, 1, 0 };
			sqe->msg_flags = flags;

			__submit_wait
		#endif
	}

	ssize_t async_recvmsg(int sockfd, struct msghdr *msg, int flags) {
		#if !defined(IORING_OP_RECVMSG)
			return recv(sockfd, msg, flags);
		#else
			__submit_prelude

			(*sqe){ IORING_OP_RECVMSG, sockfd, msg, 1, 0 };
			sqe->msg_flags = flags;

			__submit_wait
		#endif
	}

	ssize_t async_send(int sockfd, const void *buf, size_t len, int flags) {
		#if !defined(IORING_OP_SEND)
			return send( sockfd, buf, len, flags );
		#else
			__submit_prelude

			(*sqe){ IORING_OP_SEND, sockfd };
			sqe->addr = (uint64_t)buf;
			sqe->len = len;
			sqe->msg_flags = flags;

			__submit_wait
		#endif
	}

	ssize_t async_recv(int sockfd, void *buf, size_t len, int flags) {
		#if !defined(IORING_OP_RECV)
			return recv( sockfd, buf, len, flags );
		#else
			__submit_prelude

			(*sqe){ IORING_OP_RECV, sockfd };
			sqe->addr = (uint64_t)buf;
			sqe->len = len;
			sqe->msg_flags = flags;

			__submit_wait
		#endif
	}

	int async_accept4(int sockfd, struct sockaddr *addr, socklen_t *addrlen, int flags) {
		#if !defined(IORING_OP_ACCEPT)
			__SOCKADDR_ARG _addr;
			_addr.__sockaddr__ = addr;
			return accept4( sockfd, _addr, addrlen, flags );
		#else
			__submit_prelude

			(*sqe){ IORING_OP_ACCEPT, sockfd };
			sqe->addr = addr;
			sqe->addr2 = addrlen;
			sqe->accept_flags = flags;

			__submit_wait
		#endif
	}

	int async_connect(int sockfd, const struct sockaddr *addr, socklen_t addrlen) {
		#if !defined(IORING_OP_CONNECT)
			__CONST_SOCKADDR_ARG _addr;
			_addr.__sockaddr__ = addr;
			return connect( sockfd, _addr, addrlen );
		#else
			__submit_prelude

			(*sqe){ IORING_OP_CONNECT, sockfd };
			sqe->addr = (uint64_t)addr;
			sqe->off = addrlen;

			__submit_wait
		#endif
	}

	int async_fallocate(int fd, int mode, uint64_t offset, uint64_t len) {
		#if !defined(IORING_OP_FALLOCATE)
			return fallocate( fd, mode, offset, len );
		#else
			__submit_prelude

			(*sqe){ IORING_OP_FALLOCATE, fd };
			sqe->off = offset;
			sqe->len = length;
			sqe->mode = mode;

			__submit_wait
		#endif
	}

	int async_fadvise(int fd, uint64_t offset, uint64_t len, int advice) {
		#if !defined(IORING_OP_FADVISE)
			return posix_fadvise( fd, offset, len, advice );
		#else
			__submit_prelude

			(*sqe){ IORING_OP_FADVISE, fd };
			sqe->off = (uint64_t)offset;
			sqe->len = length;
			sqe->fadvise_advice = advice;

			__submit_wait
		#endif
	}

	int async_madvise(void *addr, size_t length, int advice) {
		#if !defined(IORING_OP_MADVISE)
			return madvise( addr, length, advice );
		#else
			__submit_prelude

			(*sqe){ IORING_OP_MADVISE, 0 };
			sqe->addr = (uint64_t)addr;
			sqe->len = length;
			sqe->fadvise_advice = advice;

			__submit_wait
		#endif
	}

	int async_openat(int dirfd, const char *pathname, int flags, mode_t mode) {
		#if !defined(IORING_OP_OPENAT)
			return openat( dirfd, pathname, flags, mode );
		#else
			__submit_prelude

			(*sqe){ IORING_OP_OPENAT, dirfd };
			sqe->addr = (uint64_t)pathname;
			sqe->open_flags = flags;
			sqe->mode = mode;

			__submit_wait
		#endif
	}

	int async_close(int fd) {
		#if !defined(IORING_OP_CLOSE)
			return close( fd );
		#else
			__submit_prelude

			(*sqe){ IORING_OP_CLOSE, fd };

			__submit_wait
		#endif
	}

	int async_statx(int dirfd, const char *pathname, int flags, unsigned int mask, struct statx *statxbuf) {
		#if !defined(IORING_OP_STATX)
			return statx( dirfd, pathname, flags, mask, statxbuf );
		#else
			__submit_prelude

			(*sqe){ IORING_OP_STATX, dirfd };
			sqe->addr = (uint64_t)pathname;
			sqe->statx_flags = flags;
			sqe->len = mask;
			sqe->off = (uint64_t)statxbuf;

			__submit_wait
		#endif
	}


	ssize_t async_read(int fd, void *buf, size_t count) {
		#if !defined(IORING_OP_READ)
			return read( fd, buf, count );
		#else
			__submit_prelude

			(*sqe){ IORING_OP_READ, fd, buf, count, 0 };

			__submit_wait
		#endif
	}

	ssize_t async_write(int fd, void *buf, size_t count) {
		#if !defined(IORING_OP_WRITE)
			return read( fd, buf, count );
		#else
			__submit_prelude

			(*sqe){ IORING_OP_WRITE, fd, buf, count, 0 };

			__submit_wait
		#endif
	}

//-----------------------------------------------------------------------------
// Check if a function is asynchronous

// Macro magic to reduce the size of the following switch case
	#define IS_DEFINED_APPLY(f, ...) f(__VA_ARGS__)
	#define IS_DEFINED_SECOND(first, second, ...) second
	#define IS_DEFINED_TEST(expansion) _CFA_IO_FEATURE_##expansion
	#define IS_DEFINED(macro) IS_DEFINED_APPLY( IS_DEFINED_SECOND,IS_DEFINED_TEST(macro) false, true)

	bool is_async( fptr_t func ) {

		if( /*func == (fptr_t)preadv2 || */
			func == (fptr_t)async_preadv2 )
			#define _CFA_IO_FEATURE_IORING_OP_READV ,
			return IS_DEFINED(IORING_OP_READV);

		if( /*func == (fptr_t)pwritev2 || */
		      func == (fptr_t)async_pwritev2 )
			#define _CFA_IO_FEATURE_IORING_OP_WRITEV ,
			return IS_DEFINED(IORING_OP_WRITEV);

		if( /*func == (fptr_t)fsync || */
		      func == (fptr_t)async_fsync )
			#define _CFA_IO_FEATURE_IORING_OP_FSYNC ,
			return IS_DEFINED(IORING_OP_FSYNC);

		if( /*func == (fptr_t)ync_file_range || */
		      func == (fptr_t)async_sync_file_range )
			#define _CFA_IO_FEATURE_IORING_OP_SYNC_FILE_RANGE ,
			return IS_DEFINED(IORING_OP_SYNC_FILE_RANGE);

		if( /*func == (fptr_t)sendmsg || */
		      func == (fptr_t)async_sendmsg )
			#define _CFA_IO_FEATURE_IORING_OP_SENDMSG ,
			return IS_DEFINED(IORING_OP_SENDMSG);

		if( /*func == (fptr_t)recvmsg || */
		      func == (fptr_t)async_recvmsg )
			#define _CFA_IO_FEATURE_IORING_OP_RECVMSG ,
			return IS_DEFINED(IORING_OP_RECVMSG);

		if( /*func == (fptr_t)send || */
			func == (fptr_t)async_send )
			#define _CFA_IO_FEATURE_IORING_OP_SEND ,
			return IS_DEFINED(IORING_OP_SEND);

		if( /*func == (fptr_t)recv || */
			func == (fptr_t)async_recv )
			#define _CFA_IO_FEATURE_IORING_OP_RECV ,
			return IS_DEFINED(IORING_OP_RECV);

		if( /*func == (fptr_t)accept4 || */
			func == (fptr_t)async_accept4 )
			#define _CFA_IO_FEATURE_IORING_OP_ACCEPT ,
			return IS_DEFINED(IORING_OP_ACCEPT);

		if( /*func == (fptr_t)connect || */
			func == (fptr_t)async_connect )
			#define _CFA_IO_FEATURE_IORING_OP_CONNECT ,
			return IS_DEFINED(IORING_OP_CONNECT);

		if( /*func == (fptr_t)fallocate || */
			func == (fptr_t)async_fallocate )
			#define _CFA_IO_FEATURE_IORING_OP_FALLOCATE ,
			return IS_DEFINED(IORING_OP_FALLOCATE);

		if( /*func == (fptr_t)fadvise || */
			func == (fptr_t)async_fadvise )
			#define _CFA_IO_FEATURE_IORING_OP_FADVISE ,
			return IS_DEFINED(IORING_OP_FADVISE);

		if( /*func == (fptr_t)madvise || */
			func == (fptr_t)async_madvise )
			#define _CFA_IO_FEATURE_IORING_OP_MADVISE ,
			return IS_DEFINED(IORING_OP_MADVISE);

		if( /*func == (fptr_t)openat || */
			func == (fptr_t)async_openat )
			#define _CFA_IO_FEATURE_IORING_OP_OPENAT ,
			return IS_DEFINED(IORING_OP_OPENAT);

		if( /*func == (fptr_t)close || */
			func == (fptr_t)async_close )
			#define _CFA_IO_FEATURE_IORING_OP_CLOSE ,
			return IS_DEFINED(IORING_OP_CLOSE);

		if( /*func == (fptr_t)statx || */
			func == (fptr_t)async_statx )
			#define _CFA_IO_FEATURE_IORING_OP_STATX ,
			return IS_DEFINED(IORING_OP_STATX);

		if( /*func == (fptr_t)read || */
		      func == (fptr_t)async_read )
			#define _CFA_IO_FEATURE_IORING_OP_READ ,
			return IS_DEFINED(IORING_OP_READ);

		if( /*func == (fptr_t)write || */
		      func == (fptr_t)async_write )
			#define _CFA_IO_FEATURE_IORING_OP_WRITE ,
			return IS_DEFINED(IORING_OP_WRITE);

		return false;
	}

#endif