Context Navigation

iocall.cfa @ 76de075

ADTarm-ehast-experimentalenumforall-pointer-decayjacob/cs343-translationnew-astnew-ast-unique-exprpthread-emulationqualifiedEnum

Last change on this file since 76de075 was 5877b3e, checked in by Thierry Delisle <tdelisle@…>, 4 years ago
Added missing include
Property mode set to `100644`
File size: 12.4 KB

Rev	Line
[31bb2e1]	1	//
	2	// Cforall Version 1.0.0 Copyright (C) 2020 University of Waterloo
	3	//
	4	// The contents of this file are covered under the licence agreement in the
	5	// file "LICENCE" distributed with Cforall.
	6	//
	7	// iocall.cfa --
	8	//
	9	// Author : Thierry Delisle
	10	// Created On : Wed Jul 1 14:51:00 2020
	11	// Last Modified By :
	12	// Last Modified On :
	13	// Update Count :
	14	//
	15
[5877b3e]	16	#include "bits/defs.hfa"
	17
[4b84e35]	18	//=============================================================================================
	19	// I/O uring backend
	20	//=============================================================================================
	21
[31bb2e1]	22	#if defined(HAVE_LINUX_IO_URING_H)
	23	#include <stdint.h>
	24	#include <linux/io_uring.h>
	25
	26	#include "kernel_private.hfa"
	27
	28	extern [* struct io_uring_sqe, uint32_t] __submit_alloc( struct __io_data & ring, uint64_t data );
	29	extern void __submit( struct __io_data & ring, uint32_t idx );
	30
[4b84e35]	31	static inline void ?{}(struct io_uring_sqe & this, uint8_t opcode, int fd) {
	32	this.opcode = opcode;
	33	#if !defined(IOSQE_ASYNC)
	34	this.flags = 0;
	35	#else
	36	this.flags = IOSQE_ASYNC;
	37	#endif
	38	this.ioprio = 0;
	39	this.fd = fd;
	40	this.off = 0;
	41	this.addr = 0;
	42	this.len = 0;
	43	this.rw_flags = 0;
	44	this.__pad2[0] = this.__pad2[1] = this.__pad2[2] = 0;
	45	}
	46
	47	static inline void ?{}(struct io_uring_sqe & this, uint8_t opcode, int fd, void * addr, uint32_t len, uint64_t off ) {
	48	(this){ opcode, fd };
	49	this.off = off;
	50	this.addr = (uint64_t)(uintptr_t)addr;
	51	this.len = len;
	52	}
[31bb2e1]	53
	54	#define __submit_prelude \
	55	__io_user_data_t data = { 0, active_thread() }; \
	56	struct __io_data & ring = *data.thrd->curr_cluster->io; \
	57	struct io_uring_sqe * sqe; \
	58	uint32_t idx; \
	59	[sqe, idx] = __submit_alloc( ring, (uint64_t)(uintptr_t)&data );
	60
	61	#define __submit_wait \
	62	/__cfaabi_bits_print_safe( STDERR_FILENO, "Preparing user data %p for %p\n", &data, data.thrd );/ \
	63	verify( sqe->user_data == (uint64_t)(uintptr_t)&data ); \
	64	__submit( ring, idx ); \
	65	park( __cfaabi_dbg_ctx ); \
	66	return data.result;
[4b84e35]	67	#endif
[31bb2e1]	68
[4b84e35]	69	//=============================================================================================
	70	// I/O Forwards
	71	//=============================================================================================
[31bb2e1]	72
[4b84e35]	73	// Some forward declarations
	74	#include <unistd.h>
	75
	76	extern "C" {
	77	#include <sys/types.h>
	78	#include <sys/socket.h>
	79	#include <sys/syscall.h>
	80
	81	#if defined(HAVE_PREADV2)
	82	struct iovec;
	83	extern ssize_t preadv2 (int fd, const struct iovec *iov, int iovcnt, off_t offset, int flags);
	84	#endif
	85	#if defined(HAVE_PWRITEV2)
	86	struct iovec;
	87	extern ssize_t pwritev2(int fd, const struct iovec *iov, int iovcnt, off_t offset, int flags);
	88	#endif
	89
	90	extern int fsync(int fd);
	91	extern int sync_file_range(int fd, int64_t offset, int64_t nbytes, unsigned int flags);
	92
	93	struct msghdr;
	94	struct sockaddr;
	95	extern ssize_t sendmsg(int sockfd, const struct msghdr *msg, int flags);
	96	extern ssize_t recvmsg(int sockfd, struct msghdr *msg, int flags);
	97	extern ssize_t send(int sockfd, const void *buf, size_t len, int flags);
	98	extern ssize_t recv(int sockfd, void *buf, size_t len, int flags);
	99	extern int accept4(int sockfd, struct sockaddr addr, socklen_t addrlen, int flags);
	100	extern int connect(int sockfd, const struct sockaddr *addr, socklen_t addrlen);
	101
	102	extern int fallocate(int fd, int mode, uint64_t offset, uint64_t len);
	103	extern int posix_fadvise(int fd, uint64_t offset, uint64_t len, int advice);
	104	extern int madvise(void *addr, size_t length, int advice);
	105
	106	extern int openat(int dirfd, const char *pathname, int flags, mode_t mode);
	107	extern int close(int fd);
	108
	109	extern ssize_t read (int fd, void *buf, size_t count);
	110	}
[31bb2e1]	111
[4b84e35]	112	//=============================================================================================
	113	// I/O Interface
	114	//=============================================================================================
[31bb2e1]	115
[4b84e35]	116	//-----------------------------------------------------------------------------
	117	// Asynchronous operations
	118	#if defined(HAVE_PREADV2)
	119	ssize_t cfa_preadv2(int fd, const struct iovec *iov, int iovcnt, off_t offset, int flags) {
	120	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_READV)
	121	return preadv2(fd, iov, iovcnt, offset, flags);
[31bb2e1]	122	#else
	123	__submit_prelude
	124
[4b84e35]	125	(*sqe){ IORING_OP_READV, fd, iov, iovcnt, offset };
[31bb2e1]	126
	127	__submit_wait
	128	#endif
	129	}
[4b84e35]	130	#endif
[31bb2e1]	131
[4b84e35]	132	#if defined(HAVE_PWRITEV2)
	133	ssize_t cfa_pwritev2(int fd, const struct iovec *iov, int iovcnt, off_t offset, int flags) {
	134	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_WRITEV)
	135	return pwritev2(fd, iov, iovcnt, offset, flags);
[31bb2e1]	136	#else
	137	__submit_prelude
	138
[4b84e35]	139	(*sqe){ IORING_OP_WRITEV, fd, iov, iovcnt, offset };
[31bb2e1]	140
	141	__submit_wait
	142	#endif
	143	}
[4b84e35]	144	#endif
[31bb2e1]	145
[4b84e35]	146	int cfa_fsync(int fd) {
	147	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_FSYNC)
	148	return fsync(fd);
	149	#else
	150	__submit_prelude
[31bb2e1]	151
[4b84e35]	152	(*sqe){ IORING_OP_FSYNC, fd };
[31bb2e1]	153
[4b84e35]	154	__submit_wait
	155	#endif
	156	}
[31bb2e1]	157
[4b84e35]	158	int cfa_sync_file_range(int fd, int64_t offset, int64_t nbytes, unsigned int flags) {
	159	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_SYNC_FILE_RANGE)
	160	return sync_file_range(fd, offset, nbytes, flags);
	161	#else
	162	__submit_prelude
[31bb2e1]	163
[4b84e35]	164	(*sqe){ IORING_OP_SYNC_FILE_RANGE, fd };
	165	sqe->off = offset;
	166	sqe->len = nbytes;
	167	sqe->sync_range_flags = flags;
[31bb2e1]	168
[4b84e35]	169	__submit_wait
	170	#endif
	171	}
[31bb2e1]	172
	173
[4b84e35]	174	ssize_t cfa_sendmsg(int sockfd, const struct msghdr *msg, int flags) {
	175	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_SENDMSG)
	176	return sendmsg(sockfd, msg, flags);
	177	#else
	178	__submit_prelude
[31bb2e1]	179
[4b84e35]	180	(*sqe){ IORING_OP_SENDMSG, sockfd, msg, 1, 0 };
	181	sqe->msg_flags = flags;
[31bb2e1]	182
[4b84e35]	183	__submit_wait
	184	#endif
	185	}
[31bb2e1]	186
[4b84e35]	187	ssize_t cfa_recvmsg(int sockfd, struct msghdr *msg, int flags) {
	188	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_RECVMSG)
	189	return recvmsg(sockfd, msg, flags);
	190	#else
	191	__submit_prelude
[31bb2e1]	192
[4b84e35]	193	(*sqe){ IORING_OP_RECVMSG, sockfd, msg, 1, 0 };
	194	sqe->msg_flags = flags;
[31bb2e1]	195
[4b84e35]	196	__submit_wait
	197	#endif
	198	}
[31bb2e1]	199
[4b84e35]	200	ssize_t cfa_send(int sockfd, const void *buf, size_t len, int flags) {
	201	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_SEND)
	202	return send( sockfd, buf, len, flags );
	203	#else
	204	__submit_prelude
[31bb2e1]	205
[4b84e35]	206	(*sqe){ IORING_OP_SEND, sockfd };
	207	sqe->addr = (uint64_t)buf;
	208	sqe->len = len;
	209	sqe->msg_flags = flags;
[31bb2e1]	210
[4b84e35]	211	__submit_wait
	212	#endif
	213	}
[31bb2e1]	214
[4b84e35]	215	ssize_t cfa_recv(int sockfd, void *buf, size_t len, int flags) {
	216	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_RECV)
	217	return recv( sockfd, buf, len, flags );
	218	#else
	219	__submit_prelude
[31bb2e1]	220
[4b84e35]	221	(*sqe){ IORING_OP_RECV, sockfd };
	222	sqe->addr = (uint64_t)buf;
	223	sqe->len = len;
	224	sqe->msg_flags = flags;
[31bb2e1]	225
[4b84e35]	226	__submit_wait
	227	#endif
	228	}
[31bb2e1]	229
[4b84e35]	230	int cfa_accept4(int sockfd, struct sockaddr addr, socklen_t addrlen, int flags) {
	231	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_ACCEPT)
	232	return accept4( sockfd, addr, addrlen, flags );
	233	#else
	234	__submit_prelude
[31bb2e1]	235
[4b84e35]	236	(*sqe){ IORING_OP_ACCEPT, sockfd };
	237	sqe->addr = addr;
	238	sqe->addr2 = addrlen;
	239	sqe->accept_flags = flags;
[31bb2e1]	240
[4b84e35]	241	__submit_wait
	242	#endif
	243	}
[31bb2e1]	244
[4b84e35]	245	int cfa_connect(int sockfd, const struct sockaddr *addr, socklen_t addrlen) {
	246	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_CONNECT)
	247	return connect( sockfd, addr, addrlen );
	248	#else
	249	__submit_prelude
[31bb2e1]	250
[4b84e35]	251	(*sqe){ IORING_OP_CONNECT, sockfd };
	252	sqe->addr = (uint64_t)addr;
	253	sqe->off = addrlen;
[31bb2e1]	254
[4b84e35]	255	__submit_wait
	256	#endif
	257	}
[31bb2e1]	258
[4b84e35]	259	int cfa_fallocate(int fd, int mode, uint64_t offset, uint64_t len) {
	260	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_FALLOCATE)
	261	return fallocate( fd, mode, offset, len );
	262	#else
	263	__submit_prelude
[31bb2e1]	264
[4b84e35]	265	(*sqe){ IORING_OP_FALLOCATE, fd };
	266	sqe->off = offset;
	267	sqe->len = length;
	268	sqe->mode = mode;
[31bb2e1]	269
[4b84e35]	270	__submit_wait
	271	#endif
	272	}
[31bb2e1]	273
[4b84e35]	274	int cfa_fadvise(int fd, uint64_t offset, uint64_t len, int advice) {
	275	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_FADVISE)
	276	return posix_fadvise( fd, offset, len, advice );
	277	#else
	278	__submit_prelude
[31bb2e1]	279
[4b84e35]	280	(*sqe){ IORING_OP_FADVISE, fd };
	281	sqe->off = (uint64_t)offset;
	282	sqe->len = length;
	283	sqe->fadvise_advice = advice;
[31bb2e1]	284
[4b84e35]	285	__submit_wait
	286	#endif
	287	}
[31bb2e1]	288
[4b84e35]	289	int cfa_madvise(void *addr, size_t length, int advice) {
	290	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_MADVISE)
	291	return madvise( addr, length, advice );
	292	#else
	293	__submit_prelude
[31bb2e1]	294
[4b84e35]	295	(*sqe){ IORING_OP_MADVISE, 0 };
	296	sqe->addr = (uint64_t)addr;
	297	sqe->len = length;
	298	sqe->fadvise_advice = advice;
[31bb2e1]	299
[4b84e35]	300	__submit_wait
	301	#endif
	302	}
[31bb2e1]	303
[4b84e35]	304	int cfa_openat(int dirfd, const char *pathname, int flags, mode_t mode) {
	305	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_OPENAT)
	306	return openat( dirfd, pathname, flags, mode );
	307	#else
	308	__submit_prelude
[31bb2e1]	309
[4b84e35]	310	(*sqe){ IORING_OP_OPENAT, dirfd };
	311	sqe->addr = (uint64_t)pathname;
	312	sqe->open_flags = flags;
	313	sqe->mode = mode;
[31bb2e1]	314
[4b84e35]	315	__submit_wait
	316	#endif
	317	}
[31bb2e1]	318
[4b84e35]	319	int cfa_close(int fd) {
	320	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_CLOSE)
	321	return close( fd );
	322	#else
	323	__submit_prelude
[31bb2e1]	324
[4b84e35]	325	(*sqe){ IORING_OP_CLOSE, fd };
	326
	327	__submit_wait
	328	#endif
	329	}
[31bb2e1]	330
	331
[4b84e35]	332	ssize_t cfa_read(int fd, void *buf, size_t count) {
	333	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_READ)
	334	return read( fd, buf, count );
	335	#else
	336	__submit_prelude
	337
	338	(*sqe){ IORING_OP_READ, fd, buf, count, 0 };
	339
	340	__submit_wait
	341	#endif
	342	}
	343
	344	ssize_t cfa_write(int fd, void *buf, size_t count) {
	345	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_WRITE)
	346	return read( fd, buf, count );
	347	#else
	348	__submit_prelude
	349
	350	(*sqe){ IORING_OP_WRITE, fd, buf, count, 0 };
	351
	352	__submit_wait
	353	#endif
	354	}
	355
	356	//-----------------------------------------------------------------------------
	357	// Check if a function is asynchronous
	358
	359	// Macro magic to reduce the size of the following switch case
	360	#define IS_DEFINED_APPLY(f, ...) f(__VA_ARGS__)
	361	#define IS_DEFINED_SECOND(first, second, ...) second
	362	#define IS_DEFINED_TEST(expansion) _CFA_IO_FEATURE_##expansion
	363	#define IS_DEFINED(macro) IS_DEFINED_APPLY( IS_DEFINED_SECOND,IS_DEFINED_TEST(macro) false, true)
	364
	365	bool has_user_level_blocking( fptr_t func ) {
	366	#if defined(HAVE_LINUX_IO_URING_H)
	367	#if defined(HAVE_PREADV2)
	368	if( /func == (fptr_t)preadv2 \|\| /
	369	func == (fptr_t)cfa_preadv2 )
	370	#define _CFA_IO_FEATURE_IORING_OP_READV ,
	371	return IS_DEFINED(IORING_OP_READV);
[31bb2e1]	372	#endif
	373
[4b84e35]	374	#if defined(HAVE_PWRITEV2)
	375	if( /func == (fptr_t)pwritev2 \|\| /
	376	func == (fptr_t)cfa_pwritev2 )
	377	#define _CFA_IO_FEATURE_IORING_OP_WRITEV ,
	378	return IS_DEFINED(IORING_OP_WRITEV);
[31bb2e1]	379	#endif
	380
[4b84e35]	381	if( /func == (fptr_t)fsync \|\| /
	382	func == (fptr_t)cfa_fsync )
	383	#define _CFA_IO_FEATURE_IORING_OP_FSYNC ,
	384	return IS_DEFINED(IORING_OP_FSYNC);
	385
	386	if( /func == (fptr_t)ync_file_range \|\| /
	387	func == (fptr_t)cfa_sync_file_range )
	388	#define _CFA_IO_FEATURE_IORING_OP_SYNC_FILE_RANGE ,
	389	return IS_DEFINED(IORING_OP_SYNC_FILE_RANGE);
	390
	391	if( /func == (fptr_t)sendmsg \|\| /
	392	func == (fptr_t)cfa_sendmsg )
	393	#define _CFA_IO_FEATURE_IORING_OP_SENDMSG ,
	394	return IS_DEFINED(IORING_OP_SENDMSG);
	395
	396	if( /func == (fptr_t)recvmsg \|\| /
	397	func == (fptr_t)cfa_recvmsg )
	398	#define _CFA_IO_FEATURE_IORING_OP_RECVMSG ,
	399	return IS_DEFINED(IORING_OP_RECVMSG);
	400
	401	if( /func == (fptr_t)send \|\| /
	402	func == (fptr_t)cfa_send )
	403	#define _CFA_IO_FEATURE_IORING_OP_SEND ,
	404	return IS_DEFINED(IORING_OP_SEND);
	405
	406	if( /func == (fptr_t)recv \|\| /
	407	func == (fptr_t)cfa_recv )
	408	#define _CFA_IO_FEATURE_IORING_OP_RECV ,
	409	return IS_DEFINED(IORING_OP_RECV);
	410
	411	if( /func == (fptr_t)accept4 \|\| /
	412	func == (fptr_t)cfa_accept4 )
	413	#define _CFA_IO_FEATURE_IORING_OP_ACCEPT ,
	414	return IS_DEFINED(IORING_OP_ACCEPT);
	415
	416	if( /func == (fptr_t)connect \|\| /
	417	func == (fptr_t)cfa_connect )
	418	#define _CFA_IO_FEATURE_IORING_OP_CONNECT ,
	419	return IS_DEFINED(IORING_OP_CONNECT);
	420
	421	if( /func == (fptr_t)fallocate \|\| /
	422	func == (fptr_t)cfa_fallocate )
	423	#define _CFA_IO_FEATURE_IORING_OP_FALLOCATE ,
	424	return IS_DEFINED(IORING_OP_FALLOCATE);
	425
	426	if( /func == (fptr_t)posix_fadvise \|\| /
	427	func == (fptr_t)cfa_fadvise )
	428	#define _CFA_IO_FEATURE_IORING_OP_FADVISE ,
	429	return IS_DEFINED(IORING_OP_FADVISE);
	430
	431	if( /func == (fptr_t)madvise \|\| /
	432	func == (fptr_t)cfa_madvise )
	433	#define _CFA_IO_FEATURE_IORING_OP_MADVISE ,
	434	return IS_DEFINED(IORING_OP_MADVISE);
	435
	436	if( /func == (fptr_t)openat \|\| /
	437	func == (fptr_t)cfa_openat )
	438	#define _CFA_IO_FEATURE_IORING_OP_OPENAT ,
	439	return IS_DEFINED(IORING_OP_OPENAT);
	440
	441	if( /func == (fptr_t)close \|\| /
	442	func == (fptr_t)cfa_close )
	443	#define _CFA_IO_FEATURE_IORING_OP_CLOSE ,
	444	return IS_DEFINED(IORING_OP_CLOSE);
	445
	446	if( /func == (fptr_t)read \|\| /
	447	func == (fptr_t)cfa_read )
	448	#define _CFA_IO_FEATURE_IORING_OP_READ ,
	449	return IS_DEFINED(IORING_OP_READ);
	450
	451	if( /func == (fptr_t)write \|\| /
	452	func == (fptr_t)cfa_write )
	453	#define _CFA_IO_FEATURE_IORING_OP_WRITE ,
	454	return IS_DEFINED(IORING_OP_WRITE);
	455	#endif
	456
	457	return false;
	458	}

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

Original Format