Context Navigation

iocall.cfa@ ccb776b

Visit:

ADT arm-eh ast-experimental enum forall-pointer-decay jacob/cs343-translation new-ast new-ast-unique-expr pthread-emulation qualifiedEnum

Last change on this file since ccb776b was 365e423, checked in by Thierry Delisle <tdelisle@…>, 5 years ago
Merge branch 'master' of plg.uwaterloo.ca:software/cfa/cfa-cc
Property mode set to `100644`
File size: 14.0 KB

Rev	Line
[31bb2e1]	1	//
	2	// Cforall Version 1.0.0 Copyright (C) 2020 University of Waterloo
	3	//
	4	// The contents of this file are covered under the licence agreement in the
	5	// file "LICENCE" distributed with Cforall.
	6	//
	7	// iocall.cfa --
	8	//
	9	// Author : Thierry Delisle
	10	// Created On : Wed Jul 1 14:51:00 2020
	11	// Last Modified By :
	12	// Last Modified On :
	13	// Update Count :
	14	//
	15
[5877b3e]	16	#include "bits/defs.hfa"
	17
[4b84e35]	18	//=============================================================================================
	19	// I/O uring backend
	20	//=============================================================================================
	21
[31bb2e1]	22	#if defined(HAVE_LINUX_IO_URING_H)
	23	#include <stdint.h>
	24	#include <linux/io_uring.h>
	25
	26	#include "kernel_private.hfa"
	27
	28	extern [* struct io_uring_sqe, uint32_t] __submit_alloc( struct __io_data & ring, uint64_t data );
	29	extern void __submit( struct __io_data & ring, uint32_t idx );
	30
[4b84e35]	31	static inline void ?{}(struct io_uring_sqe & this, uint8_t opcode, int fd) {
	32	this.opcode = opcode;
	33	#if !defined(IOSQE_ASYNC)
	34	this.flags = 0;
	35	#else
	36	this.flags = IOSQE_ASYNC;
	37	#endif
	38	this.ioprio = 0;
	39	this.fd = fd;
	40	this.off = 0;
	41	this.addr = 0;
	42	this.len = 0;
	43	this.rw_flags = 0;
	44	this.__pad2[0] = this.__pad2[1] = this.__pad2[2] = 0;
	45	}
	46
	47	static inline void ?{}(struct io_uring_sqe & this, uint8_t opcode, int fd, void * addr, uint32_t len, uint64_t off ) {
	48	(this){ opcode, fd };
	49	this.off = off;
	50	this.addr = (uint64_t)(uintptr_t)addr;
	51	this.len = len;
	52	}
[31bb2e1]	53
	54	#define __submit_prelude \
	55	__io_user_data_t data = { 0, active_thread() }; \
	56	struct __io_data & ring = *data.thrd->curr_cluster->io; \
	57	struct io_uring_sqe * sqe; \
	58	uint32_t idx; \
	59	[sqe, idx] = __submit_alloc( ring, (uint64_t)(uintptr_t)&data );
	60
	61	#define __submit_wait \
	62	/__cfaabi_bits_print_safe( STDERR_FILENO, "Preparing user data %p for %p\n", &data, data.thrd );/ \
	63	verify( sqe->user_data == (uint64_t)(uintptr_t)&data ); \
	64	__submit( ring, idx ); \
	65	park( __cfaabi_dbg_ctx ); \
	66	return data.result;
[4b84e35]	67	#endif
[31bb2e1]	68
[4b84e35]	69	//=============================================================================================
	70	// I/O Forwards
	71	//=============================================================================================
[31bb2e1]	72
[4b84e35]	73	// Some forward declarations
	74	#include <unistd.h>
	75
	76	extern "C" {
	77	#include <sys/types.h>
	78	#include <sys/socket.h>
	79	#include <sys/syscall.h>
	80
	81	#if defined(HAVE_PREADV2)
	82	struct iovec;
	83	extern ssize_t preadv2 (int fd, const struct iovec *iov, int iovcnt, off_t offset, int flags);
	84	#endif
	85	#if defined(HAVE_PWRITEV2)
	86	struct iovec;
	87	extern ssize_t pwritev2(int fd, const struct iovec *iov, int iovcnt, off_t offset, int flags);
	88	#endif
	89
	90	extern int fsync(int fd);
	91	extern int sync_file_range(int fd, int64_t offset, int64_t nbytes, unsigned int flags);
	92
	93	struct msghdr;
	94	struct sockaddr;
	95	extern ssize_t sendmsg(int sockfd, const struct msghdr *msg, int flags);
	96	extern ssize_t recvmsg(int sockfd, struct msghdr *msg, int flags);
	97	extern ssize_t send(int sockfd, const void *buf, size_t len, int flags);
	98	extern ssize_t recv(int sockfd, void *buf, size_t len, int flags);
	99	extern int accept4(int sockfd, struct sockaddr addr, socklen_t addrlen, int flags);
	100	extern int connect(int sockfd, const struct sockaddr *addr, socklen_t addrlen);
	101
	102	extern int fallocate(int fd, int mode, uint64_t offset, uint64_t len);
	103	extern int posix_fadvise(int fd, uint64_t offset, uint64_t len, int advice);
	104	extern int madvise(void *addr, size_t length, int advice);
	105
	106	extern int openat(int dirfd, const char *pathname, int flags, mode_t mode);
	107	extern int close(int fd);
	108
	109	extern ssize_t read (int fd, void *buf, size_t count);
[0a92c78]	110
	111	extern ssize_t splice(int fd_in, loff_t off_in, int fd_out, loff_t off_out, size_t len, unsigned int flags);
	112	extern ssize_t tee(int fd_in, int fd_out, size_t len, unsigned int flags);
[4b84e35]	113	}
[31bb2e1]	114
[4b84e35]	115	//=============================================================================================
	116	// I/O Interface
	117	//=============================================================================================
[31bb2e1]	118
[4b84e35]	119	//-----------------------------------------------------------------------------
	120	// Asynchronous operations
	121	#if defined(HAVE_PREADV2)
	122	ssize_t cfa_preadv2(int fd, const struct iovec *iov, int iovcnt, off_t offset, int flags) {
	123	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_READV)
	124	return preadv2(fd, iov, iovcnt, offset, flags);
[31bb2e1]	125	#else
	126	__submit_prelude
	127
[4b84e35]	128	(*sqe){ IORING_OP_READV, fd, iov, iovcnt, offset };
[31bb2e1]	129
	130	__submit_wait
	131	#endif
	132	}
[20ab637]	133
	134	ssize_t cfa_preadv2_fixed(int fd, const struct iovec *iov, int iovcnt, off_t offset, int flags) {
	135	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_READV)
	136	return preadv2(fd, iov, iovcnt, offset, flags);
	137	#else
	138	__submit_prelude
	139
	140	(*sqe){ IORING_OP_READV, fd, iov, iovcnt, offset };
	141	sqe->flags \|= IOSQE_FIXED_FILE;
	142
	143	__submit_wait
	144	#endif
	145	}
[4b84e35]	146	#endif
[31bb2e1]	147
[4b84e35]	148	#if defined(HAVE_PWRITEV2)
	149	ssize_t cfa_pwritev2(int fd, const struct iovec *iov, int iovcnt, off_t offset, int flags) {
	150	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_WRITEV)
	151	return pwritev2(fd, iov, iovcnt, offset, flags);
[31bb2e1]	152	#else
	153	__submit_prelude
	154
[4b84e35]	155	(*sqe){ IORING_OP_WRITEV, fd, iov, iovcnt, offset };
[31bb2e1]	156
	157	__submit_wait
	158	#endif
	159	}
[4b84e35]	160	#endif
[31bb2e1]	161
[4b84e35]	162	int cfa_fsync(int fd) {
	163	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_FSYNC)
	164	return fsync(fd);
	165	#else
	166	__submit_prelude
[31bb2e1]	167
[4b84e35]	168	(*sqe){ IORING_OP_FSYNC, fd };
[31bb2e1]	169
[4b84e35]	170	__submit_wait
	171	#endif
	172	}
[31bb2e1]	173
[4b84e35]	174	int cfa_sync_file_range(int fd, int64_t offset, int64_t nbytes, unsigned int flags) {
	175	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_SYNC_FILE_RANGE)
	176	return sync_file_range(fd, offset, nbytes, flags);
	177	#else
	178	__submit_prelude
[31bb2e1]	179
[4b84e35]	180	(*sqe){ IORING_OP_SYNC_FILE_RANGE, fd };
	181	sqe->off = offset;
	182	sqe->len = nbytes;
	183	sqe->sync_range_flags = flags;
[31bb2e1]	184
[4b84e35]	185	__submit_wait
	186	#endif
	187	}
[31bb2e1]	188
	189
[4b84e35]	190	ssize_t cfa_sendmsg(int sockfd, const struct msghdr *msg, int flags) {
	191	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_SENDMSG)
	192	return sendmsg(sockfd, msg, flags);
	193	#else
	194	__submit_prelude
[31bb2e1]	195
[4b84e35]	196	(*sqe){ IORING_OP_SENDMSG, sockfd, msg, 1, 0 };
	197	sqe->msg_flags = flags;
[31bb2e1]	198
[4b84e35]	199	__submit_wait
	200	#endif
	201	}
[31bb2e1]	202
[4b84e35]	203	ssize_t cfa_recvmsg(int sockfd, struct msghdr *msg, int flags) {
	204	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_RECVMSG)
	205	return recvmsg(sockfd, msg, flags);
	206	#else
	207	__submit_prelude
[31bb2e1]	208
[4b84e35]	209	(*sqe){ IORING_OP_RECVMSG, sockfd, msg, 1, 0 };
	210	sqe->msg_flags = flags;
[31bb2e1]	211
[4b84e35]	212	__submit_wait
	213	#endif
	214	}
[31bb2e1]	215
[4b84e35]	216	ssize_t cfa_send(int sockfd, const void *buf, size_t len, int flags) {
	217	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_SEND)
	218	return send( sockfd, buf, len, flags );
	219	#else
	220	__submit_prelude
[31bb2e1]	221
[4b84e35]	222	(*sqe){ IORING_OP_SEND, sockfd };
	223	sqe->addr = (uint64_t)buf;
	224	sqe->len = len;
	225	sqe->msg_flags = flags;
[31bb2e1]	226
[4b84e35]	227	__submit_wait
	228	#endif
	229	}
[31bb2e1]	230
[4b84e35]	231	ssize_t cfa_recv(int sockfd, void *buf, size_t len, int flags) {
	232	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_RECV)
	233	return recv( sockfd, buf, len, flags );
	234	#else
	235	__submit_prelude
[31bb2e1]	236
[4b84e35]	237	(*sqe){ IORING_OP_RECV, sockfd };
	238	sqe->addr = (uint64_t)buf;
	239	sqe->len = len;
	240	sqe->msg_flags = flags;
[31bb2e1]	241
[4b84e35]	242	__submit_wait
	243	#endif
	244	}
[31bb2e1]	245
[4b84e35]	246	int cfa_accept4(int sockfd, struct sockaddr addr, socklen_t addrlen, int flags) {
	247	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_ACCEPT)
	248	return accept4( sockfd, addr, addrlen, flags );
	249	#else
	250	__submit_prelude
[31bb2e1]	251
[4b84e35]	252	(*sqe){ IORING_OP_ACCEPT, sockfd };
	253	sqe->addr = addr;
	254	sqe->addr2 = addrlen;
	255	sqe->accept_flags = flags;
[31bb2e1]	256
[4b84e35]	257	__submit_wait
	258	#endif
	259	}
[31bb2e1]	260
[4b84e35]	261	int cfa_connect(int sockfd, const struct sockaddr *addr, socklen_t addrlen) {
	262	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_CONNECT)
	263	return connect( sockfd, addr, addrlen );
	264	#else
	265	__submit_prelude
[31bb2e1]	266
[4b84e35]	267	(*sqe){ IORING_OP_CONNECT, sockfd };
	268	sqe->addr = (uint64_t)addr;
	269	sqe->off = addrlen;
[31bb2e1]	270
[4b84e35]	271	__submit_wait
	272	#endif
	273	}
[31bb2e1]	274
[4b84e35]	275	int cfa_fallocate(int fd, int mode, uint64_t offset, uint64_t len) {
	276	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_FALLOCATE)
	277	return fallocate( fd, mode, offset, len );
	278	#else
	279	__submit_prelude
[31bb2e1]	280
[4b84e35]	281	(*sqe){ IORING_OP_FALLOCATE, fd };
	282	sqe->off = offset;
	283	sqe->len = length;
	284	sqe->mode = mode;
[31bb2e1]	285
[4b84e35]	286	__submit_wait
	287	#endif
	288	}
[31bb2e1]	289
[4b84e35]	290	int cfa_fadvise(int fd, uint64_t offset, uint64_t len, int advice) {
	291	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_FADVISE)
	292	return posix_fadvise( fd, offset, len, advice );
	293	#else
	294	__submit_prelude
[31bb2e1]	295
[4b84e35]	296	(*sqe){ IORING_OP_FADVISE, fd };
	297	sqe->off = (uint64_t)offset;
	298	sqe->len = length;
	299	sqe->fadvise_advice = advice;
[31bb2e1]	300
[4b84e35]	301	__submit_wait
	302	#endif
	303	}
[31bb2e1]	304
[4b84e35]	305	int cfa_madvise(void *addr, size_t length, int advice) {
	306	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_MADVISE)
	307	return madvise( addr, length, advice );
	308	#else
	309	__submit_prelude
[31bb2e1]	310
[4b84e35]	311	(*sqe){ IORING_OP_MADVISE, 0 };
	312	sqe->addr = (uint64_t)addr;
	313	sqe->len = length;
	314	sqe->fadvise_advice = advice;
[31bb2e1]	315
[4b84e35]	316	__submit_wait
	317	#endif
	318	}
[31bb2e1]	319
[4b84e35]	320	int cfa_openat(int dirfd, const char *pathname, int flags, mode_t mode) {
	321	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_OPENAT)
	322	return openat( dirfd, pathname, flags, mode );
	323	#else
	324	__submit_prelude
[31bb2e1]	325
[4b84e35]	326	(*sqe){ IORING_OP_OPENAT, dirfd };
	327	sqe->addr = (uint64_t)pathname;
	328	sqe->open_flags = flags;
	329	sqe->mode = mode;
[31bb2e1]	330
[4b84e35]	331	__submit_wait
	332	#endif
	333	}
[31bb2e1]	334
[4b84e35]	335	int cfa_close(int fd) {
	336	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_CLOSE)
	337	return close( fd );
	338	#else
	339	__submit_prelude
[31bb2e1]	340
[4b84e35]	341	(*sqe){ IORING_OP_CLOSE, fd };
	342
	343	__submit_wait
	344	#endif
	345	}
[31bb2e1]	346
[4b84e35]	347	ssize_t cfa_read(int fd, void *buf, size_t count) {
	348	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_READ)
	349	return read( fd, buf, count );
	350	#else
	351	__submit_prelude
	352
	353	(*sqe){ IORING_OP_READ, fd, buf, count, 0 };
	354
	355	__submit_wait
	356	#endif
	357	}
	358
	359	ssize_t cfa_write(int fd, void *buf, size_t count) {
	360	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_WRITE)
	361	return read( fd, buf, count );
	362	#else
	363	__submit_prelude
	364
	365	(*sqe){ IORING_OP_WRITE, fd, buf, count, 0 };
	366
	367	__submit_wait
	368	#endif
	369	}
	370
[0a92c78]	371	ssize_t cfa_splice(int fd_in, loff_t off_in, int fd_out, loff_t off_out, size_t len, unsigned int flags) {
	372	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_SPLICE)
	373	return splice( fd_in, off_in, fd_out, off_out, len, flags );
	374	#else
	375	__submit_prelude
	376
	377	(*sqe){ IORING_OP_SPLICE, fd_out, 0p, len, off_out };
	378	sqe->splice_fd_in = fd_in;
	379	sqe->splice_off_in = off_in;
	380	sqe->splice_flags = flags;
	381
	382	__submit_wait
	383	#endif
	384	}
	385
	386	ssize_t cfa_tee(int fd_in, int fd_out, size_t len, unsigned int flags) {
	387	#if !defined(HAVE_LINUX_IO_URING_H) \|\| !defined(IORING_OP_TEE)
	388	return tee( fd_in, fd_out, len, flags );
	389	#else
	390	__submit_prelude
	391
	392	(*sqe){ IORING_OP_TEE, fd_out, 0p, len, 0 };
	393	sqe->splice_fd_in = fd_in;
	394	sqe->splice_flags = flags;
	395
	396	__submit_wait
	397	#endif
	398	}
	399
[4b84e35]	400	//-----------------------------------------------------------------------------
	401	// Check if a function is asynchronous
	402
	403	// Macro magic to reduce the size of the following switch case
	404	#define IS_DEFINED_APPLY(f, ...) f(__VA_ARGS__)
	405	#define IS_DEFINED_SECOND(first, second, ...) second
	406	#define IS_DEFINED_TEST(expansion) _CFA_IO_FEATURE_##expansion
	407	#define IS_DEFINED(macro) IS_DEFINED_APPLY( IS_DEFINED_SECOND,IS_DEFINED_TEST(macro) false, true)
	408
	409	bool has_user_level_blocking( fptr_t func ) {
	410	#if defined(HAVE_LINUX_IO_URING_H)
	411	#if defined(HAVE_PREADV2)
	412	if( /func == (fptr_t)preadv2 \|\| /
	413	func == (fptr_t)cfa_preadv2 )
	414	#define _CFA_IO_FEATURE_IORING_OP_READV ,
	415	return IS_DEFINED(IORING_OP_READV);
[31bb2e1]	416	#endif
	417
[4b84e35]	418	#if defined(HAVE_PWRITEV2)
	419	if( /func == (fptr_t)pwritev2 \|\| /
	420	func == (fptr_t)cfa_pwritev2 )
	421	#define _CFA_IO_FEATURE_IORING_OP_WRITEV ,
	422	return IS_DEFINED(IORING_OP_WRITEV);
[31bb2e1]	423	#endif
	424
[4b84e35]	425	if( /func == (fptr_t)fsync \|\| /
	426	func == (fptr_t)cfa_fsync )
	427	#define _CFA_IO_FEATURE_IORING_OP_FSYNC ,
	428	return IS_DEFINED(IORING_OP_FSYNC);
	429
	430	if( /func == (fptr_t)ync_file_range \|\| /
	431	func == (fptr_t)cfa_sync_file_range )
	432	#define _CFA_IO_FEATURE_IORING_OP_SYNC_FILE_RANGE ,
	433	return IS_DEFINED(IORING_OP_SYNC_FILE_RANGE);
	434
	435	if( /func == (fptr_t)sendmsg \|\| /
	436	func == (fptr_t)cfa_sendmsg )
	437	#define _CFA_IO_FEATURE_IORING_OP_SENDMSG ,
	438	return IS_DEFINED(IORING_OP_SENDMSG);
	439
	440	if( /func == (fptr_t)recvmsg \|\| /
	441	func == (fptr_t)cfa_recvmsg )
	442	#define _CFA_IO_FEATURE_IORING_OP_RECVMSG ,
	443	return IS_DEFINED(IORING_OP_RECVMSG);
	444
	445	if( /func == (fptr_t)send \|\| /
	446	func == (fptr_t)cfa_send )
	447	#define _CFA_IO_FEATURE_IORING_OP_SEND ,
	448	return IS_DEFINED(IORING_OP_SEND);
	449
	450	if( /func == (fptr_t)recv \|\| /
	451	func == (fptr_t)cfa_recv )
	452	#define _CFA_IO_FEATURE_IORING_OP_RECV ,
	453	return IS_DEFINED(IORING_OP_RECV);
	454
	455	if( /func == (fptr_t)accept4 \|\| /
	456	func == (fptr_t)cfa_accept4 )
	457	#define _CFA_IO_FEATURE_IORING_OP_ACCEPT ,
	458	return IS_DEFINED(IORING_OP_ACCEPT);
	459
	460	if( /func == (fptr_t)connect \|\| /
	461	func == (fptr_t)cfa_connect )
	462	#define _CFA_IO_FEATURE_IORING_OP_CONNECT ,
	463	return IS_DEFINED(IORING_OP_CONNECT);
	464
	465	if( /func == (fptr_t)fallocate \|\| /
	466	func == (fptr_t)cfa_fallocate )
	467	#define _CFA_IO_FEATURE_IORING_OP_FALLOCATE ,
	468	return IS_DEFINED(IORING_OP_FALLOCATE);
	469
	470	if( /func == (fptr_t)posix_fadvise \|\| /
	471	func == (fptr_t)cfa_fadvise )
	472	#define _CFA_IO_FEATURE_IORING_OP_FADVISE ,
	473	return IS_DEFINED(IORING_OP_FADVISE);
	474
	475	if( /func == (fptr_t)madvise \|\| /
	476	func == (fptr_t)cfa_madvise )
	477	#define _CFA_IO_FEATURE_IORING_OP_MADVISE ,
	478	return IS_DEFINED(IORING_OP_MADVISE);
	479
	480	if( /func == (fptr_t)openat \|\| /
	481	func == (fptr_t)cfa_openat )
	482	#define _CFA_IO_FEATURE_IORING_OP_OPENAT ,
	483	return IS_DEFINED(IORING_OP_OPENAT);
	484
	485	if( /func == (fptr_t)close \|\| /
	486	func == (fptr_t)cfa_close )
	487	#define _CFA_IO_FEATURE_IORING_OP_CLOSE ,
	488	return IS_DEFINED(IORING_OP_CLOSE);
	489
	490	if( /func == (fptr_t)read \|\| /
	491	func == (fptr_t)cfa_read )
	492	#define _CFA_IO_FEATURE_IORING_OP_READ ,
	493	return IS_DEFINED(IORING_OP_READ);
	494
	495	if( /func == (fptr_t)write \|\| /
	496	func == (fptr_t)cfa_write )
	497	#define _CFA_IO_FEATURE_IORING_OP_WRITE ,
	498	return IS_DEFINED(IORING_OP_WRITE);
[0a92c78]	499
	500	if( /func == (fptr_t)splice \|\| /
	501	func == (fptr_t)cfa_splice )
	502	#define _CFA_IO_FEATURE_IORING_OP_SPLICE ,
	503	return IS_DEFINED(IORING_OP_SPLICE);
	504
	505	if( /func == (fptr_t)tee \|\| /
	506	func == (fptr_t)cfa_tee )
	507	#define _CFA_IO_FEATURE_IORING_OP_TEE ,
	508	return IS_DEFINED(IORING_OP_TEE);
[4b84e35]	509	#endif
	510
	511	return false;
	512	}

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

Original Format