source: libcfa/src/concurrency/iocall.cfa@ 59f74a2

ADT arm-eh ast-experimental enum forall-pointer-decay jacob/cs343-translation new-ast new-ast-unique-expr pthread-emulation qualifiedEnum
Last change on this file since 59f74a2 was 20ab637, checked in by Thierry Delisle <tdelisle@…>, 5 years ago

Added quick and dirty support for fixed files reads.
Added support for kernel side polling.
  • Property mode set to 100644
File size: 12.7 KB
RevLine 
[31bb2e1]1//
2// Cforall Version 1.0.0 Copyright (C) 2020 University of Waterloo
3//
4// The contents of this file are covered under the licence agreement in the
5// file "LICENCE" distributed with Cforall.
6//
7// iocall.cfa --
8//
9// Author : Thierry Delisle
10// Created On : Wed Jul 1 14:51:00 2020
11// Last Modified By :
12// Last Modified On :
13// Update Count :
14//
15
[5877b3e]16#include "bits/defs.hfa"
17
[4b84e35]18//=============================================================================================
19// I/O uring backend
20//=============================================================================================
21
[31bb2e1]22#if defined(HAVE_LINUX_IO_URING_H)
23 #include <stdint.h>
24 #include <linux/io_uring.h>
25
26 #include "kernel_private.hfa"
27
28 extern [* struct io_uring_sqe, uint32_t] __submit_alloc( struct __io_data & ring, uint64_t data );
29 extern void __submit( struct __io_data & ring, uint32_t idx );
30
[4b84e35]31 static inline void ?{}(struct io_uring_sqe & this, uint8_t opcode, int fd) {
32 this.opcode = opcode;
33 #if !defined(IOSQE_ASYNC)
34 this.flags = 0;
35 #else
36 this.flags = IOSQE_ASYNC;
37 #endif
38 this.ioprio = 0;
39 this.fd = fd;
40 this.off = 0;
41 this.addr = 0;
42 this.len = 0;
43 this.rw_flags = 0;
44 this.__pad2[0] = this.__pad2[1] = this.__pad2[2] = 0;
45 }
46
47 static inline void ?{}(struct io_uring_sqe & this, uint8_t opcode, int fd, void * addr, uint32_t len, uint64_t off ) {
48 (this){ opcode, fd };
49 this.off = off;
50 this.addr = (uint64_t)(uintptr_t)addr;
51 this.len = len;
52 }
[31bb2e1]53
54 #define __submit_prelude \
55 __io_user_data_t data = { 0, active_thread() }; \
56 struct __io_data & ring = *data.thrd->curr_cluster->io; \
57 struct io_uring_sqe * sqe; \
58 uint32_t idx; \
59 [sqe, idx] = __submit_alloc( ring, (uint64_t)(uintptr_t)&data );
60
61 #define __submit_wait \
62 /*__cfaabi_bits_print_safe( STDERR_FILENO, "Preparing user data %p for %p\n", &data, data.thrd );*/ \
63 verify( sqe->user_data == (uint64_t)(uintptr_t)&data ); \
64 __submit( ring, idx ); \
65 park( __cfaabi_dbg_ctx ); \
66 return data.result;
[4b84e35]67#endif
[31bb2e1]68
[4b84e35]69//=============================================================================================
70// I/O Forwards
71//=============================================================================================
[31bb2e1]72
[4b84e35]73// Some forward declarations
74#include <unistd.h>
75
76extern "C" {
77 #include <sys/types.h>
78 #include <sys/socket.h>
79 #include <sys/syscall.h>
80
81#if defined(HAVE_PREADV2)
82 struct iovec;
83 extern ssize_t preadv2 (int fd, const struct iovec *iov, int iovcnt, off_t offset, int flags);
84#endif
85#if defined(HAVE_PWRITEV2)
86 struct iovec;
87 extern ssize_t pwritev2(int fd, const struct iovec *iov, int iovcnt, off_t offset, int flags);
88#endif
89
90 extern int fsync(int fd);
91 extern int sync_file_range(int fd, int64_t offset, int64_t nbytes, unsigned int flags);
92
93 struct msghdr;
94 struct sockaddr;
95 extern ssize_t sendmsg(int sockfd, const struct msghdr *msg, int flags);
96 extern ssize_t recvmsg(int sockfd, struct msghdr *msg, int flags);
97 extern ssize_t send(int sockfd, const void *buf, size_t len, int flags);
98 extern ssize_t recv(int sockfd, void *buf, size_t len, int flags);
99 extern int accept4(int sockfd, struct sockaddr *addr, socklen_t *addrlen, int flags);
100 extern int connect(int sockfd, const struct sockaddr *addr, socklen_t addrlen);
101
102 extern int fallocate(int fd, int mode, uint64_t offset, uint64_t len);
103 extern int posix_fadvise(int fd, uint64_t offset, uint64_t len, int advice);
104 extern int madvise(void *addr, size_t length, int advice);
105
106 extern int openat(int dirfd, const char *pathname, int flags, mode_t mode);
107 extern int close(int fd);
108
109 extern ssize_t read (int fd, void *buf, size_t count);
110}
[31bb2e1]111
[4b84e35]112//=============================================================================================
113// I/O Interface
114//=============================================================================================
[31bb2e1]115
[4b84e35]116//-----------------------------------------------------------------------------
117// Asynchronous operations
118#if defined(HAVE_PREADV2)
119 ssize_t cfa_preadv2(int fd, const struct iovec *iov, int iovcnt, off_t offset, int flags) {
120 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_READV)
121 return preadv2(fd, iov, iovcnt, offset, flags);
[31bb2e1]122 #else
123 __submit_prelude
124
[4b84e35]125 (*sqe){ IORING_OP_READV, fd, iov, iovcnt, offset };
[31bb2e1]126
127 __submit_wait
128 #endif
129 }
[20ab637]130
131 ssize_t cfa_preadv2_fixed(int fd, const struct iovec *iov, int iovcnt, off_t offset, int flags) {
132 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_READV)
133 return preadv2(fd, iov, iovcnt, offset, flags);
134 #else
135 __submit_prelude
136
137 (*sqe){ IORING_OP_READV, fd, iov, iovcnt, offset };
138 sqe->flags |= IOSQE_FIXED_FILE;
139
140 __submit_wait
141 #endif
142 }
[4b84e35]143#endif
[31bb2e1]144
[4b84e35]145#if defined(HAVE_PWRITEV2)
146 ssize_t cfa_pwritev2(int fd, const struct iovec *iov, int iovcnt, off_t offset, int flags) {
147 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_WRITEV)
148 return pwritev2(fd, iov, iovcnt, offset, flags);
[31bb2e1]149 #else
150 __submit_prelude
151
[4b84e35]152 (*sqe){ IORING_OP_WRITEV, fd, iov, iovcnt, offset };
[31bb2e1]153
154 __submit_wait
155 #endif
156 }
[4b84e35]157#endif
[31bb2e1]158
[4b84e35]159int cfa_fsync(int fd) {
160 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_FSYNC)
161 return fsync(fd);
162 #else
163 __submit_prelude
[31bb2e1]164
[4b84e35]165 (*sqe){ IORING_OP_FSYNC, fd };
[31bb2e1]166
[4b84e35]167 __submit_wait
168 #endif
169}
[31bb2e1]170
[4b84e35]171int cfa_sync_file_range(int fd, int64_t offset, int64_t nbytes, unsigned int flags) {
172 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_SYNC_FILE_RANGE)
173 return sync_file_range(fd, offset, nbytes, flags);
174 #else
175 __submit_prelude
[31bb2e1]176
[4b84e35]177 (*sqe){ IORING_OP_SYNC_FILE_RANGE, fd };
178 sqe->off = offset;
179 sqe->len = nbytes;
180 sqe->sync_range_flags = flags;
[31bb2e1]181
[4b84e35]182 __submit_wait
183 #endif
184}
[31bb2e1]185
186
[4b84e35]187ssize_t cfa_sendmsg(int sockfd, const struct msghdr *msg, int flags) {
188 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_SENDMSG)
189 return sendmsg(sockfd, msg, flags);
190 #else
191 __submit_prelude
[31bb2e1]192
[4b84e35]193 (*sqe){ IORING_OP_SENDMSG, sockfd, msg, 1, 0 };
194 sqe->msg_flags = flags;
[31bb2e1]195
[4b84e35]196 __submit_wait
197 #endif
198}
[31bb2e1]199
[4b84e35]200ssize_t cfa_recvmsg(int sockfd, struct msghdr *msg, int flags) {
201 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_RECVMSG)
202 return recvmsg(sockfd, msg, flags);
203 #else
204 __submit_prelude
[31bb2e1]205
[4b84e35]206 (*sqe){ IORING_OP_RECVMSG, sockfd, msg, 1, 0 };
207 sqe->msg_flags = flags;
[31bb2e1]208
[4b84e35]209 __submit_wait
210 #endif
211}
[31bb2e1]212
[4b84e35]213ssize_t cfa_send(int sockfd, const void *buf, size_t len, int flags) {
214 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_SEND)
215 return send( sockfd, buf, len, flags );
216 #else
217 __submit_prelude
[31bb2e1]218
[4b84e35]219 (*sqe){ IORING_OP_SEND, sockfd };
220 sqe->addr = (uint64_t)buf;
221 sqe->len = len;
222 sqe->msg_flags = flags;
[31bb2e1]223
[4b84e35]224 __submit_wait
225 #endif
226}
[31bb2e1]227
[4b84e35]228ssize_t cfa_recv(int sockfd, void *buf, size_t len, int flags) {
229 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_RECV)
230 return recv( sockfd, buf, len, flags );
231 #else
232 __submit_prelude
[31bb2e1]233
[4b84e35]234 (*sqe){ IORING_OP_RECV, sockfd };
235 sqe->addr = (uint64_t)buf;
236 sqe->len = len;
237 sqe->msg_flags = flags;
[31bb2e1]238
[4b84e35]239 __submit_wait
240 #endif
241}
[31bb2e1]242
[4b84e35]243int cfa_accept4(int sockfd, struct sockaddr *addr, socklen_t *addrlen, int flags) {
244 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_ACCEPT)
245 return accept4( sockfd, addr, addrlen, flags );
246 #else
247 __submit_prelude
[31bb2e1]248
[4b84e35]249 (*sqe){ IORING_OP_ACCEPT, sockfd };
250 sqe->addr = addr;
251 sqe->addr2 = addrlen;
252 sqe->accept_flags = flags;
[31bb2e1]253
[4b84e35]254 __submit_wait
255 #endif
256}
[31bb2e1]257
[4b84e35]258int cfa_connect(int sockfd, const struct sockaddr *addr, socklen_t addrlen) {
259 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_CONNECT)
260 return connect( sockfd, addr, addrlen );
261 #else
262 __submit_prelude
[31bb2e1]263
[4b84e35]264 (*sqe){ IORING_OP_CONNECT, sockfd };
265 sqe->addr = (uint64_t)addr;
266 sqe->off = addrlen;
[31bb2e1]267
[4b84e35]268 __submit_wait
269 #endif
270}
[31bb2e1]271
[4b84e35]272int cfa_fallocate(int fd, int mode, uint64_t offset, uint64_t len) {
273 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_FALLOCATE)
274 return fallocate( fd, mode, offset, len );
275 #else
276 __submit_prelude
[31bb2e1]277
[4b84e35]278 (*sqe){ IORING_OP_FALLOCATE, fd };
279 sqe->off = offset;
280 sqe->len = length;
281 sqe->mode = mode;
[31bb2e1]282
[4b84e35]283 __submit_wait
284 #endif
285}
[31bb2e1]286
[4b84e35]287int cfa_fadvise(int fd, uint64_t offset, uint64_t len, int advice) {
288 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_FADVISE)
289 return posix_fadvise( fd, offset, len, advice );
290 #else
291 __submit_prelude
[31bb2e1]292
[4b84e35]293 (*sqe){ IORING_OP_FADVISE, fd };
294 sqe->off = (uint64_t)offset;
295 sqe->len = length;
296 sqe->fadvise_advice = advice;
[31bb2e1]297
[4b84e35]298 __submit_wait
299 #endif
300}
[31bb2e1]301
[4b84e35]302int cfa_madvise(void *addr, size_t length, int advice) {
303 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_MADVISE)
304 return madvise( addr, length, advice );
305 #else
306 __submit_prelude
[31bb2e1]307
[4b84e35]308 (*sqe){ IORING_OP_MADVISE, 0 };
309 sqe->addr = (uint64_t)addr;
310 sqe->len = length;
311 sqe->fadvise_advice = advice;
[31bb2e1]312
[4b84e35]313 __submit_wait
314 #endif
315}
[31bb2e1]316
[4b84e35]317int cfa_openat(int dirfd, const char *pathname, int flags, mode_t mode) {
318 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_OPENAT)
319 return openat( dirfd, pathname, flags, mode );
320 #else
321 __submit_prelude
[31bb2e1]322
[4b84e35]323 (*sqe){ IORING_OP_OPENAT, dirfd };
324 sqe->addr = (uint64_t)pathname;
325 sqe->open_flags = flags;
326 sqe->mode = mode;
[31bb2e1]327
[4b84e35]328 __submit_wait
329 #endif
330}
[31bb2e1]331
[4b84e35]332int cfa_close(int fd) {
333 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_CLOSE)
334 return close( fd );
335 #else
336 __submit_prelude
[31bb2e1]337
[4b84e35]338 (*sqe){ IORING_OP_CLOSE, fd };
339
340 __submit_wait
341 #endif
342}
[31bb2e1]343
344
[4b84e35]345ssize_t cfa_read(int fd, void *buf, size_t count) {
346 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_READ)
347 return read( fd, buf, count );
348 #else
349 __submit_prelude
350
351 (*sqe){ IORING_OP_READ, fd, buf, count, 0 };
352
353 __submit_wait
354 #endif
355}
356
357ssize_t cfa_write(int fd, void *buf, size_t count) {
358 #if !defined(HAVE_LINUX_IO_URING_H) || !defined(IORING_OP_WRITE)
359 return read( fd, buf, count );
360 #else
361 __submit_prelude
362
363 (*sqe){ IORING_OP_WRITE, fd, buf, count, 0 };
364
365 __submit_wait
366 #endif
367}
368
369//-----------------------------------------------------------------------------
370// Check if a function is asynchronous
371
372// Macro magic to reduce the size of the following switch case
373#define IS_DEFINED_APPLY(f, ...) f(__VA_ARGS__)
374#define IS_DEFINED_SECOND(first, second, ...) second
375#define IS_DEFINED_TEST(expansion) _CFA_IO_FEATURE_##expansion
376#define IS_DEFINED(macro) IS_DEFINED_APPLY( IS_DEFINED_SECOND,IS_DEFINED_TEST(macro) false, true)
377
378bool has_user_level_blocking( fptr_t func ) {
379 #if defined(HAVE_LINUX_IO_URING_H)
380 #if defined(HAVE_PREADV2)
381 if( /*func == (fptr_t)preadv2 || */
382 func == (fptr_t)cfa_preadv2 )
383 #define _CFA_IO_FEATURE_IORING_OP_READV ,
384 return IS_DEFINED(IORING_OP_READV);
[31bb2e1]385 #endif
386
[4b84e35]387 #if defined(HAVE_PWRITEV2)
388 if( /*func == (fptr_t)pwritev2 || */
389 func == (fptr_t)cfa_pwritev2 )
390 #define _CFA_IO_FEATURE_IORING_OP_WRITEV ,
391 return IS_DEFINED(IORING_OP_WRITEV);
[31bb2e1]392 #endif
393
[4b84e35]394 if( /*func == (fptr_t)fsync || */
395 func == (fptr_t)cfa_fsync )
396 #define _CFA_IO_FEATURE_IORING_OP_FSYNC ,
397 return IS_DEFINED(IORING_OP_FSYNC);
398
399 if( /*func == (fptr_t)ync_file_range || */
400 func == (fptr_t)cfa_sync_file_range )
401 #define _CFA_IO_FEATURE_IORING_OP_SYNC_FILE_RANGE ,
402 return IS_DEFINED(IORING_OP_SYNC_FILE_RANGE);
403
404 if( /*func == (fptr_t)sendmsg || */
405 func == (fptr_t)cfa_sendmsg )
406 #define _CFA_IO_FEATURE_IORING_OP_SENDMSG ,
407 return IS_DEFINED(IORING_OP_SENDMSG);
408
409 if( /*func == (fptr_t)recvmsg || */
410 func == (fptr_t)cfa_recvmsg )
411 #define _CFA_IO_FEATURE_IORING_OP_RECVMSG ,
412 return IS_DEFINED(IORING_OP_RECVMSG);
413
414 if( /*func == (fptr_t)send || */
415 func == (fptr_t)cfa_send )
416 #define _CFA_IO_FEATURE_IORING_OP_SEND ,
417 return IS_DEFINED(IORING_OP_SEND);
418
419 if( /*func == (fptr_t)recv || */
420 func == (fptr_t)cfa_recv )
421 #define _CFA_IO_FEATURE_IORING_OP_RECV ,
422 return IS_DEFINED(IORING_OP_RECV);
423
424 if( /*func == (fptr_t)accept4 || */
425 func == (fptr_t)cfa_accept4 )
426 #define _CFA_IO_FEATURE_IORING_OP_ACCEPT ,
427 return IS_DEFINED(IORING_OP_ACCEPT);
428
429 if( /*func == (fptr_t)connect || */
430 func == (fptr_t)cfa_connect )
431 #define _CFA_IO_FEATURE_IORING_OP_CONNECT ,
432 return IS_DEFINED(IORING_OP_CONNECT);
433
434 if( /*func == (fptr_t)fallocate || */
435 func == (fptr_t)cfa_fallocate )
436 #define _CFA_IO_FEATURE_IORING_OP_FALLOCATE ,
437 return IS_DEFINED(IORING_OP_FALLOCATE);
438
439 if( /*func == (fptr_t)posix_fadvise || */
440 func == (fptr_t)cfa_fadvise )
441 #define _CFA_IO_FEATURE_IORING_OP_FADVISE ,
442 return IS_DEFINED(IORING_OP_FADVISE);
443
444 if( /*func == (fptr_t)madvise || */
445 func == (fptr_t)cfa_madvise )
446 #define _CFA_IO_FEATURE_IORING_OP_MADVISE ,
447 return IS_DEFINED(IORING_OP_MADVISE);
448
449 if( /*func == (fptr_t)openat || */
450 func == (fptr_t)cfa_openat )
451 #define _CFA_IO_FEATURE_IORING_OP_OPENAT ,
452 return IS_DEFINED(IORING_OP_OPENAT);
453
454 if( /*func == (fptr_t)close || */
455 func == (fptr_t)cfa_close )
456 #define _CFA_IO_FEATURE_IORING_OP_CLOSE ,
457 return IS_DEFINED(IORING_OP_CLOSE);
458
459 if( /*func == (fptr_t)read || */
460 func == (fptr_t)cfa_read )
461 #define _CFA_IO_FEATURE_IORING_OP_READ ,
462 return IS_DEFINED(IORING_OP_READ);
463
464 if( /*func == (fptr_t)write || */
465 func == (fptr_t)cfa_write )
466 #define _CFA_IO_FEATURE_IORING_OP_WRITE ,
467 return IS_DEFINED(IORING_OP_WRITE);
468 #endif
469
470 return false;
471}
Note: See TracBrowser for help on using the repository browser.