Changeset 53ee27e
- Timestamp:
- Aug 4, 2020, 5:13:51 PM (3 years ago)
- Branches:
- ADT, arm-eh, ast-experimental, enum, forall-pointer-decay, jacob/cs343-translation, master, new-ast, new-ast-unique-expr, pthread-emulation, qualifiedEnum
- Children:
- eafec07
- Parents:
- 3f850d7 (diff), 21b0a23 (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the(diff)
links above to see all the changes relative to each parent. - Files:
-
- 12 added
- 50 edited
Legend:
- Unmodified
- Added
- Removed
-
benchmark/Makefile.in
r3f850d7 r53ee27e 349 349 AUTOMAKE_OPTIONS = foreign # do not require all the GNU file names 350 350 ACLOCAL_AMFLAGS = -I automake 351 CFACOMPILE = $(CFACC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CFAFLAGS) $(CFAFLAGS) $(AM_CFLAGS) $(CFLAGS) 352 LTCFACOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ 351 AM_T_CFA = $(am__t_CFA_@AM_T@) 352 am__t_CFA_ = 353 am__t_CFA_0 = 354 am__t_CFA_1 = /usr/bin/time --quiet -f "$@ %E" # trailling space is necessary 355 CFACOMPILE = $(AM_T_CFA)$(CFACC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CFAFLAGS) $(CFAFLAGS) $(AM_CFLAGS) $(CFLAGS) 356 LTCFACOMPILE = $(AM_T_CFA)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ 353 357 $(LIBTOOLFLAGS) --mode=compile $(CFACC) $(DEFS) \ 354 358 $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CFAFLAGS) $(AM_CFLAGS) $(CFAFLAGS) $(CFLAGS) -
benchmark/io/readv.cfa
r3f850d7 r53ee27e 12 12 } 13 13 14 #include <errno.h> 14 15 #include <unistd.h> 15 16 16 17 #include <clock.hfa> 18 #include <iofwd.hfa> 17 19 #include <kernel.hfa> 18 20 #include <thread.hfa> … … 23 25 24 26 extern bool traceHeapOn(); 25 extern ssize_t cfa_preadv2(int fd, const struct iovec *iov, int iovcnt, off_t offset, int flags);26 extern ssize_t cfa_preadv2_fixed(int fd, const struct iovec *iov, int iovcnt, off_t offset, int flags);27 extern void register_fixed_files( cluster &, int *, unsigned count );28 27 29 28 int fd; … … 31 30 volatile size_t count = 0; 32 31 33 unsigned long int buflen = 5 0;32 unsigned long int buflen = 512; 34 33 bool fixed_file = false; 35 34 … … 40 39 41 40 int do_read(int fd, struct iovec * iov) { 41 // extern ssize_t cfa_preadv2(int, const struct iovec *, int, off_t, int, int = 0, Duration = -1`s, io_cancellation * = 0p, io_context * = 0p); 42 int sflags = 0; 42 43 if(fixed_file) { 43 return cfa_preadv2_fixed(fd, iov, 1, 0, 0);44 sflags |= CFA_IO_FIXED_FD1; 44 45 } 45 else { 46 return cfa_preadv2(fd, iov, 1, 0, 0); 47 } 46 return cfa_preadv2(fd, iov, 1, 0, 0, sflags, -1`s, 0p, 0p); 48 47 } 49 48 … … 52 51 /* paranoid */ assert( true == __atomic_load_n(&run, __ATOMIC_RELAXED) ); 53 52 54 char data[buflen];53 __attribute__((aligned(512))) char data[buflen]; 55 54 struct iovec iov = { data, buflen }; 56 55 57 56 while(__atomic_load_n(&run, __ATOMIC_RELAXED)) { 58 57 int r = do_read(fd, &iov); 59 if(r < 0) abort("%s\n", strerror( -r));58 if(r < 0) abort("%s\n", strerror(errno)); 60 59 61 60 __atomic_fetch_add( &count, 1, __ATOMIC_SEQ_CST ); … … 65 64 int main(int argc, char * argv[]) { 66 65 BENCH_DECL 67 unsigned flags = 0; 66 unsigned num_io = 1; 67 io_context_params params; 68 68 int file_flags = 0; 69 69 unsigned sublen = 16; … … 74 74 BENCH_OPT_LONG 75 75 {"bufsize", required_argument, 0, 'b'}, 76 {"userthread", no_argument , 0, 'u'},77 76 {"submitthread", no_argument , 0, 's'}, 78 77 {"eagersubmit", no_argument , 0, 'e'}, 79 78 {"kpollsubmit", no_argument , 0, 'k'}, 80 79 {"kpollcomplete", no_argument , 0, 'i'}, 80 {"fixed-files", no_argument , 0, 'f'}, 81 {"open-direct", no_argument , 0, 'o'}, 81 82 {"submitlength", required_argument, 0, 'l'}, 82 83 {0, 0, 0, 0} … … 84 85 85 86 int idx = 0; 86 int opt = getopt_long(argc, argv, BENCH_OPT_SHORT "b: usekil:", options, &idx);87 int opt = getopt_long(argc, argv, BENCH_OPT_SHORT "b:sekil:", options, &idx); 87 88 88 89 const char * arg = optarg ? optarg : ""; … … 100 101 } 101 102 break; 102 case 'u':103 flags |= CFA_CLUSTER_IO_POLLER_USER_THREAD;104 break;105 103 case 's': 106 flags |= CFA_CLUSTER_IO_POLLER_THREAD_SUBMITS;104 params.poller_submits = true; 107 105 break; 108 106 case 'e': 109 flags |= CFA_CLUSTER_IO_EAGER_SUBMITS;107 params.eager_submits = true; 110 108 break; 111 109 case 'k': 112 flags |= CFA_CLUSTER_IO_KERNEL_POLL_SUBMITS; 110 params.poll_submit = true; 111 case 'f': 113 112 fixed_file = true; 114 113 break; 115 114 case 'i': 116 flags |= CFA_CLUSTER_IO_KERNEL_POLL_COMPLETES; 115 params.poll_complete = true; 116 case 'o': 117 117 file_flags |= O_DIRECT; 118 118 break; … … 123 123 goto usage; 124 124 } 125 flags |= (sublen << CFA_CLUSTER_IO_BUFFLEN_OFFSET);125 // flags |= (sublen << CFA_CLUSTER_IO_BUFFLEN_OFFSET); 126 126 break; 127 127 default: /* ? */ … … 150 150 { 151 151 Time start, end; 152 BenchCluster cl = { flags, CFA_STATS_READY_Q | CFA_STATS_IO };152 BenchCluster cl = { num_io, params, CFA_STATS_READY_Q | CFA_STATS_IO }; 153 153 154 154 if(fixed_file) { … … 179 179 printf("\nDone\n"); 180 180 } 181 printf("Readers closed\n"); 181 182 } 182 183 printf("Took %'ld ms\n", (end - start)`ms); -
libcfa/Makefile.in
r3f850d7 r53ee27e 218 218 AMTAR = @AMTAR@ 219 219 AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@ 220 AM_T = @AM_T@ 220 221 AR = @AR@ 221 222 ARCHITECTURE = @ARCHITECTURE@ -
libcfa/configure
r3f850d7 r53ee27e 701 701 CFA_PREFIX 702 702 CFA_NAME 703 AM_T 703 704 BUILDLIB_FALSE 704 705 BUILDLIB_TRUE … … 3187 3188 BUILDLIB_FALSE= 3188 3189 fi 3190 3191 3192 AM_T='$(T)' 3189 3193 3190 3194 … … 17017 17021 17018 17022 17023 17019 17024 for ac_header in linux/io_uring.h 17020 17025 do : … … 19295 19300 19296 19301 19302 19303 fi 19304 19305 19306 19307 # check support for various io_uring flags 19308 19309 ac_fn_c_check_decl "$LINENO" "IOSQE_FIXED_FILE" "ac_cv_have_decl_IOSQE_FIXED_FILE" "#include <linux/io_uring.h> 19310 " 19311 if test "x$ac_cv_have_decl_IOSQE_FIXED_FILE" = xyes; then : 19312 $as_echo "#define CFA_HAVE_IOSQE_FIXED_FILE 1" >>confdefs.h 19313 19314 fi 19315 19316 19317 ac_fn_c_check_decl "$LINENO" "IOSQE_IO_DRAIN" "ac_cv_have_decl_IOSQE_IO_DRAIN" "#include <linux/io_uring.h> 19318 " 19319 if test "x$ac_cv_have_decl_IOSQE_IO_DRAIN" = xyes; then : 19320 $as_echo "#define CFA_HAVE_IOSQE_IO_DRAIN 1" >>confdefs.h 19321 19322 fi 19323 19324 19325 ac_fn_c_check_decl "$LINENO" "IOSQE_ASYNC" "ac_cv_have_decl_IOSQE_ASYNC" "#include <linux/io_uring.h> 19326 " 19327 if test "x$ac_cv_have_decl_IOSQE_ASYNC" = xyes; then : 19328 $as_echo "#define CFA_HAVE_IOSQE_ASYNC 1" >>confdefs.h 19329 19330 fi 19331 19332 19333 ac_fn_c_check_decl "$LINENO" "IOSQE_IO_LINK" "ac_cv_have_decl_IOSQE_IO_LINK" "#include <linux/io_uring.h> 19334 " 19335 if test "x$ac_cv_have_decl_IOSQE_IO_LINK" = xyes; then : 19336 $as_echo "#define CFA_HAVE_IOSQE_IO_LINK 1" >>confdefs.h 19337 19338 fi 19339 19340 19341 ac_fn_c_check_decl "$LINENO" "IOSQE_IO_HARDLINK" "ac_cv_have_decl_IOSQE_IO_HARDLINK" "#include <linux/io_uring.h> 19342 " 19343 if test "x$ac_cv_have_decl_IOSQE_IO_HARDLINK" = xyes; then : 19344 $as_echo "#define CFA_HAVE_IOSQE_IO_HARDLINK 1" >>confdefs.h 19345 19346 fi 19347 19348 19349 ac_fn_c_check_decl "$LINENO" "SPLICE_F_FD_IN_FIXED" "ac_cv_have_decl_SPLICE_F_FD_IN_FIXED" "#include <linux/io_uring.h> 19350 " 19351 if test "x$ac_cv_have_decl_SPLICE_F_FD_IN_FIXED" = xyes; then : 19352 $as_echo "#define CFA_HAVE_SPLICE_F_FD_IN_FIXED 1" >>confdefs.h 19297 19353 19298 19354 fi -
libcfa/configure.ac
r3f850d7 r53ee27e 105 105 AM_CONDITIONAL([BUILDLIB], [test "x${CONFIG_BUILDLIB}" = "xyes"]) 106 106 107 AM_T='$(T)' 108 AC_SUBST(AM_T) 109 107 110 #============================================================================== 108 111 #Trasforming cc1 will break compilation … … 129 132 #io_uring 5.6 and later uses probes 130 133 define(ioring_ops, [IORING_OP_NOP,IORING_OP_READV,IORING_OP_WRITEV,IORING_OP_FSYNC,IORING_OP_READ_FIXED,IORING_OP_WRITE_FIXED,IORING_OP_POLL_ADD,IORING_OP_POLL_REMOVE,IORING_OP_SYNC_FILE_RANGE,IORING_OP_SENDMSG,IORING_OP_RECVMSG,IORING_OP_TIMEOUT,IORING_OP_TIMEOUT_REMOVE,IORING_OP_ACCEPT,IORING_OP_ASYNC_CANCEL,IORING_OP_LINK_TIMEOUT,IORING_OP_CONNECT,IORING_OP_FALLOCATE,IORING_OP_OPENAT,IORING_OP_CLOSE,IORING_OP_FILES_UPDATE,IORING_OP_STATX,IORING_OP_READ,IORING_OP_WRITE,IORING_OP_FADVISE,IORING_OP_MADVISE,IORING_OP_SEND,IORING_OP_RECV,IORING_OP_OPENAT2,IORING_OP_EPOLL_CTL,IORING_OP_SPLICE,IORING_OP_PROVIDE_BUFFERS,IORING_OP_REMOVE_BUFFER]) 134 define(ioring_flags, [IOSQE_FIXED_FILE,IOSQE_IO_DRAIN,IOSQE_ASYNC,IOSQE_IO_LINK,IOSQE_IO_HARDLINK,SPLICE_F_FD_IN_FIXED]) 131 135 132 136 define(ioring_from_decls, [ … … 166 170 ioring_from_decls 167 171 ]) 172 173 # check support for various io_uring flags 174 m4_foreach([op], [ioring_flags], [ 175 AC_CHECK_DECL(op, [AC_DEFINE([CFA_HAVE_]op)], [], [[#include <linux/io_uring.h>]]) 176 ]) 168 177 ]) 169 178 AC_CHECK_FUNCS([preadv2 pwritev2]) -
libcfa/prelude/Makefile.in
r3f850d7 r53ee27e 180 180 AMTAR = @AMTAR@ 181 181 AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@ 182 AM_T = @AM_T@ 182 183 AR = @AR@ 183 184 ARCHITECTURE = @ARCHITECTURE@ -
libcfa/prelude/defines.hfa.in
r3f850d7 r53ee27e 50 50 #undef CFA_HAVE_IORING_OP_REMOVE_BUFFER 51 51 52 #undef CFA_HAVE_IOSQE_FIXED_FILE 53 #undef CFA_HAVE_IOSQE_IO_DRAIN 54 #undef CFA_HAVE_IOSQE_ASYNC 55 #undef CFA_HAVE_IOSQE_IO_LINK 56 #undef CFA_HAVE_IOSQE_IO_HARDLINK 57 #undef CFA_HAVE_SPLICE_F_FD_IN_FIXED 58 52 59 #undef HAVE_PREADV2 53 60 #undef HAVE_PWRITEV2 -
libcfa/src/Makefile.am
r3f850d7 r53ee27e 50 50 51 51 # not all platforms support concurrency, add option do disable it 52 thread_headers_nosrc = concurrency/invoke.h 53 thread_headers = concurrency/coroutine.hfa concurrency/thread.hfa concurrency/kernel.hfa concurrency/monitor.hfa concurrency/mutex.hfa 54 thread_libsrc = concurrency/CtxSwitch-@ARCHITECTURE@.S concurrency/alarm.cfa concurrency/invoke.c concurrency/io.cfa concurrency/iocall.cfa concurrency/preemption.cfa concurrency/ready_queue.cfa concurrency/stats.cfa ${thread_headers:.hfa=.cfa} 52 thread_headers_nosrc = bits/random.hfa concurrency/invoke.h concurrency/kernel/fwd.hfa 53 54 thread_headers = concurrency/coroutine.hfa concurrency/thread.hfa concurrency/kernel.hfa \ 55 concurrency/monitor.hfa concurrency/mutex.hfa 56 57 thread_libsrc = concurrency/CtxSwitch-@ARCHITECTURE@.S concurrency/alarm.cfa \ 58 concurrency/invoke.c concurrency/io.cfa concurrency/iocall.cfa \ 59 concurrency/io/setup.cfa \ 60 concurrency/kernel/startup.cfa concurrency/preemption.cfa \ 61 concurrency/ready_queue.cfa concurrency/stats.cfa \ 62 ${thread_headers:.hfa=.cfa} 55 63 else 56 64 headers = -
libcfa/src/Makefile.in
r3f850d7 r53ee27e 166 166 concurrency/CtxSwitch-@ARCHITECTURE@.S concurrency/alarm.cfa \ 167 167 concurrency/invoke.c concurrency/io.cfa concurrency/iocall.cfa \ 168 concurrency/io/setup.cfa concurrency/kernel/startup.cfa \ 168 169 concurrency/preemption.cfa concurrency/ready_queue.cfa \ 169 170 concurrency/stats.cfa concurrency/coroutine.cfa \ … … 177 178 @BUILDLIB_TRUE@ concurrency/alarm.lo concurrency/invoke.lo \ 178 179 @BUILDLIB_TRUE@ concurrency/io.lo concurrency/iocall.lo \ 180 @BUILDLIB_TRUE@ concurrency/io/setup.lo \ 181 @BUILDLIB_TRUE@ concurrency/kernel/startup.lo \ 179 182 @BUILDLIB_TRUE@ concurrency/preemption.lo \ 180 183 @BUILDLIB_TRUE@ concurrency/ready_queue.lo concurrency/stats.lo \ … … 249 252 concurrency/coroutine.hfa concurrency/thread.hfa \ 250 253 concurrency/kernel.hfa concurrency/monitor.hfa \ 251 concurrency/mutex.hfa concurrency/invoke.h 254 concurrency/mutex.hfa bits/random.hfa concurrency/invoke.h \ 255 concurrency/kernel/fwd.hfa 252 256 HEADERS = $(nobase_cfa_include_HEADERS) 253 257 am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP) … … 277 281 AMTAR = @AMTAR@ 278 282 AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@ 283 AM_T = @AM_T@ 279 284 AR = @AR@ 280 285 ARCHITECTURE = @ARCHITECTURE@ … … 421 426 AUTOMAKE_OPTIONS = foreign subdir-objects 422 427 ACLOCAL_AMFLAGS = -I automake 423 CFACOMPILE = $(CFACC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CFAFLAGS) $(CFAFLAGS) $(AM_CFLAGS) $(CFLAGS) 424 LTCFACOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ 428 AM_T_CFA = $(am__t_CFA_@AM_T@) 429 am__t_CFA_ = 430 am__t_CFA_0 = 431 am__t_CFA_1 = /usr/bin/time --quiet -f "$@ %E" # trailling space is necessary 432 CFACOMPILE = $(AM_T_CFA)$(CFACC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CFAFLAGS) $(CFAFLAGS) $(AM_CFLAGS) $(CFLAGS) 433 LTCFACOMPILE = $(AM_T_CFA)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ 425 434 $(LIBTOOLFLAGS) --mode=compile $(CFACC) $(DEFS) \ 426 435 $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CFAFLAGS) $(AM_CFLAGS) $(CFAFLAGS) $(CFLAGS) … … 483 492 484 493 # not all platforms support concurrency, add option do disable it 485 @BUILDLIB_TRUE@thread_headers_nosrc = concurrency/invoke.h494 @BUILDLIB_TRUE@thread_headers_nosrc = bits/random.hfa concurrency/invoke.h concurrency/kernel/fwd.hfa 486 495 @BUILDLIB_FALSE@thread_headers = 487 @BUILDLIB_TRUE@thread_headers = concurrency/coroutine.hfa concurrency/thread.hfa concurrency/kernel.hfa concurrency/monitor.hfa concurrency/mutex.hfa 488 @BUILDLIB_TRUE@thread_libsrc = concurrency/CtxSwitch-@ARCHITECTURE@.S concurrency/alarm.cfa concurrency/invoke.c concurrency/io.cfa concurrency/iocall.cfa concurrency/preemption.cfa concurrency/ready_queue.cfa concurrency/stats.cfa ${thread_headers:.hfa=.cfa} 496 @BUILDLIB_TRUE@thread_headers = concurrency/coroutine.hfa concurrency/thread.hfa concurrency/kernel.hfa \ 497 @BUILDLIB_TRUE@ concurrency/monitor.hfa concurrency/mutex.hfa 498 499 @BUILDLIB_TRUE@thread_libsrc = concurrency/CtxSwitch-@ARCHITECTURE@.S concurrency/alarm.cfa \ 500 @BUILDLIB_TRUE@ concurrency/invoke.c concurrency/io.cfa concurrency/iocall.cfa \ 501 @BUILDLIB_TRUE@ concurrency/io/setup.cfa \ 502 @BUILDLIB_TRUE@ concurrency/kernel/startup.cfa concurrency/preemption.cfa \ 503 @BUILDLIB_TRUE@ concurrency/ready_queue.cfa concurrency/stats.cfa \ 504 @BUILDLIB_TRUE@ ${thread_headers:.hfa=.cfa} 505 489 506 490 507 #---------------------------------------------------------------------------------------------------------------- … … 625 642 concurrency/iocall.lo: concurrency/$(am__dirstamp) \ 626 643 concurrency/$(DEPDIR)/$(am__dirstamp) 644 concurrency/io/$(am__dirstamp): 645 @$(MKDIR_P) concurrency/io 646 @: > concurrency/io/$(am__dirstamp) 647 concurrency/io/$(DEPDIR)/$(am__dirstamp): 648 @$(MKDIR_P) concurrency/io/$(DEPDIR) 649 @: > concurrency/io/$(DEPDIR)/$(am__dirstamp) 650 concurrency/io/setup.lo: concurrency/io/$(am__dirstamp) \ 651 concurrency/io/$(DEPDIR)/$(am__dirstamp) 652 concurrency/kernel/$(am__dirstamp): 653 @$(MKDIR_P) concurrency/kernel 654 @: > concurrency/kernel/$(am__dirstamp) 655 concurrency/kernel/$(DEPDIR)/$(am__dirstamp): 656 @$(MKDIR_P) concurrency/kernel/$(DEPDIR) 657 @: > concurrency/kernel/$(DEPDIR)/$(am__dirstamp) 658 concurrency/kernel/startup.lo: concurrency/kernel/$(am__dirstamp) \ 659 concurrency/kernel/$(DEPDIR)/$(am__dirstamp) 627 660 concurrency/preemption.lo: concurrency/$(am__dirstamp) \ 628 661 concurrency/$(DEPDIR)/$(am__dirstamp) … … 651 684 -rm -f concurrency/*.$(OBJEXT) 652 685 -rm -f concurrency/*.lo 686 -rm -f concurrency/io/*.$(OBJEXT) 687 -rm -f concurrency/io/*.lo 688 -rm -f concurrency/kernel/*.$(OBJEXT) 689 -rm -f concurrency/kernel/*.lo 653 690 -rm -f containers/*.$(OBJEXT) 654 691 -rm -f containers/*.lo … … 717 754 -rm -rf bits/.libs bits/_libs 718 755 -rm -rf concurrency/.libs concurrency/_libs 756 -rm -rf concurrency/io/.libs concurrency/io/_libs 757 -rm -rf concurrency/kernel/.libs concurrency/kernel/_libs 719 758 -rm -rf containers/.libs containers/_libs 720 759 install-nobase_cfa_includeHEADERS: $(nobase_cfa_include_HEADERS) … … 862 901 -rm -f concurrency/$(DEPDIR)/$(am__dirstamp) 863 902 -rm -f concurrency/$(am__dirstamp) 903 -rm -f concurrency/io/$(DEPDIR)/$(am__dirstamp) 904 -rm -f concurrency/io/$(am__dirstamp) 905 -rm -f concurrency/kernel/$(DEPDIR)/$(am__dirstamp) 906 -rm -f concurrency/kernel/$(am__dirstamp) 864 907 -rm -f containers/$(DEPDIR)/$(am__dirstamp) 865 908 -rm -f containers/$(am__dirstamp) -
libcfa/src/bits/debug.hfa
r3f850d7 r53ee27e 15 15 16 16 #pragma once 17 18 #include <assert.h> 17 19 18 20 #ifdef __CFA_DEBUG__ -
libcfa/src/bits/defs.hfa
r3f850d7 r53ee27e 16 16 #pragma once 17 17 18 #include <stdbool.h>19 #include <stddef.h>20 18 #include <stdint.h> 21 19 … … 54 52 return ( (unsigned long long)lo)|( ((unsigned long long)hi)<<32 ); 55 53 } 56 57 // #define __CFA_NO_BIT_TEST_AND_SET__58 59 #if defined( __i386 )60 static inline bool __atomic_bts(volatile unsigned long int * target, unsigned long int bit ) {61 #if defined(__CFA_NO_BIT_TEST_AND_SET__)62 unsigned long int mask = 1ul << bit;63 unsigned long int ret = __atomic_fetch_or(target, mask, (int)__ATOMIC_RELAXED);64 return (ret & mask) != 0;65 #else66 int result = 0;67 asm volatile(68 "LOCK btsl %[bit], %[target]\n\t"69 : "=@ccc" (result)70 : [target] "m" (*target), [bit] "r" (bit)71 );72 return result != 0;73 #endif74 }75 76 static inline bool __atomic_btr(volatile unsigned long int * target, unsigned long int bit ) {77 #if defined(__CFA_NO_BIT_TEST_AND_SET__)78 unsigned long int mask = 1ul << bit;79 unsigned long int ret = __atomic_fetch_and(target, ~mask, (int)__ATOMIC_RELAXED);80 return (ret & mask) != 0;81 #else82 int result = 0;83 asm volatile(84 "LOCK btrl %[bit], %[target]\n\t"85 :"=@ccc" (result)86 : [target] "m" (*target), [bit] "r" (bit)87 );88 return result != 0;89 #endif90 }91 #elif defined( __x86_64 )92 static inline bool __atomic_bts(volatile unsigned long long int * target, unsigned long long int bit ) {93 #if defined(__CFA_NO_BIT_TEST_AND_SET__)94 unsigned long long int mask = 1ul << bit;95 unsigned long long int ret = __atomic_fetch_or(target, mask, (int)__ATOMIC_RELAXED);96 return (ret & mask) != 0;97 #else98 int result = 0;99 asm volatile(100 "LOCK btsq %[bit], %[target]\n\t"101 : "=@ccc" (result)102 : [target] "m" (*target), [bit] "r" (bit)103 );104 return result != 0;105 #endif106 }107 108 static inline bool __atomic_btr(volatile unsigned long long int * target, unsigned long long int bit ) {109 #if defined(__CFA_NO_BIT_TEST_AND_SET__)110 unsigned long long int mask = 1ul << bit;111 unsigned long long int ret = __atomic_fetch_and(target, ~mask, (int)__ATOMIC_RELAXED);112 return (ret & mask) != 0;113 #else114 int result = 0;115 asm volatile(116 "LOCK btrq %[bit], %[target]\n\t"117 :"=@ccc" (result)118 : [target] "m" (*target), [bit] "r" (bit)119 );120 return result != 0;121 #endif122 }123 #elif defined( __ARM_ARCH )124 #error __atomic_bts and __atomic_btr not implemented for arm125 #else126 #error uknown hardware architecture127 #endif -
libcfa/src/bits/locks.hfa
r3f850d7 r53ee27e 130 130 pthread_mutex_init(&lock, &mattr); 131 131 132 pthread_cond_init (&cond, 0p);132 pthread_cond_init (&cond, (const pthread_condattr_t *)0p); // workaround trac#208: cast should not be required 133 133 val = 0; 134 134 } … … 164 164 165 165 #undef CHECKED 166 167 struct $thread; 168 extern void park( __cfaabi_dbg_ctx_param ); 169 extern void unpark( struct $thread * this __cfaabi_dbg_ctx_param2 ); 170 static inline struct $thread * active_thread (); 171 172 // Semaphore which only supports a single thread 173 struct single_sem { 174 struct $thread * volatile ptr; 175 }; 176 177 static inline { 178 void ?{}(single_sem & this) { 179 this.ptr = 0p; 180 } 181 182 void ^?{}(single_sem & this) {} 183 184 bool wait(single_sem & this) { 185 for() { 186 struct $thread * expected = this.ptr; 187 if(expected == 1p) { 188 if(__atomic_compare_exchange_n(&this.ptr, &expected, 0p, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) { 189 return false; 190 } 191 } 192 else { 193 /* paranoid */ verify( expected == 0p ); 194 if(__atomic_compare_exchange_n(&this.ptr, &expected, active_thread(), false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) { 195 park( __cfaabi_dbg_ctx ); 196 return true; 197 } 198 } 199 200 } 201 } 202 203 bool post(single_sem & this) { 204 for() { 205 struct $thread * expected = this.ptr; 206 if(expected == 1p) return false; 207 if(expected == 0p) { 208 if(__atomic_compare_exchange_n(&this.ptr, &expected, 1p, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) { 209 return false; 210 } 211 } 212 else { 213 if(__atomic_compare_exchange_n(&this.ptr, &expected, 0p, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) { 214 unpark( expected __cfaabi_dbg_ctx2 ); 215 return true; 216 } 217 } 218 } 219 } 220 } 166 221 #endif -
libcfa/src/concurrency/alarm.cfa
r3f850d7 r53ee27e 23 23 24 24 #include "alarm.hfa" 25 #include "kernel _private.hfa"25 #include "kernel/fwd.hfa" 26 26 #include "preemption.hfa" 27 27 -
libcfa/src/concurrency/invoke.h
r3f850d7 r53ee27e 17 17 #include "bits/defs.hfa" 18 18 #include "bits/locks.hfa" 19 #include "kernel/fwd.hfa" 19 20 20 21 #ifdef __cforall … … 25 26 #ifndef _INVOKE_H_ 26 27 #define _INVOKE_H_ 27 28 #ifdef __ARM_ARCH29 // function prototypes are only really used by these macros on ARM30 void disable_global_interrupts();31 void enable_global_interrupts();32 33 #define TL_GET( member ) ( { __typeof__( kernelTLS.member ) target; \34 disable_global_interrupts(); \35 target = kernelTLS.member; \36 enable_global_interrupts(); \37 target; } )38 #define TL_SET( member, value ) disable_global_interrupts(); \39 kernelTLS.member = value; \40 enable_global_interrupts();41 #else42 #define TL_GET( member ) kernelTLS.member43 #define TL_SET( member, value ) kernelTLS.member = value;44 #endif45 46 #ifdef __cforall47 extern "Cforall" {48 extern __attribute__((aligned(128))) thread_local struct KernelThreadData {49 struct $thread * volatile this_thread;50 struct processor * volatile this_processor;51 struct __stats_t * volatile this_stats;52 53 struct {54 volatile unsigned short disable_count;55 volatile bool enabled;56 volatile bool in_progress;57 } preemption_state;58 59 #if defined(__SIZEOF_INT128__)60 __uint128_t rand_seed;61 #else62 uint64_t rand_seed;63 #endif64 } kernelTLS __attribute__ ((tls_model ( "initial-exec" )));65 }66 #endif67 28 68 29 struct __stack_context_t { … … 98 59 99 60 enum __Coroutine_State { Halted, Start, Primed, Blocked, Ready, Active }; 100 enum __Preemption_Reason { __NO_PREEMPTION, __ALARM_PREEMPTION, __POLL_PREEMPTION, __MANUAL_PREEMPTION };101 61 102 62 struct $coroutine { -
libcfa/src/concurrency/io.cfa
r3f850d7 r53ee27e 14 14 // 15 15 16 #define __cforall_thread__ 17 16 18 #if defined(__CFA_DEBUG__) 17 19 // #define __CFA_DEBUG_PRINT_IO__ … … 19 21 #endif 20 22 21 #include "kernel.hfa" 22 #include "bitmanip.hfa" 23 24 #if !defined(CFA_HAVE_LINUX_IO_URING_H) 25 void __kernel_io_startup( cluster &, unsigned, bool ) { 26 // Nothing to do without io_uring 27 } 28 29 void __kernel_io_finish_start( cluster & ) { 30 // Nothing to do without io_uring 31 } 32 33 void __kernel_io_prepare_stop( cluster & ) { 34 // Nothing to do without io_uring 35 } 36 37 void __kernel_io_shutdown( cluster &, bool ) { 38 // Nothing to do without io_uring 39 } 40 41 #else 23 24 #if defined(CFA_HAVE_LINUX_IO_URING_H) 42 25 #define _GNU_SOURCE /* See feature_test_macros(7) */ 43 26 #include <errno.h> 27 #include <signal.h> 44 28 #include <stdint.h> 45 29 #include <string.h> 46 30 #include <unistd.h> 47 #include <sys/mman.h>48 31 49 32 extern "C" { 33 #include <sys/epoll.h> 50 34 #include <sys/syscall.h> 51 35 … … 53 37 } 54 38 55 #include "bits/signal.hfa" 56 #include "kernel_private.hfa" 57 #include "thread.hfa" 58 59 uint32_t entries_per_cluster() { 60 return 256; 61 } 62 63 static void * __io_poller_slow( void * arg ); 64 65 // Weirdly, some systems that do support io_uring don't actually define these 66 #ifdef __alpha__ 67 /* 68 * alpha is the only exception, all other architectures 69 * have common numbers for new system calls. 70 */ 71 #ifndef __NR_io_uring_setup 72 #define __NR_io_uring_setup 535 73 #endif 74 #ifndef __NR_io_uring_enter 75 #define __NR_io_uring_enter 536 76 #endif 77 #ifndef __NR_io_uring_register 78 #define __NR_io_uring_register 537 79 #endif 80 #else /* !__alpha__ */ 81 #ifndef __NR_io_uring_setup 82 #define __NR_io_uring_setup 425 83 #endif 84 #ifndef __NR_io_uring_enter 85 #define __NR_io_uring_enter 426 86 #endif 87 #ifndef __NR_io_uring_register 88 #define __NR_io_uring_register 427 89 #endif 90 #endif 91 92 // Fast poller user-thread 93 // Not using the "thread" keyword because we want to control 94 // more carefully when to start/stop it 95 struct __io_poller_fast { 96 struct __io_data * ring; 97 $thread thrd; 98 }; 99 100 void ?{}( __io_poller_fast & this, struct cluster & cltr ) { 101 this.ring = cltr.io; 102 (this.thrd){ "Fast I/O Poller", cltr }; 103 } 104 void ^?{}( __io_poller_fast & mutex this ); 105 void main( __io_poller_fast & this ); 106 static inline $thread * get_thread( __io_poller_fast & this ) { return &this.thrd; } 107 void ^?{}( __io_poller_fast & mutex this ) {} 108 109 struct __submition_data { 110 // Head and tail of the ring (associated with array) 111 volatile uint32_t * head; 112 volatile uint32_t * tail; 113 volatile uint32_t prev_head; 114 115 // The actual kernel ring which uses head/tail 116 // indexes into the sqes arrays 117 uint32_t * array; 118 119 // number of entries and mask to go with it 120 const uint32_t * num; 121 const uint32_t * mask; 122 123 // Submission flags (Not sure what for) 124 uint32_t * flags; 125 126 // number of sqes not submitted (whatever that means) 127 uint32_t * dropped; 128 129 // Like head/tail but not seen by the kernel 130 volatile uint32_t * ready; 131 uint32_t ready_cnt; 132 133 __spinlock_t lock; 134 __spinlock_t release_lock; 135 136 // A buffer of sqes (not the actual ring) 137 struct io_uring_sqe * sqes; 138 139 // The location and size of the mmaped area 140 void * ring_ptr; 141 size_t ring_sz; 142 }; 143 144 struct __completion_data { 145 // Head and tail of the ring 146 volatile uint32_t * head; 147 volatile uint32_t * tail; 148 149 // number of entries and mask to go with it 150 const uint32_t * mask; 151 const uint32_t * num; 152 153 // number of cqes not submitted (whatever that means) 154 uint32_t * overflow; 155 156 // the kernel ring 157 struct io_uring_cqe * cqes; 158 159 // The location and size of the mmaped area 160 void * ring_ptr; 161 size_t ring_sz; 162 }; 163 164 struct __io_data { 165 struct __submition_data submit_q; 166 struct __completion_data completion_q; 167 uint32_t ring_flags; 168 int cltr_flags; 169 int fd; 170 semaphore submit; 171 volatile bool done; 172 struct { 173 struct { 174 __processor_id_t id; 175 void * stack; 176 pthread_t kthrd; 177 volatile bool blocked; 178 } slow; 179 __io_poller_fast fast; 180 __bin_sem_t sem; 181 } poller; 182 }; 183 184 //============================================================================================= 185 // I/O Startup / Shutdown logic 186 //============================================================================================= 187 void __kernel_io_startup( cluster & this, unsigned io_flags, bool main_cluster ) { 188 if( (io_flags & CFA_CLUSTER_IO_POLLER_THREAD_SUBMITS) && (io_flags & CFA_CLUSTER_IO_EAGER_SUBMITS) ) { 189 abort("CFA_CLUSTER_IO_POLLER_THREAD_SUBMITS and CFA_CLUSTER_IO_EAGER_SUBMITS cannot be mixed\n"); 190 } 191 192 this.io = malloc(); 193 194 // Step 1 : call to setup 195 struct io_uring_params params; 196 memset(¶ms, 0, sizeof(params)); 197 if( io_flags & CFA_CLUSTER_IO_KERNEL_POLL_SUBMITS ) params.flags |= IORING_SETUP_SQPOLL; 198 if( io_flags & CFA_CLUSTER_IO_KERNEL_POLL_COMPLETES ) params.flags |= IORING_SETUP_IOPOLL; 199 200 uint32_t nentries = entries_per_cluster(); 201 202 int fd = syscall(__NR_io_uring_setup, nentries, ¶ms ); 203 if(fd < 0) { 204 abort("KERNEL ERROR: IO_URING SETUP - %s\n", strerror(errno)); 205 } 206 207 // Step 2 : mmap result 208 memset( this.io, 0, sizeof(struct __io_data) ); 209 struct __submition_data & sq = this.io->submit_q; 210 struct __completion_data & cq = this.io->completion_q; 211 212 // calculate the right ring size 213 sq.ring_sz = params.sq_off.array + (params.sq_entries * sizeof(unsigned) ); 214 cq.ring_sz = params.cq_off.cqes + (params.cq_entries * sizeof(struct io_uring_cqe)); 215 216 // Requires features 217 #if defined(IORING_FEAT_SINGLE_MMAP) 218 // adjust the size according to the parameters 219 if ((params.features & IORING_FEAT_SINGLE_MMAP) != 0) { 220 cq.ring_sz = sq.ring_sz = max(cq.ring_sz, sq.ring_sz); 221 } 222 #endif 223 224 // mmap the Submit Queue into existence 225 sq.ring_ptr = mmap(0, sq.ring_sz, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd, IORING_OFF_SQ_RING); 226 if (sq.ring_ptr == (void*)MAP_FAILED) { 227 abort("KERNEL ERROR: IO_URING MMAP1 - %s\n", strerror(errno)); 228 } 229 230 // Requires features 231 #if defined(IORING_FEAT_SINGLE_MMAP) 232 // mmap the Completion Queue into existence (may or may not be needed) 233 if ((params.features & IORING_FEAT_SINGLE_MMAP) != 0) { 234 cq.ring_ptr = sq.ring_ptr; 235 } 236 else 237 #endif 238 { 239 // We need multiple call to MMAP 240 cq.ring_ptr = mmap(0, cq.ring_sz, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd, IORING_OFF_CQ_RING); 241 if (cq.ring_ptr == (void*)MAP_FAILED) { 242 munmap(sq.ring_ptr, sq.ring_sz); 243 abort("KERNEL ERROR: IO_URING MMAP2 - %s\n", strerror(errno)); 244 } 245 } 246 247 // mmap the submit queue entries 248 size_t size = params.sq_entries * sizeof(struct io_uring_sqe); 249 sq.sqes = (struct io_uring_sqe *)mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd, IORING_OFF_SQES); 250 if (sq.sqes == (struct io_uring_sqe *)MAP_FAILED) { 251 munmap(sq.ring_ptr, sq.ring_sz); 252 if (cq.ring_ptr != sq.ring_ptr) munmap(cq.ring_ptr, cq.ring_sz); 253 abort("KERNEL ERROR: IO_URING MMAP3 - %s\n", strerror(errno)); 254 } 255 256 // Get the pointers from the kernel to fill the structure 257 // submit queue 258 sq.head = (volatile uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.head); 259 sq.tail = (volatile uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.tail); 260 sq.mask = ( const uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.ring_mask); 261 sq.num = ( const uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.ring_entries); 262 sq.flags = ( uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.flags); 263 sq.dropped = ( uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.dropped); 264 sq.array = ( uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.array); 265 sq.prev_head = *sq.head; 266 267 { 268 const uint32_t num = *sq.num; 269 for( i; num ) { 270 sq.sqes[i].user_data = 0ul64; 271 } 272 } 273 274 (sq.lock){}; 275 (sq.release_lock){}; 276 277 if( io_flags & ( CFA_CLUSTER_IO_POLLER_THREAD_SUBMITS | CFA_CLUSTER_IO_EAGER_SUBMITS ) ) { 278 /* paranoid */ verify( is_pow2( io_flags >> CFA_CLUSTER_IO_BUFFLEN_OFFSET ) || ((io_flags >> CFA_CLUSTER_IO_BUFFLEN_OFFSET) < 8) ); 279 sq.ready_cnt = max(io_flags >> CFA_CLUSTER_IO_BUFFLEN_OFFSET, 8); 280 sq.ready = alloc_align( 64, sq.ready_cnt ); 281 for(i; sq.ready_cnt) { 282 sq.ready[i] = -1ul32; 283 } 284 } 285 else { 286 sq.ready_cnt = 0; 287 sq.ready = 0p; 288 } 289 290 // completion queue 291 cq.head = (volatile uint32_t *)(((intptr_t)cq.ring_ptr) + params.cq_off.head); 292 cq.tail = (volatile uint32_t *)(((intptr_t)cq.ring_ptr) + params.cq_off.tail); 293 cq.mask = ( const uint32_t *)(((intptr_t)cq.ring_ptr) + params.cq_off.ring_mask); 294 cq.num = ( const uint32_t *)(((intptr_t)cq.ring_ptr) + params.cq_off.ring_entries); 295 cq.overflow = ( uint32_t *)(((intptr_t)cq.ring_ptr) + params.cq_off.overflow); 296 cq.cqes = (struct io_uring_cqe *)(((intptr_t)cq.ring_ptr) + params.cq_off.cqes); 297 298 // some paranoid checks 299 /* paranoid */ verifyf( (*cq.mask) == ((*cq.num) - 1ul32), "IO_URING Expected mask to be %u (%u entries), was %u", (*cq.num) - 1ul32, *cq.num, *cq.mask ); 300 /* paranoid */ verifyf( (*cq.num) >= nentries, "IO_URING Expected %u entries, got %u", nentries, *cq.num ); 301 /* paranoid */ verifyf( (*cq.head) == 0, "IO_URING Expected head to be 0, got %u", *cq.head ); 302 /* paranoid */ verifyf( (*cq.tail) == 0, "IO_URING Expected tail to be 0, got %u", *cq.tail ); 303 304 /* paranoid */ verifyf( (*sq.mask) == ((*sq.num) - 1ul32), "IO_URING Expected mask to be %u (%u entries), was %u", (*sq.num) - 1ul32, *sq.num, *sq.mask ); 305 /* paranoid */ verifyf( (*sq.num) >= nentries, "IO_URING Expected %u entries, got %u", nentries, *sq.num ); 306 /* paranoid */ verifyf( (*sq.head) == 0, "IO_URING Expected head to be 0, got %u", *sq.head ); 307 /* paranoid */ verifyf( (*sq.tail) == 0, "IO_URING Expected tail to be 0, got %u", *sq.tail ); 308 309 // Update the global ring info 310 this.io->ring_flags = params.flags; 311 this.io->cltr_flags = io_flags; 312 this.io->fd = fd; 313 this.io->done = false; 314 (this.io->submit){ min(*sq.num, *cq.num) }; 315 316 if(!main_cluster) { 317 __kernel_io_finish_start( this ); 318 } 319 } 320 321 void __kernel_io_finish_start( cluster & this ) { 322 if( this.io->cltr_flags & CFA_CLUSTER_IO_POLLER_USER_THREAD ) { 323 __cfadbg_print_safe(io_core, "Kernel I/O : Creating fast poller for cluter %p\n", &this); 324 (this.io->poller.fast){ this }; 325 __thrd_start( this.io->poller.fast, main ); 326 } 327 328 // Create the poller thread 329 __cfadbg_print_safe(io_core, "Kernel I/O : Creating slow poller for cluster %p\n", &this); 330 this.io->poller.slow.blocked = false; 331 this.io->poller.slow.stack = __create_pthread( &this.io->poller.slow.kthrd, __io_poller_slow, &this ); 332 } 333 334 void __kernel_io_prepare_stop( cluster & this ) { 335 __cfadbg_print_safe(io_core, "Kernel I/O : Stopping pollers for cluster\n", &this); 336 // Notify the poller thread of the shutdown 337 __atomic_store_n(&this.io->done, true, __ATOMIC_SEQ_CST); 338 339 // Stop the IO Poller 340 sigval val = { 1 }; 341 pthread_sigqueue( this.io->poller.slow.kthrd, SIGUSR1, val ); 342 post( this.io->poller.sem ); 343 344 // Wait for the poller thread to finish 345 pthread_join( this.io->poller.slow.kthrd, 0p ); 346 free( this.io->poller.slow.stack ); 347 348 __cfadbg_print_safe(io_core, "Kernel I/O : Slow poller stopped for cluster\n", &this); 349 350 if( this.io->cltr_flags & CFA_CLUSTER_IO_POLLER_USER_THREAD ) { 351 with( this.io->poller.fast ) { 352 /* paranoid */ verify( this.nprocessors == 0 || &this == mainCluster ); 353 /* paranoid */ verify( !ready_mutate_islocked() ); 354 355 // We need to adjust the clean-up based on where the thread is 356 if( thrd.state == Ready || thrd.preempted != __NO_PREEMPTION ) { 357 358 ready_schedule_lock( (struct __processor_id_t *)active_processor() ); 359 360 // This is the tricky case 361 // The thread was preempted and now it is on the ready queue 362 // The thread should be the last on the list 363 /* paranoid */ verify( thrd.link.next != 0p ); 364 365 // Remove the thread from the ready queue of this cluster 366 __attribute__((unused)) bool removed = remove_head( &this, &thrd ); 367 /* paranoid */ verify( removed ); 368 thrd.link.next = 0p; 369 thrd.link.prev = 0p; 370 __cfaabi_dbg_debug_do( thrd.unpark_stale = true ); 371 372 // Fixup the thread state 373 thrd.state = Blocked; 374 thrd.ticket = 0; 375 thrd.preempted = __NO_PREEMPTION; 376 377 ready_schedule_unlock( (struct __processor_id_t *)active_processor() ); 378 379 // Pretend like the thread was blocked all along 380 } 381 // !!! This is not an else if !!! 382 if( thrd.state == Blocked ) { 383 384 // This is the "easy case" 385 // The thread is parked and can easily be moved to active cluster 386 verify( thrd.curr_cluster != active_cluster() || thrd.curr_cluster == mainCluster ); 387 thrd.curr_cluster = active_cluster(); 388 389 // unpark the fast io_poller 390 unpark( &thrd __cfaabi_dbg_ctx2 ); 391 } 392 else { 393 394 // The thread is in a weird state 395 // I don't know what to do here 396 abort("Fast poller thread is in unexpected state, cannot clean-up correctly\n"); 397 } 398 399 } 400 401 ^(this.io->poller.fast){}; 402 403 __cfadbg_print_safe(io_core, "Kernel I/O : Fast poller stopped for cluster\n", &this); 404 } 405 } 406 407 void __kernel_io_shutdown( cluster & this, bool main_cluster ) { 408 if(!main_cluster) { 409 __kernel_io_prepare_stop( this ); 410 } 411 412 // Shutdown the io rings 413 struct __submition_data & sq = this.io->submit_q; 414 struct __completion_data & cq = this.io->completion_q; 415 416 // unmap the submit queue entries 417 munmap(sq.sqes, (*sq.num) * sizeof(struct io_uring_sqe)); 418 419 // unmap the Submit Queue ring 420 munmap(sq.ring_ptr, sq.ring_sz); 421 422 // unmap the Completion Queue ring, if it is different 423 if (cq.ring_ptr != sq.ring_ptr) { 424 munmap(cq.ring_ptr, cq.ring_sz); 425 } 426 427 // close the file descriptor 428 close(this.io->fd); 429 430 free( this.io->submit_q.ready ); // Maybe null, doesn't matter 431 free( this.io ); 432 } 433 434 int __io_uring_enter( struct __io_data & ring, unsigned to_submit, bool get, sigset_t * mask ) { 39 #include "stats.hfa" 40 #include "kernel.hfa" 41 #include "kernel/fwd.hfa" 42 #include "io/types.hfa" 43 44 //============================================================================================= 45 // I/O Syscall 46 //============================================================================================= 47 static int __io_uring_enter( struct __io_data & ring, unsigned to_submit, bool get ) { 435 48 bool need_sys_to_submit = false; 436 49 bool need_sys_to_complete = false; 437 unsigned min_complete = 0;438 50 unsigned flags = 0; 439 440 51 441 52 TO_SUBMIT: … … 451 62 } 452 63 453 TO_COMPLETE:454 64 if( get && !(ring.ring_flags & IORING_SETUP_SQPOLL) ) { 455 65 flags |= IORING_ENTER_GETEVENTS; 456 if( mask ) {457 need_sys_to_complete = true;458 min_complete = 1;459 break TO_COMPLETE;460 }461 66 if( (ring.ring_flags & IORING_SETUP_IOPOLL) ) { 462 67 need_sys_to_complete = true; … … 466 71 int ret = 0; 467 72 if( need_sys_to_submit || need_sys_to_complete ) { 468 ret = syscall( __NR_io_uring_enter, ring.fd, to_submit, min_complete, flags, mask, _NSIG / 8);73 ret = syscall( __NR_io_uring_enter, ring.fd, to_submit, 0, flags, 0p, _NSIG / 8); 469 74 if( ret < 0 ) { 470 75 switch((int)errno) { … … 490 95 static uint32_t __release_consumed_submission( struct __io_data & ring ); 491 96 492 static inline void process(struct io_uring_cqe & cqe , struct __processor_id_t * id) {97 static inline void process(struct io_uring_cqe & cqe ) { 493 98 struct __io_user_data_t * data = (struct __io_user_data_t *)(uintptr_t)cqe.user_data; 494 99 __cfadbg_print_safe( io, "Kernel I/O : Syscall completed : cqe %p, result %d for %p\n", data, cqe.res, data->thrd ); 495 100 496 101 data->result = cqe.res; 497 if(!id) { unpark( data->thrd __cfaabi_dbg_ctx2 ); } 498 else { __unpark( id, data->thrd __cfaabi_dbg_ctx2 ); } 102 unpark( data->thrd __cfaabi_dbg_ctx2 ); 499 103 } 500 104 501 105 // Process a single completion message from the io_uring 502 106 // This is NOT thread-safe 503 static [int, bool] __drain_io( & struct __io_data ring , * sigset_t mask) {107 static [int, bool] __drain_io( & struct __io_data ring ) { 504 108 /* paranoid */ verify( !kernelTLS.preemption_state.enabled ); 505 109 506 110 unsigned to_submit = 0; 507 if( ring. cltr_flags & CFA_CLUSTER_IO_POLLER_THREAD_SUBMITS) {111 if( ring.poller_submits ) { 508 112 // If the poller thread also submits, then we need to aggregate the submissions which are ready 509 113 to_submit = __collect_submitions( ring ); 510 114 } 511 115 512 int ret = __io_uring_enter(ring, to_submit, true , mask);116 int ret = __io_uring_enter(ring, to_submit, true); 513 117 if( ret < 0 ) { 514 118 return [0, true]; … … 547 151 /* paranoid */ verify(&cqe); 548 152 549 process( cqe, !mask ? (struct __processor_id_t *)0p : &ring.poller.slow.id ); 550 } 551 552 // Allow new submissions to happen 553 // V(ring.submit, count); 153 process( cqe ); 154 } 554 155 555 156 // Mark to the kernel that the cqe has been seen … … 561 162 } 562 163 563 static void * __io_poller_slow( void * arg ) { 564 #if !defined( __CFA_NO_STATISTICS__ ) 565 __stats_t local_stats; 566 __init_stats( &local_stats ); 567 kernelTLS.this_stats = &local_stats; 568 #endif 569 570 cluster * cltr = (cluster *)arg; 571 struct __io_data & ring = *cltr->io; 572 573 ring.poller.slow.id.id = doregister( &ring.poller.slow.id ); 574 575 sigset_t mask; 576 sigfillset(&mask); 577 if ( pthread_sigmask( SIG_BLOCK, &mask, 0p ) == -1 ) { 578 abort( "KERNEL ERROR: IO_URING - pthread_sigmask" ); 579 } 580 581 sigdelset( &mask, SIGUSR1 ); 582 583 verify( (*ring.submit_q.head) == (*ring.submit_q.tail) ); 584 verify( (*ring.completion_q.head) == (*ring.completion_q.tail) ); 585 586 __cfadbg_print_safe(io_core, "Kernel I/O : Slow poller for ring %p ready\n", &ring); 587 588 if( ring.cltr_flags & CFA_CLUSTER_IO_POLLER_USER_THREAD ) { 589 while(!__atomic_load_n(&ring.done, __ATOMIC_SEQ_CST)) { 590 591 __atomic_store_n( &ring.poller.slow.blocked, true, __ATOMIC_SEQ_CST ); 592 593 // In the user-thread approach drain and if anything was drained, 594 // batton pass to the user-thread 595 int count; 596 bool again; 597 [count, again] = __drain_io( ring, &mask ); 598 599 __atomic_store_n( &ring.poller.slow.blocked, false, __ATOMIC_SEQ_CST ); 600 601 // Update statistics 602 __STATS__( true, 603 io.complete_q.completed_avg.val += count; 604 io.complete_q.completed_avg.slow_cnt += 1; 605 ) 606 607 if(again) { 608 __cfadbg_print_safe(io_core, "Kernel I/O : Moving to ring %p to fast poller\n", &ring); 609 __unpark( &ring.poller.slow.id, &ring.poller.fast.thrd __cfaabi_dbg_ctx2 ); 610 wait( ring.poller.sem ); 611 } 612 } 613 } 614 else { 615 while(!__atomic_load_n(&ring.done, __ATOMIC_SEQ_CST)) { 616 //In the naive approach, just poll the io completion queue directly 617 int count; 618 bool again; 619 [count, again] = __drain_io( ring, &mask ); 620 621 // Update statistics 622 __STATS__( true, 623 io.complete_q.completed_avg.val += count; 624 io.complete_q.completed_avg.slow_cnt += 1; 625 ) 626 } 627 } 628 629 __cfadbg_print_safe(io_core, "Kernel I/O : Slow poller for ring %p stopping\n", &ring); 630 631 unregister( &ring.poller.slow.id ); 632 633 #if !defined(__CFA_NO_STATISTICS__) 634 __tally_stats(cltr->stats, &local_stats); 635 #endif 636 637 return 0p; 638 } 639 640 void main( __io_poller_fast & this ) { 641 verify( this.ring->cltr_flags & CFA_CLUSTER_IO_POLLER_USER_THREAD ); 642 643 // Start parked 644 park( __cfaabi_dbg_ctx ); 645 646 __cfadbg_print_safe(io_core, "Kernel I/O : Fast poller for ring %p ready\n", &this.ring); 164 void main( $io_ctx_thread & this ) { 165 epoll_event ev; 166 __ioctx_register( this, ev ); 167 168 __cfadbg_print_safe(io_core, "Kernel I/O : IO poller %p for ring %p ready\n", &this, &this.ring); 647 169 648 170 int reset = 0; 649 650 171 // Then loop until we need to start 651 while(!__atomic_load_n(&this.ring->done, __ATOMIC_SEQ_CST)) { 652 172 while(!__atomic_load_n(&this.done, __ATOMIC_SEQ_CST)) { 653 173 // Drain the io 654 174 int count; 655 175 bool again; 656 176 disable_interrupts(); 657 [count, again] = __drain_io( *this.ring , 0p);177 [count, again] = __drain_io( *this.ring ); 658 178 659 179 if(!again) reset++; … … 672 192 // We didn't get anything baton pass to the slow poller 673 193 else { 674 __cfadbg_print_safe(io_core, "Kernel I/O : Moving to ring %p to slow poller\n", &this.ring);194 __cfadbg_print_safe(io_core, "Kernel I/O : Parking io poller %p\n", &this.self); 675 195 reset = 0; 676 196 677 // wake up the slow poller 678 post( this.ring->poller.sem ); 679 680 // park this thread 681 park( __cfaabi_dbg_ctx ); 197 // block this thread 198 __ioctx_prepare_block( this, ev ); 199 wait( this.sem ); 682 200 } 683 201 } 684 202 685 203 __cfadbg_print_safe(io_core, "Kernel I/O : Fast poller for ring %p stopping\n", &this.ring); 686 }687 688 static inline void __wake_poller( struct __io_data & ring ) __attribute__((artificial));689 static inline void __wake_poller( struct __io_data & ring ) {690 if(!__atomic_load_n( &ring.poller.slow.blocked, __ATOMIC_SEQ_CST)) return;691 692 sigval val = { 1 };693 pthread_sigqueue( ring.poller.slow.kthrd, SIGUSR1, val );694 204 } 695 205 … … 806 316 } 807 317 808 void __submit( struct __io_data & ring, uint32_t idx ) { 318 void __submit( struct io_context * ctx, uint32_t idx ) __attribute__((nonnull (1))) { 319 __io_data & ring = *ctx->thrd.ring; 809 320 // Get now the data we definetely need 810 321 uint32_t * const tail = ring.submit_q.tail; 811 const uint32_t mask = *ring.submit_q.mask;322 const uint32_t mask = *ring.submit_q.mask; 812 323 813 324 // There are 2 submission schemes, check which one we are using 814 if( ring. cltr_flags & CFA_CLUSTER_IO_POLLER_THREAD_SUBMITS) {325 if( ring.poller_submits ) { 815 326 // If the poller thread submits, then we just need to add this to the ready array 816 327 __submit_to_ready_array( ring, idx, mask ); 817 328 818 __wake_poller( ring);329 post( ctx->thrd.sem ); 819 330 820 331 __cfadbg_print_safe( io, "Kernel I/O : Added %u to ready for %p\n", idx, active_thread() ); 821 332 } 822 else if( ring. cltr_flags & CFA_CLUSTER_IO_EAGER_SUBMITS) {333 else if( ring.eager_submits ) { 823 334 uint32_t picked = __submit_to_ready_array( ring, idx, mask ); 824 335 … … 849 360 // We got the lock 850 361 unsigned to_submit = __collect_submitions( ring ); 851 int ret = __io_uring_enter( ring, to_submit, false , 0p);362 int ret = __io_uring_enter( ring, to_submit, false ); 852 363 if( ret < 0 ) { 853 364 unlock(ring.submit_q.lock); … … 892 403 893 404 // Submit however, many entries need to be submitted 894 int ret = __io_uring_enter( ring, 1, false , 0p);405 int ret = __io_uring_enter( ring, 1, false ); 895 406 if( ret < 0 ) { 896 407 switch((int)errno) { … … 958 469 return count; 959 470 } 960 961 //=============================================================================================962 // I/O Submissions963 //=============================================================================================964 965 void register_fixed_files( cluster & cl, int * files, unsigned count ) {966 int ret = syscall( __NR_io_uring_register, cl.io->fd, IORING_REGISTER_FILES, files, count );967 if( ret < 0 ) {968 abort( "KERNEL ERROR: IO_URING SYSCALL - (%d) %s\n", (int)errno, strerror(errno) );969 }970 971 __cfadbg_print_safe( io_core, "Kernel I/O : Performed io_register for %p, returned %d\n", active_thread(), ret );972 }973 471 #endif -
libcfa/src/concurrency/iocall.cfa
r3f850d7 r53ee27e 14 14 // 15 15 16 #define __cforall_thread__ 17 16 18 #include "bits/defs.hfa" 19 #include "kernel.hfa" 17 20 18 21 //============================================================================================= … … 21 24 22 25 #if defined(CFA_HAVE_LINUX_IO_URING_H) 26 #include <assert.h> 23 27 #include <stdint.h> 28 #include <errno.h> 24 29 #include <linux/io_uring.h> 25 30 26 #include "kernel_private.hfa" 31 #include "kernel/fwd.hfa" 32 #include "io/types.hfa" 27 33 28 34 extern [* struct io_uring_sqe, uint32_t] __submit_alloc( struct __io_data & ring, uint64_t data ); 29 extern void __submit( struct __io_data & ring, uint32_t idx);35 extern void __submit( struct io_context * ctx, uint32_t idx ) __attribute__((nonnull (1))); 30 36 31 37 static inline void ?{}(struct io_uring_sqe & this, uint8_t opcode, int fd) { … … 52 58 } 53 59 60 static inline io_context * __get_io_context( void ) { 61 cluster * cltr = active_cluster(); 62 /* paranoid */ verifyf( cltr, "No active cluster for io operation\n"); 63 assertf( cltr->io.cnt > 0, "Cluster %p has no default io contexts and no context was specified\n", cltr ); 64 /* paranoid */ verifyf( cltr->io.ctxs, "default io contexts for cluster %p are missing\n", cltr); 65 return &cltr->io.ctxs[ __tls_rand() % cltr->io.cnt ]; 66 } 67 68 69 #if defined(CFA_HAVE_IOSQE_FIXED_FILE) && defined(CFA_HAVE_IOSQE_IO_DRAIN) && defined(CFA_HAVE_IOSQE_ASYNC) 70 #define REGULAR_FLAGS (IOSQE_FIXED_FILE | IOSQE_IO_DRAIN | IOSQE_ASYNC) 71 #elif defined(CFA_HAVE_IOSQE_FIXED_FILE) && defined(CFA_HAVE_IOSQE_ASYNC) 72 #define REGULAR_FLAGS (IOSQE_FIXED_FILE | IOSQE_ASYNC) 73 #elif defined(CFA_HAVE_IOSQE_FIXED_FILE) && defined(CFA_HAVE_IOSQE_IO_DRAIN) 74 #define REGULAR_FLAGS (IOSQE_FIXED_FILE | IOSQE_IO_DRAIN) 75 #elif defined(CFA_HAVE_IOSQE_IO_DRAIN) && defined(CFA_HAVE_IOSQE_ASYNC) 76 #define REGULAR_FLAGS (IOSQE_IO_DRAIN | IOSQE_ASYNC) 77 #elif defined(CFA_HAVE_IOSQE_FIXED_FILE) 78 #define REGULAR_FLAGS (IOSQE_FIXED_FILE) 79 #elif defined(CFA_HAVE_IOSQE_IO_DRAIN) 80 #define REGULAR_FLAGS (IOSQE_IO_DRAIN) 81 #elif defined(CFA_HAVE_IOSQE_ASYNC) 82 #define REGULAR_FLAGS (IOSQE_ASYNC) 83 #else 84 #define REGULAR_FLAGS (0) 85 #endif 86 87 #if defined(CFA_HAVE_IOSQE_IO_LINK) && defined(CFA_HAVE_IOSQE_IO_HARDLINK) 88 #define LINK_FLAGS (IOSQE_IO_LINK | IOSQE_IO_HARDLINK) 89 #elif defined(CFA_HAVE_IOSQE_IO_LINK) 90 #define LINK_FLAGS (IOSQE_IO_LINK) 91 #elif defined(CFA_HAVE_IOSQE_IO_HARDLINK) 92 #define LINK_FLAGS (IOSQE_IO_HARDLINK) 93 #else 94 #define LINK_FLAGS (0) 95 #endif 96 97 #if defined(CFA_HAVE_SPLICE_F_FD_IN_FIXED) 98 #define SPLICE_FLAGS (SPLICE_F_FD_IN_FIXED) 99 #else 100 #define SPLICE_FLAGS (0) 101 #endif 102 103 54 104 #define __submit_prelude \ 105 if( 0 != (submit_flags & LINK_FLAGS) ) { errno = ENOTSUP; return -1; } \ 106 (void)timeout; (void)cancellation; \ 107 if( !context ) context = __get_io_context(); \ 55 108 __io_user_data_t data = { 0, active_thread() }; \ 56 struct __io_data & ring = * data.thrd->curr_cluster->io; \109 struct __io_data & ring = *context->thrd.ring; \ 57 110 struct io_uring_sqe * sqe; \ 58 111 uint32_t idx; \ 59 [sqe, idx] = __submit_alloc( ring, (uint64_t)(uintptr_t)&data ); 112 [sqe, idx] = __submit_alloc( ring, (uint64_t)(uintptr_t)&data ); \ 113 sqe->flags = REGULAR_FLAGS & submit_flags; 60 114 61 115 #define __submit_wait \ 62 116 /*__cfaabi_bits_print_safe( STDERR_FILENO, "Preparing user data %p for %p\n", &data, data.thrd );*/ \ 63 117 verify( sqe->user_data == (uint64_t)(uintptr_t)&data ); \ 64 __submit( ring, idx ); \118 __submit( context, idx ); \ 65 119 park( __cfaabi_dbg_ctx ); \ 120 if( data.result < 0 ) { \ 121 errno = -data.result; \ 122 return -1; \ 123 } \ 66 124 return data.result; 67 125 #endif … … 70 128 // I/O Forwards 71 129 //============================================================================================= 130 #include <time.hfa> 72 131 73 132 // Some forward declarations … … 121 180 // Asynchronous operations 122 181 #if defined(HAVE_PREADV2) 123 ssize_t cfa_preadv2(int fd, const struct iovec *iov, int iovcnt, off_t offset, int flags ) {182 ssize_t cfa_preadv2(int fd, const struct iovec *iov, int iovcnt, off_t offset, int flags, int submit_flags, Duration timeout, io_cancellation * cancellation, io_context * context) { 124 183 #if !defined(CFA_HAVE_LINUX_IO_URING_H) || !defined(CFA_HAVE_IORING_OP_READV) 125 184 return preadv2(fd, iov, iovcnt, offset, flags); … … 132 191 #endif 133 192 } 134 135 ssize_t cfa_preadv2_fixed(int fd, const struct iovec *iov, int iovcnt, off_t offset, int flags) { 136 #if !defined(CFA_HAVE_LINUX_IO_URING_H) || !defined(CFA_HAVE_IORING_OP_READV) 137 return preadv2(fd, iov, iovcnt, offset, flags); 193 #endif 194 195 #if defined(HAVE_PWRITEV2) 196 ssize_t cfa_pwritev2(int fd, const struct iovec *iov, int iovcnt, off_t offset, int flags, int submit_flags, Duration timeout, io_cancellation * cancellation, io_context * context) { 197 #if !defined(CFA_HAVE_LINUX_IO_URING_H) || !defined(CFA_HAVE_IORING_OP_WRITEV) 198 return pwritev2(fd, iov, iovcnt, offset, flags); 138 199 #else 139 200 __submit_prelude 140 201 141 (*sqe){ IORING_OP_READV, fd, iov, iovcnt, offset }; 142 sqe->flags |= IOSQE_FIXED_FILE; 202 (*sqe){ IORING_OP_WRITEV, fd, iov, iovcnt, offset }; 143 203 144 204 __submit_wait … … 147 207 #endif 148 208 149 #if defined(HAVE_PWRITEV2) 150 ssize_t cfa_pwritev2(int fd, const struct iovec *iov, int iovcnt, off_t offset, int flags) { 151 #if !defined(CFA_HAVE_LINUX_IO_URING_H) || !defined(CFA_HAVE_IORING_OP_WRITEV) 152 return pwritev2(fd, iov, iovcnt, offset, flags); 153 #else 154 __submit_prelude 155 156 (*sqe){ IORING_OP_WRITEV, fd, iov, iovcnt, offset }; 157 158 __submit_wait 159 #endif 160 } 161 #endif 162 163 int cfa_fsync(int fd) { 209 int cfa_fsync(int fd, int submit_flags, Duration timeout, io_cancellation * cancellation, io_context * context) { 164 210 #if !defined(CFA_HAVE_LINUX_IO_URING_H) || !defined(CFA_HAVE_IORING_OP_FSYNC) 165 211 return fsync(fd); … … 173 219 } 174 220 175 int cfa_sync_file_range(int fd, int64_t offset, int64_t nbytes, unsigned int flags ) {221 int cfa_sync_file_range(int fd, int64_t offset, int64_t nbytes, unsigned int flags, int submit_flags, Duration timeout, io_cancellation * cancellation, io_context * context) { 176 222 #if !defined(CFA_HAVE_LINUX_IO_URING_H) || !defined(CFA_HAVE_IORING_OP_SYNC_FILE_RANGE) 177 223 return sync_file_range(fd, offset, nbytes, flags); … … 189 235 190 236 191 ssize_t cfa_sendmsg(int sockfd, const struct msghdr *msg, int flags ) {237 ssize_t cfa_sendmsg(int sockfd, const struct msghdr *msg, int flags, int submit_flags, Duration timeout, io_cancellation * cancellation, io_context * context) { 192 238 #if !defined(CFA_HAVE_LINUX_IO_URING_H) || !defined(CFA_HAVE_IORING_OP_SENDMSG) 193 239 return sendmsg(sockfd, msg, flags); … … 202 248 } 203 249 204 ssize_t cfa_recvmsg(int sockfd, struct msghdr *msg, int flags ) {250 ssize_t cfa_recvmsg(int sockfd, struct msghdr *msg, int flags, int submit_flags, Duration timeout, io_cancellation * cancellation, io_context * context) { 205 251 #if !defined(CFA_HAVE_LINUX_IO_URING_H) || !defined(CFA_HAVE_IORING_OP_RECVMSG) 206 252 return recvmsg(sockfd, msg, flags); … … 215 261 } 216 262 217 ssize_t cfa_send(int sockfd, const void *buf, size_t len, int flags ) {263 ssize_t cfa_send(int sockfd, const void *buf, size_t len, int flags, int submit_flags, Duration timeout, io_cancellation * cancellation, io_context * context) { 218 264 #if !defined(CFA_HAVE_LINUX_IO_URING_H) || !defined(CFA_HAVE_IORING_OP_SEND) 219 265 return send( sockfd, buf, len, flags ); … … 230 276 } 231 277 232 ssize_t cfa_recv(int sockfd, void *buf, size_t len, int flags ) {278 ssize_t cfa_recv(int sockfd, void *buf, size_t len, int flags, int submit_flags, Duration timeout, io_cancellation * cancellation, io_context * context) { 233 279 #if !defined(CFA_HAVE_LINUX_IO_URING_H) || !defined(CFA_HAVE_IORING_OP_RECV) 234 280 return recv( sockfd, buf, len, flags ); … … 245 291 } 246 292 247 int cfa_accept4(int sockfd, struct sockaddr *addr, socklen_t *addrlen, int flags ) {293 int cfa_accept4(int sockfd, struct sockaddr *addr, socklen_t *addrlen, int flags, int submit_flags, Duration timeout, io_cancellation * cancellation, io_context * context) { 248 294 #if !defined(CFA_HAVE_LINUX_IO_URING_H) || !defined(CFA_HAVE_IORING_OP_ACCEPT) 249 295 return accept4( sockfd, addr, addrlen, flags ); … … 260 306 } 261 307 262 int cfa_connect(int sockfd, const struct sockaddr *addr, socklen_t addrlen ) {308 int cfa_connect(int sockfd, const struct sockaddr *addr, socklen_t addrlen, int submit_flags, Duration timeout, io_cancellation * cancellation, io_context * context) { 263 309 #if !defined(CFA_HAVE_LINUX_IO_URING_H) || !defined(CFA_HAVE_IORING_OP_CONNECT) 264 310 return connect( sockfd, addr, addrlen ); … … 274 320 } 275 321 276 int cfa_fallocate(int fd, int mode, uint64_t offset, uint64_t len ) {322 int cfa_fallocate(int fd, int mode, uint64_t offset, uint64_t len, int submit_flags, Duration timeout, io_cancellation * cancellation, io_context * context) { 277 323 #if !defined(CFA_HAVE_LINUX_IO_URING_H) || !defined(CFA_HAVE_IORING_OP_FALLOCATE) 278 324 return fallocate( fd, mode, offset, len ); … … 291 337 } 292 338 293 int cfa_fadvise(int fd, uint64_t offset, uint64_t len, int advice ) {339 int cfa_fadvise(int fd, uint64_t offset, uint64_t len, int advice, int submit_flags, Duration timeout, io_cancellation * cancellation, io_context * context) { 294 340 #if !defined(CFA_HAVE_LINUX_IO_URING_H) || !defined(CFA_HAVE_IORING_OP_FADVISE) 295 341 return posix_fadvise( fd, offset, len, advice ); … … 306 352 } 307 353 308 int cfa_madvise(void *addr, size_t length, int advice ) {354 int cfa_madvise(void *addr, size_t length, int advice, int submit_flags, Duration timeout, io_cancellation * cancellation, io_context * context) { 309 355 #if !defined(CFA_HAVE_LINUX_IO_URING_H) || !defined(CFA_HAVE_IORING_OP_MADVISE) 310 356 return madvise( addr, length, advice ); … … 321 367 } 322 368 323 int cfa_openat(int dirfd, const char *pathname, int flags, mode_t mode ) {369 int cfa_openat(int dirfd, const char *pathname, int flags, mode_t mode, int submit_flags, Duration timeout, io_cancellation * cancellation, io_context * context) { 324 370 #if !defined(CFA_HAVE_LINUX_IO_URING_H) || !defined(CFA_HAVE_IORING_OP_OPENAT) 325 371 return openat( dirfd, pathname, flags, mode ); … … 336 382 } 337 383 338 int cfa_close(int fd ) {384 int cfa_close(int fd, int submit_flags, Duration timeout, io_cancellation * cancellation, io_context * context) { 339 385 #if !defined(CFA_HAVE_LINUX_IO_URING_H) || !defined(CFA_HAVE_IORING_OP_CLOSE) 340 386 return close( fd ); … … 350 396 // Forward declare in case it is not supported 351 397 struct statx; 352 int cfa_statx(int dirfd, const char *pathname, int flags, unsigned int mask, struct statx *statxbuf ) {398 int cfa_statx(int dirfd, const char *pathname, int flags, unsigned int mask, struct statx *statxbuf, int submit_flags, Duration timeout, io_cancellation * cancellation, io_context * context) { 353 399 #if !defined(CFA_HAVE_LINUX_IO_URING_H) || !defined(CFA_HAVE_IORING_OP_STATX) 354 400 #if defined(__NR_statx) … … 362 408 363 409 (*sqe){ IORING_OP_STATX, dirfd, pathname, mask, (uint64_t)statxbuf }; 364 sqe-> flags = flags;365 366 __submit_wait 367 #endif 368 } 369 370 ssize_t cfa_read(int fd, void *buf, size_t count ) {410 sqe->statx_flags = flags; 411 412 __submit_wait 413 #endif 414 } 415 416 ssize_t cfa_read(int fd, void *buf, size_t count, int submit_flags, Duration timeout, io_cancellation * cancellation, io_context * context) { 371 417 #if !defined(CFA_HAVE_LINUX_IO_URING_H) || !defined(CFA_HAVE_IORING_OP_READ) 372 418 return read( fd, buf, count ); … … 380 426 } 381 427 382 ssize_t cfa_write(int fd, void *buf, size_t count ) {428 ssize_t cfa_write(int fd, void *buf, size_t count, int submit_flags, Duration timeout, io_cancellation * cancellation, io_context * context) { 383 429 #if !defined(CFA_HAVE_LINUX_IO_URING_H) || !defined(CFA_HAVE_IORING_OP_WRITE) 384 430 return read( fd, buf, count ); … … 392 438 } 393 439 394 ssize_t cfa_splice(int fd_in, loff_t *off_in, int fd_out, loff_t *off_out, size_t len, unsigned int flags ) {440 ssize_t cfa_splice(int fd_in, loff_t *off_in, int fd_out, loff_t *off_out, size_t len, unsigned int flags, int submit_flags, Duration timeout, io_cancellation * cancellation, io_context * context) { 395 441 #if !defined(CFA_HAVE_LINUX_IO_URING_H) || !defined(CFA_HAVE_IORING_OP_SPLICE) 396 442 return splice( fd_in, off_in, fd_out, off_out, len, flags ); … … 413 459 sqe->splice_off_in = (uint64_t)-1; 414 460 } 415 sqe->splice_flags = flags; 416 417 __submit_wait 418 #endif 419 } 420 421 ssize_t cfa_splice(int fd_in, loff_t *off_in, int fd_out, loff_t *off_out, size_t len, unsigned int flags, int in_flags, int out_flags) { 422 #if !defined(CFA_HAVE_LINUX_IO_URING_H) || !defined(CFA_HAVE_IORING_OP_SPLICE) 423 return splice( fd_in, off_in, fd_out, off_out, len, flags ); 424 #else 425 __submit_prelude 426 427 (*sqe){ IORING_OP_SPLICE, fd_out }; 428 if( off_out ) { 429 sqe->off = *off_out; 430 } 431 else { 432 sqe->off = (uint64_t)-1; 433 } 434 sqe->len = len; 435 sqe->splice_fd_in = fd_in; 436 if( off_in ) { 437 sqe->splice_off_in = *off_in; 438 } 439 else { 440 sqe->splice_off_in = (uint64_t)-1; 441 } 442 sqe->splice_flags = flags | out_flags; 443 sqe->flags = in_flags; 444 445 __submit_wait 446 #endif 447 } 448 449 ssize_t cfa_tee(int fd_in, int fd_out, size_t len, unsigned int flags) { 461 sqe->splice_flags = flags | (SPLICE_FLAGS & submit_flags); 462 463 __submit_wait 464 #endif 465 } 466 467 ssize_t cfa_tee(int fd_in, int fd_out, size_t len, unsigned int flags, int submit_flags, Duration timeout, io_cancellation * cancellation, io_context * context) { 450 468 #if !defined(CFA_HAVE_LINUX_IO_URING_H) || !defined(CFA_HAVE_IORING_OP_TEE) 451 469 return tee( fd_in, fd_out, len, flags ); … … 455 473 (*sqe){ IORING_OP_TEE, fd_out, 0p, len, 0 }; 456 474 sqe->splice_fd_in = fd_in; 457 sqe->splice_flags = flags;475 sqe->splice_flags = flags | (SPLICE_FLAGS & submit_flags); 458 476 459 477 __submit_wait … … 562 580 563 581 if( /*func == (fptr_t)splice || */ 564 func == (fptr_t)(ssize_t (*)(int, loff_t *, int, loff_t *, size_t, unsigned int))cfa_splice, 565 func == (fptr_t)(ssize_t (*)(int, loff_t *, int, loff_t *, size_t, unsigned int, int, int))cfa_splice ) 582 func == (fptr_t)cfa_splice ) 566 583 #define _CFA_IO_FEATURE_CFA_HAVE_IORING_OP_SPLICE , 567 584 return IS_DEFINED(CFA_HAVE_IORING_OP_SPLICE); -
libcfa/src/concurrency/iofwd.hfa
r3f850d7 r53ee27e 19 19 extern "C" { 20 20 #include <sys/types.h> 21 #if CFA_HAVE_LINUX_IO_URING_H 22 #include <linux/io_uring.h> 23 #endif 21 24 } 22 25 #include "bits/defs.hfa" 26 #include "time.hfa" 27 28 #if defined(CFA_HAVE_IOSQE_FIXED_FILE) 29 #define CFA_IO_FIXED_FD1 IOSQE_FIXED_FILE 30 #endif 31 #if defined(CFA_HAVE_SPLICE_F_FD_IN_FIXED) 32 #define CFA_IO_FIXED_FD2 SPLICE_F_FD_IN_FIXED 33 #endif 34 #if defined(CFA_HAVE_IOSQE_IO_DRAIN) 35 #define CFA_IO_DRAIN IOSQE_IO_DRAIN 36 #endif 37 #if defined(CFA_HAVE_IOSQE_ASYNC) 38 #define CFA_IO_ASYNC IOSQE_ASYNC 39 #endif 40 41 struct cluster; 42 struct io_context; 43 struct io_cancellation; 23 44 24 45 struct iovec; … … 27 48 struct statx; 28 49 29 extern ssize_t cfa_preadv2(int fd, const struct iovec *iov, int iovcnt, off_t offset, int flags );30 extern ssize_t cfa_pwritev2(int fd, const struct iovec *iov, int iovcnt, off_t offset, int flags );31 extern int cfa_fsync(int fd );32 extern int cfa_sync_file_range(int fd, int64_t offset, int64_t nbytes, unsigned int flags );33 extern ssize_t cfa_sendmsg(int sockfd, const struct msghdr *msg, int flags );34 extern ssize_t cfa_recvmsg(int sockfd, struct msghdr *msg, int flags );35 extern ssize_t cfa_send(int sockfd, const void *buf, size_t len, int flags );36 extern ssize_t cfa_recv(int sockfd, void *buf, size_t len, int flags );37 extern int cfa_accept4(int sockfd, struct sockaddr *addr, socklen_t *addrlen, int flags );38 extern int cfa_connect(int sockfd, const struct sockaddr *addr, socklen_t addrlen );39 extern int cfa_fallocate(int fd, int mode, uint64_t offset, uint64_t len );40 extern int cfa_fadvise(int fd, uint64_t offset, uint64_t len, int advice );41 extern int cfa_madvise(void *addr, size_t length, int advice );42 extern int cfa_openat(int dirfd, const char *pathname, int flags, mode_t mode );43 extern int cfa_close(int fd );44 extern int cfa_statx(int dirfd, const char *pathname, int flags, unsigned int mask, struct statx *statxbuf );45 extern ssize_t cfa_read(int fd, void *buf, size_t count );46 extern ssize_t cfa_write(int fd, void *buf, size_t count );47 extern ssize_t cfa_splice(int fd_in, loff_t *off_in, int fd_out, loff_t *off_out, size_t len, unsigned int flags );48 extern ssize_t cfa_tee(int fd_in, int fd_out, size_t len, unsigned int flags );50 extern ssize_t cfa_preadv2(int fd, const struct iovec *iov, int iovcnt, off_t offset, int flags, int submit_flags = 0, Duration timeout = -1`s, io_cancellation * cancellation = 0p, io_context * context = 0p); 51 extern ssize_t cfa_pwritev2(int fd, const struct iovec *iov, int iovcnt, off_t offset, int flags, int submit_flags = 0, Duration timeout = -1`s, io_cancellation * cancellation = 0p, io_context * context = 0p); 52 extern int cfa_fsync(int fd, int submit_flags = 0, Duration timeout = -1`s, io_cancellation * cancellation = 0p, io_context * context = 0p); 53 extern int cfa_sync_file_range(int fd, int64_t offset, int64_t nbytes, unsigned int flags, int submit_flags = 0, Duration timeout = -1`s, io_cancellation * cancellation = 0p, io_context * context = 0p); 54 extern ssize_t cfa_sendmsg(int sockfd, const struct msghdr *msg, int flags, int submit_flags = 0, Duration timeout = -1`s, io_cancellation * cancellation = 0p, io_context * context = 0p); 55 extern ssize_t cfa_recvmsg(int sockfd, struct msghdr *msg, int flags, int submit_flags = 0, Duration timeout = -1`s, io_cancellation * cancellation = 0p, io_context * context = 0p); 56 extern ssize_t cfa_send(int sockfd, const void *buf, size_t len, int flags, int submit_flags = 0, Duration timeout = -1`s, io_cancellation * cancellation = 0p, io_context * context = 0p); 57 extern ssize_t cfa_recv(int sockfd, void *buf, size_t len, int flags, int submit_flags = 0, Duration timeout = -1`s, io_cancellation * cancellation = 0p, io_context * context = 0p); 58 extern int cfa_accept4(int sockfd, struct sockaddr *addr, socklen_t *addrlen, int flags, int submit_flags = 0, Duration timeout = -1`s, io_cancellation * cancellation = 0p, io_context * context = 0p); 59 extern int cfa_connect(int sockfd, const struct sockaddr *addr, socklen_t addrlen, int submit_flags = 0, Duration timeout = -1`s, io_cancellation * cancellation = 0p, io_context * context = 0p); 60 extern int cfa_fallocate(int fd, int mode, uint64_t offset, uint64_t len, int submit_flags = 0, Duration timeout = -1`s, io_cancellation * cancellation = 0p, io_context * context = 0p); 61 extern int cfa_fadvise(int fd, uint64_t offset, uint64_t len, int advice, int submit_flags = 0, Duration timeout = -1`s, io_cancellation * cancellation = 0p, io_context * context = 0p); 62 extern int cfa_madvise(void *addr, size_t length, int advice, int submit_flags = 0, Duration timeout = -1`s, io_cancellation * cancellation = 0p, io_context * context = 0p); 63 extern int cfa_openat(int dirfd, const char *pathname, int flags, mode_t mode, int submit_flags = 0, Duration timeout = -1`s, io_cancellation * cancellation = 0p, io_context * context = 0p); 64 extern int cfa_close(int fd, int submit_flags = 0, Duration timeout = -1`s, io_cancellation * cancellation = 0p, io_context * context = 0p); 65 extern int cfa_statx(int dirfd, const char *pathname, int flags, unsigned int mask, struct statx *statxbuf, int submit_flags = 0, Duration timeout = -1`s, io_cancellation * cancellation = 0p, io_context * context = 0p); 66 extern ssize_t cfa_read(int fd, void *buf, size_t count, int submit_flags = 0, Duration timeout = -1`s, io_cancellation * cancellation = 0p, io_context * context = 0p); 67 extern ssize_t cfa_write(int fd, void *buf, size_t count, int submit_flags = 0, Duration timeout = -1`s, io_cancellation * cancellation = 0p, io_context * context = 0p); 68 extern ssize_t cfa_splice(int fd_in, loff_t *off_in, int fd_out, loff_t *off_out, size_t len, unsigned int flags, int submit_flags = 0, Duration timeout = -1`s, io_cancellation * cancellation = 0p, io_context * context = 0p); 69 extern ssize_t cfa_tee(int fd_in, int fd_out, size_t len, unsigned int flags, int submit_flags = 0, Duration timeout = -1`s, io_cancellation * cancellation = 0p, io_context * context = 0p); 49 70 50 71 //----------------------------------------------------------------------------- 51 72 // Check if a function is blocks a only the user thread 52 73 bool has_user_level_blocking( fptr_t func ); 74 75 //----------------------------------------------------------------------------- 76 void register_fixed_files( io_context & ctx , int * files, unsigned count ); 77 void register_fixed_files( cluster & cltr, int * files, unsigned count ); -
libcfa/src/concurrency/kernel.cfa
r3f850d7 r53ee27e 18 18 19 19 //C Includes 20 #include <stddef.h>21 20 #include <errno.h> 22 #include <string.h>23 21 #include <stdio.h> 24 #include <fenv.h>25 22 #include <signal.h> 26 23 #include <unistd.h> 27 #include <limits.h> // PTHREAD_STACK_MIN28 #include <sys/mman.h> // mprotect29 extern "C" {30 #include <sys/resource.h>31 }32 24 33 25 //CFA Includes 34 #include "time.hfa"35 26 #include "kernel_private.hfa" 36 27 #include "preemption.hfa" 37 #include "startup.hfa"38 28 39 29 //Private includes … … 45 35 // Some assembly required 46 36 #if defined( __i386 ) 47 #define CtxGet( ctx ) \48 __asm__ volatile ( \49 "movl %%esp,%0\n"\50 "movl %%ebp,%1\n"\51 : "=rm" (ctx.SP),\52 "=rm" (ctx.FP) \53 )54 55 37 // mxcr : SSE Status and Control bits (control bits are preserved across function calls) 56 38 // fcw : X87 FPU control word (preserved across function calls) … … 74 56 75 57 #elif defined( __x86_64 ) 76 #define CtxGet( ctx ) \77 __asm__ volatile ( \78 "movq %%rsp,%0\n"\79 "movq %%rbp,%1\n"\80 : "=rm" (ctx.SP),\81 "=rm" (ctx.FP) \82 )83 84 58 #define __x87_store \ 85 59 uint32_t __mxcr; \ … … 102 76 103 77 #elif defined( __ARM_ARCH ) 104 #define CtxGet( ctx ) __asm__ ( \105 "mov %0,%%sp\n" \106 "mov %1,%%r11\n" \107 : "=rm" (ctx.SP), "=rm" (ctx.FP) )108 78 #else 109 79 #error unknown hardware architecture 110 80 #endif 111 81 112 //----------------------------------------------------------------------------- 113 //Start and stop routine for the kernel, declared first to make sure they run first 114 static void __kernel_startup (void) __attribute__(( constructor( STARTUP_PRIORITY_KERNEL ) )); 115 static void __kernel_shutdown(void) __attribute__(( destructor ( STARTUP_PRIORITY_KERNEL ) )); 82 extern $thread * mainThread; 83 extern processor * mainProcessor; 116 84 117 85 //----------------------------------------------------------------------------- … … 120 88 static bool __has_next_thread(cluster * this); 121 89 static void __run_thread(processor * this, $thread * dst); 122 static bool __wake_proc(processor *);123 90 static bool __wake_one(struct __processor_id_t * id, cluster * cltr); 124 91 static void __halt(processor * this); 125 126 //----------------------------------------------------------------------------- 127 // Kernel storage 128 KERNEL_STORAGE(cluster, mainCluster); 129 KERNEL_STORAGE(processor, mainProcessor); 130 KERNEL_STORAGE($thread, mainThread); 131 KERNEL_STORAGE(__stack_t, mainThreadCtx); 132 KERNEL_STORAGE(__scheduler_RWLock_t, __scheduler_lock); 133 #if !defined(__CFA_NO_STATISTICS__) 134 KERNEL_STORAGE(__stats_t, mainProcStats); 135 #endif 136 137 cluster * mainCluster; 138 processor * mainProcessor; 139 $thread * mainThread; 140 __scheduler_RWLock_t * __scheduler_lock; 141 142 extern "C" { 143 struct { __dllist_t(cluster) list; __spinlock_t lock; } __cfa_dbg_global_clusters; 144 } 145 146 size_t __page_size = 0; 147 148 //----------------------------------------------------------------------------- 149 // Global state 150 thread_local struct KernelThreadData kernelTLS __attribute__ ((tls_model ( "initial-exec" ))) @= { 151 NULL, // cannot use 0p 152 NULL, 153 NULL, 154 { 1, false, false }, 155 }; 156 157 //----------------------------------------------------------------------------- 158 // Struct to steal stack 159 struct current_stack_info_t { 160 __stack_t * storage; // pointer to stack object 161 void * base; // base of stack 162 void * limit; // stack grows towards stack limit 163 void * context; // address of cfa_context_t 164 }; 165 166 void ?{}( current_stack_info_t & this ) { 167 __stack_context_t ctx; 168 CtxGet( ctx ); 169 this.base = ctx.FP; 170 171 rlimit r; 172 getrlimit( RLIMIT_STACK, &r); 173 size_t size = r.rlim_cur; 174 175 this.limit = (void *)(((intptr_t)this.base) - size); 176 this.context = &storage_mainThreadCtx; 177 } 178 179 //----------------------------------------------------------------------------- 180 // Main thread construction 181 182 void ?{}( $coroutine & this, current_stack_info_t * info) with( this ) { 183 stack.storage = info->storage; 184 with(*stack.storage) { 185 limit = info->limit; 186 base = info->base; 187 } 188 __attribute__((may_alias)) intptr_t * istorage = (intptr_t*) &stack.storage; 189 *istorage |= 0x1; 190 name = "Main Thread"; 191 state = Start; 192 starter = 0p; 193 last = 0p; 194 cancellation = 0p; 195 } 196 197 void ?{}( $thread & this, current_stack_info_t * info) with( this ) { 198 ticket = 1; 199 state = Start; 200 self_cor{ info }; 201 curr_cor = &self_cor; 202 curr_cluster = mainCluster; 203 self_mon.owner = &this; 204 self_mon.recursion = 1; 205 self_mon_p = &self_mon; 206 link.next = 0p; 207 link.prev = 0p; 208 209 node.next = 0p; 210 node.prev = 0p; 211 doregister(curr_cluster, this); 212 213 monitors{ &self_mon_p, 1, (fptr_t)0 }; 214 } 215 216 //----------------------------------------------------------------------------- 217 // Processor coroutine 218 void ?{}(processorCtx_t & this) { 219 220 } 221 222 // Construct the processor context of non-main processors 223 static void ?{}(processorCtx_t & this, processor * proc, current_stack_info_t * info) { 224 (this.__cor){ info }; 225 this.proc = proc; 226 } 227 228 static void * __invoke_processor(void * arg); 229 230 static init(processor & this, const char name[], cluster & _cltr) with( this ) { 231 this.name = name; 232 this.cltr = &_cltr; 233 id = -1u; 234 destroyer = 0p; 235 do_terminate = false; 236 preemption_alarm = 0p; 237 pending_preemption = false; 238 239 #if !defined(__CFA_NO_STATISTICS__) 240 print_stats = 0; 241 print_halts = false; 242 #endif 243 244 int target = __atomic_add_fetch( &cltr->nprocessors, 1u, __ATOMIC_SEQ_CST ); 245 246 id = doregister((__processor_id_t*)&this); 247 248 // Lock the RWlock so no-one pushes/pops while we are changing the queue 249 uint_fast32_t last_size = ready_mutate_lock(); 250 251 // Adjust the ready queue size 252 ready_queue_grow( cltr, target ); 253 254 // Unlock the RWlock 255 ready_mutate_unlock( last_size ); 256 257 __cfadbg_print_safe(runtime_core, "Kernel : core %p created\n", &this); 258 } 259 260 // Not a ctor, it just preps the destruction but should not destroy members 261 void deinit(processor & this) { 262 263 int target = __atomic_sub_fetch( &this.cltr->nprocessors, 1u, __ATOMIC_SEQ_CST ); 264 265 // Lock the RWlock so no-one pushes/pops while we are changing the queue 266 uint_fast32_t last_size = ready_mutate_lock(); 267 268 // Adjust the ready queue size 269 ready_queue_shrink( this.cltr, target ); 270 271 // Make sure we aren't on the idle queue 272 unsafe_remove( this.cltr->idles, &this ); 273 274 // Unlock the RWlock 275 ready_mutate_unlock( last_size ); 276 277 // Finally we don't need the read_lock any more 278 unregister((__processor_id_t*)&this); 279 } 280 281 void ?{}(processor & this, const char name[], cluster & _cltr) { 282 ( this.idle ){}; 283 ( this.terminated ){ 0 }; 284 ( this.runner ){}; 285 init( this, name, _cltr ); 286 287 __cfadbg_print_safe(runtime_core, "Kernel : Starting core %p\n", &this); 288 289 this.stack = __create_pthread( &this.kernel_thread, __invoke_processor, (void *)&this ); 290 291 } 292 293 void ^?{}(processor & this) with( this ){ 294 if( ! __atomic_load_n(&do_terminate, __ATOMIC_ACQUIRE) ) { 295 __cfadbg_print_safe(runtime_core, "Kernel : core %p signaling termination\n", &this); 296 297 __atomic_store_n(&do_terminate, true, __ATOMIC_RELAXED); 298 __wake_proc( &this ); 299 300 P( terminated ); 301 verify( kernelTLS.this_processor != &this); 302 } 303 304 int err = pthread_join( kernel_thread, 0p ); 305 if( err != 0 ) abort("KERNEL ERROR: joining processor %p caused error %s\n", &this, strerror(err)); 306 307 free( this.stack ); 308 309 deinit( this ); 310 } 311 312 void ?{}(cluster & this, const char name[], Duration preemption_rate, unsigned io_flags) with( this ) { 313 this.name = name; 314 this.preemption_rate = preemption_rate; 315 this.nprocessors = 0; 316 ready_queue{}; 317 318 #if !defined(__CFA_NO_STATISTICS__) 319 print_stats = 0; 320 stats = alloc(); 321 __init_stats( stats ); 322 #endif 323 324 threads{ __get }; 325 326 doregister(this); 327 328 // Lock the RWlock so no-one pushes/pops while we are changing the queue 329 uint_fast32_t last_size = ready_mutate_lock(); 330 331 // Adjust the ready queue size 332 ready_queue_grow( &this, 0 ); 333 334 // Unlock the RWlock 335 ready_mutate_unlock( last_size ); 336 337 338 __kernel_io_startup( this, io_flags, &this == mainCluster ); 339 } 340 341 void ^?{}(cluster & this) { 342 __kernel_io_shutdown( this, &this == mainCluster ); 343 344 // Lock the RWlock so no-one pushes/pops while we are changing the queue 345 uint_fast32_t last_size = ready_mutate_lock(); 346 347 // Adjust the ready queue size 348 ready_queue_shrink( &this, 0 ); 349 350 // Unlock the RWlock 351 ready_mutate_unlock( last_size ); 352 353 #if !defined(__CFA_NO_STATISTICS__) 354 if( 0 != this.print_stats ) { 355 __print_stats( this.stats, this.print_stats, true, this.name, (void*)&this ); 356 } 357 free( this.stats ); 358 #endif 359 360 unregister(this); 361 } 92 bool __wake_proc(processor *); 362 93 363 94 //============================================================================================= … … 550 281 } 551 282 552 // KERNEL_ONLY553 // Context invoker for processors554 // This is the entry point for processors (kernel threads)555 // It effectively constructs a coroutine by stealing the pthread stack556 static void * __invoke_processor(void * arg) {557 #if !defined( __CFA_NO_STATISTICS__ )558 __stats_t local_stats;559 __init_stats( &local_stats );560 kernelTLS.this_stats = &local_stats;561 #endif562 563 processor * proc = (processor *) arg;564 kernelTLS.this_processor = proc;565 kernelTLS.this_thread = 0p;566 kernelTLS.preemption_state.[enabled, disable_count] = [false, 1];567 // SKULLDUGGERY: We want to create a context for the processor coroutine568 // which is needed for the 2-step context switch. However, there is no reason569 // to waste the perfectly valid stack create by pthread.570 current_stack_info_t info;571 __stack_t ctx;572 info.storage = &ctx;573 (proc->runner){ proc, &info };574 575 __cfaabi_dbg_print_safe("Coroutine : created stack %p\n", get_coroutine(proc->runner)->stack.storage);576 577 //Set global state578 kernelTLS.this_thread = 0p;579 580 //We now have a proper context from which to schedule threads581 __cfadbg_print_safe(runtime_core, "Kernel : core %p created (%p, %p)\n", proc, &proc->runner, &ctx);582 583 // SKULLDUGGERY: Since the coroutine doesn't have its own stack, we can't584 // resume it to start it like it normally would, it will just context switch585 // back to here. Instead directly call the main since we already are on the586 // appropriate stack.587 get_coroutine(proc->runner)->state = Active;588 main( proc->runner );589 get_coroutine(proc->runner)->state = Halted;590 591 // Main routine of the core returned, the core is now fully terminated592 __cfadbg_print_safe(runtime_core, "Kernel : core %p main ended (%p)\n", proc, &proc->runner);593 594 #if !defined(__CFA_NO_STATISTICS__)595 __tally_stats(proc->cltr->stats, &local_stats);596 if( 0 != proc->print_stats ) {597 __print_stats( &local_stats, proc->print_stats, true, proc->name, (void*)proc );598 }599 #endif600 601 return 0p;602 }603 604 static void Abort( int ret, const char func[] ) {605 if ( ret ) { // pthread routines return errno values606 abort( "%s : internal error, error(%d) %s.", func, ret, strerror( ret ) );607 } // if608 } // Abort609 610 void * __create_pthread( pthread_t * pthread, void * (*start)(void *), void * arg ) {611 pthread_attr_t attr;612 613 Abort( pthread_attr_init( &attr ), "pthread_attr_init" ); // initialize attribute614 615 size_t stacksize;616 // default stack size, normally defined by shell limit617 Abort( pthread_attr_getstacksize( &attr, &stacksize ), "pthread_attr_getstacksize" );618 assert( stacksize >= PTHREAD_STACK_MIN );619 620 void * stack;621 __cfaabi_dbg_debug_do(622 stack = memalign( __page_size, stacksize + __page_size );623 // pthread has no mechanism to create the guard page in user supplied stack.624 if ( mprotect( stack, __page_size, PROT_NONE ) == -1 ) {625 abort( "mprotect : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );626 } // if627 );628 __cfaabi_dbg_no_debug_do(629 stack = malloc( stacksize );630 );631 632 Abort( pthread_attr_setstack( &attr, stack, stacksize ), "pthread_attr_setstack" );633 634 Abort( pthread_create( pthread, &attr, start, arg ), "pthread_create" );635 return stack;636 }637 638 // KERNEL_ONLY639 static void __kernel_first_resume( processor * this ) {640 $thread * src = mainThread;641 $coroutine * dst = get_coroutine(this->runner);642 643 verify( ! kernelTLS.preemption_state.enabled );644 645 kernelTLS.this_thread->curr_cor = dst;646 __stack_prepare( &dst->stack, 65000 );647 __cfactx_start(main, dst, this->runner, __cfactx_invoke_coroutine);648 649 verify( ! kernelTLS.preemption_state.enabled );650 651 dst->last = &src->self_cor;652 dst->starter = dst->starter ? dst->starter : &src->self_cor;653 654 // make sure the current state is still correct655 /* paranoid */ verify(src->state == Ready);656 657 // context switch to specified coroutine658 verify( dst->context.SP );659 __cfactx_switch( &src->context, &dst->context );660 // when __cfactx_switch returns we are back in the src coroutine661 662 mainThread->curr_cor = &mainThread->self_cor;663 664 // make sure the current state has been update665 /* paranoid */ verify(src->state == Active);666 667 verify( ! kernelTLS.preemption_state.enabled );668 }669 670 // KERNEL_ONLY671 static void __kernel_last_resume( processor * this ) {672 $coroutine * src = &mainThread->self_cor;673 $coroutine * dst = get_coroutine(this->runner);674 675 verify( ! kernelTLS.preemption_state.enabled );676 verify( dst->starter == src );677 verify( dst->context.SP );678 679 // SKULLDUGGERY in debug the processors check that the680 // stack is still within the limit of the stack limits after running a thread.681 // that check doesn't make sense if we context switch to the processor using the682 // coroutine semantics. Since this is a special case, use the current context683 // info to populate these fields.684 __cfaabi_dbg_debug_do(685 __stack_context_t ctx;686 CtxGet( ctx );687 mainThread->context.SP = ctx.SP;688 mainThread->context.FP = ctx.FP;689 )690 691 // context switch to the processor692 __cfactx_switch( &src->context, &dst->context );693 }694 695 283 //----------------------------------------------------------------------------- 696 284 // Scheduler routines … … 834 422 835 423 //============================================================================================= 836 // Kernel Setup logic837 //=============================================================================================838 //-----------------------------------------------------------------------------839 // Kernel boot procedures840 static void __kernel_startup(void) {841 verify( ! kernelTLS.preemption_state.enabled );842 __cfadbg_print_safe(runtime_core, "Kernel : Starting\n");843 844 __page_size = sysconf( _SC_PAGESIZE );845 846 __cfa_dbg_global_clusters.list{ __get };847 __cfa_dbg_global_clusters.lock{};848 849 // Initialize the global scheduler lock850 __scheduler_lock = (__scheduler_RWLock_t*)&storage___scheduler_lock;851 (*__scheduler_lock){};852 853 // Initialize the main cluster854 mainCluster = (cluster *)&storage_mainCluster;855 (*mainCluster){"Main Cluster"};856 857 __cfadbg_print_safe(runtime_core, "Kernel : Main cluster ready\n");858 859 // Start by initializing the main thread860 // SKULLDUGGERY: the mainThread steals the process main thread861 // which will then be scheduled by the mainProcessor normally862 mainThread = ($thread *)&storage_mainThread;863 current_stack_info_t info;864 info.storage = (__stack_t*)&storage_mainThreadCtx;865 (*mainThread){ &info };866 867 __cfadbg_print_safe(runtime_core, "Kernel : Main thread ready\n");868 869 870 871 // Construct the processor context of the main processor872 void ?{}(processorCtx_t & this, processor * proc) {873 (this.__cor){ "Processor" };874 this.__cor.starter = 0p;875 this.proc = proc;876 }877 878 void ?{}(processor & this) with( this ) {879 ( this.idle ){};880 ( this.terminated ){ 0 };881 ( this.runner ){};882 init( this, "Main Processor", *mainCluster );883 kernel_thread = pthread_self();884 885 runner{ &this };886 __cfadbg_print_safe(runtime_core, "Kernel : constructed main processor context %p\n", &runner);887 }888 889 // Initialize the main processor and the main processor ctx890 // (the coroutine that contains the processing control flow)891 mainProcessor = (processor *)&storage_mainProcessor;892 (*mainProcessor){};893 894 //initialize the global state variables895 kernelTLS.this_processor = mainProcessor;896 kernelTLS.this_thread = mainThread;897 898 #if !defined( __CFA_NO_STATISTICS__ )899 kernelTLS.this_stats = (__stats_t *)& storage_mainProcStats;900 __init_stats( kernelTLS.this_stats );901 #endif902 903 // Enable preemption904 kernel_start_preemption();905 906 // Add the main thread to the ready queue907 // once resume is called on mainProcessor->runner the mainThread needs to be scheduled like any normal thread908 __schedule_thread((__processor_id_t *)mainProcessor, mainThread);909 910 // SKULLDUGGERY: Force a context switch to the main processor to set the main thread's context to the current UNIX911 // context. Hence, the main thread does not begin through __cfactx_invoke_thread, like all other threads. The trick here is that912 // mainThread is on the ready queue when this call is made.913 __kernel_first_resume( kernelTLS.this_processor );914 915 916 // THE SYSTEM IS NOW COMPLETELY RUNNING917 918 919 // Now that the system is up, finish creating systems that need threading920 __kernel_io_finish_start( *mainCluster );921 922 923 __cfadbg_print_safe(runtime_core, "Kernel : Started\n--------------------------------------------------\n\n");924 925 verify( ! kernelTLS.preemption_state.enabled );926 enable_interrupts( __cfaabi_dbg_ctx );927 verify( TL_GET( preemption_state.enabled ) );928 }929 930 static void __kernel_shutdown(void) {931 //Before we start shutting things down, wait for systems that need threading to shutdown932 __kernel_io_prepare_stop( *mainCluster );933 934 /* paranoid */ verify( TL_GET( preemption_state.enabled ) );935 disable_interrupts();936 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );937 938 __cfadbg_print_safe(runtime_core, "\n--------------------------------------------------\nKernel : Shutting down\n");939 940 // SKULLDUGGERY: Notify the mainProcessor it needs to terminates.941 // When its coroutine terminates, it return control to the mainThread942 // which is currently here943 __atomic_store_n(&mainProcessor->do_terminate, true, __ATOMIC_RELEASE);944 __kernel_last_resume( kernelTLS.this_processor );945 mainThread->self_cor.state = Halted;946 947 // THE SYSTEM IS NOW COMPLETELY STOPPED948 949 // Disable preemption950 kernel_stop_preemption();951 952 // Destroy the main processor and its context in reverse order of construction953 // These were manually constructed so we need manually destroy them954 void ^?{}(processor & this) with( this ){955 deinit( this );956 957 /* paranoid */ verify( this.do_terminate == true );958 __cfaabi_dbg_print_safe("Kernel : destroyed main processor context %p\n", &runner);959 }960 961 ^(*mainProcessor){};962 963 // Final step, destroy the main thread since it is no longer needed964 965 // Since we provided a stack to this taxk it will not destroy anything966 /* paranoid */ verify(mainThread->self_cor.stack.storage == (__stack_t*)(((uintptr_t)&storage_mainThreadCtx)| 0x1));967 ^(*mainThread){};968 969 ^(*mainCluster){};970 971 ^(*__scheduler_lock){};972 973 ^(__cfa_dbg_global_clusters.list){};974 ^(__cfa_dbg_global_clusters.lock){};975 976 __cfadbg_print_safe(runtime_core, "Kernel : Shutdown complete\n");977 }978 979 //=============================================================================================980 424 // Kernel Idle Sleep 981 425 //============================================================================================= … … 997 441 998 442 // Unconditionnaly wake a thread 999 staticbool __wake_proc(processor * this) {443 bool __wake_proc(processor * this) { 1000 444 __cfadbg_print_safe(runtime_core, "Kernel : waking Processor %p\n", this); 1001 445 … … 1075 519 1076 520 void kernel_abort_msg( void * kernel_data, char * abort_text, int abort_text_size ) { 1077 $thread * thrd = kernel_data;521 $thread * thrd = ( $thread * ) kernel_data; 1078 522 1079 523 if(thrd) { … … 1170 614 1171 615 return thrd != 0p; 1172 }1173 1174 //-----------------------------------------------------------------------------1175 // Global Queues1176 void doregister( cluster & cltr ) {1177 lock ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);1178 push_front( __cfa_dbg_global_clusters.list, cltr );1179 unlock ( __cfa_dbg_global_clusters.lock );1180 }1181 1182 void unregister( cluster & cltr ) {1183 lock ( __cfa_dbg_global_clusters.lock __cfaabi_dbg_ctx2);1184 remove( __cfa_dbg_global_clusters.list, cltr );1185 unlock( __cfa_dbg_global_clusters.lock );1186 }1187 1188 void doregister( cluster * cltr, $thread & thrd ) {1189 lock (cltr->thread_list_lock __cfaabi_dbg_ctx2);1190 cltr->nthreads += 1;1191 push_front(cltr->threads, thrd);1192 unlock (cltr->thread_list_lock);1193 }1194 1195 void unregister( cluster * cltr, $thread & thrd ) {1196 lock (cltr->thread_list_lock __cfaabi_dbg_ctx2);1197 remove(cltr->threads, thrd );1198 cltr->nthreads -= 1;1199 unlock(cltr->thread_list_lock);1200 616 } 1201 617 -
libcfa/src/concurrency/kernel.hfa
r3f850d7 r53ee27e 16 16 #pragma once 17 17 18 #include <stdbool.h>19 #include <stdint.h>20 21 18 #include "invoke.h" 22 19 #include "time_t.hfa" … … 26 23 27 24 extern "C" { 28 #include <pthread.h> 29 #include <semaphore.h> 25 #include <bits/pthreadtypes.h> 30 26 } 31 27 … … 129 125 struct __io_data; 130 126 131 #define CFA_CLUSTER_IO_POLLER_USER_THREAD (1 << 0) // 0x01 132 #define CFA_CLUSTER_IO_POLLER_THREAD_SUBMITS (1 << 1) // 0x02 133 #define CFA_CLUSTER_IO_EAGER_SUBMITS (1 << 2) // 0x04 134 #define CFA_CLUSTER_IO_KERNEL_POLL_SUBMITS (1 << 3) // 0x08 135 #define CFA_CLUSTER_IO_KERNEL_POLL_COMPLETES (1 << 4) // 0x10 136 #define CFA_CLUSTER_IO_BUFFLEN_OFFSET 16 137 127 // IO poller user-thread 128 // Not using the "thread" keyword because we want to control 129 // more carefully when to start/stop it 130 struct $io_ctx_thread { 131 struct __io_data * ring; 132 single_sem sem; 133 volatile bool done; 134 $thread self; 135 }; 136 137 138 struct io_context { 139 $io_ctx_thread thrd; 140 }; 141 142 struct io_context_params { 143 int num_entries; 144 int num_ready; 145 int submit_aff; 146 bool eager_submits:1; 147 bool poller_submits:1; 148 bool poll_submit:1; 149 bool poll_complete:1; 150 }; 151 152 void ?{}(io_context_params & this); 153 154 void ?{}(io_context & this, struct cluster & cl); 155 void ?{}(io_context & this, struct cluster & cl, const io_context_params & params); 156 void ^?{}(io_context & this); 157 158 struct io_cancellation { 159 uint32_t target; 160 }; 161 162 static inline void ?{}(io_cancellation & this) { this.target = -1u; } 163 static inline void ^?{}(io_cancellation & this) {} 164 bool cancel(io_cancellation & this); 138 165 139 166 //----------------------------------------------------------------------------- … … 206 233 } node; 207 234 208 struct __io_data * io; 235 struct { 236 io_context * ctxs; 237 unsigned cnt; 238 } io; 209 239 210 240 #if !defined(__CFA_NO_STATISTICS__) … … 215 245 extern Duration default_preemption(); 216 246 217 void ?{} (cluster & this, const char name[], Duration preemption_rate, unsigned flags);247 void ?{} (cluster & this, const char name[], Duration preemption_rate, unsigned num_io, const io_context_params & io_params); 218 248 void ^?{}(cluster & this); 219 249 220 static inline void ?{} (cluster & this) { this{"Anonymous Cluster", default_preemption(), 0}; } 221 static inline void ?{} (cluster & this, Duration preemption_rate) { this{"Anonymous Cluster", preemption_rate, 0}; } 222 static inline void ?{} (cluster & this, const char name[]) { this{name, default_preemption(), 0}; } 223 static inline void ?{} (cluster & this, unsigned flags) { this{"Anonymous Cluster", default_preemption(), flags}; } 224 static inline void ?{} (cluster & this, Duration preemption_rate, unsigned flags) { this{"Anonymous Cluster", preemption_rate, flags}; } 225 static inline void ?{} (cluster & this, const char name[], unsigned flags) { this{name, default_preemption(), flags}; } 250 static inline void ?{} (cluster & this) { io_context_params default_params; this{"Anonymous Cluster", default_preemption(), 1, default_params}; } 251 static inline void ?{} (cluster & this, Duration preemption_rate) { io_context_params default_params; this{"Anonymous Cluster", preemption_rate, 1, default_params}; } 252 static inline void ?{} (cluster & this, const char name[]) { io_context_params default_params; this{name, default_preemption(), 1, default_params}; } 253 static inline void ?{} (cluster & this, unsigned num_io) { io_context_params default_params; this{"Anonymous Cluster", default_preemption(), num_io, default_params}; } 254 static inline void ?{} (cluster & this, Duration preemption_rate, unsigned num_io) { io_context_params default_params; this{"Anonymous Cluster", preemption_rate, num_io, default_params}; } 255 static inline void ?{} (cluster & this, const char name[], unsigned num_io) { io_context_params default_params; this{name, default_preemption(), num_io, default_params}; } 256 static inline void ?{} (cluster & this, const io_context_params & io_params) { this{"Anonymous Cluster", default_preemption(), 1, io_params}; } 257 static inline void ?{} (cluster & this, Duration preemption_rate, const io_context_params & io_params) { this{"Anonymous Cluster", preemption_rate, 1, io_params}; } 258 static inline void ?{} (cluster & this, const char name[], const io_context_params & io_params) { this{name, default_preemption(), 1, io_params}; } 259 static inline void ?{} (cluster & this, unsigned num_io, const io_context_params & io_params) { this{"Anonymous Cluster", default_preemption(), num_io, io_params}; } 260 static inline void ?{} (cluster & this, Duration preemption_rate, unsigned num_io, const io_context_params & io_params) { this{"Anonymous Cluster", preemption_rate, num_io, io_params}; } 261 static inline void ?{} (cluster & this, const char name[], unsigned num_io, const io_context_params & io_params) { this{name, default_preemption(), num_io, io_params}; } 226 262 227 263 static inline [cluster *&, cluster *& ] __get( cluster & this ) __attribute__((const)) { return this.node.[next, prev]; } -
libcfa/src/concurrency/kernel_private.hfa
r3f850d7 r53ee27e 22 22 #include "stats.hfa" 23 23 24 #include "bits/random.hfa"25 26 27 24 //----------------------------------------------------------------------------- 28 25 // Scheduler … … 53 50 54 51 55 struct event_kernel_t {56 alarm_list_t alarms;57 __spinlock_t lock;58 };59 60 extern event_kernel_t * event_kernel;61 62 struct __cfa_kernel_preemption_state_t {63 bool enabled;64 bool in_progress;65 unsigned short disable_count;66 };67 68 extern volatile thread_local __cfa_kernel_preemption_state_t preemption_state __attribute__ ((tls_model ( "initial-exec" )));69 70 52 extern cluster * mainCluster; 71 53 … … 84 66 void __unpark( struct __processor_id_t *, $thread * thrd __cfaabi_dbg_ctx_param2 ); 85 67 86 //----------------------------------------------------------------------------- 87 // I/O 88 void __kernel_io_startup ( cluster &, unsigned, bool ); 89 void __kernel_io_finish_start( cluster & ); 90 void __kernel_io_prepare_stop( cluster & ); 91 void __kernel_io_shutdown ( cluster &, bool ); 68 static inline bool __post(single_sem & this, struct __processor_id_t * id) { 69 for() { 70 struct $thread * expected = this.ptr; 71 if(expected == 1p) return false; 72 if(expected == 0p) { 73 if(__atomic_compare_exchange_n(&this.ptr, &expected, 1p, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) { 74 return false; 75 } 76 } 77 else { 78 if(__atomic_compare_exchange_n(&this.ptr, &expected, 0p, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) { 79 __unpark( id, expected __cfaabi_dbg_ctx2 ); 80 return true; 81 } 82 } 83 } 84 } 92 85 93 86 //----------------------------------------------------------------------------- 94 87 // Utils 95 #define KERNEL_STORAGE(T,X) __attribute((aligned(__alignof__(T)))) static char storage_##X[sizeof(T)]96 97 static inline uint64_t __tls_rand() {98 #if defined(__SIZEOF_INT128__)99 return __lehmer64( kernelTLS.rand_seed );100 #else101 return __xorshift64( kernelTLS.rand_seed );102 #endif103 }104 105 106 void doregister( struct cluster & cltr );107 void unregister( struct cluster & cltr );108 109 88 void doregister( struct cluster * cltr, struct $thread & thrd ); 110 89 void unregister( struct cluster * cltr, struct $thread & thrd ); 90 91 //----------------------------------------------------------------------------- 92 // I/O 93 void ^?{}(io_context & this, bool ); 111 94 112 95 //======================================================================= … … 280 263 void ready_queue_shrink(struct cluster * cltr, int target); 281 264 282 //-----------------------------------------------------------------------283 // IO user data284 struct __io_user_data_t {285 int32_t result;286 $thread * thrd;287 };288 289 //-----------------------------------------------------------------------290 // Statics call at the end of each thread to register statistics291 #if !defined(__CFA_NO_STATISTICS__)292 static inline struct __stats_t * __tls_stats() {293 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );294 /* paranoid */ verify( kernelTLS.this_stats );295 return kernelTLS.this_stats;296 }297 298 #define __STATS__(in_kernel, ...) { \299 if( !(in_kernel) ) disable_interrupts(); \300 with( *__tls_stats() ) { \301 __VA_ARGS__ \302 } \303 if( !(in_kernel) ) enable_interrupts( __cfaabi_dbg_ctx ); \304 }305 #else306 #define __STATS__(in_kernel, ...)307 #endif308 265 309 266 // Local Variables: // -
libcfa/src/concurrency/preemption.cfa
r3f850d7 r53ee27e 26 26 27 27 #include "bits/signal.hfa" 28 #include "kernel_private.hfa" 28 29 29 30 #if !defined(__CFA_DEFAULT_PREEMPTION__) … … 293 294 // Startup routine to activate preemption 294 295 // Called from kernel_startup 295 void kernel_start_preemption() {296 void __kernel_alarm_startup() { 296 297 __cfaabi_dbg_print_safe( "Kernel : Starting preemption\n" ); 297 298 … … 315 316 // Shutdown routine to deactivate preemption 316 317 // Called from kernel_shutdown 317 void kernel_stop_preemption() {318 void __kernel_alarm_shutdown() { 318 319 __cfaabi_dbg_print_safe( "Kernel : Preemption stopping\n" ); 319 320 … … 481 482 sigset_t oldset; 482 483 int ret; 483 ret = pthread_sigmask(0, 0p, &oldset);484 ret = pthread_sigmask(0, ( const sigset_t * ) 0p, &oldset); // workaround trac#208: cast should be unnecessary 484 485 if(ret != 0) { abort("ERROR sigprocmask returned %d", ret); } 485 486 -
libcfa/src/concurrency/preemption.hfa
r3f850d7 r53ee27e 16 16 #pragma once 17 17 18 #include "bits/locks.hfa" 18 19 #include "alarm.hfa" 19 #include "kernel_private.hfa"20 20 21 void kernel_start_preemption(); 22 void kernel_stop_preemption(); 21 struct event_kernel_t { 22 alarm_list_t alarms; 23 __spinlock_t lock; 24 }; 25 26 extern event_kernel_t * event_kernel; 27 23 28 void update_preemption( processor * this, Duration duration ); 24 29 -
libcfa/src/concurrency/thread.hfa
r3f850d7 r53ee27e 84 84 85 85 //----------------------------------------------------------------------------- 86 // Thread getters87 static inline struct $thread * active_thread () { return TL_GET( this_thread ); }88 89 //-----------------------------------------------------------------------------90 86 // Scheduler API 91 87 … … 106 102 bool force_yield( enum __Preemption_Reason ); 107 103 108 static inline void yield() {109 force_yield(__MANUAL_PREEMPTION);110 }111 112 // Yield: yield N times113 static inline void yield( unsigned times ) {114 for( times ) {115 yield();116 }117 }118 119 104 //---------- 120 105 // sleep: force thread to block and be rescheduled after Duration duration -
libcfa/src/containers/list.hfa
r3f850d7 r53ee27e 22 22 \ 23 23 static inline NODE& $tempcv_e2n(ELEM &node) { \ 24 return node; \24 return ( NODE & ) node; \ 25 25 } \ 26 26 \ … … 187 187 $next_link(singleton_to_insert) = $next_link(list_pos); 188 188 if ($next_link(list_pos).is_terminator) { 189 dlist(Tnode, Telem) *list = $next_link(list_pos).terminator;189 dlist(Tnode, Telem) *list = ( dlist(Tnode, Telem) * ) $next_link(list_pos).terminator; 190 190 $dlinks(Telem) *list_links = & list->$links; 191 191 $mgd_link(Telem) *list_last = & list_links->prev; … … 210 210 $prev_link(singleton_to_insert) = $prev_link(list_pos); 211 211 if ($prev_link(list_pos).is_terminator) { 212 dlist(Tnode, Telem) *list = $prev_link(list_pos).terminator;212 dlist(Tnode, Telem) *list = ( dlist(Tnode, Telem) * ) $prev_link(list_pos).terminator; 213 213 $dlinks(Telem) *list_links = & list->$links; 214 214 $mgd_link(Telem) *list_first = & list_links->next; … … 275 275 276 276 if ( $prev_link(list_pos).is_terminator ) { 277 dlist(Tnode, Telem) * tgt_before = $prev_link(list_pos).terminator;277 dlist(Tnode, Telem) * tgt_before = ( dlist(Tnode, Telem) * ) $prev_link(list_pos).terminator; 278 278 $dlinks(Telem) * links_before = & tgt_before->$links; 279 279 &incoming_from_prev = & links_before->next; … … 285 285 286 286 if ( $next_link(list_pos).is_terminator ) { 287 dlist(Tnode, Telem) * tgt_after = $next_link(list_pos).terminator;287 dlist(Tnode, Telem) * tgt_after = ( dlist(Tnode, Telem) * ) $next_link(list_pos).terminator; 288 288 $dlinks(Telem) * links_after = & tgt_after->$links; 289 289 &incoming_from_next = & links_after->prev; -
libcfa/src/exception.hfa
r3f850d7 r53ee27e 10 10 // Created On : Thu Apr 7 10:25:00 2020 11 11 // Last Modified By : Andrew Beach 12 // Last Modified On : Tue May 19 14:17:00 202013 // Update Count : 212 // Last Modified On : Tue Aug 4 16:22:00 2020 13 // Update Count : 3 14 14 // 15 15 … … 18 18 // ----------------------------------------------------------------------------------------------- 19 19 20 // All internals helper macros begin with an underscore. 21 #define _CLOSE(...) __VA_ARGS__ } 22 #define _GLUE2(left, right) left##right 23 #define _GLUE3(left, middle, right) left##middle##right 24 #define _EXC_DISPATCH(to, ...) to(__VA_ARGS__,__cfaehm_base_exception_t,) 25 26 // FWD_TRIVIAL_EXCEPTION(exception_name); 20 // TRIVIAL_EXCEPTION_DECLARATION(exception_name); 27 21 // Declare a trivial exception, one that adds no fields or features. 28 22 // This will make the exception visible and may go in a .hfa or .cfa file. 29 #define FWD_TRIVIAL_EXCEPTION(...) _EXC_DISPATCH(_FWD_TRIVIAL_EXCEPTION, __VA_ARGS__) 30 // INST_TRIVIAL_EXCEPTION(exception_name); 23 #define TRIVIAL_EXCEPTION_DECLARATION(...) \ 24 _EXC_DISPATCH(_TRIVIAL_EXCEPTION_DECLARATION, __VA_ARGS__) 25 26 // TRIVIAL_EXCEPTION_INSTANCE(exception_name); 31 27 // Create the trival exception. This must be used exactly once and should be used in a .cfa file, 32 28 // as it creates the unique instance of the virtual table. 33 #define INST_TRIVIAL_EXCEPTION(...) _EXC_DISPATCH(_INST_TRIVIAL_EXCEPTION, __VA_ARGS__) 29 #define TRIVIAL_EXCEPTION_INSTANCE(...) _EXC_DISPATCH(_TRIVIAL_EXCEPTION_INSTANCE, __VA_ARGS__) 30 34 31 // TRIVIAL_EXCEPTION(exception_name[, parent_name]); 35 32 // Does both of the above, a short hand if the exception is only used in one .cfa file. … … 37 34 // base exception. This feature may be removed or changed. 38 35 #define TRIVIAL_EXCEPTION(...) \ 39 _EXC_DISPATCH(_FWD_TRIVIAL_EXCEPTION, __VA_ARGS__); \ 40 _EXC_DISPATCH(_INST_TRIVIAL_EXCEPTION, __VA_ARGS__) 41 #define _FWD_TRIVIAL_EXCEPTION(exception_name, parent_name, ...) \ 36 _EXC_DISPATCH(_TRIVIAL_EXCEPTION_DECLARATION, __VA_ARGS__); \ 37 _EXC_DISPATCH(_TRIVIAL_EXCEPTION_INSTANCE, __VA_ARGS__) 38 39 // FORALL_TRIVIAL_EXCEPTION(exception_name, (assertions...), (parameters...)); 40 // Forward declare a polymorphic but otherwise trivial exception type. You must provide the entire 41 // assertion list (exactly what would go in the forall clause) and parameters list (only the 42 // parameter names from the assertion list, same order and comma seperated). This should be 43 // visible where ever use the exception. This just generates the polymorphic framework, see 44 // POLY_VTABLE_DECLARATION to allow instantiations. 45 #define FORALL_TRIVIAL_EXCEPTION(exception_name, assertions, parameters) \ 46 _FORALL_TRIVIAL_EXCEPTION(exception_name, __cfaehm_base_exception_t, assertions, parameters, ) 47 48 // FORALL_TRIVIAL_INSTANCE(exception_name, (assertions...), (parameters...)) 49 // Create the forall trivial exception. The assertion list and parameters must match. 50 // There must be exactly one use of this in a program for each exception type. This just 51 // generates the polymorphic framework, see POLY_VTABLE_INSTANCE to allow instantiations. 52 #define FORALL_TRIVIAL_INSTANCE(exception_name, assertions, parameters) \ 53 _FORALL_CTOR0_INSTANCE(exception_name, assertions, parameters) 54 55 // DATA_EXCEPTION(exception_name)(fields...); 56 // Forward declare an exception that adds fields but no features. The added fields go in the 57 // second argument list. The virtual table instance must be provided later (see VTABLE_INSTANCE). 58 #define DATA_EXCEPTION(...) _EXC_DISPATCH(_DATA_EXCEPTION, __VA_ARGS__) 59 60 // FORALL_DATA_EXCEPTION(exception_name, (assertions...), (parameters...))(fields...); 61 // Define a polymorphic exception that adds fields but no additional features. The assertion list 62 // and matching parameters must match. Then you can give the list of fields. This should be 63 // visible where ever you use the exception. This just generates the polymorphic framework, see 64 // POLY_VTABLE_DECLARATION to allow instantiations. 65 #define FORALL_DATA_EXCEPTION(exception_name, assertions, parameters) \ 66 _FORALL_DATA_EXCEPTION(exception_name, __cfaehm_base_exception_t, assertions, parameters, ) 67 68 // FORALL_DATA_INSTANCE(exception_name, (assertions...), (parameters...)) 69 // Create a polymorphic data exception. The assertion list and parameters must match. This should 70 // appear once in each program. This just generates the polymorphic framework, see 71 // POLY_VTABLE_INSTANCE to allow instantiations. 72 #define FORALL_DATA_INSTANCE(exception_name, assertions, parameters) \ 73 _FORALL_CTOR0_INSTANCE(exception_name, assertions, parameters) 74 75 // VTABLE_DECLARATION(exception_name)([new_features...]); 76 // Declare a virtual table type for an exception with exception_name. You may also add features 77 // (fields on the virtual table) by including them in the second list. 78 #define VTABLE_DECLARATION(...) _EXC_DISPATCH(_VTABLE_DECLARATION, __VA_ARGS__) 79 80 // VTABLE_INSTANCE(exception_name)(msg [, others...]); 81 // Create the instance of the virtual table. There must be exactly one instance of a virtual table 82 // for each exception type. This fills in most of the fields of the virtual table (uses ?=? and 83 // ^?{}) but you must provide the message function and any other fields added in the declaration. 84 #define VTABLE_INSTANCE(...) _EXC_DISPATCH(_VTABLE_INSTANCE, __VA_ARGS__) 85 86 // FORALL_VTABLE_DECLARATION(exception_name, (assertions...), (parameters...))([new_features...]); 87 // Declare a polymorphic virtual table type for an exception with exception_name, the given 88 // assertions and parameters. You may also add features (fields on the virtual table). This just 89 // generates the polymorphic framework, see POLY_VTABLE_DECLARATION to allow instantiations. 90 #define FORALL_VTABLE_DECLARATION(exception_name, assertions, parameters) \ 91 _FORALL_VTABLE_DECLARATION(exception_name, __cfaehm_base_exception_t, assertions, parameters, ) 92 93 // POLY_VTABLE_DECLARATION(exception_name, types...); 94 // Declares that an instantiation for this exception exists for the given types. This should be 95 // visible anywhere you use the instantiation of the exception is used. 96 #define POLY_VTABLE_DECLARATION(exception_name, ...) \ 97 void mark_exception(exception_name(__VA_ARGS__) *); \ 98 extern VTABLE_TYPE(exception_name)(__VA_ARGS__) VTABLE_NAME(exception_name) 99 100 // POLY_VTABLE_INSTANCE(exception_name, types...)(msg [, others...]); 101 // Creates an instantiation for the given exception for the given types. This should occur only 102 // once in the entire program. You must fill in all features, message and any others given in the 103 // initial declaration. 104 #define POLY_VTABLE_INSTANCE(exception_name, ...) \ 105 _POLY_VTABLE_INSTANCE(exception_name, __cfaehm_base_exception_t, __VA_ARGS__) 106 107 // VTABLE_TYPE(exception_name) | VTABLE_NAME(exception_name) 108 // Get the name of the vtable type or the name of the vtable instance for an exception type. 109 #define VTABLE_TYPE(exception_name) struct _GLUE2(exception_name,_vtable) 110 #define VTABLE_NAME(exception_name) _GLUE3(_,exception_name,_vtable_instance) 111 112 // VTABLE_FIELD(exception_name); 113 // FORALL_VTABLE_FIELD(exception_name, (parameters-or-types)); 114 // The declaration of the virtual table field. Should be the first declaration in a virtual type. 115 #define VTABLE_FIELD(exception_name) VTABLE_TYPE(exception_name) const * virtual_table 116 #define FORALL_VTABLE_FIELD(exception_name, parameters) \ 117 VTABLE_TYPE(exception_name) parameters const * virtual_table 118 119 // VTABLE_INIT(object_reference, exception_name); 120 // Sets a virtual table field on an object to the virtual table instance for the type. 121 #define VTABLE_INIT(this, exception_name) (this).virtual_table = &VTABLE_NAME(exception_name) 122 123 // VTABLE_ASSERTION(exception_name, (parameters...)) 124 // The assertion that there is an instantiation of the vtable for the exception and types. 125 #define VTABLE_ASSERTION(exception_name, parameters) \ 126 { VTABLE_TYPE(exception_name) parameters VTABLE_NAME(exception_name); } 127 128 // All internal helper macros begin with an underscore. 129 #define _CLOSE(...) __VA_ARGS__ } 130 #define _GLUE2(left, right) left##right 131 #define _GLUE3(left, middle, right) left##middle##right 132 #define _EXC_DISPATCH(to, ...) to(__VA_ARGS__,__cfaehm_base_exception_t,) 133 #define _UNPACK(...) __VA_ARGS__ 134 135 #define _TRIVIAL_EXCEPTION_DECLARATION(exception_name, parent_name, ...) \ 42 136 _VTABLE_DECLARATION(exception_name, parent_name)(); \ 43 137 struct exception_name { \ … … 46 140 void ?{}(exception_name & this); \ 47 141 const char * _GLUE2(exception_name,_msg)(exception_name * this) 48 #define _INST_TRIVIAL_EXCEPTION(exception_name, parent_name, ...) \ 142 143 #define _TRIVIAL_EXCEPTION_INSTANCE(exception_name, parent_name, ...) \ 49 144 void ?{}(exception_name & this) { \ 50 145 VTABLE_INIT(this, exception_name); \ … … 55 150 _VTABLE_INSTANCE(exception_name, parent_name,)(_GLUE2(exception_name,_msg)) 56 151 57 // DATA_EXCEPTION(exception_name)(fields...); 58 // Forward declare an exception that adds fields but no features. The added fields go in the 59 // second argument list. The virtual table instance must be provided later (see VTABLE_INSTANCE). 60 #define DATA_EXCEPTION(...) _EXC_DISPATCH(_DATA_EXCEPTION, __VA_ARGS__) 152 #define _FORALL_TRIVIAL_EXCEPTION(exception_name, parent_name, assertions, \ 153 parameters, parent_parameters) \ 154 _FORALL_VTABLE_DECLARATION(exception_name, parent_name, assertions, \ 155 parameters, parent_parameters)(); \ 156 forall assertions struct exception_name { \ 157 FORALL_VTABLE_FIELD(exception_name, parameters); \ 158 }; \ 159 _FORALL_CTOR0_DECLARATION(exception_name, assertions, parameters) 160 161 #define _FORALL_CTOR0_DECLARATION(exception_name, assertions, parameters) \ 162 forall(_UNPACK assertions | VTABLE_ASSERTION(exception_name, assertions, parameters) ) \ 163 /*| { VTABLE_TYPE(exception_name) parameters VTABLE_NAME(exception_name); } ) */ \ 164 void ?{}(exception_name parameters & this) 165 166 #define _FORALL_CTOR0_INSTANCE(exception_name, assertions, parameters) \ 167 _FORALL_CTOR0_DECLARATION(exception_name, assertions, parameters) { \ 168 VTABLE_INIT(this, exception_name); \ 169 } 170 61 171 #define _DATA_EXCEPTION(exception_name, parent_name, ...) \ 62 172 _VTABLE_DECLARATION(exception_name, parent_name)(); \ 63 struct exception_name { VTABLE_FIELD(exception_name); _CLOSE 64 65 // VTABLE_DECLARATION(exception_name)([new_features...]); 66 // Declare a virtual table type for an exception with exception_name. You may also add features 67 // (fields on the virtual table) by including them in the second list. 68 #define VTABLE_DECLARATION(...) _EXC_DISPATCH(_VTABLE_DECLARATION, __VA_ARGS__) 173 struct exception_name { \ 174 VTABLE_FIELD(exception_name); \ 175 _CLOSE 176 177 #define _FORALL_DATA_EXCEPTION(exception_name, parent_name, \ 178 assertions, parameters, parent_parameters) \ 179 _FORALL_VTABLE_DECLARATION(exception_name, parent_name, \ 180 assertions, parameters, parent_parameters)(); \ 181 _FORALL_CTOR0_DECLARATION(exception_name, assertions, parameters); \ 182 forall assertions struct exception_name { \ 183 FORALL_VTABLE_FIELD(exception_name, parameters); \ 184 _CLOSE 185 69 186 #define _VTABLE_DECLARATION(exception_name, parent_name, ...) \ 70 187 struct exception_name; \ … … 80 197 _CLOSE 81 198 82 // VTABLE_INSTANCE(exception_name)(msg [, others...]);83 // Create the instance of the virtual table. There must be exactly one instance of a virtual table84 // for each exception type. This fills in most of the fields of the virtual table (uses ?=? and85 // ^?{}) but you must provide the message function and any other fields added in the declaration.86 #define VTABLE_INSTANCE(...) _EXC_DISPATCH(_VTABLE_INSTANCE, __VA_ARGS__)87 199 #define _VTABLE_INSTANCE(exception_name, parent_name, ...) \ 88 200 void mark_exception(exception_name *) {} \ … … 95 207 _CLOSE 96 208 97 // VTABLE_TYPE(exception_name) | VTABLE_NAME(exception_name) 98 // Get the name of the vtable type or the name of the vtable instance for an exception type. 99 #define VTABLE_TYPE(exception_name) struct _GLUE2(exception_name,_vtable) 100 #define VTABLE_NAME(exception_name) _GLUE3(_,exception_name,_vtable_instance) 101 102 // VTABLE_FIELD(exception_name); 103 // The declaration of the virtual table field. Should be the first declaration in a virtual type. 104 #define VTABLE_FIELD(exception_name) VTABLE_TYPE(exception_name) const * virtual_table 105 106 // VTABLE_INIT(object_reference, exception_name); 107 // Sets a virtual table field on an object to the virtual table instance for the type. 108 #define VTABLE_INIT(this, exception_name) (this).virtual_table = &VTABLE_NAME(exception_name) 209 #define _FORALL_VTABLE_DECLARATION(exception_name, parent_name, assertions, \ 210 parameters, parent_parameters) \ 211 forall assertions struct exception_name; \ 212 forall assertions VTABLE_TYPE(exception_name) { \ 213 VTABLE_TYPE(parent_name) parent_parameters const * parent; \ 214 size_t size; \ 215 void (*copy)(exception_name parameters * this, exception_name parameters * other); \ 216 void (*free)(exception_name parameters & this); \ 217 const char * (*msg)(exception_name parameters * this); \ 218 _CLOSE 219 220 #define _POLY_VTABLE_INSTANCE(exception_name, parent_name, ...) \ 221 void mark_exception(exception_name(__VA_ARGS__) *) {} \ 222 void _GLUE2(exception_name,_copy)( \ 223 exception_name(__VA_ARGS__) * this, exception_name(__VA_ARGS__) * other) { \ 224 *this = *other; \ 225 } \ 226 VTABLE_TYPE(exception_name)(__VA_ARGS__) VTABLE_NAME(exception_name) @= { \ 227 &VTABLE_NAME(parent_name), sizeof(exception_name(__VA_ARGS__)), \ 228 _GLUE2(exception_name,_copy), ^?{}, \ 229 _CLOSE -
libcfa/src/heap.cfa
r3f850d7 r53ee27e 10 10 // Created On : Tue Dec 19 21:58:35 2017 11 11 // Last Modified By : Peter A. Buhr 12 // Last Modified On : Mon Jul 20 23:00:32 202013 // Update Count : 8 0812 // Last Modified On : Mon Aug 3 19:01:22 2020 13 // Update Count : 828 14 14 // 15 15 … … 222 222 223 223 // Bucket size must be multiple of 16. 224 // Powers of 2 are common allocation sizes, so make powers of 2 generate the minimum required size. 224 // Smaller multiples of 16 and powers of 2 are common allocation sizes, so make them generate the minimum required bucket size. 225 // malloc(0) returns 0p, so no bucket is necessary for 0 bytes returning an address that can be freed. 225 226 static const unsigned int bucketSizes[] @= { // different bucket sizes 226 16 , 32, 48, 64 + sizeof(HeapManager.Storage), // 4227 96 , 112, 128 + sizeof(HeapManager.Storage), // 3227 16 + sizeof(HeapManager.Storage), 32 + sizeof(HeapManager.Storage), 48 + sizeof(HeapManager.Storage), 64 + sizeof(HeapManager.Storage), // 4 228 96 + sizeof(HeapManager.Storage), 112 + sizeof(HeapManager.Storage), 128 + sizeof(HeapManager.Storage), // 3 228 229 160, 192, 224, 256 + sizeof(HeapManager.Storage), // 4 229 230 320, 384, 448, 512 + sizeof(HeapManager.Storage), // 4 … … 434 435 #endif // __CFA_DEBUG__ 435 436 header = realHeader( header ); // backup from fake to real header 437 } else { 438 alignment = 0; 436 439 } // if 437 440 } // fakeHeader … … 481 484 unlock( extlock ); 482 485 errno = ENOMEM; 483 return 0p; 486 // return 0p; 487 abort( "no memory" ); 484 488 } // if 485 489 #ifdef __STATISTICS__ … … 550 554 551 555 block = (HeapManager.Storage *)extend( tsize ); // mutual exclusion on call 552 if ( unlikely( block == 0p ) ) return 0p;556 // if ( unlikely( block == 0p ) ) return 0p; 553 557 #if BUCKETLOCK == SPINLOCK 554 558 } else { … … 746 750 747 751 static inline void * mallocNoStats( size_t size ) { // necessary for malloc statistics 748 //assert( heapManager.heapBegin != 0 ); 749 if ( unlikely( heapManager.heapBegin == 0p ) ) heapManager{}; // called before memory_startup ? 752 verify( heapManager.heapBegin != 0 ); // called before memory_startup ? 753 if ( size == 0 ) return 0p; // 0 BYTE ALLOCATION RETURNS NULL POINTER 754 750 755 #if __SIZEOF_POINTER__ == 8 751 756 verify( size < ((typeof(size_t))1 << 48) ); 752 757 #endif // __SIZEOF_POINTER__ == 8 753 void * addr = doMalloc( size ); 754 if ( unlikely( addr == 0p ) ) errno = ENOMEM; // POSIX 755 return addr; 758 return doMalloc( size ); 756 759 } // mallocNoStats 757 760 … … 759 762 static inline void * callocNoStats( size_t dim, size_t elemSize ) { 760 763 size_t size = dim * elemSize; 764 if ( size == 0 ) return 0p; // 0 BYTE ALLOCATION RETURNS NULL POINTER 761 765 char * addr = (char *)mallocNoStats( size ); 762 if ( unlikely( addr == 0p ) ) return 0p;763 766 764 767 HeapManager.Storage.Header * header; 765 768 HeapManager.FreeHeader * freeElem; 766 769 size_t bsize, alignment; 767 bool mapped __attribute__(( unused )) = headers( "calloc", addr, header, freeElem, bsize, alignment ); 770 #ifndef __CFA_DEBUG__ 771 bool mapped = 772 #endif // __CFA_DEBUG__ 773 headers( "calloc", addr, header, freeElem, bsize, alignment ); 768 774 #ifndef __CFA_DEBUG__ 769 775 // Mapped storage is zero filled, but in debug mode mapped memory is scrubbed in doMalloc, so it has to be reset to zero. 770 776 if ( ! mapped ) 771 777 #endif // __CFA_DEBUG__ 772 // Zero entire data space even when > than size => realloc without a new allocation and zero fill works. 773 // <-------00000000000000000000000000000000000000000000000000000> bsize (bucket size) 778 // <-------0000000000000000000000000000UUUUUUUUUUUUUUUUUUUUUUUUU> bsize (bucket size) U => undefined 774 779 // `-header`-addr `-size 775 memset( addr, '\0', bsize - sizeof(HeapManager.Storage) );// set to zeros780 memset( addr, '\0', size ); // set to zeros 776 781 777 782 header->kind.real.blockSize |= 2; // mark as zero filled … … 781 786 782 787 static inline void * memalignNoStats( size_t alignment, size_t size ) { // necessary for malloc statistics 788 if ( size == 0 ) return 0p; // 0 BYTE ALLOCATION RETURNS NULL POINTER 789 783 790 #ifdef __CFA_DEBUG__ 784 791 checkAlign( alignment ); // check alignment … … 798 805 // add sizeof(Storage) for fake header 799 806 char * addr = (char *)mallocNoStats( size + alignment - libAlign() + sizeof(HeapManager.Storage) ); 800 if ( unlikely( addr == 0p ) ) return addr;801 807 802 808 // address in the block of the "next" alignment address … … 805 811 // address of header from malloc 806 812 HeapManager.Storage.Header * realHeader = headerAddr( addr ); 813 realHeader->kind.real.size = size; // correct size to eliminate above alignment offset 807 814 // address of fake header * before* the alignment location 808 815 HeapManager.Storage.Header * fakeHeader = headerAddr( user ); … … 818 825 static inline void * cmemalignNoStats( size_t alignment, size_t dim, size_t elemSize ) { 819 826 size_t size = dim * elemSize; 827 if ( size == 0 ) return 0p; // 0 BYTE ALLOCATION RETURNS NULL POINTER 820 828 char * addr = (char *)memalignNoStats( alignment, size ); 821 if ( unlikely( addr == 0p ) ) return 0p; 829 822 830 HeapManager.Storage.Header * header; 823 831 HeapManager.FreeHeader * freeElem; … … 889 897 890 898 // If size is equal to 0, either NULL or a pointer suitable to be passed to free() is returned. 891 if ( unlikely( size == 0 ) ) { free( oaddr ); return mallocNoStats( size ); } // special cases899 if ( unlikely( size == 0 ) ) { free( oaddr ); return 0p; } // special cases 892 900 if ( unlikely( oaddr == 0p ) ) return mallocNoStats( size ); 893 901 … … 901 909 if ( oalign == 0 && size <= odsize && odsize <= size * 2 ) { // allow 50% wasted storage for smaller size 902 910 header->kind.real.blockSize &= -2; // no alignment and turn off 0 fill 903 if ( size != odsize ) header->kind.real.size = size;// reset allocation size911 header->kind.real.size = size; // reset allocation size 904 912 return oaddr; 905 913 } // if … … 907 915 // change size, DO NOT preserve STICKY PROPERTIES. 908 916 free( oaddr ); 909 void * naddr = mallocNoStats( size ); // create new area 910 return naddr; 917 return mallocNoStats( size ); // create new area 911 918 } // resize 912 919 … … 921 928 922 929 // If size is equal to 0, either NULL or a pointer suitable to be passed to free() is returned. 923 if ( unlikely( size == 0 ) ) { free( oaddr ); return mallocNoStats( size ); } // special cases930 if ( unlikely( size == 0 ) ) { free( oaddr ); return 0p; } // special cases 924 931 if ( unlikely( oaddr == 0p ) ) return mallocNoStats( size ); 925 932 … … 930 937 931 938 size_t odsize = dataStorage( bsize, oaddr, header ); // data storage available in bucket 932 if ( size <= odsize && odsize <= size * 2 ) { // allow up to 50% wasted storage in smaller size 933 if ( size != odsize ) header->kind.real.size = size; // reset allocation size 939 size_t osize = header->kind.real.size; // old allocation size 940 bool ozfill = (header->kind.real.blockSize & 2) != 0; // old allocation zero filled 941 if ( unlikely( size <= odsize ) && size > odsize / 2 ) { // allow up to 50% wasted storage 942 header->kind.real.size = size; // reset allocation size 943 if ( unlikely( ozfill ) && size > osize ) { // previous request zero fill and larger ? 944 memset( (char *)oaddr + osize, (int)'\0', size - osize ); // initialize added storage 945 } // if 934 946 return oaddr; 935 947 } // if … … 938 950 939 951 void * naddr; 940 if ( unlikely( oalign != 0 ) ) { // previous request memalign? 941 if ( unlikely( header->kind.real.blockSize & 2 ) ) { // previous request zero fill 942 naddr = cmemalignNoStats( oalign, 1, size ); // create new aligned area 943 } else { 944 naddr = memalignNoStats( oalign, size ); // create new aligned area 952 if ( likely( oalign == 0 ) ) { // previous request memalign? 953 naddr = mallocNoStats( size ); // create new area 954 } else { 955 naddr = memalignNoStats( oalign, size ); // create new aligned area 956 } // if 957 958 headers( "realloc", naddr, header, freeElem, bsize, oalign ); 959 memcpy( naddr, oaddr, MIN( osize, size ) ); // copy bytes 960 free( oaddr ); 961 962 if ( unlikely( ozfill ) ) { // previous request zero fill ? 963 header->kind.real.blockSize |= 2; // mark new request as zero filled 964 if ( size > osize ) { // previous request larger ? 965 memset( (char *)naddr + osize, (int)'\0', size - osize ); // initialize added storage 945 966 } // if 946 } else { 947 if ( unlikely( header->kind.real.blockSize & 2 ) ) { // previous request zero fill 948 naddr = callocNoStats( 1, size ); // create new area 949 } else { 950 naddr = mallocNoStats( size ); // create new area 951 } // if 952 } // if 953 if ( unlikely( naddr == 0p ) ) return 0p; 954 955 headers( "realloc", naddr, header, freeElem, bsize, oalign ); 956 size_t ndsize = dataStorage( bsize, naddr, header ); // data storage avilable in bucket 957 // To preserve prior fill, the entire bucket must be copied versus the size. 958 memcpy( naddr, oaddr, MIN( odsize, ndsize ) ); // copy bytes 959 free( oaddr ); 967 } // if 960 968 return naddr; 961 969 } // realloc … … 1007 1015 if ( alignment < sizeof(void *) || ! libPow2( alignment ) ) return EINVAL; // check alignment 1008 1016 * memptr = memalign( alignment, size ); 1009 if ( unlikely( * memptr == 0p ) ) return ENOMEM;1010 1017 return 0; 1011 1018 } // posix_memalign … … 1205 1212 1206 1213 // If size is equal to 0, either NULL or a pointer suitable to be passed to free() is returned. 1207 if ( unlikely( size == 0 ) ) { free( oaddr ); return memalignNoStats( nalign, size ); } // special cases1214 if ( unlikely( size == 0 ) ) { free( oaddr ); return 0p; } // special cases 1208 1215 if ( unlikely( oaddr == 0p ) ) return memalignNoStats( nalign, size ); 1209 1210 1216 1211 1217 if ( unlikely( nalign == 0 ) ) nalign = libAlign(); // reset alignment to minimum … … 1234 1240 // change size 1235 1241 1236 void * naddr; 1237 if ( unlikely( header->kind.real.blockSize & 2 ) ) { // previous request zero fill 1238 naddr = cmemalignNoStats( nalign, 1, size ); // create new aligned area 1239 } else { 1240 naddr = memalignNoStats( nalign, size ); // create new aligned area 1241 } // if 1242 1242 void * naddr = memalignNoStats( nalign, size ); // create new aligned area 1243 1243 free( oaddr ); 1244 1244 return naddr; … … 1257 1257 size_t bsize, oalign = 0; 1258 1258 headers( "realloc", oaddr, header, freeElem, bsize, oalign ); 1259 size_t odsize = dataStorage( bsize, oaddr, header ); // data storage available in bucket1260 1259 1261 1260 if ( oalign <= nalign && (uintptr_t)oaddr % nalign == 0 ) { // <= alignment and new alignment happens to match … … 1273 1272 #endif // __STATISTICS__ 1274 1273 1274 size_t osize = header->kind.real.size; // old allocation size 1275 bool ozfill = (header->kind.real.blockSize & 2) != 0; // old allocation zero filled 1276 1275 1277 // If size is equal to 0, either NULL or a pointer suitable to be passed to free() is returned. 1276 if ( unlikely( size == 0 ) ) { free( oaddr ); return memalignNoStats( nalign, size ); } // special cases1278 if ( unlikely( size == 0 ) ) { free( oaddr ); return 0p; } // special cases 1277 1279 if ( unlikely( oaddr == 0p ) ) return memalignNoStats( nalign, size ); 1278 1280 … … 1285 1287 1286 1288 headers( "realloc", naddr, header, freeElem, bsize, oalign ); 1287 size_t ndsize = dataStorage( bsize, naddr, header ); // data storage available in bucket 1288 // To preserve prior fill, the entire bucket must be copied versus the size. 1289 memcpy( naddr, oaddr, MIN( odsize, ndsize ) ); // copy bytes 1289 memcpy( naddr, oaddr, MIN( osize, size ) ); // copy bytes 1290 1290 free( oaddr ); 1291 1292 if ( unlikely( ozfill ) ) { // previous request zero fill ? 1293 header->kind.real.blockSize |= 2; // mark new request as zero filled 1294 if ( size > osize ) { // previous request larger ? 1295 memset( (char *)naddr + osize, (int)'\0', size - osize ); // initialize added storage 1296 } // if 1297 } // if 1291 1298 return naddr; 1292 1299 } // realloc -
libcfa/src/iostream.hfa
r3f850d7 r53ee27e 363 363 _Istream_Cstr excl( const char scanset[], char * s ) { return (_Istream_Cstr){ s, scanset, -1, { .flags.inex : true } }; } 364 364 _Istream_Cstr & excl( const char scanset[], _Istream_Cstr & fmt ) { fmt.scanset = scanset; fmt.flags.inex = true; return fmt; } 365 _Istream_Cstr ignore( c onst char s[] ) { return (_Istream_Cstr)@{ s, 0p, -1, { .flags.ignore : true } }; }365 _Istream_Cstr ignore( char s[] ) { return (_Istream_Cstr)@{ s, 0p, -1, { .flags.ignore : true } }; } 366 366 _Istream_Cstr & ignore( _Istream_Cstr & fmt ) { fmt.flags.ignore = true; return fmt; } 367 367 _Istream_Cstr wdi( unsigned int w, char s[] ) { return (_Istream_Cstr)@{ s, 0p, w, { .all : 0 } }; } -
libcfa/src/stdlib.hfa
r3f850d7 r53ee27e 10 10 // Created On : Thu Jan 28 17:12:35 2016 11 11 // Last Modified By : Peter A. Buhr 12 // Last Modified On : T ue Jul 21 07:58:05202013 // Update Count : 4 7512 // Last Modified On : Thu Jul 30 16:14:58 2020 13 // Update Count : 490 14 14 // 15 15 … … 71 71 T * resize( T * ptr, size_t size ) { // CFA resize, eliminate return-type cast 72 72 $RE_SPECIALS( ptr, size, malloc, memalign ); 73 return (T *)(void *)resize( (void *)ptr, size ); // CFA resize 73 if ( _Alignof(T) <= libAlign() ) return (T *)(void *)resize( (void *)ptr, size ); // CFA resize 74 else return (T *)(void *)resize( (void *)ptr, _Alignof(T), size ); // CFA resize 74 75 } // resize 75 76 76 77 T * realloc( T * ptr, size_t size ) { // CFA realloc, eliminate return-type cast 77 78 $RE_SPECIALS( ptr, size, malloc, memalign ); 78 return (T *)(void *)realloc( (void *)ptr, size ); // C realloc 79 if ( _Alignof(T) <= libAlign() ) return (T *)(void *)realloc( (void *)ptr, size ); // C realloc 80 else return (T *)(void *)realloc( (void *)ptr, _Alignof(T), size ); // CFA realloc 79 81 } // realloc 80 82 … … 121 123 forall( dtype S | sized(S) ) 122 124 T * alloc( S ptr[], size_t dim = 1 ) { // singleton/array resize 123 size_t len = malloc_usable_size( ptr ); // current bucket size 124 if ( sizeof(T) * dim > len ) { // not enough space ? 125 T * temp = alloc( dim ); // new storage 126 free( ptr ); // free old storage 127 return temp; 128 } else { 129 return (T *)ptr; 130 } // if 131 } // alloc 132 133 T * alloc( T ptr[], size_t dim, bool copy = true ) { 125 return resize( (T *)ptr, dim * sizeof(T) ); // CFA resize 126 } // alloc 127 128 T * alloc( T ptr[], size_t dim = 1, bool copy = true ) { 134 129 if ( copy ) { 135 130 return realloc( ptr, dim * sizeof(T) ); // CFA realloc … … 168 163 memset( (char *)nptr + osize, (int)fill, nsize - osize ); // initialize added storage 169 164 } // if 170 return (T *)nptr;165 return nptr; 171 166 } // alloc_set 172 167 … … 181 176 } // for 182 177 } // if 183 return (T *)nptr;178 return nptr; 184 179 } // alloc_align_set 185 180 } // distribution … … 195 190 196 191 T * alloc_align( T * ptr, size_t align ) { // aligned realloc array 197 return (T *)(void *)realloc( (void *)ptr, align, sizeof(T) ); // CFA realloc192 return (T *)(void *)realloc( (void *)ptr, align, sizeof(T) ); // CFA C realloc 198 193 } // alloc_align 199 194 … … 232 227 size_t osize = malloc_size( ptr ); // current allocation 233 228 size_t nsize = dim * sizeof(T); // new allocation 234 T * nptr = realloc( ptr, align, nsize ); // CFA realloc229 T * nptr = alloc_align( ptr, align, nsize ); 235 230 if ( nsize > osize ) { // larger ? 236 231 memset( (char *)nptr + osize, (int)fill, nsize - osize ); // initialize added storage 237 232 } // if 238 return (T *)nptr;233 return nptr; 239 234 } // alloc_align_set 240 235 … … 243 238 size_t nsize = dim * sizeof(T); // new allocation 244 239 size_t ndim = nsize / sizeof(T); // new dimension 245 T * nptr = realloc( ptr, align, nsize ); // CFA realloc240 T * nptr = alloc_align( ptr, align, nsize ); 246 241 if ( ndim > odim ) { // larger ? 247 242 for ( i; odim ~ ndim ) { … … 249 244 } // for 250 245 } // if 251 return (T *)nptr;246 return nptr; 252 247 } // alloc_align_set 253 248 } // distribution -
longrun_tests/Makefile.in
r3f850d7 r53ee27e 483 483 AUTOMAKE_OPTIONS = foreign # do not require all the GNU file names 484 484 ACLOCAL_AMFLAGS = -I automake 485 CFACOMPILE = $(CFACC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CFAFLAGS) $(CFAFLAGS) $(AM_CFLAGS) $(CFLAGS) 486 LTCFACOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ 485 AM_T_CFA = $(am__t_CFA_@AM_T@) 486 am__t_CFA_ = 487 am__t_CFA_0 = 488 am__t_CFA_1 = /usr/bin/time --quiet -f "$@ %E" # trailling space is necessary 489 CFACOMPILE = $(AM_T_CFA)$(CFACC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CFAFLAGS) $(CFAFLAGS) $(AM_CFLAGS) $(CFLAGS) 490 LTCFACOMPILE = $(AM_T_CFA)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ 487 491 $(LIBTOOLFLAGS) --mode=compile $(CFACC) $(DEFS) \ 488 492 $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CFAFLAGS) $(AM_CFLAGS) $(CFAFLAGS) $(CFLAGS) -
src/Common/ScopedMap.h
r3f850d7 r53ee27e 93 93 94 94 reference operator* () { return *it; } 95 pointer operator-> () { return it.operator->(); }95 pointer operator-> () const { return it.operator->(); } 96 96 97 97 iterator& operator++ () { -
src/Concurrency/Keywords.cc
r3f850d7 r53ee27e 510 510 new CastExpr( 511 511 new VariableExpr( func->get_functionType()->get_parameters().front() ), 512 func->get_functionType()->get_parameters().front()->get_type()->stripReferences()->clone() 512 func->get_functionType()->get_parameters().front()->get_type()->stripReferences()->clone(), 513 false 513 514 ) 514 515 ) … … 888 889 new SingleInit( new UntypedExpr( 889 890 new NameExpr( "get_monitor" ), 890 { new CastExpr( new VariableExpr( args.front() ), arg_type ) }891 { new CastExpr( new VariableExpr( args.front() ), arg_type, false ) } 891 892 )) 892 893 ); … … 909 910 { 910 911 new SingleInit( new AddressExpr( new VariableExpr( monitors ) ) ), 911 new SingleInit( new CastExpr( new VariableExpr( func ), generic_func->clone() ) )912 new SingleInit( new CastExpr( new VariableExpr( func ), generic_func->clone(), false ) ) 912 913 }, 913 914 noDesignators, … … 946 947 return new SingleInit( new UntypedExpr( 947 948 new NameExpr( "get_monitor" ), 948 { new CastExpr( new VariableExpr( var ), type ) }949 { new CastExpr( new VariableExpr( var ), type, false ) } 949 950 ) ); 950 951 }) … … 970 971 new SingleInit( new VariableExpr( monitors ) ), 971 972 new SingleInit( new ConstantExpr( Constant::from_ulong( args.size() ) ) ), 972 new SingleInit( new CastExpr( new VariableExpr( func ), generic_func->clone() ) )973 new SingleInit( new CastExpr( new VariableExpr( func ), generic_func->clone(), false ) ) 973 974 }, 974 975 noDesignators, -
src/Concurrency/Waitfor.cc
r3f850d7 r53ee27e 384 384 decl_monitor 385 385 ) 386 ) 386 ), 387 false 387 388 ); 388 389 … … 408 409 new CompoundStmt({ 409 410 makeAccStatement( acceptables, index, "is_dtor", detectIsDtor( clause.target.function ) , indexer ), 410 makeAccStatement( acceptables, index, "func" , new CastExpr( clause.target.function, fptr_t ), indexer ),411 makeAccStatement( acceptables, index, "func" , new CastExpr( clause.target.function, fptr_t, false ) , indexer ), 411 412 makeAccStatement( acceptables, index, "data" , new VariableExpr( monitors ) , indexer ), 412 413 makeAccStatement( acceptables, index, "size" , new ConstantExpr( Constant::from_ulong( clause.target.arguments.size() ) ), indexer ), … … 531 532 decl_mask 532 533 ) 533 ) 534 ), 535 false 534 536 ), 535 537 timeout -
src/Parser/ExpressionNode.cc
r3f850d7 r53ee27e 427 427 if ( str[1] == '8' ) goto Default; // utf-8 characters => array of char 428 428 // lookup type of associated typedef 429 strtype = new TypeInstType( Type::Qualifiers( Type::Const), "char16_t", false );429 strtype = new TypeInstType( Type::Qualifiers( ), "char16_t", false ); 430 430 break; 431 431 case 'U': 432 strtype = new TypeInstType( Type::Qualifiers( Type::Const), "char32_t", false );432 strtype = new TypeInstType( Type::Qualifiers( ), "char32_t", false ); 433 433 break; 434 434 case 'L': 435 strtype = new TypeInstType( Type::Qualifiers( Type::Const), "wchar_t", false );435 strtype = new TypeInstType( Type::Qualifiers( ), "wchar_t", false ); 436 436 break; 437 437 Default: // char default string type 438 438 default: 439 strtype = new BasicType( Type::Qualifiers( Type::Const), BasicType::Char );439 strtype = new BasicType( Type::Qualifiers( ), BasicType::Char ); 440 440 } // switch 441 441 ArrayType * at = new ArrayType( noQualifiers, strtype, -
src/ResolvExpr/AlternativeFinder.cc
r3f850d7 r53ee27e 1216 1216 unify( castExpr->result, alt.expr->result, alt.env, needAssertions, 1217 1217 haveAssertions, openVars, indexer ); 1218 Cost thisCost = castCost( alt.expr->result, castExpr->result, alt.expr->get_lvalue(), 1219 indexer, alt.env ); 1218 Cost thisCost = 1219 castExpr->isGenerated 1220 ? conversionCost( alt.expr->result, castExpr->result, alt.expr->get_lvalue(), indexer, alt.env ) 1221 : castCost( alt.expr->result, castExpr->result, alt.expr->get_lvalue(), indexer, alt.env ); 1220 1222 PRINT( 1221 1223 std::cerr << "working on cast with result: " << castExpr->result << std::endl; … … 1698 1700 1699 1701 // unification run for side-effects 1700 unify( toType, alt.expr->result, newEnv, need, have, openVars, indexer ); 1702 bool canUnify = unify( toType, alt.expr->result, newEnv, need, have, openVars, indexer ); 1703 (void) canUnify; 1701 1704 // xxx - do some inspecting on this line... why isn't result bound to initAlt.type? 1702 1705 1703 Cost thisCost = c astCost( alt.expr->result, toType, alt.expr->get_lvalue(),1706 Cost thisCost = computeConversionCost( alt.expr->result, toType, alt.expr->get_lvalue(), 1704 1707 indexer, newEnv ); 1708 1709 PRINT( 1710 Cost legacyCost = castCost( alt.expr->result, toType, alt.expr->get_lvalue(), 1711 indexer, newEnv ); 1712 std::cerr << "Considering initialization:"; 1713 std::cerr << std::endl << " FROM: "; alt.expr->result->print(std::cerr); 1714 std::cerr << std::endl << " TO: "; toType ->print(std::cerr); 1715 std::cerr << std::endl << " Unification " << (canUnify ? "succeeded" : "failed"); 1716 std::cerr << std::endl << " Legacy cost " << legacyCost; 1717 std::cerr << std::endl << " New cost " << thisCost; 1718 std::cerr << std::endl; 1719 ) 1720 1705 1721 if ( thisCost != Cost::infinity ) { 1706 1722 // count one safe conversion for each value that is thrown away -
src/ResolvExpr/ConversionCost.cc
r3f850d7 r53ee27e 10 10 // Created On : Sun May 17 07:06:19 2015 11 11 // Last Modified By : Andrew Beach 12 // Last Modified On : Mon Aug 12 10:21:00 201913 // Update Count : 2 712 // Last Modified On : Wed Jul 29 16:11:00 2020 13 // Update Count : 28 14 14 // 15 15 … … 392 392 void ConversionCost::postvisit( const FunctionType * ) {} 393 393 394 void ConversionCost::postvisit( const StructInstType * inst ) {395 if ( const StructInstType * destAsInst = dynamic_cast< const StructInstType * >( dest ) ) {396 if ( inst->name == destAsInst->name ) {397 cost = Cost::zero;398 } // if399 } // if400 }401 402 void ConversionCost::postvisit( const UnionInstType * inst ) {403 if ( const UnionInstType * destAsInst = dynamic_cast< const UnionInstType * >( dest ) ) {404 if ( inst->name == destAsInst->name ) {405 cost = Cost::zero;406 } // if407 } // if408 }409 410 394 void ConversionCost::postvisit( const EnumInstType * ) { 411 395 static Type::Qualifiers q; … … 681 665 } 682 666 683 void ConversionCost_new::postvisit( const ast::StructInstType * structInstType ) {684 if ( const ast::StructInstType * dstAsInst =685 dynamic_cast< const ast::StructInstType * >( dst ) ) {686 if ( structInstType->name == dstAsInst->name ) {687 cost = Cost::zero;688 }689 }690 }691 692 void ConversionCost_new::postvisit( const ast::UnionInstType * unionInstType ) {693 if ( const ast::UnionInstType * dstAsInst =694 dynamic_cast< const ast::UnionInstType * >( dst ) ) {695 if ( unionInstType->name == dstAsInst->name ) {696 cost = Cost::zero;697 }698 }699 }700 701 667 void ConversionCost_new::postvisit( const ast::EnumInstType * enumInstType ) { 702 668 (void)enumInstType; -
src/ResolvExpr/ConversionCost.h
r3f850d7 r53ee27e 10 10 // Created On : Sun May 17 09:37:28 2015 11 11 // Last Modified By : Andrew Beach 12 // Last Modified On : Thu Aug 8 16:13:00 201913 // Update Count : 612 // Last Modified On : Wed Jul 29 16:12:00 2020 13 // Update Count : 7 14 14 // 15 15 … … 51 51 void postvisit( const ReferenceType * refType ); 52 52 void postvisit( const FunctionType * functionType ); 53 void postvisit( const StructInstType * aggregateUseType );54 void postvisit( const UnionInstType * aggregateUseType );55 53 void postvisit( const EnumInstType * aggregateUseType ); 56 54 void postvisit( const TraitInstType * aggregateUseType ); … … 102 100 void postvisit( const ast::ReferenceType * refType ); 103 101 void postvisit( const ast::FunctionType * functionType ); 104 void postvisit( const ast::StructInstType * structInstType );105 void postvisit( const ast::UnionInstType * unionInstType );106 102 void postvisit( const ast::EnumInstType * enumInstType ); 107 103 void postvisit( const ast::TraitInstType * traitInstType ); -
src/SynTree/Expression.h
r3f850d7 r53ee27e 206 206 public: 207 207 Expression * arg; 208 bool isGenerated = true; // cast generated implicitly by code generation or explicit in program 208 209 // Inidicates cast is introduced by the CFA type system. 210 // true for casts that the resolver introduces to force a return type 211 // false for casts from user code 212 // false for casts from desugaring advanced CFA features into simpler CFA 213 // example 214 // int * p; // declaration 215 // (float *) p; // use, with subject cast 216 // subject cast isGenerated means we are considering an interpretation with a type mismatch 217 // subject cast not isGenerated means someone in charge wants it that way 218 bool isGenerated = true; 209 219 210 220 CastExpr( Expression * arg, bool isGenerated = true ); -
src/Virtual/ExpandCasts.cc
r3f850d7 r53ee27e 10 10 // Created On : Mon Jul 24 13:59:00 2017 11 11 // Last Modified By : Andrew Beach 12 // Last Modified On : Tue Jul 22 10:04:00 202013 // Update Count : 312 // Last Modified On : Fri Jul 31 10:29:00 2020 13 // Update Count : 4 14 14 // 15 15 … … 18 18 #include <cassert> // for assert, assertf 19 19 #include <iterator> // for back_inserter, inserter 20 #include <map> // for map, _Rb_tree_iterator, map<>::ite...21 20 #include <string> // for string, allocator, operator==, ope... 22 #include <utility> // for pair23 21 24 22 #include "Common/PassVisitor.h" // for PassVisitor 23 #include "Common/ScopedMap.h" // for ScopedMap 25 24 #include "Common/SemanticError.h" // for SemanticError 26 25 #include "SymTab/Mangler.h" // for mangleType … … 37 36 /// Maps virtual table types the instance for that type. 38 37 class VirtualTableMap final { 39 std::unordered_map<std::string, ObjectDecl *> vtable_instances;38 ScopedMap<std::string, ObjectDecl *> vtable_instances; 40 39 public: 40 void enterScope() { 41 vtable_instances.beginScope(); 42 } 43 void leaveScope() { 44 vtable_instances.endScope(); 45 } 46 41 47 ObjectDecl * insert( ObjectDecl * vtableDecl ) { 42 48 std::string const & mangledName = SymTab::Mangler::mangleType( vtableDecl->type ); … … 93 99 94 100 class VirtualCastCore { 95 VirtualTableMap vtable_instances;96 FunctionDecl *vcast_decl;97 StructDecl *pvt_decl;98 99 101 Type * pointer_to_pvt(int level_of_indirection) { 100 102 Type * type = new StructInstType( … … 108 110 public: 109 111 VirtualCastCore() : 110 vtable_instances(), vcast_decl( nullptr ), pvt_decl( nullptr )112 indexer(), vcast_decl( nullptr ), pvt_decl( nullptr ) 111 113 {} 112 114 … … 116 118 117 119 Expression * postmutate( VirtualCastExpr * castExpr ); 120 121 VirtualTableMap indexer; 122 private: 123 FunctionDecl *vcast_decl; 124 StructDecl *pvt_decl; 118 125 }; 119 126 … … 135 142 void VirtualCastCore::premutate( ObjectDecl * objectDecl ) { 136 143 if ( is_vtable_inst_name( objectDecl->get_name() ) ) { 137 if ( ObjectDecl * existing = vtable_instances.insert( objectDecl ) ) {144 if ( ObjectDecl * existing = indexer.insert( objectDecl ) ) { 138 145 std::string msg = "Repeated instance of virtual table, original found at: "; 139 146 msg += existing->location.filename; … … 222 229 223 230 const Type * vtable_type = getVirtualTableType( castExpr ); 224 ObjectDecl * table = vtable_instances.lookup( vtable_type );231 ObjectDecl * table = indexer.lookup( vtable_type ); 225 232 if ( nullptr == table ) { 226 233 SemanticError( castLocation( castExpr ), -
src/cfa.make
r3f850d7 r53ee27e 1 CFACOMPILE = $(CFACC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CFAFLAGS) $(CFAFLAGS) $(AM_CFLAGS) $(CFLAGS) 2 LTCFACOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ 1 AM_T_CFA = $(am__t_CFA_@AM_T@) 2 am__t_CFA_ = 3 am__t_CFA_0 = 4 am__t_CFA_1 = /usr/bin/time --quiet -f "$@ %E" # trailling space is necessary 5 6 7 CFACOMPILE = $(AM_T_CFA)$(CFACC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CFAFLAGS) $(CFAFLAGS) $(AM_CFLAGS) $(CFLAGS) 8 LTCFACOMPILE = $(AM_T_CFA)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ 3 9 $(LIBTOOLFLAGS) --mode=compile $(CFACC) $(DEFS) \ 4 10 $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CFAFLAGS) $(AM_CFLAGS) $(CFAFLAGS) $(CFLAGS) -
tests/.expect/castError.txt
r3f850d7 r53ee27e 1 castError.cfa:2 1:1 error: Cannot choose between 3 alternatives for expression1 castError.cfa:23:1 error: Cannot choose between 3 alternatives for expression 2 2 Explicit Cast of: 3 3 Name: f … … 35 35 36 36 37 castError.cfa:2 6:1 error: Cannot choose between 2 alternatives for expression37 castError.cfa:28:1 error: Cannot choose between 2 alternatives for expression 38 38 Generated Cast of: 39 39 Comma Expression: … … 62 62 63 63 64 castError.cfa:30:1 error: No reasonable alternatives for expression Explicit Cast of: 65 Name: sint 66 ... to: 67 instance of struct S with body 1 68 ... with parameters 69 char 70 -
tests/Makefile.in
r3f850d7 r53ee27e 355 355 AUTOMAKE_OPTIONS = foreign # do not require all the GNU file names 356 356 ACLOCAL_AMFLAGS = -I automake 357 CFACOMPILE = $(CFACC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CFAFLAGS) $(CFAFLAGS) $(AM_CFLAGS) $(CFLAGS) 358 LTCFACOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ 357 AM_T_CFA = $(am__t_CFA_@AM_T@) 358 am__t_CFA_ = 359 am__t_CFA_0 = 360 am__t_CFA_1 = /usr/bin/time --quiet -f "$@ %E" # trailling space is necessary 361 CFACOMPILE = $(AM_T_CFA)$(CFACC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CFAFLAGS) $(CFAFLAGS) $(AM_CFLAGS) $(CFLAGS) 362 LTCFACOMPILE = $(AM_T_CFA)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ 359 363 $(LIBTOOLFLAGS) --mode=compile $(CFACC) $(DEFS) \ 360 364 $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CFAFLAGS) $(AM_CFLAGS) $(CFAFLAGS) $(CFLAGS) -
tests/avltree/avl1.cfa
r3f850d7 r53ee27e 24 24 tree(K, V) * create(K key, V value) { 25 25 // infinite loop trying to resolve ... t = malloc(); 26 tree(K, V) * t = malloc(sizeof(tree(K,V)));26 tree(K, V) * t = ( tree(K, V) * ) malloc(sizeof(tree(K,V))); 27 27 (*t){ key, value }; 28 28 return t; -
tests/bugs/66.cfa
r3f850d7 r53ee27e 5 5 6 6 int main() { 7 int * next = (void*)0;7 int * next = 0p; 8 8 if( next ) { 9 9 return 1; -
tests/castError.cfa
r3f850d7 r53ee27e 14 14 // 15 15 16 forall(otype T) struct S { T p; }; 16 17 int f; 18 S(int) sint; 17 19 18 20 void f() { … … 25 27 short int v; 26 28 3, v; // implicit void cast 29 30 (S(char)) sint; 27 31 } 28 32 -
tests/concurrent/signal/block.cfa
r3f850d7 r53ee27e 82 82 if( !is_empty( cond ) ) { 83 83 84 $thread * next = front( cond );84 $thread * next = ( $thread * ) front( cond ); 85 85 86 86 if( ! signal_block( cond ) ) { -
tests/exceptions/conditional.cfa
r3f850d7 r53ee27e 17 17 }; 18 18 19 voidnum_error_msg(num_error * this) {19 const char * num_error_msg(num_error * this) { 20 20 if ( ! this->msg ) { 21 21 static const char * base = "Num Error with code: X"; -
tests/exceptions/defaults.cfa
r3f850d7 r53ee27e 13 13 } 14 14 15 c har * get_log_message(log_message * this) {15 const char * get_log_message(log_message * this) { 16 16 return this->msg; 17 17 } … … 28 28 // We can catch log: 29 29 try { 30 throwResume (log_message){ (char *)"Should be printed.\n"};30 throwResume (log_message){"Should be printed.\n"}; 31 31 } catchResume (log_message * this) { 32 32 printf("%s", this->virtual_table->msg(this)); 33 33 } 34 34 // But we don't have to: 35 throwResume (log_message){ (char *)"Should not be printed.\n"};35 throwResume (log_message){"Should not be printed.\n"}; 36 36 } 37 37 -
tests/heap.cfa
r3f850d7 r53ee27e 10 10 // Created On : Tue Nov 6 17:54:56 2018 11 11 // Last Modified By : Peter A. Buhr 12 // Last Modified On : Sun Nov 24 12:34:51 201913 // Update Count : 2812 // Last Modified On : Tue Aug 4 06:36:17 2020 13 // Update Count : 56 14 14 // 15 15 … … 75 75 size_t s = (i + 1) * 20; 76 76 char * area = (char *)malloc( s ); 77 if ( area == 0p ) abort( "malloc/free out of memory" );78 77 area[0] = '\345'; area[s - 1] = '\345'; // fill first/last 79 78 area[malloc_usable_size( area ) - 1] = '\345'; // fill ultimate byte … … 84 83 size_t s = i + 1; // +1 to make initialization simpler 85 84 locns[i] = (char *)malloc( s ); 86 if ( locns[i] == 0p ) abort( "malloc/free out of memory" );87 85 locns[i][0] = '\345'; locns[i][s - 1] = '\345'; // fill first/last 88 86 locns[i][malloc_usable_size( locns[i] ) - 1] = '\345'; // fill ultimate byte … … 100 98 size_t s = i + default_mmap_start(); // cross over point 101 99 char * area = (char *)malloc( s ); 102 if ( area == 0p ) abort( "malloc/free out of memory" );103 100 area[0] = '\345'; area[s - 1] = '\345'; // fill first/last 104 101 area[malloc_usable_size( area ) - 1] = '\345'; // fill ultimate byte … … 109 106 size_t s = i + default_mmap_start(); // cross over point 110 107 locns[i] = (char *)malloc( s ); 111 if ( locns[i] == 0p ) abort( "malloc/free out of memory" );112 108 locns[i][0] = '\345'; locns[i][s - 1] = '\345'; // fill first/last 113 109 locns[i][malloc_usable_size( locns[i] ) - 1] = '\345'; // fill ultimate byte … … 125 121 size_t s = (i + 1) * 20; 126 122 char * area = (char *)calloc( 5, s ); 127 if ( area == 0p ) abort( "calloc/free out of memory" );128 123 if ( area[0] != '\0' || area[s - 1] != '\0' || 129 area[malloc_ usable_size( area ) - 1] != '\0' ||124 area[malloc_size( area ) - 1] != '\0' || 130 125 ! malloc_zero_fill( area ) ) abort( "calloc/free corrupt storage1" ); 131 126 area[0] = '\345'; area[s - 1] = '\345'; // fill first/last … … 137 132 size_t s = i + 1; 138 133 locns[i] = (char *)calloc( 5, s ); 139 if ( locns[i] == 0p ) abort( "calloc/free out of memory" );140 134 if ( locns[i][0] != '\0' || locns[i][s - 1] != '\0' || 141 locns[i][malloc_ usable_size( locns[i] ) - 1] != '\0' ||135 locns[i][malloc_size( locns[i] ) - 1] != '\0' || 142 136 ! malloc_zero_fill( locns[i] ) ) abort( "calloc/free corrupt storage2" ); 143 137 locns[i][0] = '\345'; locns[i][s - 1] = '\345'; // fill first/last … … 156 150 size_t s = i + default_mmap_start(); // cross over point 157 151 char * area = (char *)calloc( 1, s ); 158 if ( area == 0p ) abort( "calloc/free out of memory" );159 152 if ( area[0] != '\0' || area[s - 1] != '\0' ) abort( "calloc/free corrupt storage4.1" ); 160 if ( area[malloc_ usable_size( area ) - 1] != '\0' ) abort( "calloc/free corrupt storage4.2" );153 if ( area[malloc_size( area ) - 1] != '\0' ) abort( "calloc/free corrupt storage4.2" ); 161 154 if ( ! malloc_zero_fill( area ) ) abort( "calloc/free corrupt storage4.3" ); 162 155 area[0] = '\345'; area[s - 1] = '\345'; // fill first/last … … 168 161 size_t s = i + default_mmap_start(); // cross over point 169 162 locns[i] = (char *)calloc( 1, s ); 170 if ( locns[i] == 0p ) abort( "calloc/free out of memory" );171 163 if ( locns[i][0] != '\0' || locns[i][s - 1] != '\0' || 172 locns[i][malloc_ usable_size( locns[i] ) - 1] != '\0' ||164 locns[i][malloc_size( locns[i] ) - 1] != '\0' || 173 165 ! malloc_zero_fill( locns[i] ) ) abort( "calloc/free corrupt storage5" ); 174 166 locns[i][0] = '\345'; locns[i][s - 1] = '\345'; // fill first/last … … 188 180 for ( s; 1 ~ NoOfAllocs ) { // allocation of size 0 can return null 189 181 char * area = (char *)memalign( a, s ); 190 if ( area == 0p ) abort( "memalign/free out of memory" );191 182 //sout | i | area; 192 183 if ( (size_t)area % a != 0 || malloc_alignment( area ) != a ) { // check for initial alignment … … 206 197 size_t s = i + default_mmap_start(); // cross over point 207 198 char * area = (char *)memalign( a, s ); 208 if ( area == 0p ) abort( "memalign/free out of memory" );209 199 //sout | i | area; 210 200 if ( (size_t)area % a != 0 || malloc_alignment( area ) != a ) { // check for initial alignment … … 222 212 // initial N byte allocation 223 213 char * area = (char *)calloc( 5, i ); 224 if ( area == 0p ) abort( "calloc/realloc/free out of memory" );225 214 if ( area[0] != '\0' || area[i - 1] != '\0' || 226 area[malloc_ usable_size( area ) - 1] != '\0' ||215 area[malloc_size( area ) - 1] != '\0' || 227 216 ! malloc_zero_fill( area ) ) abort( "calloc/realloc/free corrupt storage1" ); 228 217 … … 230 219 for ( s; i ~ 256 * 1024 ~ 26 ) { // start at initial memory request 231 220 area = (char *)realloc( area, s ); // attempt to reuse storage 232 if ( area == 0p ) abort( "calloc/realloc/free out of memory" );233 221 if ( area[0] != '\0' || area[s - 1] != '\0' || 234 area[malloc_ usable_size( area ) - 1] != '\0' ||222 area[malloc_size( area ) - 1] != '\0' || 235 223 ! malloc_zero_fill( area ) ) abort( "calloc/realloc/free corrupt storage2" ); 236 224 } // for … … 244 232 size_t s = i + default_mmap_start(); // cross over point 245 233 char * area = (char *)calloc( 1, s ); 246 if ( area == 0p ) abort( "calloc/realloc/free out of memory" );234 // if ( area == 0p ) abort( "calloc/realloc/free out of memory" ); 247 235 if ( area[0] != '\0' || area[s - 1] != '\0' || 248 area[malloc_usable_size( area ) - 1] != '\0' || 249 ! malloc_zero_fill( area ) ) abort( "calloc/realloc/free corrupt storage1" ); 236 area[malloc_size( area ) - 1] != '\0' || 237 ! malloc_zero_fill( area ) ) //abort( "calloc/realloc/free corrupt storage3" ); 238 printf( "C %zd %d %d %d %d\n", s, area[0] != '\0', area[s - 1] != '\0', area[malloc_size( area ) - 1] != '\0', ! malloc_zero_fill( area ) ); 250 239 251 240 // Do not start this loop index at 0 because realloc of 0 bytes frees the storage. 252 241 for ( r; i ~ 256 * 1024 ~ 26 ) { // start at initial memory request 253 242 area = (char *)realloc( area, r ); // attempt to reuse storage 254 if ( area == 0p ) abort( "calloc/realloc/free out of memory" );243 // if ( area == 0p ) abort( "calloc/realloc/free out of memory" ); 255 244 if ( area[0] != '\0' || area[r - 1] != '\0' || 256 area[malloc_ usable_size( area ) - 1] != '\0' ||257 ! malloc_zero_fill( area ) ) abort( "calloc/realloc/free corrupt storage 2" );245 area[malloc_size( area ) - 1] != '\0' || 246 ! malloc_zero_fill( area ) ) abort( "calloc/realloc/free corrupt storage4" ); 258 247 } // for 259 248 free( area ); … … 266 255 // initial N byte allocation 267 256 char * area = (char *)memalign( a, amount ); // aligned N-byte allocation 268 if ( area == 0p ) abort( "memalign/realloc/free out of memory" ); // no storage ?257 // if ( area == 0p ) abort( "memalign/realloc/free out of memory" ); // no storage ? 269 258 //sout | alignments[a] | area; 270 259 if ( (size_t)area % a != 0 || malloc_alignment( area ) != a ) { // check for initial alignment … … 277 266 if ( area[0] != '\345' || area[s - 2] != '\345' ) abort( "memalign/realloc/free corrupt storage" ); 278 267 area = (char *)realloc( area, s ); // attempt to reuse storage 279 if ( area == 0p ) abort( "memalign/realloc/free out of memory" ); // no storage ?280 268 //sout | i | area; 281 269 if ( (size_t)area % a != 0 ) { // check for initial alignment … … 293 281 for ( s; 1 ~ limit ) { // allocation of size 0 can return null 294 282 char * area = (char *)cmemalign( a, 1, s ); 295 if ( area == 0p ) abort( "cmemalign/free out of memory" );296 283 //sout | i | area; 297 284 if ( (size_t)area % a != 0 || malloc_alignment( area ) != a ) { // check for initial alignment … … 299 286 } // if 300 287 if ( area[0] != '\0' || area[s - 1] != '\0' || 301 area[malloc_ usable_size( area ) - 1] != '\0' ||288 area[malloc_size( area ) - 1] != '\0' || 302 289 ! malloc_zero_fill( area ) ) abort( "cmemalign/free corrupt storage" ); 303 290 area[0] = '\345'; area[s - 1] = '\345'; // fill first/last byte … … 312 299 // initial N byte allocation 313 300 char * area = (char *)cmemalign( a, 1, amount ); // aligned N-byte allocation 314 if ( area == 0p ) abort( "cmemalign/realloc/free out of memory" ); // no storage ?315 301 //sout | alignments[a] | area; 316 302 if ( (size_t)area % a != 0 || malloc_alignment( area ) != a ) { // check for initial alignment … … 318 304 } // if 319 305 if ( area[0] != '\0' || area[amount - 1] != '\0' || 320 area[malloc_ usable_size( area ) - 1] != '\0' ||306 area[malloc_size( area ) - 1] != '\0' || 321 307 ! malloc_zero_fill( area ) ) abort( "cmemalign/realloc/free corrupt storage1" ); 322 308 area[0] = '\345'; area[amount - 2] = '\345'; // fill first/penultimate byte … … 326 312 if ( area[0] != '\345' || area[s - 2] != '\345' ) abort( "cmemalign/realloc/free corrupt storage2" ); 327 313 area = (char *)realloc( area, s ); // attempt to reuse storage 328 if ( area == 0p ) abort( "cmemalign/realloc/free out of memory" ); // no storage ?329 314 //sout | i | area; 330 315 if ( (size_t)area % a != 0 || malloc_alignment( area ) != a ) { // check for initial alignment 331 316 abort( "cmemalign/realloc/free bad alignment %p", area ); 332 317 } // if 333 if ( area[ s - 1] != '\0' || area[s - 1] != '\0' ||334 area[malloc_ usable_size( area ) - 1] != '\0' ||318 if ( area[0] != '\345' || area[s - 1] != '\0' || 319 area[malloc_size( area ) - 1] != '\0' || 335 320 ! malloc_zero_fill( area ) ) abort( "cmemalign/realloc/free corrupt storage3" ); 336 321 area[s - 1] = '\345'; // fill last byte … … 345 330 // initial N byte allocation 346 331 char * area = (char *)memalign( a, amount ); // aligned N-byte allocation 347 if ( area == 0p ) abort( "memalign/realloc with align/free out of memory" ); // no storage ?348 332 //sout | alignments[a] | area | endl; 349 333 if ( (size_t)area % a != 0 || malloc_alignment( area ) != a ) { // check for initial alignment … … 356 340 if ( area[0] != '\345' || area[s - 2] != '\345' ) abort( "memalign/realloc/free corrupt storage" ); 357 341 area = (char *)realloc( area, a * 2, s ); // attempt to reuse storage 358 if ( area == 0p ) abort( "memalign/realloc with align/free out of memory" ); // no storage ?359 342 //sout | i | area | endl; 360 343 if ( (size_t)area % a * 2 != 0 ) { // check for initial alignment … … 371 354 for ( size_t a = libAlign() + libAlign(); a <= limit; a += a ) { // generate powers of 2 372 355 // initial N byte allocation 373 char *area = (char *)cmemalign( a, 1, amount ); // aligned N-byte allocation 374 if ( area == 0p ) abort( "cmemalign/realloc with align/free out of memory" ); // no storage ? 356 char * area = (char *)cmemalign( a, 1, amount ); // aligned N-byte allocation 375 357 //sout | alignments[a] | area | endl; 376 358 if ( (size_t)area % a != 0 || malloc_alignment( area ) != a ) { // check for initial alignment … … 378 360 } // if 379 361 if ( area[0] != '\0' || area[amount - 1] != '\0' || 380 area[malloc_ usable_size( area ) - 1] != '\0' ||362 area[malloc_size( area ) - 1] != '\0' || 381 363 ! malloc_zero_fill( area ) ) abort( "cmemalign/realloc with align/free corrupt storage1" ); 382 364 area[0] = '\345'; area[amount - 2] = '\345'; // fill first/penultimate byte … … 386 368 if ( area[0] != '\345' || area[s - 2] != '\345' ) abort( "cmemalign/realloc with align/free corrupt storage2" ); 387 369 area = (char *)realloc( area, a * 2, s ); // attempt to reuse storage 388 if ( area == 0p ) abort( "cmemalign/realloc with align/free out of memory" ); // no storage ?389 370 //sout | i | area | endl; 390 371 if ( (size_t)area % a * 2 != 0 || malloc_alignment( area ) != a * 2 ) { // check for initial alignment 391 abort( "cmemalign/realloc with align/free bad alignment %p % jd %jd", area, malloc_alignment( area ), a * 2 );372 abort( "cmemalign/realloc with align/free bad alignment %p %zd %zd", area, malloc_alignment( area ), a * 2 ); 392 373 } // if 393 374 if ( area[s - 1] != '\0' || area[s - 1] != '\0' || 394 area[malloc_ usable_size( area ) - 1] != '\0' ||375 area[malloc_size( area ) - 1] != '\0' || 395 376 ! malloc_zero_fill( area ) ) abort( "cmemalign/realloc/free corrupt storage3" ); 396 377 area[s - 1] = '\345'; // fill last byte -
tests/io2.cfa
r3f850d7 r53ee27e 121 121 122 122 [int, int, const char *, double] t3 = { 3, 4, "a", 7.2 }; 123 sout | [ 3, 4, "a", 7.2 ];123 sout | [ 3, 4, (const char*)"a", 7.2 ]; // workaround trac#207: the const cast should not be needed 124 124 sout | t3; 125 125 sepSetTuple( sout, " " ); -
tests/searchsort.cfa
r3f850d7 r53ee27e 38 38 } // for 39 39 sout | nl; 40 for ( i; 0 ~ size ) { // C version 40 for ( i; 0 ~ size ) { // C version, returns void* 41 41 int key = size - i; 42 int * v = bsearch( &key, iarr, size, sizeof( iarr[0] ), comp );42 int * v = ( int * ) bsearch( &key, iarr, size, sizeof( iarr[0] ), comp ); 43 43 sout | key | ':' | *v | ", "; 44 44 } // for
Note: See TracChangeset
for help on using the changeset viewer.