- Timestamp:
- Jan 14, 2021, 12:23:14 PM (5 years ago)
- Branches:
- ADT, arm-eh, ast-experimental, enum, forall-pointer-decay, jacob/cs343-translation, master, new-ast-unique-expr, pthread-emulation, qualifiedEnum
- Children:
- 8e4aa05
- Parents:
- 4468a70 (diff), ec19b21 (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the(diff)links above to see all the changes relative to each parent. - Location:
- libcfa/src
- Files:
-
- 5 added
- 9 edited
-
Makefile.am (modified) (2 diffs)
-
bits/locks.hfa (modified) (3 diffs)
-
concurrency/future.hfa (added)
-
concurrency/io.cfa (modified) (23 diffs)
-
concurrency/io/call.cfa.in (modified) (4 diffs)
-
concurrency/io/setup.cfa (modified) (13 diffs)
-
concurrency/io/types.hfa (modified) (4 diffs)
-
concurrency/monitor.hfa (modified) (1 diff)
-
heap.cfa (modified) (25 diffs)
-
startup.cfa (modified) (2 diffs)
-
vec/vec.hfa (added)
-
vec/vec2.hfa (added)
-
vec/vec3.hfa (added)
-
vec/vec4.hfa (added)
Legend:
- Unmodified
- Added
- Removed
-
libcfa/src/Makefile.am
r4468a70 r342af53 58 58 concurrency/iofwd.hfa \ 59 59 containers/list.hfa \ 60 containers/stackLockFree.hfa 60 containers/stackLockFree.hfa \ 61 vec/vec.hfa \ 62 vec/vec2.hfa \ 63 vec/vec3.hfa \ 64 vec/vec4.hfa 61 65 62 66 inst_headers_src = \ … … 94 98 concurrency/clib/cfathread.h \ 95 99 concurrency/invoke.h \ 100 concurrency/future.hfa \ 96 101 concurrency/kernel/fwd.hfa 97 102 -
libcfa/src/bits/locks.hfa
r4468a70 r342af53 283 283 void ^?{}(future_t &) {} 284 284 285 void reset(future_t & this) { 286 // needs to be in 0p or 1p 287 __atomic_exchange_n( &this.ptr, 0p, __ATOMIC_SEQ_CST); 288 } 289 285 290 // check if the future is available 286 291 bool available( future_t & this ) { … … 340 345 341 346 // Mark the future as abandoned, meaning it will be deleted by the server 342 void abandon( future_t & this ) { 347 bool abandon( future_t & this ) { 348 /* paranoid */ verify( this.ptr != 3p ); 349 350 // Mark the future as abandonned 343 351 struct oneshot * got = __atomic_exchange_n( &this.ptr, 3p, __ATOMIC_SEQ_CST); 352 353 // If the future isn't already fulfilled, let the server delete it 354 if( got == 0p ) return false; 344 355 345 356 // got == 2p: the future is ready but the context hasn't fully been consumed … … 347 358 if( got == 2p ) { 348 359 while( this.ptr != 1p ) Pause(); 349 } 350 return; 360 got = 1p; 361 } 362 363 // The future is completed delete it now 364 /* paranoid */ verify( this.ptr != 1p ); 365 free( &this ); 366 return true; 351 367 } 352 368 -
libcfa/src/concurrency/io.cfa
r4468a70 r342af53 31 31 32 32 extern "C" { 33 #include <sys/epoll.h>34 33 #include <sys/syscall.h> 35 34 … … 41 40 #include "kernel/fwd.hfa" 42 41 #include "io/types.hfa" 42 43 static const char * opcodes[] = { 44 "OP_NOP", 45 "OP_READV", 46 "OP_WRITEV", 47 "OP_FSYNC", 48 "OP_READ_FIXED", 49 "OP_WRITE_FIXED", 50 "OP_POLL_ADD", 51 "OP_POLL_REMOVE", 52 "OP_SYNC_FILE_RANGE", 53 "OP_SENDMSG", 54 "OP_RECVMSG", 55 "OP_TIMEOUT", 56 "OP_TIMEOUT_REMOVE", 57 "OP_ACCEPT", 58 "OP_ASYNC_CANCEL", 59 "OP_LINK_TIMEOUT", 60 "OP_CONNECT", 61 "OP_FALLOCATE", 62 "OP_OPENAT", 63 "OP_CLOSE", 64 "OP_FILES_UPDATE", 65 "OP_STATX", 66 "OP_READ", 67 "OP_WRITE", 68 "OP_FADVISE", 69 "OP_MADVISE", 70 "OP_SEND", 71 "OP_RECV", 72 "OP_OPENAT2", 73 "OP_EPOLL_CTL", 74 "OP_SPLICE", 75 "OP_PROVIDE_BUFFERS", 76 "OP_REMOVE_BUFFERS", 77 "OP_TEE", 78 "INVALID_OP" 79 }; 43 80 44 81 // returns true of acquired as leader or second leader … … 134 171 int ret = 0; 135 172 if( need_sys_to_submit || need_sys_to_complete ) { 173 __cfadbg_print_safe(io_core, "Kernel I/O : IO_URING enter %d %u %u\n", ring.fd, to_submit, flags); 136 174 ret = syscall( __NR_io_uring_enter, ring.fd, to_submit, 0, flags, (sigset_t *)0p, _NSIG / 8); 137 175 if( ret < 0 ) { … … 157 195 static unsigned __collect_submitions( struct __io_data & ring ); 158 196 static __u32 __release_consumed_submission( struct __io_data & ring ); 159 160 static inline void process(struct io_uring_cqe & cqe ) { 197 static inline void __clean( volatile struct io_uring_sqe * sqe ); 198 199 // Process a single completion message from the io_uring 200 // This is NOT thread-safe 201 static inline void process( volatile struct io_uring_cqe & cqe ) { 161 202 struct io_future_t * future = (struct io_future_t *)(uintptr_t)cqe.user_data; 162 203 __cfadbg_print_safe( io, "Kernel I/O : Syscall completed : cqe %p, result %d for %p\n", &cqe, cqe.res, future ); … … 165 206 } 166 207 167 // Process a single completion message from the io_uring168 // This is NOT thread-safe169 208 static [int, bool] __drain_io( & struct __io_data ring ) { 170 209 /* paranoid */ verify( ! __preemption_enabled() ); … … 192 231 } 193 232 233 __atomic_thread_fence( __ATOMIC_SEQ_CST ); 234 194 235 // Release the consumed SQEs 195 236 __release_consumed_submission( ring ); … … 209 250 for(i; count) { 210 251 unsigned idx = (head + i) & mask; 211 struct io_uring_cqe & cqe = ring.completion_q.cqes[idx];252 volatile struct io_uring_cqe & cqe = ring.completion_q.cqes[idx]; 212 253 213 254 /* paranoid */ verify(&cqe); … … 218 259 // Mark to the kernel that the cqe has been seen 219 260 // Ensure that the kernel only sees the new value of the head index after the CQEs have been read. 220 __atomic_thread_fence( __ATOMIC_SEQ_CST ); 221 __atomic_fetch_add( ring.completion_q.head, count, __ATOMIC_RELAXED ); 261 __atomic_fetch_add( ring.completion_q.head, count, __ATOMIC_SEQ_CST ); 222 262 223 263 return [count, count > 0 || to_submit > 0]; … … 225 265 226 266 void main( $io_ctx_thread & this ) { 227 epoll_event ev;228 __ioctx_register( this, ev ); 229 230 __cfadbg_print_safe(io_core, "Kernel I/O : IO poller %p for ring %p ready\n", &this, &this.ring); 231 232 int reset = 0;267 __ioctx_register( this ); 268 269 __cfadbg_print_safe(io_core, "Kernel I/O : IO poller %d (%p) ready\n", this.ring->fd, &this); 270 271 const int reset_cnt = 5; 272 int reset = reset_cnt; 233 273 // Then loop until we need to start 274 LOOP: 234 275 while(!__atomic_load_n(&this.done, __ATOMIC_SEQ_CST)) { 235 276 // Drain the io … … 239 280 [count, again] = __drain_io( *this.ring ); 240 281 241 if(!again) reset ++;282 if(!again) reset--; 242 283 243 284 // Update statistics … … 249 290 250 291 // If we got something, just yield and check again 251 if(reset < 5) {292 if(reset > 1) { 252 293 yield(); 253 } 254 // We didn't get anything baton pass to the slow poller 255 else { 294 continue LOOP; 295 } 296 297 // We alread failed to find completed entries a few time. 298 if(reset == 1) { 299 // Rearm the context so it can block 300 // but don't block right away 301 // we need to retry one last time in case 302 // something completed *just now* 303 __ioctx_prepare_block( this ); 304 continue LOOP; 305 } 306 256 307 __STATS__( false, 257 308 io.complete_q.blocks += 1; 258 309 ) 259 __cfadbg_print_safe(io_core, "Kernel I/O : Parking io poller %p\n", &this.self); 260 reset = 0; 310 __cfadbg_print_safe(io_core, "Kernel I/O : Parking io poller %d (%p)\n", this.ring->fd, &this); 261 311 262 312 // block this thread 263 __ioctx_prepare_block( this, ev );264 313 wait( this.sem ); 265 } 266 } 267 268 __cfadbg_print_safe(io_core, "Kernel I/O : Fast poller for ring %p stopping\n", &this.ring); 314 315 // restore counter 316 reset = reset_cnt; 317 } 318 319 __cfadbg_print_safe(io_core, "Kernel I/O : Fast poller %d (%p) stopping\n", this.ring->fd, &this); 269 320 } 270 321 … … 289 340 // 290 341 291 [* struct io_uring_sqe, __u32] __submit_alloc( struct __io_data & ring, __u64 data ) { 342 // Allocate an submit queue entry. 343 // The kernel cannot see these entries until they are submitted, but other threads must be 344 // able to see which entries can be used and which are already un used by an other thread 345 // for convenience, return both the index and the pointer to the sqe 346 // sqe == &sqes[idx] 347 [* volatile struct io_uring_sqe, __u32] __submit_alloc( struct __io_data & ring, __u64 data ) { 292 348 /* paranoid */ verify( data != 0 ); 293 349 … … 304 360 // Look through the list starting at some offset 305 361 for(i; cnt) { 306 __u64 expected = 0;307 __u32 idx = (i + off) & mask; 308 struct io_uring_sqe * sqe = &ring.submit_q.sqes[idx];362 __u64 expected = 3; 363 __u32 idx = (i + off) & mask; // Get an index from a random 364 volatile struct io_uring_sqe * sqe = &ring.submit_q.sqes[idx]; 309 365 volatile __u64 * udata = &sqe->user_data; 310 366 367 // Allocate the entry by CASing the user_data field from 0 to the future address 311 368 if( *udata == expected && 312 369 __atomic_compare_exchange_n( udata, &expected, data, true, __ATOMIC_SEQ_CST, __ATOMIC_RELAXED ) ) … … 319 376 ) 320 377 378 // debug log 379 __cfadbg_print_safe( io, "Kernel I/O : allocated [%p, %u] for %p (%p)\n", sqe, idx, active_thread(), (void*)data ); 321 380 322 381 // Success return the data … … 325 384 verify(expected != data); 326 385 386 // This one was used 327 387 len ++; 328 388 } 329 389 330 390 block++; 391 392 abort( "Kernel I/O : all submit queue entries used, yielding\n" ); 393 331 394 yield(); 332 395 } … … 377 440 void __submit( struct io_context * ctx, __u32 idx ) __attribute__((nonnull (1))) { 378 441 __io_data & ring = *ctx->thrd.ring; 442 443 { 444 __attribute__((unused)) volatile struct io_uring_sqe * sqe = &ring.submit_q.sqes[idx]; 445 __cfadbg_print_safe( io, 446 "Kernel I/O : submitting %u (%p) for %p\n" 447 " data: %p\n" 448 " opcode: %s\n" 449 " fd: %d\n" 450 " flags: %d\n" 451 " prio: %d\n" 452 " off: %p\n" 453 " addr: %p\n" 454 " len: %d\n" 455 " other flags: %d\n" 456 " splice fd: %d\n" 457 " pad[0]: %llu\n" 458 " pad[1]: %llu\n" 459 " pad[2]: %llu\n", 460 idx, sqe, 461 active_thread(), 462 (void*)sqe->user_data, 463 opcodes[sqe->opcode], 464 sqe->fd, 465 sqe->flags, 466 sqe->ioprio, 467 sqe->off, 468 sqe->addr, 469 sqe->len, 470 sqe->accept_flags, 471 sqe->splice_fd_in, 472 sqe->__pad2[0], 473 sqe->__pad2[1], 474 sqe->__pad2[2] 475 ); 476 } 477 478 379 479 // Get now the data we definetely need 380 480 volatile __u32 * const tail = ring.submit_q.tail; … … 443 543 unlock(ring.submit_q.submit_lock); 444 544 #endif 445 if( ret < 0 ) return; 545 if( ret < 0 ) { 546 return; 547 } 446 548 447 549 // Release the consumed SQEs … … 454 556 io.submit_q.submit_avg.cnt += 1; 455 557 ) 456 } 457 else { 558 559 __cfadbg_print_safe( io, "Kernel I/O : submitted %u (among %u) for %p\n", idx, ret, active_thread() ); 560 } 561 else 562 { 458 563 // get mutual exclusion 459 564 #if defined(LEADER_LOCK) … … 463 568 #endif 464 569 465 /* paranoid */ verifyf( ring.submit_q.sqes[ idx ].user_data != 0,570 /* paranoid */ verifyf( ring.submit_q.sqes[ idx ].user_data != 3ul64, 466 571 /* paranoid */ "index %u already reclaimed\n" 467 572 /* paranoid */ "head %u, prev %u, tail %u\n" … … 490 595 } 491 596 597 /* paranoid */ verify(ret == 1); 598 492 599 // update statistics 493 600 __STATS__( false, … … 496 603 ) 497 604 605 { 606 __attribute__((unused)) volatile __u32 * const head = ring.submit_q.head; 607 __attribute__((unused)) __u32 last_idx = ring.submit_q.array[ ((*head) - 1) & mask ]; 608 __attribute__((unused)) volatile struct io_uring_sqe * sqe = &ring.submit_q.sqes[last_idx]; 609 610 __cfadbg_print_safe( io, 611 "Kernel I/O : last submitted is %u (%p)\n" 612 " data: %p\n" 613 " opcode: %s\n" 614 " fd: %d\n" 615 " flags: %d\n" 616 " prio: %d\n" 617 " off: %p\n" 618 " addr: %p\n" 619 " len: %d\n" 620 " other flags: %d\n" 621 " splice fd: %d\n" 622 " pad[0]: %llu\n" 623 " pad[1]: %llu\n" 624 " pad[2]: %llu\n", 625 last_idx, sqe, 626 (void*)sqe->user_data, 627 opcodes[sqe->opcode], 628 sqe->fd, 629 sqe->flags, 630 sqe->ioprio, 631 sqe->off, 632 sqe->addr, 633 sqe->len, 634 sqe->accept_flags, 635 sqe->splice_fd_in, 636 sqe->__pad2[0], 637 sqe->__pad2[1], 638 sqe->__pad2[2] 639 ); 640 } 641 642 __atomic_thread_fence( __ATOMIC_SEQ_CST ); 498 643 // Release the consumed SQEs 499 644 __release_consumed_submission( ring ); 645 // ring.submit_q.sqes[idx].user_data = 3ul64; 500 646 501 647 #if defined(LEADER_LOCK) … … 505 651 #endif 506 652 507 __cfadbg_print_safe( io, "Kernel I/O : Performed io_submit for %p, returned %d\n", active_thread(), ret);653 __cfadbg_print_safe( io, "Kernel I/O : submitted %u for %p\n", idx, active_thread() ); 508 654 } 509 655 } 510 656 511 657 // #define PARTIAL_SUBMIT 32 658 659 // go through the list of submissions in the ready array and moved them into 660 // the ring's submit queue 512 661 static unsigned __collect_submitions( struct __io_data & ring ) { 513 662 /* paranoid */ verify( ring.submit_q.ready != 0p ); … … 550 699 } 551 700 701 // Go through the ring's submit queue and release everything that has already been consumed 702 // by io_uring 552 703 static __u32 __release_consumed_submission( struct __io_data & ring ) { 553 704 const __u32 smask = *ring.submit_q.mask; 554 705 706 // We need to get the lock to copy the old head and new head 555 707 if( !try_lock(ring.submit_q.release_lock __cfaabi_dbg_ctx2) ) return 0; 556 __u32 chead = *ring.submit_q.head; 557 __u32 phead = ring.submit_q.prev_head; 558 ring.submit_q.prev_head = chead; 708 __attribute__((unused)) 709 __u32 ctail = *ring.submit_q.tail; // get the current tail of the queue 710 __u32 chead = *ring.submit_q.head; // get the current head of the queue 711 __u32 phead = ring.submit_q.prev_head; // get the head the last time we were here 712 ring.submit_q.prev_head = chead; // note up to were we processed 559 713 unlock(ring.submit_q.release_lock); 560 714 715 // the 3 fields are organized like this diagram 716 // except it's are ring 717 // ---+--------+--------+---- 718 // ---+--------+--------+---- 719 // ^ ^ ^ 720 // phead chead ctail 721 722 // make sure ctail doesn't wrap around and reach phead 723 /* paranoid */ verify( 724 (ctail >= chead && chead >= phead) 725 || (chead >= phead && phead >= ctail) 726 || (phead >= ctail && ctail >= chead) 727 ); 728 729 // find the range we need to clear 561 730 __u32 count = chead - phead; 731 732 // We acquired an previous-head/current-head range 733 // go through the range and release the sqes 562 734 for( i; count ) { 563 735 __u32 idx = ring.submit_q.array[ (phead + i) & smask ]; 564 ring.submit_q.sqes[ idx ].user_data = 0; 736 737 /* paranoid */ verify( 0 != ring.submit_q.sqes[ idx ].user_data ); 738 __clean( &ring.submit_q.sqes[ idx ] ); 565 739 } 566 740 return count; 567 741 } 742 743 void __sqe_clean( volatile struct io_uring_sqe * sqe ) { 744 __clean( sqe ); 745 } 746 747 static inline void __clean( volatile struct io_uring_sqe * sqe ) { 748 // If we are in debug mode, thrash the fields to make sure we catch reclamation errors 749 __cfaabi_dbg_debug_do( 750 memset(sqe, 0xde, sizeof(*sqe)); 751 sqe->opcode = (sizeof(opcodes) / sizeof(const char *)) - 1; 752 ); 753 754 // Mark the entry as unused 755 __atomic_store_n(&sqe->user_data, 3ul64, __ATOMIC_SEQ_CST); 756 } 568 757 #endif -
libcfa/src/concurrency/io/call.cfa.in
r4468a70 r342af53 74 74 ; 75 75 76 extern [* struct io_uring_sqe, __u32] __submit_alloc( struct __io_data & ring, __u64 data );76 extern [* volatile struct io_uring_sqe, __u32] __submit_alloc( struct __io_data & ring, __u64 data ); 77 77 extern void __submit( struct io_context * ctx, __u32 idx ) __attribute__((nonnull (1))); 78 78 … … 222 222 __u32 idx; 223 223 struct io_uring_sqe * sqe; 224 [sqe, idx] = __submit_alloc( ring, (__u64)(uintptr_t)&future ); 225 226 sqe->__pad2[0] = sqe->__pad2[1] = sqe->__pad2[2] = 0; 224 [(volatile struct io_uring_sqe *) sqe, idx] = __submit_alloc( ring, (__u64)(uintptr_t)&future ); 225 227 226 sqe->opcode = IORING_OP_{op}; 228 sqe->flags = sflags;{body} 227 sqe->flags = sflags; 228 sqe->ioprio = 0; 229 sqe->fd = 0; 230 sqe->off = 0; 231 sqe->addr = 0; 232 sqe->len = 0; 233 sqe->fsync_flags = 0; 234 sqe->__pad2[0] = 0; 235 sqe->__pad2[1] = 0; 236 sqe->__pad2[2] = 0;{body} 237 238 asm volatile("": : :"memory"); 229 239 230 240 verify( sqe->user_data == (__u64)(uintptr_t)&future ); … … 312 322 }), 313 323 # CFA_HAVE_IORING_OP_ACCEPT 314 Call('ACCEPT 4', 'int accept4(int sockfd, struct sockaddr *addr, socklen_t *addrlen, int flags)', {324 Call('ACCEPT', 'int accept4(int sockfd, struct sockaddr *addr, socklen_t *addrlen, int flags)', { 315 325 'fd': 'sockfd', 316 'addr': ' addr',317 'addr2': ' addrlen',326 'addr': '(__u64)addr', 327 'addr2': '(__u64)addrlen', 318 328 'accept_flags': 'flags' 319 329 }), … … 464 474 465 475 print(""" 476 //----------------------------------------------------------------------------- 477 bool cancel(io_cancellation & this) { 478 #if !defined(CFA_HAVE_LINUX_IO_URING_H) || !defined(CFA_HAVE_IORING_OP_ASYNC_CANCEL) 479 return false; 480 #else 481 io_future_t future; 482 483 io_context * context = __get_io_context(); 484 485 __u8 sflags = 0; 486 struct __io_data & ring = *context->thrd.ring; 487 488 __u32 idx; 489 volatile struct io_uring_sqe * sqe; 490 [sqe, idx] = __submit_alloc( ring, (__u64)(uintptr_t)&future ); 491 492 sqe->__pad2[0] = sqe->__pad2[1] = sqe->__pad2[2] = 0; 493 sqe->opcode = IORING_OP_ASYNC_CANCEL; 494 sqe->flags = sflags; 495 sqe->addr = this.target; 496 497 verify( sqe->user_data == (__u64)(uintptr_t)&future ); 498 __submit( context, idx ); 499 500 wait(future); 501 502 if( future.result == 0 ) return true; // Entry found 503 if( future.result == -EALREADY) return true; // Entry found but in progress 504 if( future.result == -ENOENT ) return false; // Entry not found 505 return false; 506 #endif 507 } 508 466 509 //----------------------------------------------------------------------------- 467 510 // Check if a function is has asynchronous -
libcfa/src/concurrency/io/setup.cfa
r4468a70 r342af53 52 52 #include <pthread.h> 53 53 #include <sys/epoll.h> 54 #include <sys/eventfd.h> 54 55 #include <sys/mman.h> 55 56 #include <sys/syscall.h> … … 169 170 // Main loop 170 171 while( iopoll.run ) { 172 __cfadbg_print_safe(io_core, "Kernel I/O - epoll : waiting on io_uring contexts\n"); 173 171 174 // Wait for events 172 175 int nfds = epoll_pwait( iopoll.epollfd, events, 10, -1, &mask ); 176 177 __cfadbg_print_safe(io_core, "Kernel I/O - epoll : %d io contexts events, waking up\n", nfds); 173 178 174 179 // Check if an error occured … … 181 186 $io_ctx_thread * io_ctx = ($io_ctx_thread *)(uintptr_t)events[i].data.u64; 182 187 /* paranoid */ verify( io_ctx ); 183 __cfadbg_print_safe(io_core, "Kernel I/O : Unparking io poller %p\n", io_ctx);188 __cfadbg_print_safe(io_core, "Kernel I/O - epoll : Unparking io poller %d (%p)\n", io_ctx->ring->fd, io_ctx); 184 189 #if !defined( __CFA_NO_STATISTICS__ ) 185 190 __cfaabi_tls.this_stats = io_ctx->self.curr_cluster->stats; 186 191 #endif 192 193 eventfd_t v; 194 eventfd_read(io_ctx->ring->efd, &v); 195 187 196 post( io_ctx->sem ); 188 197 } … … 233 242 $thread & thrd = this.thrd.self; 234 243 if( cluster_context ) { 244 // We are about to do weird things with the threads 245 // we don't need interrupts to complicate everything 246 disable_interrupts(); 247 248 // Get cluster info 235 249 cluster & cltr = *thrd.curr_cluster; 236 250 /* paranoid */ verify( cltr.idles.total == 0 || &cltr == mainCluster ); … … 239 253 // We need to adjust the clean-up based on where the thread is 240 254 if( thrd.state == Ready || thrd.preempted != __NO_PREEMPTION ) { 255 // This is the tricky case 256 // The thread was preempted or ready to run and now it is on the ready queue 257 // but the cluster is shutting down, so there aren't any processors to run the ready queue 258 // the solution is to steal the thread from the ready-queue and pretend it was blocked all along 241 259 242 260 ready_schedule_lock(); 243 244 // This is the tricky case 245 // The thread was preempted and now it is on the ready queue 261 // The thread should on the list 262 /* paranoid */ verify( thrd.link.next != 0p ); 263 264 // Remove the thread from the ready queue of this cluster 246 265 // The thread should be the last on the list 247 /* paranoid */ verify( thrd.link.next != 0p );248 249 // Remove the thread from the ready queue of this cluster250 266 __attribute__((unused)) bool removed = remove_head( &cltr, &thrd ); 251 267 /* paranoid */ verify( removed ); … … 263 279 } 264 280 // !!! This is not an else if !!! 281 // Ok, now the thread is blocked (whether we cheated to get here or not) 265 282 if( thrd.state == Blocked ) { 266 267 283 // This is the "easy case" 268 284 // The thread is parked and can easily be moved to active cluster … … 274 290 } 275 291 else { 276 277 292 // The thread is in a weird state 278 293 // I don't know what to do here 279 294 abort("io_context poller thread is in unexpected state, cannot clean-up correctly\n"); 280 295 } 296 297 // The weird thread kidnapping stuff is over, restore interrupts. 298 enable_interrupts( __cfaabi_dbg_ctx ); 281 299 } else { 282 300 post( this.thrd.sem ); … … 365 383 } 366 384 385 // Step 3 : Initialize the data structure 367 386 // Get the pointers from the kernel to fill the structure 368 387 // submit queue … … 379 398 const __u32 num = *sq.num; 380 399 for( i; num ) { 381 sq.sqes[i].user_data = 0ul64;400 __sqe_clean( &sq.sqes[i] ); 382 401 } 383 402 } … … 409 428 cq.cqes = (struct io_uring_cqe *)(((intptr_t)cq.ring_ptr) + params.cq_off.cqes); 410 429 430 // Step 4 : eventfd 431 int efd; 432 for() { 433 efd = eventfd(0, 0); 434 if (efd < 0) { 435 if (errno == EINTR) continue; 436 abort("KERNEL ERROR: IO_URING EVENTFD - %s\n", strerror(errno)); 437 } 438 break; 439 } 440 441 int ret; 442 for() { 443 ret = syscall( __NR_io_uring_register, fd, IORING_REGISTER_EVENTFD, &efd, 1); 444 if (ret < 0) { 445 if (errno == EINTR) continue; 446 abort("KERNEL ERROR: IO_URING EVENTFD REGISTER - %s\n", strerror(errno)); 447 } 448 break; 449 } 450 411 451 // some paranoid checks 412 452 /* paranoid */ verifyf( (*cq.mask) == ((*cq.num) - 1ul32), "IO_URING Expected mask to be %u (%u entries), was %u", (*cq.num) - 1ul32, *cq.num, *cq.mask ); … … 423 463 this.ring_flags = params.flags; 424 464 this.fd = fd; 465 this.efd = efd; 425 466 this.eager_submits = params_in.eager_submits; 426 467 this.poller_submits = params_in.poller_submits; … … 445 486 // close the file descriptor 446 487 close(this.fd); 488 close(this.efd); 447 489 448 490 free( this.submit_q.ready ); // Maybe null, doesn't matter … … 452 494 // I/O Context Sleep 453 495 //============================================================================================= 454 455 void __ioctx_register($io_ctx_thread & ctx, struct epoll_event & ev) { 456 ev.events = EPOLLIN | EPOLLONESHOT; 496 #define IOEVENTS EPOLLIN | EPOLLONESHOT 497 498 static inline void __ioctx_epoll_ctl($io_ctx_thread & ctx, int op, const char * error) { 499 struct epoll_event ev; 500 ev.events = IOEVENTS; 457 501 ev.data.u64 = (__u64)&ctx; 458 int ret = epoll_ctl(iopoll.epollfd, EPOLL_CTL_ADD, ctx.ring->fd, &ev);502 int ret = epoll_ctl(iopoll.epollfd, op, ctx.ring->efd, &ev); 459 503 if (ret < 0) { 460 abort( "KERNEL ERROR: EPOLL ADD - (%d) %s\n", (int)errno, strerror(errno) ); 461 } 462 } 463 464 void __ioctx_prepare_block($io_ctx_thread & ctx, struct epoll_event & ev) { 465 int ret = epoll_ctl(iopoll.epollfd, EPOLL_CTL_MOD, ctx.ring->fd, &ev); 466 if (ret < 0) { 467 abort( "KERNEL ERROR: EPOLL REARM - (%d) %s\n", (int)errno, strerror(errno) ); 468 } 504 abort( "KERNEL ERROR: EPOLL %s - (%d) %s\n", error, (int)errno, strerror(errno) ); 505 } 506 } 507 508 void __ioctx_register($io_ctx_thread & ctx) { 509 __ioctx_epoll_ctl(ctx, EPOLL_CTL_ADD, "ADD"); 510 } 511 512 void __ioctx_prepare_block($io_ctx_thread & ctx) { 513 __cfadbg_print_safe(io_core, "Kernel I/O - epoll : Re-arming io poller %d (%p)\n", ctx.ring->fd, &ctx); 514 __ioctx_epoll_ctl(ctx, EPOLL_CTL_MOD, "REARM"); 469 515 } 470 516 -
libcfa/src/concurrency/io/types.hfa
r4468a70 r342af53 65 65 66 66 // A buffer of sqes (not the actual ring) 67 struct io_uring_sqe * sqes;67 volatile struct io_uring_sqe * sqes; 68 68 69 69 // The location and size of the mmaped area … … 85 85 86 86 // the kernel ring 87 struct io_uring_cqe * cqes;87 volatile struct io_uring_cqe * cqes; 88 88 89 89 // The location and size of the mmaped area … … 97 97 __u32 ring_flags; 98 98 int fd; 99 int efd; 99 100 bool eager_submits:1; 100 101 bool poller_submits:1; … … 130 131 #endif 131 132 132 struct epoll_event;133 133 struct $io_ctx_thread; 134 void __ioctx_register($io_ctx_thread & ctx, struct epoll_event & ev); 135 void __ioctx_prepare_block($io_ctx_thread & ctx, struct epoll_event & ev); 134 void __ioctx_register($io_ctx_thread & ctx); 135 void __ioctx_prepare_block($io_ctx_thread & ctx); 136 void __sqe_clean( volatile struct io_uring_sqe * sqe ); 136 137 #endif 137 138 -
libcfa/src/concurrency/monitor.hfa
r4468a70 r342af53 132 132 133 133 void wait ( condition & this, uintptr_t user_info = 0 ); 134 static inline bool is_empty ( condition & this ) { return this.blocked.head == 1p; } 134 135 bool signal ( condition & this ); 135 136 bool signal_block( condition & this ); 136 static inline bool is_empty ( condition & this ) { return this.blocked.head == 1p; }137 static inline bool signal_all ( condition & this ) { bool ret = false; while(!is_empty(this)) { ret = signal(this) || ret; } return ret; } 137 138 uintptr_t front ( condition & this ); 138 139 -
libcfa/src/heap.cfa
r4468a70 r342af53 10 10 // Created On : Tue Dec 19 21:58:35 2017 11 11 // Last Modified By : Peter A. Buhr 12 // Last Modified On : Wed Dec 16 12:28:25 202013 // Update Count : 10 2312 // Last Modified On : Sun Jan 10 11:20:49 2021 13 // Update Count : 1031 14 14 // 15 15 … … 262 262 #ifdef __STATISTICS__ 263 263 // Heap statistics counters. 264 static unsigned int malloc_ calls;264 static unsigned int malloc_zero_calls, malloc_calls; 265 265 static unsigned long long int malloc_storage; 266 static unsigned int aalloc_ calls;266 static unsigned int aalloc_zero_calls, aalloc_calls; 267 267 static unsigned long long int aalloc_storage; 268 static unsigned int calloc_ calls;268 static unsigned int calloc_zero_calls, calloc_calls; 269 269 static unsigned long long int calloc_storage; 270 static unsigned int memalign_ calls;270 static unsigned int memalign_zero_calls, memalign_calls; 271 271 static unsigned long long int memalign_storage; 272 static unsigned int amemalign_ calls;272 static unsigned int amemalign_zero_calls, amemalign_calls; 273 273 static unsigned long long int amemalign_storage; 274 static unsigned int cmemalign_ calls;274 static unsigned int cmemalign_zero_calls, cmemalign_calls; 275 275 static unsigned long long int cmemalign_storage; 276 static unsigned int resize_ calls;276 static unsigned int resize_zero_calls, resize_calls; 277 277 static unsigned long long int resize_storage; 278 static unsigned int realloc_ calls;278 static unsigned int realloc_zero_calls, realloc_calls; 279 279 static unsigned long long int realloc_storage; 280 static unsigned int free_ calls;280 static unsigned int free_zero_calls, free_calls; 281 281 static unsigned long long int free_storage; 282 282 static unsigned int mmap_calls; … … 287 287 static unsigned long long int sbrk_storage; 288 288 // Statistics file descriptor (changed by malloc_stats_fd). 289 static int stat _fd = STDERR_FILENO;// default stderr289 static int stats_fd = STDERR_FILENO; // default stderr 290 290 291 291 // Use "write" because streams may be shutdown when calls are made. … … 293 293 char helpText[1024]; 294 294 __cfaabi_bits_print_buffer( STDERR_FILENO, helpText, sizeof(helpText), 295 296 " malloc: calls %u / storage %llu\n"297 " aalloc: calls %u / storage %llu\n"298 " calloc: calls %u / storage %llu\n"299 " memalign: calls %u / storage %llu\n"300 " amemalign: calls %u / storage %llu\n"301 " cmemalign: calls %u / storage %llu\n"302 " resize: calls %u / storage %llu\n"303 " realloc: calls %u / storage %llu\n"304 " free: calls %u / storage %llu\n"305 " mmap: calls %u / storage %llu\n"306 " munmap: calls %u / storage %llu\n"307 " sbrk: calls %u / storage %llu\n",308 309 310 311 312 313 314 315 316 317 318 319 295 "\nHeap statistics:\n" 296 " malloc 0-calls %'u; >0-calls %'u; storage %'llu bytes\n" 297 " aalloc 0-calls %'u; >0-calls %'u; storage %'llu bytes\n" 298 " calloc 0-calls %'u; >0-calls %'u; storage %'llu bytes\n" 299 " memalign 0-calls %'u; >0-calls %'u; storage %'llu bytes\n" 300 " amemalign 0-calls %'u; >0-calls %'u; storage %'llu bytes\n" 301 " cmemalign 0-calls %'u; >0-calls %'u; storage %'llu bytes\n" 302 " resize 0-calls %'u; >0-calls %'u; storage %'llu bytes\n" 303 " realloc 0-calls %'u; >0-calls %'u; storage %'llu bytes\n" 304 " free 0-calls %'u; >0-calls %'u; storage %'llu bytes\n" 305 " mmap calls %'u; storage %'llu bytes\n" 306 " munmap calls %'u; storage %'llu bytes\n" 307 " sbrk calls %'u; storage %'llu bytes\n", 308 malloc_zero_calls, malloc_calls, malloc_storage, 309 aalloc_zero_calls, aalloc_calls, aalloc_storage, 310 calloc_zero_calls, calloc_calls, calloc_storage, 311 memalign_zero_calls, memalign_calls, memalign_storage, 312 amemalign_zero_calls, amemalign_calls, amemalign_storage, 313 cmemalign_zero_calls, cmemalign_calls, cmemalign_storage, 314 resize_zero_calls, resize_calls, resize_storage, 315 realloc_zero_calls, realloc_calls, realloc_storage, 316 free_zero_calls, free_calls, free_storage, 317 mmap_calls, mmap_storage, 318 munmap_calls, munmap_storage, 319 sbrk_calls, sbrk_storage 320 320 ); 321 321 } // printStats … … 328 328 "<sizes>\n" 329 329 "</sizes>\n" 330 "<total type=\"malloc\" count=\"%u\" size=\"%llu\"/>\n"331 "<total type=\"aalloc\" count=\"%u\" size=\"%llu\"/>\n"332 "<total type=\"calloc\" count=\"%u\" size=\"%llu\"/>\n"333 "<total type=\"memalign\" count=\"%u\" size=\"%llu\"/>\n"334 "<total type=\"amemalign\" count=\"%u\" size=\"%llu\"/>\n"335 "<total type=\"cmemalign\" count=\"%u\" size=\"%llu\"/>\n"336 "<total type=\"resize\" count=\"%u\" size=\"%llu\"/>\n"337 "<total type=\"realloc\" count=\"%u\" size=\"%llu\"/>\n"338 "<total type=\"free\" count=\"%u\" size=\"%llu\"/>\n"339 "<total type=\"mmap\" count=\"% u\" size=\"%llu\"/>\n"340 "<total type=\"munmap\" count=\"% u\" size=\"%llu\"/>\n"341 "<total type=\"sbrk\" count=\"% u\" size=\"%llu\"/>\n"330 "<total type=\"malloc\" 0 count=\"%'u;\" >0 count=\"%'u;\" size=\"%'llu\"/> bytes\n" 331 "<total type=\"aalloc\" 0 count=\"%'u;\" >0 count=\"%'u;\" size=\"%'llu\"/> bytes\n" 332 "<total type=\"calloc\" 0 count=\"%'u;\" >0 count=\"%'u;\" size=\"%'llu\"/> bytes\n" 333 "<total type=\"memalign\" 0 count=\"%'u;\" >0 count=\"%'u;\" size=\"%'llu\"/> bytes\n" 334 "<total type=\"amemalign\" 0 count=\"%'u;\" >0 count=\"%'u;\" size=\"%'llu\"/> bytes\n" 335 "<total type=\"cmemalign\" 0 count=\"%'u;\" >0 count=\"%'u;\" size=\"%'llu\"/> bytes\n" 336 "<total type=\"resize\" 0 count=\"%'u;\" >0 count=\"%'u;\" size=\"%'llu\"/> bytes\n" 337 "<total type=\"realloc\" 0 count=\"%'u;\" >0 count=\"%'u;\" size=\"%'llu\"/> bytes\n" 338 "<total type=\"free\" 0 count=\"%'u;\" >0 count=\"%'u;\" size=\"%'llu\"/> bytes\n" 339 "<total type=\"mmap\" count=\"%'u;\" size=\"%'llu\"/> bytes\n" 340 "<total type=\"munmap\" count=\"%'u;\" size=\"%'llu\"/> bytes\n" 341 "<total type=\"sbrk\" count=\"%'u;\" size=\"%'llu\"/> bytes\n" 342 342 "</malloc>", 343 malloc_ calls, malloc_storage,344 aalloc_ calls, aalloc_storage,345 calloc_ calls, calloc_storage,346 memalign_ calls, memalign_storage,347 amemalign_ calls, amemalign_storage,348 cmemalign_ calls, cmemalign_storage,349 resize_ calls, resize_storage,350 realloc_ calls, realloc_storage,351 free_ calls, free_storage,343 malloc_zero_calls, malloc_calls, malloc_storage, 344 aalloc_zero_calls, aalloc_calls, aalloc_storage, 345 calloc_zero_calls, calloc_calls, calloc_storage, 346 memalign_zero_calls, memalign_calls, memalign_storage, 347 amemalign_zero_calls, amemalign_calls, amemalign_storage, 348 cmemalign_zero_calls, cmemalign_calls, cmemalign_storage, 349 resize_zero_calls, resize_calls, resize_storage, 350 realloc_zero_calls, realloc_calls, realloc_storage, 351 free_zero_calls, free_calls, free_storage, 352 352 mmap_calls, mmap_storage, 353 353 munmap_calls, munmap_storage, … … 466 466 } // headers 467 467 468 #ifdef __CFA_DEBUG__469 #if __SIZEOF_POINTER__ == 4470 #define MASK 0xdeadbeef471 #else472 #define MASK 0xdeadbeefdeadbeef473 #endif474 #define STRIDE size_t475 476 static void * Memset( void * addr, STRIDE size ) { // debug only477 if ( size % sizeof(STRIDE) != 0 ) abort( "Memset() : internal error, size %zd not multiple of %zd.", size, sizeof(STRIDE) );478 if ( (STRIDE)addr % sizeof(STRIDE) != 0 ) abort( "Memset() : internal error, addr %p not multiple of %zd.", addr, sizeof(STRIDE) );479 480 STRIDE * end = (STRIDE *)addr + size / sizeof(STRIDE);481 for ( STRIDE * p = (STRIDE *)addr; p < end; p += 1 ) *p = MASK;482 return addr;483 } // Memset484 #endif // __CFA_DEBUG__468 // #ifdef __CFA_DEBUG__ 469 // #if __SIZEOF_POINTER__ == 4 470 // #define MASK 0xdeadbeef 471 // #else 472 // #define MASK 0xdeadbeefdeadbeef 473 // #endif 474 // #define STRIDE size_t 475 476 // static void * Memset( void * addr, STRIDE size ) { // debug only 477 // if ( size % sizeof(STRIDE) != 0 ) abort( "Memset() : internal error, size %zd not multiple of %zd.", size, sizeof(STRIDE) ); 478 // if ( (STRIDE)addr % sizeof(STRIDE) != 0 ) abort( "Memset() : internal error, addr %p not multiple of %zd.", addr, sizeof(STRIDE) ); 479 480 // STRIDE * end = (STRIDE *)addr + size / sizeof(STRIDE); 481 // for ( STRIDE * p = (STRIDE *)addr; p < end; p += 1 ) *p = MASK; 482 // return addr; 483 // } // Memset 484 // #endif // __CFA_DEBUG__ 485 485 486 486 … … 498 498 unlock( extlock ); 499 499 __cfaabi_bits_print_nolock( STDERR_FILENO, NO_MEMORY_MSG, size ); 500 _exit( EXIT_FAILURE ); 501 } // if 500 _exit( EXIT_FAILURE ); // give up 501 } // if 502 // Make storage executable for thunks. 502 503 if ( mprotect( (char *)heapEnd + heapRemaining, increase, __map_prot ) ) { 503 504 unlock( extlock ); … … 770 771 771 772 772 static inline void * callocNoStats( size_t dim, size_t elemSize ) {773 size_t size = dim * elemSize;774 if ( unlikely( size ) == 0 ) return 0p; // 0 BYTE ALLOCATION RETURNS NULL POINTER775 char * addr = (char *)mallocNoStats( size );776 777 HeapManager.Storage.Header * header;778 HeapManager.FreeHeader * freeElem;779 size_t bsize, alignment;780 #ifndef __CFA_DEBUG__781 bool mapped =782 #endif // __CFA_DEBUG__783 headers( "calloc", addr, header, freeElem, bsize, alignment );784 #ifndef __CFA_DEBUG__785 786 // Mapped storage is zero filled, but in debug mode mapped memory is scrubbed in doMalloc, so it has to be reset to zero.787 if ( ! mapped )788 #endif // __CFA_DEBUG__789 // <-------0000000000000000000000000000UUUUUUUUUUUUUUUUUUUUUUUUU> bsize (bucket size) U => undefined790 // `-header`-addr `-size791 memset( addr, '\0', size ); // set to zeros792 793 header->kind.real.blockSize |= 2; // mark as zero filled794 return addr;795 } // callocNoStats796 797 798 773 static inline void * memalignNoStats( size_t alignment, size_t size ) { 799 774 if ( unlikely( size ) == 0 ) return 0p; // 0 BYTE ALLOCATION RETURNS NULL POINTER … … 834 809 835 810 836 static inline void * cmemalignNoStats( size_t alignment, size_t dim, size_t elemSize ) {837 size_t size = dim * elemSize;838 if ( unlikely( size ) == 0 ) return 0p; // 0 BYTE ALLOCATION RETURNS NULL POINTER839 char * addr = (char *)memalignNoStats( alignment, size );840 841 HeapManager.Storage.Header * header;842 HeapManager.FreeHeader * freeElem;843 size_t bsize;844 #ifndef __CFA_DEBUG__845 bool mapped =846 #endif // __CFA_DEBUG__847 headers( "cmemalign", addr, header, freeElem, bsize, alignment );848 849 // Mapped storage is zero filled, but in debug mode mapped memory is scrubbed in doMalloc, so it has to be reset to zero.850 #ifndef __CFA_DEBUG__851 if ( ! mapped )852 #endif // __CFA_DEBUG__853 // <-------0000000000000000000000000000UUUUUUUUUUUUUUUUUUUUUUUUU> bsize (bucket size) U => undefined854 // `-header`-addr `-size855 memset( addr, '\0', size ); // set to zeros856 857 header->kind.real.blockSize |= 2; // mark as zero filled858 return addr;859 } // cmemalignNoStats860 861 862 811 extern "C" { 863 812 // Allocates size bytes and returns a pointer to the allocated memory. The contents are undefined. If size is 0, … … 865 814 void * malloc( size_t size ) { 866 815 #ifdef __STATISTICS__ 867 __atomic_add_fetch( &malloc_calls, 1, __ATOMIC_SEQ_CST ); 868 __atomic_add_fetch( &malloc_storage, size, __ATOMIC_SEQ_CST ); 816 if ( likely( size > 0 ) ) { 817 __atomic_add_fetch( &malloc_calls, 1, __ATOMIC_SEQ_CST ); 818 __atomic_add_fetch( &malloc_storage, size, __ATOMIC_SEQ_CST ); 819 } else { 820 __atomic_add_fetch( &malloc_zero_calls, 1, __ATOMIC_SEQ_CST ); 821 } // if 869 822 #endif // __STATISTICS__ 870 823 … … 877 830 size_t size = dim * elemSize; 878 831 #ifdef __STATISTICS__ 879 __atomic_add_fetch( &aalloc_calls, 1, __ATOMIC_SEQ_CST ); 880 __atomic_add_fetch( &aalloc_storage, size, __ATOMIC_SEQ_CST ); 832 if ( likely( size > 0 ) ) { 833 __atomic_add_fetch( &aalloc_calls, 1, __ATOMIC_SEQ_CST ); 834 __atomic_add_fetch( &aalloc_storage, size, __ATOMIC_SEQ_CST ); 835 } else { 836 __atomic_add_fetch( &aalloc_zero_calls, 1, __ATOMIC_SEQ_CST ); 837 } // if 881 838 #endif // __STATISTICS__ 882 839 … … 887 844 // Same as aalloc() with memory set to zero. 888 845 void * calloc( size_t dim, size_t elemSize ) { 846 size_t size = dim * elemSize; 847 if ( unlikely( size ) == 0 ) { // 0 BYTE ALLOCATION RETURNS NULL POINTER 848 #ifdef __STATISTICS__ 849 __atomic_add_fetch( &calloc_zero_calls, 1, __ATOMIC_SEQ_CST ); 850 #endif // __STATISTICS__ 851 return 0p; 852 } // if 889 853 #ifdef __STATISTICS__ 890 854 __atomic_add_fetch( &calloc_calls, 1, __ATOMIC_SEQ_CST ); … … 892 856 #endif // __STATISTICS__ 893 857 894 return callocNoStats( dim, elemSize ); 858 char * addr = (char *)mallocNoStats( size ); 859 860 HeapManager.Storage.Header * header; 861 HeapManager.FreeHeader * freeElem; 862 size_t bsize, alignment; 863 864 #ifndef __CFA_DEBUG__ 865 bool mapped = 866 #endif // __CFA_DEBUG__ 867 headers( "calloc", addr, header, freeElem, bsize, alignment ); 868 869 #ifndef __CFA_DEBUG__ 870 // Mapped storage is zero filled, but in debug mode mapped memory is scrubbed in doMalloc, so it has to be reset to zero. 871 if ( ! mapped ) 872 #endif // __CFA_DEBUG__ 873 // <-------0000000000000000000000000000UUUUUUUUUUUUUUUUUUUUUUUUU> bsize (bucket size) U => undefined 874 // `-header`-addr `-size 875 memset( addr, '\0', size ); // set to zeros 876 877 header->kind.real.blockSize |= 2; // mark as zero filled 878 return addr; 895 879 } // calloc 896 880 … … 901 885 // call to malloc(), alloc(), calloc() or realloc(). If the area pointed to was moved, a free(oaddr) is done. 902 886 void * resize( void * oaddr, size_t size ) { 887 // If size is equal to 0, either NULL or a pointer suitable to be passed to free() is returned. 888 if ( unlikely( size == 0 ) ) { // special cases 889 #ifdef __STATISTICS__ 890 __atomic_add_fetch( &resize_zero_calls, 1, __ATOMIC_SEQ_CST ); 891 #endif // __STATISTICS__ 892 free( oaddr ); 893 return 0p; 894 } // if 903 895 #ifdef __STATISTICS__ 904 896 __atomic_add_fetch( &resize_calls, 1, __ATOMIC_SEQ_CST ); 905 897 #endif // __STATISTICS__ 906 898 907 // If size is equal to 0, either NULL or a pointer suitable to be passed to free() is returned.908 if ( unlikely( size == 0 ) ) { free( oaddr ); return 0p; } // special cases909 899 if ( unlikely( oaddr == 0p ) ) { 910 900 #ifdef __STATISTICS__ … … 918 908 size_t bsize, oalign; 919 909 headers( "resize", oaddr, header, freeElem, bsize, oalign ); 910 920 911 size_t odsize = dataStorage( bsize, oaddr, header ); // data storage available in bucket 921 922 912 // same size, DO NOT preserve STICKY PROPERTIES. 923 913 if ( oalign == libAlign() && size <= odsize && odsize <= size * 2 ) { // allow 50% wasted storage for smaller size … … 940 930 // the old and new sizes. 941 931 void * realloc( void * oaddr, size_t size ) { 932 // If size is equal to 0, either NULL or a pointer suitable to be passed to free() is returned. 933 if ( unlikely( size == 0 ) ) { // special cases 934 #ifdef __STATISTICS__ 935 __atomic_add_fetch( &realloc_zero_calls, 1, __ATOMIC_SEQ_CST ); 936 #endif // __STATISTICS__ 937 free( oaddr ); 938 return 0p; 939 } // if 942 940 #ifdef __STATISTICS__ 943 941 __atomic_add_fetch( &realloc_calls, 1, __ATOMIC_SEQ_CST ); 944 942 #endif // __STATISTICS__ 945 943 946 // If size is equal to 0, either NULL or a pointer suitable to be passed to free() is returned.947 if ( unlikely( size == 0 ) ) { free( oaddr ); return 0p; } // special cases948 944 if ( unlikely( oaddr == 0p ) ) { 949 945 #ifdef __STATISTICS__ … … 999 995 void * memalign( size_t alignment, size_t size ) { 1000 996 #ifdef __STATISTICS__ 1001 __atomic_add_fetch( &memalign_calls, 1, __ATOMIC_SEQ_CST ); 1002 __atomic_add_fetch( &memalign_storage, size, __ATOMIC_SEQ_CST ); 997 if ( likely( size > 0 ) ) { 998 __atomic_add_fetch( &memalign_calls, 1, __ATOMIC_SEQ_CST ); 999 __atomic_add_fetch( &memalign_storage, size, __ATOMIC_SEQ_CST ); 1000 } else { 1001 __atomic_add_fetch( &memalign_zero_calls, 1, __ATOMIC_SEQ_CST ); 1002 } // if 1003 1003 #endif // __STATISTICS__ 1004 1004 … … 1011 1011 size_t size = dim * elemSize; 1012 1012 #ifdef __STATISTICS__ 1013 __atomic_add_fetch( &cmemalign_calls, 1, __ATOMIC_SEQ_CST ); 1014 __atomic_add_fetch( &cmemalign_storage, size, __ATOMIC_SEQ_CST ); 1013 if ( likely( size > 0 ) ) { 1014 __atomic_add_fetch( &cmemalign_calls, 1, __ATOMIC_SEQ_CST ); 1015 __atomic_add_fetch( &cmemalign_storage, size, __ATOMIC_SEQ_CST ); 1016 } else { 1017 __atomic_add_fetch( &cmemalign_zero_calls, 1, __ATOMIC_SEQ_CST ); 1018 } // if 1015 1019 #endif // __STATISTICS__ 1016 1020 … … 1021 1025 // Same as calloc() with memory alignment. 1022 1026 void * cmemalign( size_t alignment, size_t dim, size_t elemSize ) { 1027 size_t size = dim * elemSize; 1028 if ( unlikely( size ) == 0 ) { // 0 BYTE ALLOCATION RETURNS NULL POINTER 1029 #ifdef __STATISTICS__ 1030 __atomic_add_fetch( &cmemalign_zero_calls, 1, __ATOMIC_SEQ_CST ); 1031 #endif // __STATISTICS__ 1032 return 0p; 1033 } // if 1023 1034 #ifdef __STATISTICS__ 1024 1035 __atomic_add_fetch( &cmemalign_calls, 1, __ATOMIC_SEQ_CST ); … … 1026 1037 #endif // __STATISTICS__ 1027 1038 1028 return cmemalignNoStats( alignment, dim, elemSize ); 1039 char * addr = (char *)memalignNoStats( alignment, size ); 1040 1041 HeapManager.Storage.Header * header; 1042 HeapManager.FreeHeader * freeElem; 1043 size_t bsize; 1044 1045 #ifndef __CFA_DEBUG__ 1046 bool mapped = 1047 #endif // __CFA_DEBUG__ 1048 headers( "cmemalign", addr, header, freeElem, bsize, alignment ); 1049 1050 // Mapped storage is zero filled, but in debug mode mapped memory is scrubbed in doMalloc, so it has to be reset to zero. 1051 #ifndef __CFA_DEBUG__ 1052 if ( ! mapped ) 1053 #endif // __CFA_DEBUG__ 1054 // <-------0000000000000000000000000000UUUUUUUUUUUUUUUUUUUUUUUUU> bsize (bucket size) U => undefined 1055 // `-header`-addr `-size 1056 memset( addr, '\0', size ); // set to zeros 1057 1058 header->kind.real.blockSize |= 2; // mark as zero filled 1059 return addr; 1029 1060 } // cmemalign 1030 1061 … … 1065 1096 // 0p, no operation is performed. 1066 1097 void free( void * addr ) { 1067 #ifdef __STATISTICS__1068 __atomic_add_fetch( &free_calls, 1, __ATOMIC_SEQ_CST );1069 #endif // __STATISTICS__1070 1071 1098 if ( unlikely( addr == 0p ) ) { // special case 1099 #ifdef __STATISTICS__ 1100 __atomic_add_fetch( &free_zero_calls, 1, __ATOMIC_SEQ_CST ); 1101 #endif // __STATISTICS__ 1102 1072 1103 // #ifdef __CFA_DEBUG__ 1073 1104 // if ( traceHeap() ) { … … 1182 1213 int malloc_stats_fd( int fd __attribute__(( unused )) ) { 1183 1214 #ifdef __STATISTICS__ 1184 int temp = stat _fd;1185 stat _fd = fd;1215 int temp = stats_fd; 1216 stats_fd = fd; 1186 1217 return temp; 1187 1218 #else … … 1214 1245 // The string is printed on the file stream stream. The exported string includes information about all arenas (see 1215 1246 // malloc). 1216 int malloc_info( int options, FILE * stream ) {1247 int malloc_info( int options, FILE * stream __attribute__(( unused )) ) { 1217 1248 if ( options != 0 ) { errno = EINVAL; return -1; } 1218 1249 #ifdef __STATISTICS__ … … 1243 1274 // Must have CFA linkage to overload with C linkage realloc. 1244 1275 void * resize( void * oaddr, size_t nalign, size_t size ) { 1245 #ifdef __STATISTICS__ 1246 __atomic_add_fetch( &resize_calls, 1, __ATOMIC_SEQ_CST ); 1247 #endif // __STATISTICS__ 1276 // If size is equal to 0, either NULL or a pointer suitable to be passed to free() is returned. 1277 if ( unlikely( size == 0 ) ) { // special cases 1278 #ifdef __STATISTICS__ 1279 __atomic_add_fetch( &resize_zero_calls, 1, __ATOMIC_SEQ_CST ); 1280 #endif // __STATISTICS__ 1281 free( oaddr ); 1282 return 0p; 1283 } // if 1248 1284 1249 1285 if ( unlikely( nalign < libAlign() ) ) nalign = libAlign(); // reset alignment to minimum 1250 1286 #ifdef __CFA_DEBUG__ 1251 else 1252 checkAlign( nalign ); // check alignment 1287 else checkAlign( nalign ); // check alignment 1253 1288 #endif // __CFA_DEBUG__ 1254 1289 1255 // If size is equal to 0, either NULL or a pointer suitable to be passed to free() is returned.1256 if ( unlikely( size == 0 ) ) { free( oaddr ); return 0p; } // special cases1257 1290 if ( unlikely( oaddr == 0p ) ) { 1258 1291 #ifdef __STATISTICS__ 1292 __atomic_add_fetch( &resize_calls, 1, __ATOMIC_SEQ_CST ); 1259 1293 __atomic_add_fetch( &resize_storage, size, __ATOMIC_SEQ_CST ); 1260 1294 #endif // __STATISTICS__ … … 1302 1336 1303 1337 void * realloc( void * oaddr, size_t nalign, size_t size ) { 1338 // If size is equal to 0, either NULL or a pointer suitable to be passed to free() is returned. 1339 if ( unlikely( size == 0 ) ) { // special cases 1340 #ifdef __STATISTICS__ 1341 __atomic_add_fetch( &realloc_zero_calls, 1, __ATOMIC_SEQ_CST ); 1342 #endif // __STATISTICS__ 1343 free( oaddr ); 1344 return 0p; 1345 } // if 1346 1304 1347 if ( unlikely( nalign < libAlign() ) ) nalign = libAlign(); // reset alignment to minimum 1305 1348 #ifdef __CFA_DEBUG__ 1306 else 1307 checkAlign( nalign ); // check alignment 1349 else checkAlign( nalign ); // check alignment 1308 1350 #endif // __CFA_DEBUG__ 1309 1351 1310 // If size is equal to 0, either NULL or a pointer suitable to be passed to free() is returned.1311 if ( unlikely( size == 0 ) ) { free( oaddr ); return 0p; } // special cases1312 1352 if ( unlikely( oaddr == 0p ) ) { 1313 1353 #ifdef __STATISTICS__ -
libcfa/src/startup.cfa
r4468a70 r342af53 10 10 // Created On : Tue Jul 24 16:21:57 2018 11 11 // Last Modified By : Peter A. Buhr 12 // Last Modified On : Tue Feb 4 13:03:18 202013 // Update Count : 3 012 // Last Modified On : Sat Jan 9 23:18:23 2021 13 // Update Count : 34 14 14 // 15 15 16 #include <time.h> // tzset 17 #include <locale.h> // setlocale 16 #include <time.h> // tzset 17 #include <locale.h> // setlocale 18 #include <stdlib.h> // getenv 18 19 #include "startup.hfa" 19 20 … … 22 23 void __cfaabi_appready_startup( void ) { 23 24 tzset(); // initialize time global variables 24 setlocale( LC_NUMERIC, "");25 setlocale( LC_NUMERIC, getenv("LANG") ); 25 26 #ifdef __CFA_DEBUG__ 26 27 extern void heapAppStart();
Note:
See TracChangeset
for help on using the changeset viewer.
