Changeset af7acb9 for libcfa/src
- Timestamp:
- May 8, 2020, 11:53:29 AM (4 years ago)
- Branches:
- ADT, arm-eh, ast-experimental, enum, forall-pointer-decay, jacob/cs343-translation, master, new-ast, new-ast-unique-expr, pthread-emulation, qualifiedEnum
- Children:
- 04bc1c0
- Parents:
- 6ec07e5 (diff), e68d092 (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the(diff)links above to see all the changes relative to each parent. - Location:
- libcfa/src
- Files:
-
- 1 added
- 7 edited
-
concurrency/io.cfa (modified) (17 diffs)
-
concurrency/kernel.cfa (modified) (4 diffs)
-
concurrency/kernel.hfa (modified) (4 diffs)
-
concurrency/kernel_private.hfa (modified) (2 diffs)
-
exception.hfa (added)
-
heap.cfa (modified) (18 diffs)
-
iostream.cfa (modified) (4 diffs)
-
startup.cfa (modified) (2 diffs)
Legend:
- Unmodified
- Added
- Removed
-
libcfa/src/concurrency/io.cfa
r6ec07e5 raf7acb9 20 20 21 21 #if !defined(HAVE_LINUX_IO_URING_H) 22 void __kernel_io_startup( cluster &, bool ) {22 void __kernel_io_startup( cluster &, int, bool ) { 23 23 // Nothing to do without io_uring 24 24 } … … 86 86 #endif 87 87 88 #if defined(__CFA_IO_POLLING_USER__) 89 void ?{}( __io_poller_fast & this, struct cluster & cltr ) { 90 this.ring = &cltr.io; 91 (this.thrd){ "Fast I/O Poller", cltr }; 92 } 93 void ^?{}( __io_poller_fast & mutex this ); 94 void main( __io_poller_fast & this ); 95 static inline $thread * get_thread( __io_poller_fast & this ) { return &this.thrd; } 96 void ^?{}( __io_poller_fast & mutex this ) {} 97 #endif 88 // Fast poller user-thread 89 // Not using the "thread" keyword because we want to control 90 // more carefully when to start/stop it 91 struct __io_poller_fast { 92 struct __io_data * ring; 93 bool waiting; 94 $thread thrd; 95 }; 96 97 void ?{}( __io_poller_fast & this, struct cluster & cltr ) { 98 this.ring = cltr.io; 99 this.waiting = true; 100 (this.thrd){ "Fast I/O Poller", cltr }; 101 } 102 void ^?{}( __io_poller_fast & mutex this ); 103 void main( __io_poller_fast & this ); 104 static inline $thread * get_thread( __io_poller_fast & this ) { return &this.thrd; } 105 void ^?{}( __io_poller_fast & mutex this ) {} 106 107 struct __submition_data { 108 // Head and tail of the ring (associated with array) 109 volatile uint32_t * head; 110 volatile uint32_t * tail; 111 112 // The actual kernel ring which uses head/tail 113 // indexes into the sqes arrays 114 uint32_t * array; 115 116 // number of entries and mask to go with it 117 const uint32_t * num; 118 const uint32_t * mask; 119 120 // Submission flags (Not sure what for) 121 uint32_t * flags; 122 123 // number of sqes not submitted (whatever that means) 124 uint32_t * dropped; 125 126 // Like head/tail but not seen by the kernel 127 volatile uint32_t alloc; 128 volatile uint32_t ready; 129 130 __spinlock_t lock; 131 132 // A buffer of sqes (not the actual ring) 133 struct io_uring_sqe * sqes; 134 135 // The location and size of the mmaped area 136 void * ring_ptr; 137 size_t ring_sz; 138 139 // Statistics 140 #if !defined(__CFA_NO_STATISTICS__) 141 struct { 142 struct { 143 volatile unsigned long long int val; 144 volatile unsigned long long int cnt; 145 volatile unsigned long long int block; 146 } submit_avg; 147 } stats; 148 #endif 149 }; 150 151 struct __completion_data { 152 // Head and tail of the ring 153 volatile uint32_t * head; 154 volatile uint32_t * tail; 155 156 // number of entries and mask to go with it 157 const uint32_t * mask; 158 const uint32_t * num; 159 160 // number of cqes not submitted (whatever that means) 161 uint32_t * overflow; 162 163 // the kernel ring 164 struct io_uring_cqe * cqes; 165 166 // The location and size of the mmaped area 167 void * ring_ptr; 168 size_t ring_sz; 169 170 // Statistics 171 #if !defined(__CFA_NO_STATISTICS__) 172 struct { 173 struct { 174 unsigned long long int val; 175 unsigned long long int slow_cnt; 176 unsigned long long int fast_cnt; 177 } completed_avg; 178 } stats; 179 #endif 180 }; 181 182 struct __io_data { 183 struct __submition_data submit_q; 184 struct __completion_data completion_q; 185 uint32_t ring_flags; 186 int cltr_flags; 187 int fd; 188 semaphore submit; 189 volatile bool done; 190 struct { 191 struct { 192 void * stack; 193 pthread_t kthrd; 194 } slow; 195 __io_poller_fast fast; 196 __bin_sem_t sem; 197 } poller; 198 }; 98 199 99 200 //============================================================================================= 100 201 // I/O Startup / Shutdown logic 101 202 //============================================================================================= 102 void __kernel_io_startup( cluster & this, bool main_cluster ) { 203 void __kernel_io_startup( cluster & this, int io_flags, bool main_cluster ) { 204 this.io = malloc(); 205 103 206 // Step 1 : call to setup 104 207 struct io_uring_params params; … … 113 216 114 217 // Step 2 : mmap result 115 memset( &this.io, 0, sizeof(struct io_ring));116 struct io_uring_sq & sq = this.io.submit_q;117 struct io_uring_cq & cq = this.io.completion_q;218 memset( this.io, 0, sizeof(struct __io_data) ); 219 struct __submition_data & sq = this.io->submit_q; 220 struct __completion_data & cq = this.io->completion_q; 118 221 119 222 // calculate the right ring size … … 193 296 194 297 // Update the global ring info 195 this.io.flags = params.flags; 196 this.io.fd = fd; 197 this.io.done = false; 198 (this.io.submit){ min(*sq.num, *cq.num) }; 298 this.io->ring_flags = params.flags; 299 this.io->cltr_flags = io_flags; 300 this.io->fd = fd; 301 this.io->done = false; 302 (this.io->submit){ min(*sq.num, *cq.num) }; 199 303 200 304 // Initialize statistics 201 305 #if !defined(__CFA_NO_STATISTICS__) 202 this.io.submit_q.stats.submit_avg.val = 0; 203 this.io.submit_q.stats.submit_avg.cnt = 0; 204 this.io.completion_q.stats.completed_avg.val = 0; 205 this.io.completion_q.stats.completed_avg.cnt = 0; 306 this.io->submit_q.stats.submit_avg.val = 0; 307 this.io->submit_q.stats.submit_avg.cnt = 0; 308 this.io->submit_q.stats.submit_avg.block = 0; 309 this.io->completion_q.stats.completed_avg.val = 0; 310 this.io->completion_q.stats.completed_avg.slow_cnt = 0; 311 this.io->completion_q.stats.completed_avg.fast_cnt = 0; 206 312 #endif 207 313 … … 212 318 213 319 void __kernel_io_finish_start( cluster & this ) { 214 #if defined(__CFA_IO_POLLING_USER__)320 if( this.io->cltr_flags & CFA_CLUSTER_IO_POLLER_USER_THREAD ) { 215 321 __cfadbg_print_safe(io_core, "Kernel I/O : Creating fast poller for cluter %p\n", &this); 216 (this.io .poller.fast){ this };217 __thrd_start( this.io .poller.fast, main );218 #endif322 (this.io->poller.fast){ this }; 323 __thrd_start( this.io->poller.fast, main ); 324 } 219 325 220 326 // Create the poller thread 221 327 __cfadbg_print_safe(io_core, "Kernel I/O : Creating slow poller for cluter %p\n", &this); 222 this.io .poller.slow.stack = __create_pthread( &this.io.poller.slow.kthrd, __io_poller_slow, &this );328 this.io->poller.slow.stack = __create_pthread( &this.io->poller.slow.kthrd, __io_poller_slow, &this ); 223 329 } 224 330 … … 226 332 __cfadbg_print_safe(io_core, "Kernel I/O : Stopping pollers for cluster\n", &this); 227 333 // Notify the poller thread of the shutdown 228 __atomic_store_n(&this.io .done, true, __ATOMIC_SEQ_CST);334 __atomic_store_n(&this.io->done, true, __ATOMIC_SEQ_CST); 229 335 230 336 // Stop the IO Poller 231 337 sigval val = { 1 }; 232 pthread_sigqueue( this.io.poller.slow.kthrd, SIGUSR1, val ); 233 #if defined(__CFA_IO_POLLING_USER__) 234 post( this.io.poller.sem ); 235 #endif 338 pthread_sigqueue( this.io->poller.slow.kthrd, SIGUSR1, val ); 339 post( this.io->poller.sem ); 236 340 237 341 // Wait for the poller thread to finish 238 pthread_join( this.io .poller.slow.kthrd, 0p );239 free( this.io .poller.slow.stack );342 pthread_join( this.io->poller.slow.kthrd, 0p ); 343 free( this.io->poller.slow.stack ); 240 344 241 345 __cfadbg_print_safe(io_core, "Kernel I/O : Slow poller stopped for cluster\n", &this); 242 346 243 #if defined(__CFA_IO_POLLING_USER__) 347 if( this.io->cltr_flags & CFA_CLUSTER_IO_POLLER_USER_THREAD ) { 348 with( this.io->poller.fast ) { 349 /* paranoid */ verify( waiting ); // The thread shouldn't be in a system call 350 /* paranoid */ verify( this.procs.head == 0p || &this == mainCluster ); 351 /* paranoid */ verify( this.idles.head == 0p || &this == mainCluster ); 352 353 // We need to adjust the clean-up based on where the thread is 354 if( thrd.preempted != __NO_PREEMPTION ) { 355 356 // This is the tricky case 357 // The thread was preempted and now it is on the ready queue 358 /* paranoid */ verify( thrd.state == Active ); // The thread better be in this state 359 /* paranoid */ verify( thrd.next == 1p ); // The thread should be the last on the list 360 /* paranoid */ verify( this.ready_queue.head == &thrd ); // The thread should be the only thing on the list 361 362 // Remove the thread from the ready queue of this cluster 363 this.ready_queue.head = 1p; 364 thrd.next = 0p; 365 366 // Fixup the thread state 367 thrd.state = Blocked; 368 thrd.preempted = __NO_PREEMPTION; 369 370 // Pretend like the thread was blocked all along 371 } 372 // !!! This is not an else if !!! 373 if( thrd.state == Blocked ) { 374 375 // This is the "easy case" 376 // The thread is parked and can easily be moved to active cluster 377 verify( thrd.curr_cluster != active_cluster() || thrd.curr_cluster == mainCluster ); 378 thrd.curr_cluster = active_cluster(); 379 244 380 // unpark the fast io_poller 245 unpark( &this.io.poller.fast.thrd __cfaabi_dbg_ctx2 ); 246 247 ^(this.io.poller.fast){}; 381 unpark( &thrd __cfaabi_dbg_ctx2 ); 382 } 383 else { 384 385 // The thread is in a weird state 386 // I don't know what to do here 387 abort("Fast poller thread is in unexpected state, cannot clean-up correctly\n"); 388 } 389 390 } 391 392 ^(this.io->poller.fast){}; 248 393 249 394 __cfadbg_print_safe(io_core, "Kernel I/O : Fast poller stopped for cluster\n", &this); 250 #endif395 } 251 396 } 252 397 … … 259 404 #if !defined(__CFA_NO_STATISTICS__) 260 405 if(this.print_stats) { 261 __cfaabi_bits_print_safe( STDERR_FILENO, 262 "----- I/O uRing Stats -----\n" 263 "- total submit calls : %llu\n" 264 "- avg submit : %lf\n" 265 "- total wait calls : %llu\n" 266 "- avg completion/wait : %lf\n", 267 this.io.submit_q.stats.submit_avg.cnt, 268 ((double)this.io.submit_q.stats.submit_avg.val) / this.io.submit_q.stats.submit_avg.cnt, 269 this.io.completion_q.stats.completed_avg.cnt, 270 ((double)this.io.completion_q.stats.completed_avg.val) / this.io.completion_q.stats.completed_avg.cnt 271 ); 406 with(this.io->submit_q.stats, this.io->completion_q.stats) { 407 __cfaabi_bits_print_safe( STDERR_FILENO, 408 "----- I/O uRing Stats -----\n" 409 "- total submit calls : %'15llu\n" 410 "- avg submit : %'18.2lf\n" 411 "- pre-submit block %% : %'18.2lf\n" 412 "- total wait calls : %'15llu (%'llu slow, %'llu fast)\n" 413 "- avg completion/wait : %'18.2lf\n", 414 submit_avg.cnt, 415 ((double)submit_avg.val) / submit_avg.cnt, 416 (100.0 * submit_avg.block) / submit_avg.cnt, 417 completed_avg.slow_cnt + completed_avg.fast_cnt, 418 completed_avg.slow_cnt, completed_avg.fast_cnt, 419 ((double)completed_avg.val) / (completed_avg.slow_cnt + completed_avg.fast_cnt) 420 ); 421 } 272 422 } 273 423 #endif 274 424 275 425 // Shutdown the io rings 276 struct io_uring_sq & sq = this.io.submit_q;277 struct io_uring_cq & cq = this.io.completion_q;426 struct __submition_data & sq = this.io->submit_q; 427 struct __completion_data & cq = this.io->completion_q; 278 428 279 429 // unmap the submit queue entries … … 289 439 290 440 // close the file descriptor 291 close(this.io.fd); 441 close(this.io->fd); 442 443 free( this.io ); 292 444 } 293 445 … … 302 454 // Process a single completion message from the io_uring 303 455 // This is NOT thread-safe 304 static int __drain_io( struct io_ring& ring, sigset_t * mask, int waitcnt, bool in_kernel ) {456 static int __drain_io( struct __io_data & ring, sigset_t * mask, int waitcnt, bool in_kernel ) { 305 457 int ret = syscall( __NR_io_uring_enter, ring.fd, 0, waitcnt, IORING_ENTER_GETEVENTS, mask, _NSIG / 8); 306 458 if( ret < 0 ) { … … 320 472 // Nothing was new return 0 321 473 if (head == tail) { 322 #if !defined(__CFA_NO_STATISTICS__)323 ring.completion_q.stats.completed_avg.cnt += 1;324 #endif325 474 return 0; 326 475 } … … 348 497 __atomic_fetch_add( ring.completion_q.head, count, __ATOMIC_RELAXED ); 349 498 350 // Update statistics351 #if !defined(__CFA_NO_STATISTICS__)352 ring.completion_q.stats.completed_avg.val += count;353 ring.completion_q.stats.completed_avg.cnt += 1;354 #endif355 356 499 return count; 357 500 } … … 359 502 static void * __io_poller_slow( void * arg ) { 360 503 cluster * cltr = (cluster *)arg; 361 struct io_ring & ring =cltr->io;504 struct __io_data & ring = *cltr->io; 362 505 363 506 sigset_t mask; … … 372 515 verify( (*ring.completion_q.head) == (*ring.completion_q.tail) ); 373 516 374 while(!__atomic_load_n(&ring.done, __ATOMIC_SEQ_CST)) { 375 #if defined(__CFA_IO_POLLING_USER__) 376 517 __cfadbg_print_safe(io_core, "Kernel I/O : Slow poller for ring %p ready\n", &ring); 518 519 if( ring.cltr_flags & CFA_CLUSTER_IO_POLLER_USER_THREAD ) { 520 while(!__atomic_load_n(&ring.done, __ATOMIC_SEQ_CST)) { 377 521 // In the user-thread approach drain and if anything was drained, 378 522 // batton pass to the user-thread 379 523 int count = __drain_io( ring, &mask, 1, true ); 524 525 // Update statistics 526 #if !defined(__CFA_NO_STATISTICS__) 527 ring.completion_q.stats.completed_avg.val += count; 528 ring.completion_q.stats.completed_avg.slow_cnt += 1; 529 #endif 530 380 531 if(count > 0) { 381 532 __cfadbg_print_safe(io_core, "Kernel I/O : Moving to ring %p to fast poller\n", &ring); … … 383 534 wait( ring.poller.sem ); 384 535 } 385 386 #else 387 536 } 537 } 538 else { 539 while(!__atomic_load_n(&ring.done, __ATOMIC_SEQ_CST)) { 388 540 //In the naive approach, just poll the io completion queue directly 389 __drain_io( ring, &mask, 1, true ); 390 541 int count = __drain_io( ring, &mask, 1, true ); 542 543 // Update statistics 544 #if !defined(__CFA_NO_STATISTICS__) 545 ring.completion_q.stats.completed_avg.val += count; 546 ring.completion_q.stats.completed_avg.slow_cnt += 1; 547 #endif 548 } 549 } 550 551 __cfadbg_print_safe(io_core, "Kernel I/O : Slow poller for ring %p stopping\n", &ring); 552 553 return 0p; 554 } 555 556 void main( __io_poller_fast & this ) { 557 verify( this.ring->cltr_flags & CFA_CLUSTER_IO_POLLER_USER_THREAD ); 558 559 // Start parked 560 park( __cfaabi_dbg_ctx ); 561 562 __cfadbg_print_safe(io_core, "Kernel I/O : Fast poller for ring %p ready\n", &this.ring); 563 564 int reset = 0; 565 566 // Then loop until we need to start 567 while(!__atomic_load_n(&this.ring->done, __ATOMIC_SEQ_CST)) { 568 // Drain the io 569 this.waiting = false; 570 int count = __drain_io( *this.ring, 0p, 0, false ); 571 reset += count > 0 ? 1 : 0; 572 573 // Update statistics 574 #if !defined(__CFA_NO_STATISTICS__) 575 this.ring->completion_q.stats.completed_avg.val += count; 576 this.ring->completion_q.stats.completed_avg.fast_cnt += 1; 391 577 #endif 392 } 393 394 return 0p; 395 } 396 397 #if defined(__CFA_IO_POLLING_USER__) 398 void main( __io_poller_fast & this ) { 399 // Start parked 400 park( __cfaabi_dbg_ctx ); 401 402 // Then loop until we need to start 403 while(!__atomic_load_n(&this.ring->done, __ATOMIC_SEQ_CST)) { 404 // Drain the io 405 if(0 > __drain_io( *this.ring, 0p, 0, false )) { 406 // If we got something, just yield and check again 407 yield(); 408 } 409 else { 410 // We didn't get anything baton pass to the slow poller 411 __cfadbg_print_safe(io_core, "Kernel I/O : Moving to ring %p to slow poller\n", &this.ring); 412 post( this.ring->poller.sem ); 413 park( __cfaabi_dbg_ctx ); 414 } 578 579 this.waiting = true; 580 if(reset < 5) { 581 // If we got something, just yield and check again 582 yield(); 415 583 } 416 } 417 #endif 584 else { 585 // We didn't get anything baton pass to the slow poller 586 __cfadbg_print_safe(io_core, "Kernel I/O : Moving to ring %p to slow poller\n", &this.ring); 587 post( this.ring->poller.sem ); 588 park( __cfaabi_dbg_ctx ); 589 reset = 0; 590 } 591 } 592 593 __cfadbg_print_safe(io_core, "Kernel I/O : Fast poller for ring %p stopping\n", &this.ring); 594 } 418 595 419 596 //============================================================================================= … … 445 622 // 446 623 447 static inline [* struct io_uring_sqe, uint32_t] __submit_alloc( struct io_ring& ring ) {624 static inline [* struct io_uring_sqe, uint32_t] __submit_alloc( struct __io_data & ring ) { 448 625 // Wait for a spot to be available 449 P(ring.submit); 626 __attribute__((unused)) bool blocked = P(ring.submit); 627 #if !defined(__CFA_NO_STATISTICS__) 628 __atomic_fetch_add( &ring.submit_q.stats.submit_avg.block, blocked ? 1ul64 : 0ul64, __ATOMIC_RELAXED ); 629 #endif 450 630 451 631 // Allocate the sqe … … 463 643 } 464 644 465 static inline void __submit( struct io_ring& ring, uint32_t idx ) {645 static inline void __submit( struct __io_data & ring, uint32_t idx ) { 466 646 // get mutual exclusion 467 647 lock(ring.submit_q.lock __cfaabi_dbg_ctx2); … … 524 704 525 705 #define __submit_prelude \ 526 struct io_ring & ring =active_cluster()->io; \706 struct __io_data & ring = *active_cluster()->io; \ 527 707 struct io_uring_sqe * sqe; \ 528 708 uint32_t idx; \ -
libcfa/src/concurrency/kernel.cfa
r6ec07e5 raf7acb9 256 256 } 257 257 258 void ?{}(cluster & this, const char name[], Duration preemption_rate ) with( this ) {258 void ?{}(cluster & this, const char name[], Duration preemption_rate, int io_flags) with( this ) { 259 259 this.name = name; 260 260 this.preemption_rate = preemption_rate; … … 270 270 threads{ __get }; 271 271 272 __kernel_io_startup( this, &this == mainCluster );272 __kernel_io_startup( this, io_flags, &this == mainCluster ); 273 273 274 274 doregister(this); … … 993 993 void ^?{}(semaphore & this) {} 994 994 995 voidP(semaphore & this) with( this ){995 bool P(semaphore & this) with( this ){ 996 996 lock( lock __cfaabi_dbg_ctx2 ); 997 997 count -= 1; … … 1003 1003 unlock( lock ); 1004 1004 park( __cfaabi_dbg_ctx ); 1005 return true; 1005 1006 } 1006 1007 else { 1007 1008 unlock( lock ); 1009 return false; 1008 1010 } 1009 1011 } -
libcfa/src/concurrency/kernel.hfa
r6ec07e5 raf7acb9 38 38 void ?{}(semaphore & this, int count = 1); 39 39 void ^?{}(semaphore & this); 40 voidP (semaphore & this);40 bool P (semaphore & this); 41 41 bool V (semaphore & this); 42 42 bool V (semaphore & this, unsigned count); … … 114 114 //----------------------------------------------------------------------------- 115 115 // I/O 116 #if defined(HAVE_LINUX_IO_URING_H) 117 struct io_uring_sq { 118 // Head and tail of the ring (associated with array) 119 volatile uint32_t * head; 120 volatile uint32_t * tail; 116 struct __io_data; 121 117 122 // The actual kernel ring which uses head/tail 123 // indexes into the sqes arrays 124 uint32_t * array; 125 126 // number of entries and mask to go with it 127 const uint32_t * num; 128 const uint32_t * mask; 129 130 // Submission flags (Not sure what for) 131 uint32_t * flags; 132 133 // number of sqes not submitted (whatever that means) 134 uint32_t * dropped; 135 136 // Like head/tail but not seen by the kernel 137 volatile uint32_t alloc; 138 volatile uint32_t ready; 139 140 __spinlock_t lock; 141 142 // A buffer of sqes (not the actual ring) 143 struct io_uring_sqe * sqes; 144 145 // The location and size of the mmaped area 146 void * ring_ptr; 147 size_t ring_sz; 148 149 // Statistics 150 #if !defined(__CFA_NO_STATISTICS__) 151 struct { 152 struct { 153 unsigned long long int val; 154 unsigned long long int cnt; 155 } submit_avg; 156 } stats; 157 #endif 158 }; 159 160 struct io_uring_cq { 161 // Head and tail of the ring 162 volatile uint32_t * head; 163 volatile uint32_t * tail; 164 165 // number of entries and mask to go with it 166 const uint32_t * mask; 167 const uint32_t * num; 168 169 // number of cqes not submitted (whatever that means) 170 uint32_t * overflow; 171 172 // the kernel ring 173 struct io_uring_cqe * cqes; 174 175 // The location and size of the mmaped area 176 void * ring_ptr; 177 size_t ring_sz; 178 179 // Statistics 180 #if !defined(__CFA_NO_STATISTICS__) 181 struct { 182 struct { 183 unsigned long long int val; 184 unsigned long long int cnt; 185 } completed_avg; 186 } stats; 187 #endif 188 }; 189 190 #if defined(__CFA_IO_POLLING_USER__) 191 struct __io_poller_fast { 192 struct io_ring * ring; 193 $thread thrd; 194 }; 195 #endif 196 197 struct io_ring { 198 struct io_uring_sq submit_q; 199 struct io_uring_cq completion_q; 200 uint32_t flags; 201 int fd; 202 semaphore submit; 203 volatile bool done; 204 struct { 205 struct { 206 void * stack; 207 pthread_t kthrd; 208 } slow; 209 #if defined(__CFA_IO_POLLING_USER__) 210 __io_poller_fast fast; 211 __bin_sem_t sem; 212 #endif 213 } poller; 214 }; 215 #endif 118 #define CFA_CLUSTER_IO_POLLER_USER_THREAD 1 << 0 119 // #define CFA_CLUSTER_IO_POLLER_KERNEL_SIDE 1 << 1 216 120 217 121 //----------------------------------------------------------------------------- … … 247 151 } node; 248 152 249 #if defined(HAVE_LINUX_IO_URING_H) 250 struct io_ring io; 251 #endif 153 struct __io_data * io; 252 154 253 155 #if !defined(__CFA_NO_STATISTICS__) … … 257 159 extern Duration default_preemption(); 258 160 259 void ?{} (cluster & this, const char name[], Duration preemption_rate );161 void ?{} (cluster & this, const char name[], Duration preemption_rate, int flags); 260 162 void ^?{}(cluster & this); 261 163 262 static inline void ?{} (cluster & this) { this{"Anonymous Cluster", default_preemption()}; } 263 static inline void ?{} (cluster & this, Duration preemption_rate) { this{"Anonymous Cluster", preemption_rate}; } 264 static inline void ?{} (cluster & this, const char name[]) { this{name, default_preemption()}; } 164 static inline void ?{} (cluster & this) { this{"Anonymous Cluster", default_preemption(), 0}; } 165 static inline void ?{} (cluster & this, Duration preemption_rate) { this{"Anonymous Cluster", preemption_rate, 0}; } 166 static inline void ?{} (cluster & this, const char name[]) { this{name, default_preemption(), 0}; } 167 static inline void ?{} (cluster & this, int flags) { this{"Anonymous Cluster", default_preemption(), flags}; } 168 static inline void ?{} (cluster & this, Duration preemption_rate, int flags) { this{"Anonymous Cluster", preemption_rate, flags}; } 169 static inline void ?{} (cluster & this, const char name[], int flags) { this{name, default_preemption(), flags}; } 265 170 266 171 static inline [cluster *&, cluster *& ] __get( cluster & this ) __attribute__((const)) { return this.node.[next, prev]; } -
libcfa/src/concurrency/kernel_private.hfa
r6ec07e5 raf7acb9 59 59 extern volatile thread_local __cfa_kernel_preemption_state_t preemption_state __attribute__ ((tls_model ( "initial-exec" ))); 60 60 61 extern cluster * mainCluster; 62 61 63 //----------------------------------------------------------------------------- 62 64 // Threads … … 75 77 //----------------------------------------------------------------------------- 76 78 // I/O 77 void __kernel_io_startup ( cluster &, bool );79 void __kernel_io_startup ( cluster &, int, bool ); 78 80 void __kernel_io_finish_start( cluster & ); 79 81 void __kernel_io_prepare_stop( cluster & ); -
libcfa/src/heap.cfa
r6ec07e5 raf7acb9 10 10 // Created On : Tue Dec 19 21:58:35 2017 11 11 // Last Modified By : Peter A. Buhr 12 // Last Modified On : Sat Apr 18 17:43:15202013 // Update Count : 7 1812 // Last Modified On : Wed May 6 17:29:26 2020 13 // Update Count : 727 14 14 // 15 15 … … 19 19 #include <errno.h> // errno 20 20 #include <string.h> // memset, memcpy 21 #include <limits.h> // ULONG_MAX 21 22 extern "C" { 22 23 #include <sys/mman.h> // mmap, munmap 23 24 } // extern "C" 24 25 25 // #comment TD : Many of these should be merged into math I believe26 26 #include "bits/align.hfa" // libPow2 27 27 #include "bits/defs.hfa" // likely, unlikely … … 30 30 //#include "stdlib.hfa" // bsearchl 31 31 #include "malloc.h" 32 #include "bitmanip.hfa" // ceiling 32 33 33 34 #define MIN(x, y) (y > x ? x : y) … … 81 82 }; 82 83 84 size_t default_heap_expansion() __attribute__(( weak )) { 85 return __CFA_DEFAULT_HEAP_EXPANSION__; 86 } // default_heap_expansion 87 83 88 size_t default_mmap_start() __attribute__(( weak )) { 84 89 return __CFA_DEFAULT_MMAP_START__; 85 90 } // default_mmap_start 86 87 size_t default_heap_expansion() __attribute__(( weak )) {88 return __CFA_DEFAULT_HEAP_EXPANSION__;89 } // default_heap_expansion90 91 91 92 … … 360 361 361 362 362 static inline bool setHeapExpand( size_t value ) {363 if ( heapExpand < pageSize ) return true;364 heapExpand = value;365 return false;366 } // setHeapExpand367 368 369 363 // thunk problem 370 364 size_t Bsearchl( unsigned int key, const unsigned int * vals, size_t dim ) { … … 383 377 384 378 static inline bool setMmapStart( size_t value ) { // true => mmapped, false => sbrk 385 if ( value < pageSize || bucketSizes[NoBucketSizes - 1] < value ) return true;379 if ( value < pageSize || bucketSizes[NoBucketSizes - 1] < value ) return false; 386 380 mmapStart = value; // set global 387 381 … … 390 384 assert( maxBucketsUsed < NoBucketSizes ); // subscript failure ? 391 385 assert( mmapStart <= bucketSizes[maxBucketsUsed] ); // search failure ? 392 return false;386 return true; 393 387 } // setMmapStart 394 388 … … 449 443 450 444 #ifdef __CFA_DEBUG__ 451 checkHeader( addr < heapBegin || header < (HeapManager.Storage.Header *)heapBegin, name, addr );// bad low address ?445 checkHeader( addr < heapBegin, name, addr ); // bad low address ? 452 446 #endif // __CFA_DEBUG__ 453 447 … … 508 502 // along with the block and is a multiple of the alignment size. 509 503 510 if ( unlikely( size > ~0ul- sizeof(HeapManager.Storage) ) ) return 0p;504 if ( unlikely( size > ULONG_MAX - sizeof(HeapManager.Storage) ) ) return 0p; 511 505 size_t tsize = size + sizeof(HeapManager.Storage); 512 506 if ( likely( tsize < mmapStart ) ) { // small size => sbrk … … 560 554 block->header.kind.real.home = freeElem; // pointer back to free list of apropriate size 561 555 } else { // large size => mmap 562 if ( unlikely( size > ~0ul- pageSize ) ) return 0p;556 if ( unlikely( size > ULONG_MAX - pageSize ) ) return 0p; 563 557 tsize = libCeiling( tsize, pageSize ); // must be multiple of page size 564 558 #ifdef __STATISTICS__ … … 702 696 #endif // FASTLOOKUP 703 697 704 if ( setMmapStart( default_mmap_start() ) ) {698 if ( ! setMmapStart( default_mmap_start() ) ) { 705 699 abort( "HeapManager : internal error, mmap start initialization failure." ); 706 700 } // if … … 708 702 709 703 char * end = (char *)sbrk( 0 ); 710 sbrk( (char *)libCeiling( (long unsigned int)end, libAlign() ) - end ); // move start of heap to multiple of alignment 711 heapBegin = heapEnd = sbrk( 0 ); // get new start point 704 heapBegin = heapEnd = sbrk( (char *)libCeiling( (long unsigned int)end, libAlign() ) - end ); // move start of heap to multiple of alignment 712 705 } // HeapManager 713 706 … … 735 728 //assert( heapManager.heapBegin != 0 ); 736 729 //heapManager{}; 737 if ( heapManager.heapBegin == 0p ) heapManager{}; 730 if ( heapManager.heapBegin == 0p ) heapManager{}; // sanity check 738 731 } // memory_startup 739 732 … … 863 856 #endif // __STATISTICS__ 864 857 865 size_t size = dim * elemSize; 866 char * addr = (char *)mallocNoStats( size ); 867 if ( unlikely( addr == 0p ) ) return 0p; 868 869 HeapManager.Storage.Header * header; 870 HeapManager.FreeHeader * freeElem; 871 size_t bsize, alignment; 872 headers( "aalloc", addr, header, freeElem, bsize, alignment ); 873 874 header->kind.real.blockSize |= 2; // mark as zero filled 875 return addr; 858 return mallocNoStats( dim * elemSize ); 876 859 } // aalloc 877 860 … … 914 897 915 898 // change size, DO NOT preserve STICKY PROPERTIES. 899 free( oaddr ); 916 900 void * naddr = mallocNoStats( size ); // create new area 917 free( oaddr );918 901 return naddr; 919 902 } // resize … … 988 971 #endif // __STATISTICS__ 989 972 990 size_t size = dim * elemSize; 991 char * addr = (char *)memalignNoStats( alignment, size ); 992 if ( unlikely( addr == 0p ) ) return 0p; 993 HeapManager.Storage.Header * header; 994 HeapManager.FreeHeader * freeElem; 995 size_t bsize; 996 headers( "amemalign", addr, header, freeElem, bsize, alignment ); 997 998 header->kind.real.blockSize |= 2; // mark as zero filled 999 return addr; 973 return memalignNoStats( alignment, dim * elemSize ); 1000 974 } // amemalign 1001 975 … … 1043 1017 1044 1018 // Frees the memory space pointed to by ptr, which must have been returned by a previous call to malloc(), calloc() 1045 // or realloc(). Otherwise, or if free(ptr) has already been called before, undefined behavio r occurs. If ptr is1019 // or realloc(). Otherwise, or if free(ptr) has already been called before, undefined behaviour occurs. If ptr is 1046 1020 // 0p, no operation is performed. 1047 1021 void free( void * addr ) { … … 1170 1144 1171 1145 1172 // Adjusts parameters that control the behavio r of the memory-allocation functions (see malloc). The param argument1146 // Adjusts parameters that control the behaviour of the memory-allocation functions (see malloc). The param argument 1173 1147 // specifies the parameter to be modified, and value specifies the new value for that parameter. 1174 1148 int mallopt( int option, int value ) { 1175 1149 choose( option ) { 1176 1150 case M_TOP_PAD: 1177 if ( setHeapExpand( value ) )return 1;1151 heapExpand = ceiling( value, pageSize ); return 1; 1178 1152 case M_MMAP_THRESHOLD: 1179 1153 if ( setMmapStart( value ) ) return 1; 1154 break; 1180 1155 } // switch 1181 1156 return 0; // error, unsupported -
libcfa/src/iostream.cfa
r6ec07e5 raf7acb9 10 10 // Created On : Wed May 27 17:56:53 2015 11 11 // Last Modified By : Peter A. Buhr 12 // Last Modified On : Thu Apr 30 10:50:31202013 // Update Count : 10 0112 // Last Modified On : Sat May 2 18:30:25 2020 13 // Update Count : 1017 14 14 // 15 15 … … 583 583 if ( fmt.flags.pc && fmt.pc > 64 ) fmt.pc -= 64; else { fmt.flags.pc = false; fmt.pc = 0; } \ 584 584 if ( fmt.flags.left ) { \ 585 fmt2.wd = fmt.wd; \ 586 if ( fmt2.wd <= 64 ) { \ 587 fmt2.wd = 64; \ 588 } else { \ 589 if ( fmt.pc > 0 ) { \ 590 fmt2.wd -= fmt.pc; \ 591 } else { \ 592 fmt2.wd = fmt.wd - high1( msig ); \ 593 } /* if */ \ 594 if ( ! fmt.flags.nobsdp ) fmt2.wd -= 2; \ 595 if ( fmt2.wd < 0 ) fmt2.wd = 0; \ 596 fmt2.flags.left = true; \ 597 fmt.wd = 0; \ 598 } /* if */ \ 599 printf( "left %d %d %x\n", f.wd, f.pc, f.all ); \ 600 printf( "left %d %d %x\n", fmt.wd, fmt.pc, fmt.all ); \ 585 fmt.flags.left = false; \ 586 fmt.wd = 0; \ 587 /* printf( "L %llo %llo %llo %d %d '%c' %x\n", msig, lsig, fmt.val, fmt.wd, fmt.pc, fmt.base, fmt.all ); */ \ 588 fmt2.flags.left = true; \ 589 int msigd = high1( msig ); \ 590 fmt2.wd = f.wd - (fmt.pc > msigd ? fmt.pc : msigd); \ 591 if ( ! fmt.flags.nobsdp ) fmt2.wd -= 2; /* compensate for 0b base specifier */ \ 592 if ( (int)fmt2.wd < 64 ) fmt2.wd = 64; /* cast deals with negative value */ \ 593 fmt2.flags.pc = true; fmt2.pc = 64; \ 601 594 } else { \ 602 595 if ( fmt.wd > 64 ) fmt.wd -= 64; \ 603 /* if ( ! fmt.flags.nobsdp && fmt.pc < fmt.wd ) fmt.wd -= 1; */ \ 596 else fmt.wd = 1; \ 597 /* printf( "R %llo %llo %llo %d %d '%c' %x\n", msig, lsig, fmt.val, fmt.wd, fmt.pc, fmt.base, fmt.all ); */ \ 604 598 fmt2.wd = 64; \ 605 /* printf( "R %llo %llo %llo %llo %d %d '%c' %x\n", msig, lsig, fmt.val, fmt2.val, fmt.wd, fmt.pc, fmt.base, fmt.all ); */ \606 599 } /* if */ \ 607 /* printf( " \nC %llo %d %d '%c' %x\n", fmt2.val, fmt2.wd, fmt2.pc, fmt2.base, fmt2.all ); */ \600 /* printf( "C %llo %d %d '%c' %x\n", fmt2.val, fmt2.wd, fmt2.pc, fmt2.base, fmt2.all ); */ \ 608 601 (ostype &)(os | fmt | "" | fmt2); \ 609 602 } else if ( f.base == 'o' ) { \ … … 614 607 fmt.flags.left = false; \ 615 608 fmt.wd = 0; \ 616 /* if ( fmt2.val > 1 && fmt.flags.pc && fmt.pc > 0 ) fmt.pc -= 1; */ \617 609 /* printf( "L %llo %llo %llo %d %d '%c' %x %llo %d %d '%c' %x\n", msig, lsig, fmt.val, fmt.wd, fmt.pc, fmt.base, fmt.all, fmt2.val, fmt2.wd, fmt2.pc, fmt2.base, fmt2.all ); */ \ 618 610 (ostype &)(os | fmt | "" | fmt2); \ 619 611 sepOff( os ); \ 620 612 fmt2.flags.left = true; \ 621 int msigd = ceiling( high1( fmt.val ), 3 ) + 1; \ 622 fmt2.wd = f.wd - (fmt.pc > msigd ? fmt.pc : msigd) - 1; \ 613 int msigd = ceiling( high1( fmt.val ), 3 ); \ 614 fmt2.wd = f.wd - (fmt.pc > msigd ? fmt.pc : msigd); \ 615 if ( ! fmt.flags.nobsdp ) fmt2.wd -= 1; /* compensate for 0 base specifier */ \ 623 616 if ( (int)fmt2.wd < 21 ) fmt2.wd = 21; /* cast deals with negative value */ \ 624 617 fmt2.flags.pc = true; fmt2.pc = 21; \ 625 618 } else { \ 626 619 if ( fmt.wd > 22 ) fmt.wd -= 22; \ 627 /* compensate for leading 0 */ \ 628 /*if ( ! fmt.flags.nobsdp && fmt.pc < fmt.wd ) fmt.wd -= 1;*/ \ 629 fmt2.wd = 1; \ 620 else fmt.wd = 1; \ 630 621 /* printf( "R %llo %llo %llo %d %d '%c' %x %llo %d %d '%c' %x\n", msig, lsig, fmt.val, fmt.wd, fmt.pc, fmt.base, fmt.all, fmt2.val, fmt2.wd, fmt2.pc, fmt2.base, fmt2.all ); */ \ 631 622 (ostype &)(os | fmt | "" | fmt2); \ … … 636 627 /* printf( "\nC %llo %d %d '%c' %x\n", fmt2.val, fmt2.wd, fmt2.pc, fmt2.base, fmt2.all ); */ \ 637 628 (ostype &)(os | fmt2); \ 638 } else { \639 if ( fmt.flags.pc && fmt.pc > 16 ) fmt.pc -= 16; \629 } else { /* f.base == 'x' | f.base == 'X' */ \ 630 if ( fmt.flags.pc && fmt.pc > 16 ) fmt.pc -= 16; else { fmt.flags.pc = false; fmt.pc = 0; } \ 640 631 if ( fmt.flags.left ) { \ 641 fmt2.wd = fmt.wd; \ 642 if ( fmt2.wd <= 16 ) { \ 643 fmt2.wd = 16; \ 644 } else { \ 645 if ( fmt.pc > 0 ) { \ 646 fmt2.wd -= fmt.pc; \ 647 } else { \ 648 fmt2.wd = fmt.wd - ceiling2( high1( msig ), 4 ) / 4; \ 649 } /* if */ \ 650 if ( ! fmt.flags.nobsdp ) fmt2.wd -= 2; \ 651 if ( fmt2.wd < 0 ) fmt2.wd = 0; \ 652 fmt2.flags.left = true; \ 653 fmt.wd = 0; \ 654 } /* if */ \ 632 fmt.flags.left = false; \ 633 fmt.wd = 0; \ 634 /* printf( "L %llo %llo %llo %d %d '%c' %x\n", msig, lsig, fmt.val, fmt.wd, fmt.pc, fmt.base, fmt.all ); */ \ 635 fmt2.flags.left = true; \ 636 int msigd = high1( msig ); \ 637 fmt2.wd = f.wd - (fmt.pc > msigd ? fmt.pc : msigd); \ 638 if ( ! fmt.flags.nobsdp ) fmt2.wd -= 2; /* compensate for 0x base specifier */ \ 639 if ( (int)fmt2.wd < 16 ) fmt2.wd = 16; /* cast deals with negative value */ \ 640 fmt2.flags.pc = true; fmt2.pc = 16; \ 655 641 } else { \ 642 if ( fmt.wd > 16 ) fmt.wd -= 16; \ 643 else fmt.wd = 1; \ 644 /* printf( "R %llo %llo %llo %d %d '%c' %x\n", msig, lsig, fmt.val, fmt.wd, fmt.pc, fmt.base, fmt.all ); */ \ 656 645 fmt2.wd = 16; \ 657 if ( fmt.wd > 16 ) fmt.wd -= 16; \658 646 } /* if */ \ 659 fmt2.pc = 16; fmt2.flags.pc = true;\647 /* printf( "C %llo %d %d '%c' %x\n", fmt2.val, fmt2.wd, fmt2.pc, fmt2.base, fmt2.all ); */ \ 660 648 (ostype &)(os | fmt | "" | fmt2); \ 661 649 } /* if */ \ -
libcfa/src/startup.cfa
r6ec07e5 raf7acb9 14 14 // 15 15 16 #include <time.h> // tzset 16 #include <time.h> // tzset 17 #include <locale.h> // setlocale 17 18 #include "startup.hfa" 18 19 … … 21 22 void __cfaabi_appready_startup( void ) { 22 23 tzset(); // initialize time global variables 24 setlocale(LC_NUMERIC, ""); 23 25 #ifdef __CFA_DEBUG__ 24 26 extern void heapAppStart();
Note: See TracChangeset
for help on using the changeset viewer.
