Changeset 3e2b9c9
- Timestamp:
- Aug 3, 2020, 1:59:13 PM (4 years ago)
- Branches:
- ADT, arm-eh, ast-experimental, enum, forall-pointer-decay, jacob/cs343-translation, master, new-ast, new-ast-unique-expr, pthread-emulation, qualifiedEnum
- Children:
- 95789be
- Parents:
- e660761
- Location:
- libcfa/src
- Files:
-
- 2 added
- 11 edited
Legend:
- Unmodified
- Added
- Removed
-
libcfa/src/Makefile.am
re660761 r3e2b9c9 51 51 # not all platforms support concurrency, add option do disable it 52 52 thread_headers_nosrc = concurrency/invoke.h 53 thread_headers = concurrency/coroutine.hfa concurrency/thread.hfa concurrency/kernel.hfa concurrency/monitor.hfa concurrency/mutex.hfa 54 thread_libsrc = concurrency/CtxSwitch-@ARCHITECTURE@.S concurrency/alarm.cfa concurrency/invoke.c concurrency/io.cfa concurrency/iocall.cfa concurrency/preemption.cfa concurrency/ready_queue.cfa concurrency/stats.cfa ${thread_headers:.hfa=.cfa} 53 54 thread_headers = concurrency/coroutine.hfa concurrency/thread.hfa concurrency/kernel.hfa \ 55 concurrency/monitor.hfa concurrency/mutex.hfa 56 57 thread_libsrc = concurrency/CtxSwitch-@ARCHITECTURE@.S concurrency/alarm.cfa \ 58 concurrency/invoke.c concurrency/io.cfa concurrency/iocall.cfa \ 59 concurrency/io/setup.cfa \ 60 concurrency/kernel/startup.cfa concurrency/preemption.cfa \ 61 concurrency/ready_queue.cfa concurrency/stats.cfa \ 62 ${thread_headers:.hfa=.cfa} 55 63 else 56 64 headers = -
libcfa/src/bits/debug.hfa
re660761 r3e2b9c9 15 15 16 16 #pragma once 17 18 #include <assert.h> 17 19 18 20 #ifdef __CFA_DEBUG__ -
libcfa/src/concurrency/invoke.h
re660761 r3e2b9c9 17 17 #include "bits/defs.hfa" 18 18 #include "bits/locks.hfa" 19 #include "kernel/fwd.hfa" 19 20 20 21 #ifdef __cforall … … 25 26 #ifndef _INVOKE_H_ 26 27 #define _INVOKE_H_ 27 28 #ifdef __ARM_ARCH29 // function prototypes are only really used by these macros on ARM30 void disable_global_interrupts();31 void enable_global_interrupts();32 33 #define TL_GET( member ) ( { __typeof__( kernelTLS.member ) target; \34 disable_global_interrupts(); \35 target = kernelTLS.member; \36 enable_global_interrupts(); \37 target; } )38 #define TL_SET( member, value ) disable_global_interrupts(); \39 kernelTLS.member = value; \40 enable_global_interrupts();41 #else42 #define TL_GET( member ) kernelTLS.member43 #define TL_SET( member, value ) kernelTLS.member = value;44 #endif45 46 #ifdef __cforall47 extern "Cforall" {48 extern __attribute__((aligned(128))) thread_local struct KernelThreadData {49 struct $thread * volatile this_thread;50 struct processor * volatile this_processor;51 struct __stats_t * volatile this_stats;52 53 struct {54 volatile unsigned short disable_count;55 volatile bool enabled;56 volatile bool in_progress;57 } preemption_state;58 59 #if defined(__SIZEOF_INT128__)60 __uint128_t rand_seed;61 #else62 uint64_t rand_seed;63 #endif64 } kernelTLS __attribute__ ((tls_model ( "initial-exec" )));65 }66 #endif67 28 68 29 struct __stack_context_t { … … 98 59 99 60 enum __Coroutine_State { Halted, Start, Primed, Blocked, Ready, Active }; 100 enum __Preemption_Reason { __NO_PREEMPTION, __ALARM_PREEMPTION, __POLL_PREEMPTION, __MANUAL_PREEMPTION };101 61 102 62 struct $coroutine { -
libcfa/src/concurrency/io.cfa
re660761 r3e2b9c9 14 14 // 15 15 16 #define __cforall_thread__ 17 16 18 #if defined(__CFA_DEBUG__) 17 19 // #define __CFA_DEBUG_PRINT_IO__ … … 19 21 #endif 20 22 21 #include "kernel_private.hfa" 22 #include "bitmanip.hfa" 23 24 #if !defined(CFA_HAVE_LINUX_IO_URING_H) 25 void __kernel_io_startup() { 26 // Nothing to do without io_uring 27 } 28 29 void __kernel_io_shutdown() { 30 // Nothing to do without io_uring 31 } 32 33 void ?{}(io_context & this, struct cluster & cl) {} 34 void ?{}(io_context & this, struct cluster & cl, const io_context_params & params) {} 35 36 void ^?{}(io_context & this) {} 37 void ^?{}(io_context & this, bool cluster_context) {} 38 39 #else 23 24 #if defined(CFA_HAVE_LINUX_IO_URING_H) 40 25 #define _GNU_SOURCE /* See feature_test_macros(7) */ 41 26 #include <errno.h> 27 #include <signal.h> 42 28 #include <stdint.h> 43 29 #include <string.h> … … 46 32 extern "C" { 47 33 #include <sys/epoll.h> 48 #include <sys/mman.h>49 34 #include <sys/syscall.h> 50 35 … … 52 37 } 53 38 54 #include "bits/signal.hfa" 55 #include "kernel_private.hfa" 56 #include "thread.hfa" 57 58 void ?{}(io_context_params & this) { 59 this.num_entries = 256; 60 this.num_ready = 256; 61 this.submit_aff = -1; 62 this.eager_submits = false; 63 this.poller_submits = false; 64 this.poll_submit = false; 65 this.poll_complete = false; 66 } 67 68 static void * __io_poller_slow( void * arg ); 69 70 // Weirdly, some systems that do support io_uring don't actually define these 71 #ifdef __alpha__ 72 /* 73 * alpha is the only exception, all other architectures 74 * have common numbers for new system calls. 75 */ 76 #ifndef __NR_io_uring_setup 77 #define __NR_io_uring_setup 535 78 #endif 79 #ifndef __NR_io_uring_enter 80 #define __NR_io_uring_enter 536 81 #endif 82 #ifndef __NR_io_uring_register 83 #define __NR_io_uring_register 537 84 #endif 85 #else /* !__alpha__ */ 86 #ifndef __NR_io_uring_setup 87 #define __NR_io_uring_setup 425 88 #endif 89 #ifndef __NR_io_uring_enter 90 #define __NR_io_uring_enter 426 91 #endif 92 #ifndef __NR_io_uring_register 93 #define __NR_io_uring_register 427 94 #endif 95 #endif 96 97 struct __submition_data { 98 // Head and tail of the ring (associated with array) 99 volatile uint32_t * head; 100 volatile uint32_t * tail; 101 volatile uint32_t prev_head; 102 103 // The actual kernel ring which uses head/tail 104 // indexes into the sqes arrays 105 uint32_t * array; 106 107 // number of entries and mask to go with it 108 const uint32_t * num; 109 const uint32_t * mask; 110 111 // Submission flags (Not sure what for) 112 uint32_t * flags; 113 114 // number of sqes not submitted (whatever that means) 115 uint32_t * dropped; 116 117 // Like head/tail but not seen by the kernel 118 volatile uint32_t * ready; 119 uint32_t ready_cnt; 120 121 __spinlock_t lock; 122 __spinlock_t release_lock; 123 124 // A buffer of sqes (not the actual ring) 125 struct io_uring_sqe * sqes; 126 127 // The location and size of the mmaped area 128 void * ring_ptr; 129 size_t ring_sz; 130 }; 131 132 struct __completion_data { 133 // Head and tail of the ring 134 volatile uint32_t * head; 135 volatile uint32_t * tail; 136 137 // number of entries and mask to go with it 138 const uint32_t * mask; 139 const uint32_t * num; 140 141 // number of cqes not submitted (whatever that means) 142 uint32_t * overflow; 143 144 // the kernel ring 145 struct io_uring_cqe * cqes; 146 147 // The location and size of the mmaped area 148 void * ring_ptr; 149 size_t ring_sz; 150 }; 151 152 struct __io_data { 153 struct __submition_data submit_q; 154 struct __completion_data completion_q; 155 uint32_t ring_flags; 156 int fd; 157 bool eager_submits:1; 158 bool poller_submits:1; 159 }; 160 161 //============================================================================================= 162 // I/O Startup / Shutdown logic + Master Poller 163 //============================================================================================= 164 165 // IO Master poller loop forward 166 static void * iopoll_loop( __attribute__((unused)) void * args ); 167 168 static struct { 169 pthread_t thrd; // pthread handle to io poller thread 170 void * stack; // pthread stack for io poller thread 171 int epollfd; // file descriptor to the epoll instance 172 volatile bool run; // Whether or not to continue 173 } iopoll; 174 175 void __kernel_io_startup(void) { 176 __cfaabi_dbg_print_safe( "Kernel : Creating EPOLL instance\n" ); 177 178 iopoll.epollfd = epoll_create1(0); 179 if (iopoll.epollfd == -1) { 180 abort( "internal error, epoll_create1\n"); 181 } 182 183 __cfaabi_dbg_print_safe( "Kernel : Starting io poller thread\n" ); 184 185 iopoll.run = true; 186 iopoll.stack = __create_pthread( &iopoll.thrd, iopoll_loop, 0p ); 187 } 188 189 void __kernel_io_shutdown(void) { 190 // Notify the io poller thread of the shutdown 191 iopoll.run = false; 192 sigval val = { 1 }; 193 pthread_sigqueue( iopoll.thrd, SIGUSR1, val ); 194 195 // Wait for the io poller thread to finish 196 197 pthread_join( iopoll.thrd, 0p ); 198 free( iopoll.stack ); 199 200 int ret = close(iopoll.epollfd); 201 if (ret == -1) { 202 abort( "internal error, close epoll\n"); 203 } 204 205 // Io polling is now fully stopped 206 207 __cfaabi_dbg_print_safe( "Kernel : IO poller stopped\n" ); 208 } 209 210 static void * iopoll_loop( __attribute__((unused)) void * args ) { 211 __processor_id_t id; 212 id.id = doregister(&id); 213 __cfaabi_dbg_print_safe( "Kernel : IO poller thread starting\n" ); 214 215 // Block signals to control when they arrive 216 sigset_t mask; 217 sigfillset(&mask); 218 if ( pthread_sigmask( SIG_BLOCK, &mask, 0p ) == -1 ) { 219 abort( "internal error, pthread_sigmask" ); 220 } 221 222 sigdelset( &mask, SIGUSR1 ); 223 224 // Create sufficient events 225 struct epoll_event events[10]; 226 // Main loop 227 while( iopoll.run ) { 228 // Wait for events 229 int nfds = epoll_pwait( iopoll.epollfd, events, 10, -1, &mask ); 230 231 // Check if an error occured 232 if (nfds == -1) { 233 if( errno == EINTR ) continue; 234 abort( "internal error, pthread_sigmask" ); 235 } 236 237 for(i; nfds) { 238 $io_ctx_thread * io_ctx = ($io_ctx_thread *)(uintptr_t)events[i].data.u64; 239 /* paranoid */ verify( io_ctx ); 240 __cfadbg_print_safe(io_core, "Kernel I/O : Unparking io poller %p\n", io_ctx); 241 #if !defined( __CFA_NO_STATISTICS__ ) 242 kernelTLS.this_stats = io_ctx->self.curr_cluster->stats; 243 #endif 244 __post( io_ctx->sem, &id ); 245 } 246 } 247 248 __cfaabi_dbg_print_safe( "Kernel : IO poller thread stopping\n" ); 249 unregister(&id); 250 return 0p; 251 } 252 253 //============================================================================================= 254 // I/O Context Constrution/Destruction 255 //============================================================================================= 256 257 void ?{}($io_ctx_thread & this, struct cluster & cl) { (this.self){ "IO Poller", cl }; } 258 void main( $io_ctx_thread & this ); 259 static inline $thread * get_thread( $io_ctx_thread & this ) { return &this.self; } 260 void ^?{}( $io_ctx_thread & mutex this ) {} 261 262 static void __io_create ( __io_data & this, const io_context_params & params_in ); 263 static void __io_destroy( __io_data & this ); 264 265 void ?{}(io_context & this, struct cluster & cl, const io_context_params & params) { 266 (this.thrd){ cl }; 267 this.thrd.ring = malloc(); 268 __cfadbg_print_safe(io_core, "Kernel I/O : Creating ring for io_context %p\n", &this); 269 __io_create( *this.thrd.ring, params ); 270 271 __cfadbg_print_safe(io_core, "Kernel I/O : Starting poller thread for io_context %p\n", &this); 272 this.thrd.done = false; 273 __thrd_start( this.thrd, main ); 274 275 __cfadbg_print_safe(io_core, "Kernel I/O : io_context %p ready\n", &this); 276 } 277 278 void ?{}(io_context & this, struct cluster & cl) { 279 io_context_params params; 280 (this){ cl, params }; 281 } 282 283 void ^?{}(io_context & this, bool cluster_context) { 284 __cfadbg_print_safe(io_core, "Kernel I/O : tearing down io_context %p\n", &this); 285 286 // Notify the thread of the shutdown 287 __atomic_store_n(&this.thrd.done, true, __ATOMIC_SEQ_CST); 288 289 // If this is an io_context within a cluster, things get trickier 290 $thread & thrd = this.thrd.self; 291 if( cluster_context ) { 292 cluster & cltr = *thrd.curr_cluster; 293 /* paranoid */ verify( cltr.nprocessors == 0 || &cltr == mainCluster ); 294 /* paranoid */ verify( !ready_mutate_islocked() ); 295 296 // We need to adjust the clean-up based on where the thread is 297 if( thrd.state == Ready || thrd.preempted != __NO_PREEMPTION ) { 298 299 ready_schedule_lock( (struct __processor_id_t *)active_processor() ); 300 301 // This is the tricky case 302 // The thread was preempted and now it is on the ready queue 303 // The thread should be the last on the list 304 /* paranoid */ verify( thrd.link.next != 0p ); 305 306 // Remove the thread from the ready queue of this cluster 307 __attribute__((unused)) bool removed = remove_head( &cltr, &thrd ); 308 /* paranoid */ verify( removed ); 309 thrd.link.next = 0p; 310 thrd.link.prev = 0p; 311 __cfaabi_dbg_debug_do( thrd.unpark_stale = true ); 312 313 // Fixup the thread state 314 thrd.state = Blocked; 315 thrd.ticket = 0; 316 thrd.preempted = __NO_PREEMPTION; 317 318 ready_schedule_unlock( (struct __processor_id_t *)active_processor() ); 319 320 // Pretend like the thread was blocked all along 321 } 322 // !!! This is not an else if !!! 323 if( thrd.state == Blocked ) { 324 325 // This is the "easy case" 326 // The thread is parked and can easily be moved to active cluster 327 verify( thrd.curr_cluster != active_cluster() || thrd.curr_cluster == mainCluster ); 328 thrd.curr_cluster = active_cluster(); 329 330 // unpark the fast io_poller 331 unpark( &thrd __cfaabi_dbg_ctx2 ); 332 } 333 else { 334 335 // The thread is in a weird state 336 // I don't know what to do here 337 abort("io_context poller thread is in unexpected state, cannot clean-up correctly\n"); 338 } 339 } else { 340 unpark( &thrd __cfaabi_dbg_ctx2 ); 341 } 342 343 ^(this.thrd){}; 344 __cfadbg_print_safe(io_core, "Kernel I/O : Stopped poller thread for io_context %p\n", &this); 345 346 __io_destroy( *this.thrd.ring ); 347 __cfadbg_print_safe(io_core, "Kernel I/O : Destroyed ring for io_context %p\n", &this); 348 349 free(this.thrd.ring); 350 } 351 352 void ^?{}(io_context & this) { 353 ^(this){ false }; 354 } 355 356 static void __io_create( __io_data & this, const io_context_params & params_in ) { 357 // Step 1 : call to setup 358 struct io_uring_params params; 359 memset(¶ms, 0, sizeof(params)); 360 if( params_in.poll_submit ) params.flags |= IORING_SETUP_SQPOLL; 361 if( params_in.poll_complete ) params.flags |= IORING_SETUP_IOPOLL; 362 363 uint32_t nentries = params_in.num_entries; 364 365 int fd = syscall(__NR_io_uring_setup, nentries, ¶ms ); 366 if(fd < 0) { 367 abort("KERNEL ERROR: IO_URING SETUP - %s\n", strerror(errno)); 368 } 369 370 // Step 2 : mmap result 371 memset( &this, 0, sizeof(struct __io_data) ); 372 struct __submition_data & sq = this.submit_q; 373 struct __completion_data & cq = this.completion_q; 374 375 // calculate the right ring size 376 sq.ring_sz = params.sq_off.array + (params.sq_entries * sizeof(unsigned) ); 377 cq.ring_sz = params.cq_off.cqes + (params.cq_entries * sizeof(struct io_uring_cqe)); 378 379 // Requires features 380 #if defined(IORING_FEAT_SINGLE_MMAP) 381 // adjust the size according to the parameters 382 if ((params.features & IORING_FEAT_SINGLE_MMAP) != 0) { 383 cq.ring_sz = sq.ring_sz = max(cq.ring_sz, sq.ring_sz); 384 } 385 #endif 386 387 // mmap the Submit Queue into existence 388 sq.ring_ptr = mmap(0, sq.ring_sz, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd, IORING_OFF_SQ_RING); 389 if (sq.ring_ptr == (void*)MAP_FAILED) { 390 abort("KERNEL ERROR: IO_URING MMAP1 - %s\n", strerror(errno)); 391 } 392 393 // Requires features 394 #if defined(IORING_FEAT_SINGLE_MMAP) 395 // mmap the Completion Queue into existence (may or may not be needed) 396 if ((params.features & IORING_FEAT_SINGLE_MMAP) != 0) { 397 cq.ring_ptr = sq.ring_ptr; 398 } 399 else 400 #endif 401 { 402 // We need multiple call to MMAP 403 cq.ring_ptr = mmap(0, cq.ring_sz, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd, IORING_OFF_CQ_RING); 404 if (cq.ring_ptr == (void*)MAP_FAILED) { 405 munmap(sq.ring_ptr, sq.ring_sz); 406 abort("KERNEL ERROR: IO_URING MMAP2 - %s\n", strerror(errno)); 407 } 408 } 409 410 // mmap the submit queue entries 411 size_t size = params.sq_entries * sizeof(struct io_uring_sqe); 412 sq.sqes = (struct io_uring_sqe *)mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd, IORING_OFF_SQES); 413 if (sq.sqes == (struct io_uring_sqe *)MAP_FAILED) { 414 munmap(sq.ring_ptr, sq.ring_sz); 415 if (cq.ring_ptr != sq.ring_ptr) munmap(cq.ring_ptr, cq.ring_sz); 416 abort("KERNEL ERROR: IO_URING MMAP3 - %s\n", strerror(errno)); 417 } 418 419 // Get the pointers from the kernel to fill the structure 420 // submit queue 421 sq.head = (volatile uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.head); 422 sq.tail = (volatile uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.tail); 423 sq.mask = ( const uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.ring_mask); 424 sq.num = ( const uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.ring_entries); 425 sq.flags = ( uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.flags); 426 sq.dropped = ( uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.dropped); 427 sq.array = ( uint32_t *)(((intptr_t)sq.ring_ptr) + params.sq_off.array); 428 sq.prev_head = *sq.head; 429 430 { 431 const uint32_t num = *sq.num; 432 for( i; num ) { 433 sq.sqes[i].user_data = 0ul64; 434 } 435 } 436 437 (sq.lock){}; 438 (sq.release_lock){}; 439 440 if( params_in.poller_submits || params_in.eager_submits ) { 441 /* paranoid */ verify( is_pow2( params_in.num_ready ) || (params_in.num_ready < 8) ); 442 sq.ready_cnt = max( params_in.num_ready, 8 ); 443 sq.ready = alloc_align( 64, sq.ready_cnt ); 444 for(i; sq.ready_cnt) { 445 sq.ready[i] = -1ul32; 446 } 447 } 448 else { 449 sq.ready_cnt = 0; 450 sq.ready = 0p; 451 } 452 453 // completion queue 454 cq.head = (volatile uint32_t *)(((intptr_t)cq.ring_ptr) + params.cq_off.head); 455 cq.tail = (volatile uint32_t *)(((intptr_t)cq.ring_ptr) + params.cq_off.tail); 456 cq.mask = ( const uint32_t *)(((intptr_t)cq.ring_ptr) + params.cq_off.ring_mask); 457 cq.num = ( const uint32_t *)(((intptr_t)cq.ring_ptr) + params.cq_off.ring_entries); 458 cq.overflow = ( uint32_t *)(((intptr_t)cq.ring_ptr) + params.cq_off.overflow); 459 cq.cqes = (struct io_uring_cqe *)(((intptr_t)cq.ring_ptr) + params.cq_off.cqes); 460 461 // some paranoid checks 462 /* paranoid */ verifyf( (*cq.mask) == ((*cq.num) - 1ul32), "IO_URING Expected mask to be %u (%u entries), was %u", (*cq.num) - 1ul32, *cq.num, *cq.mask ); 463 /* paranoid */ verifyf( (*cq.num) >= nentries, "IO_URING Expected %u entries, got %u", nentries, *cq.num ); 464 /* paranoid */ verifyf( (*cq.head) == 0, "IO_URING Expected head to be 0, got %u", *cq.head ); 465 /* paranoid */ verifyf( (*cq.tail) == 0, "IO_URING Expected tail to be 0, got %u", *cq.tail ); 466 467 /* paranoid */ verifyf( (*sq.mask) == ((*sq.num) - 1ul32), "IO_URING Expected mask to be %u (%u entries), was %u", (*sq.num) - 1ul32, *sq.num, *sq.mask ); 468 /* paranoid */ verifyf( (*sq.num) >= nentries, "IO_URING Expected %u entries, got %u", nentries, *sq.num ); 469 /* paranoid */ verifyf( (*sq.head) == 0, "IO_URING Expected head to be 0, got %u", *sq.head ); 470 /* paranoid */ verifyf( (*sq.tail) == 0, "IO_URING Expected tail to be 0, got %u", *sq.tail ); 471 472 // Update the global ring info 473 this.ring_flags = params.flags; 474 this.fd = fd; 475 this.eager_submits = params_in.eager_submits; 476 this.poller_submits = params_in.poller_submits; 477 } 478 479 void __io_destroy( __io_data & this ) { 480 // Shutdown the io rings 481 struct __submition_data & sq = this.submit_q; 482 struct __completion_data & cq = this.completion_q; 483 484 // unmap the submit queue entries 485 munmap(sq.sqes, (*sq.num) * sizeof(struct io_uring_sqe)); 486 487 // unmap the Submit Queue ring 488 munmap(sq.ring_ptr, sq.ring_sz); 489 490 // unmap the Completion Queue ring, if it is different 491 if (cq.ring_ptr != sq.ring_ptr) { 492 munmap(cq.ring_ptr, cq.ring_sz); 493 } 494 495 // close the file descriptor 496 close(this.fd); 497 498 free( this.submit_q.ready ); // Maybe null, doesn't matter 499 } 500 501 int __io_uring_enter( struct __io_data & ring, unsigned to_submit, bool get ) { 39 #include "stats.hfa" 40 #include "kernel.hfa" 41 #include "kernel/fwd.hfa" 42 #include "io/types.hfa" 43 44 //============================================================================================= 45 // I/O Syscall 46 //============================================================================================= 47 static int __io_uring_enter( struct __io_data & ring, unsigned to_submit, bool get ) { 502 48 bool need_sys_to_submit = false; 503 49 bool need_sys_to_complete = false; … … 618 164 void main( $io_ctx_thread & this ) { 619 165 epoll_event ev; 620 ev.events = EPOLLIN | EPOLLONESHOT; 621 ev.data.u64 = (uint64_t)&this; 622 int ret = epoll_ctl(iopoll.epollfd, EPOLL_CTL_ADD, this.ring->fd, &ev); 623 if (ret < 0) { 624 abort( "KERNEL ERROR: EPOLL ADD - (%d) %s\n", (int)errno, strerror(errno) ); 625 } 166 __ioctx_register( this, ev ); 626 167 627 168 __cfadbg_print_safe(io_core, "Kernel I/O : IO poller %p for ring %p ready\n", &this, &this.ring); … … 654 195 reset = 0; 655 196 656 // wake up the slow poller 657 ret = epoll_ctl(iopoll.epollfd, EPOLL_CTL_MOD, this.ring->fd, &ev); 658 if (ret < 0) { 659 abort( "KERNEL ERROR: EPOLL REARM - (%d) %s\n", (int)errno, strerror(errno) ); 660 } 661 662 // park this thread 197 // block this thread 198 __ioctx_prepare_block( this, ev ); 663 199 wait( this.sem ); 664 200 } … … 933 469 return count; 934 470 } 935 936 //=============================================================================================937 // I/O Submissions938 //=============================================================================================939 940 void register_fixed_files( io_context & ctx, int * files, unsigned count ) {941 int ret = syscall( __NR_io_uring_register, ctx.thrd.ring->fd, IORING_REGISTER_FILES, files, count );942 if( ret < 0 ) {943 abort( "KERNEL ERROR: IO_URING SYSCALL - (%d) %s\n", (int)errno, strerror(errno) );944 }945 946 __cfadbg_print_safe( io_core, "Kernel I/O : Performed io_register for %p, returned %d\n", active_thread(), ret );947 }948 949 void register_fixed_files( cluster & cltr, int * files, unsigned count ) {950 for(i; cltr.io.cnt) {951 register_fixed_files( cltr.io.ctxs[i], files, count );952 }953 }954 471 #endif -
libcfa/src/concurrency/iocall.cfa
re660761 r3e2b9c9 14 14 // 15 15 16 #define __cforall_thread__ 17 16 18 #include "bits/defs.hfa" 17 19 … … 21 23 22 24 #if defined(CFA_HAVE_LINUX_IO_URING_H) 25 #include <assert.h> 23 26 #include <stdint.h> 24 27 #include <errno.h> 25 28 #include <linux/io_uring.h> 26 29 27 #include "kernel_private.hfa" 30 #include "kernel.hfa" 31 #include "kernel/fwd.hfa" 32 #include "io/types.hfa" 28 33 29 34 extern [* struct io_uring_sqe, uint32_t] __submit_alloc( struct __io_data & ring, uint64_t data ); … … 53 58 } 54 59 60 static inline io_context * __get_io_context( void ) { 61 cluster * cltr = active_cluster(); 62 /* paranoid */ verifyf( cltr, "No active cluster for io operation\n"); 63 assertf( cltr->io.cnt > 0, "Cluster %p has no default io contexts and no context was specified\n", cltr ); 64 /* paranoid */ verifyf( cltr->io.ctxs, "default io contexts for cluster %p are missing\n", cltr); 65 return &cltr->io.ctxs[ __tls_rand() % cltr->io.cnt ]; 66 } 55 67 56 68 -
libcfa/src/concurrency/kernel.hfa
re660761 r3e2b9c9 23 23 24 24 extern "C" { 25 #include < pthread.h>25 #include <bits/pthreadtypes.h> 26 26 } 27 27 -
libcfa/src/concurrency/kernel/fwd.hfa
re660761 r3e2b9c9 14 14 // 15 15 16 #pragma once 17 16 18 #include "bits/defs.hfa" 17 19 #include "bits/debug.hfa" 18 20 19 #if !defined(__cforall_thread__)20 # error non-thread source file includes kernel/fwd.hfa21 #ifdef __cforall 22 #include "bits/random.hfa" 21 23 #endif 22 24 … … 25 27 struct cluster; 26 28 29 enum __Preemption_Reason { __NO_PREEMPTION, __ALARM_PREEMPTION, __POLL_PREEMPTION, __MANUAL_PREEMPTION }; 30 31 #define KERNEL_STORAGE(T,X) __attribute((aligned(__alignof__(T)))) static char storage_##X[sizeof(T)] 32 27 33 #ifdef __cforall 28 34 extern "C" { 29 35 extern "Cforall" { 30 36 extern __attribute__((aligned(128))) thread_local struct KernelThreadData { 31 37 struct $thread * volatile this_thread; … … 45 51 #endif 46 52 } kernelTLS __attribute__ ((tls_model ( "initial-exec" ))); 53 54 static inline uint64_t __tls_rand() { 55 #if defined(__SIZEOF_INT128__) 56 return __lehmer64( kernelTLS.rand_seed ); 57 #else 58 return __xorshift64( kernelTLS.rand_seed ); 59 #endif 60 } 47 61 } 48 62 49 50 51 52 63 #ifdef __ARM_ARCH 64 // function prototypes are only really used by these macros on ARM 65 void disable_global_interrupts(); 66 void enable_global_interrupts(); 53 67 54 55 56 57 58 59 60 61 62 63 64 65 68 #define TL_GET( member ) ( { __typeof__( kernelTLS.member ) target; \ 69 disable_global_interrupts(); \ 70 target = kernelTLS.member; \ 71 enable_global_interrupts(); \ 72 target; } ) 73 #define TL_SET( member, value ) disable_global_interrupts(); \ 74 kernelTLS.member = value; \ 75 enable_global_interrupts(); 76 #else 77 #define TL_GET( member ) kernelTLS.member 78 #define TL_SET( member, value ) kernelTLS.member = value; 79 #endif 66 80 67 68 81 extern void disable_interrupts(); 82 extern void enable_interrupts_noPoll(); 69 83 extern void enable_interrupts( __cfaabi_dbg_ctx_param ); 70 84 71 enum __Preemption_Reason { __NO_PREEMPTION, __ALARM_PREEMPTION, __POLL_PREEMPTION, __MANUAL_PREEMPTION }; 85 extern "Cforall" { 86 extern void park( __cfaabi_dbg_ctx_param ); 87 extern void unpark( struct $thread * this __cfaabi_dbg_ctx_param2 ); 88 static inline struct $thread * active_thread () { return TL_GET( this_thread ); } 72 89 73 extern "Cforall" { 74 extern void park( __cfaabi_dbg_ctx_param ); 75 extern void unpark( struct $thread * this __cfaabi_dbg_ctx_param2 ); 76 static inline struct $thread * active_thread () { return TL_GET( this_thread ); } 90 extern bool force_yield( enum __Preemption_Reason ); 77 91 78 extern bool force_yield( enum __Preemption_Reason ); 79 } 92 static inline void yield() { 93 force_yield(__MANUAL_PREEMPTION); 94 } 95 96 // Yield: yield N times 97 static inline void yield( unsigned times ) { 98 for( times ) { 99 yield(); 100 } 101 } 102 103 //----------------------------------------------------------------------- 104 // Statics call at the end of each thread to register statistics 105 #if !defined(__CFA_NO_STATISTICS__) 106 static inline struct __stats_t * __tls_stats() { 107 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled ); 108 /* paranoid */ verify( kernelTLS.this_stats ); 109 return kernelTLS.this_stats; 110 } 111 112 #define __STATS__(in_kernel, ...) { \ 113 if( !(in_kernel) ) disable_interrupts(); \ 114 with( *__tls_stats() ) { \ 115 __VA_ARGS__ \ 116 } \ 117 if( !(in_kernel) ) enable_interrupts( __cfaabi_dbg_ctx ); \ 118 } 119 #else 120 #define __STATS__(in_kernel, ...) 121 #endif 122 } 80 123 } 81 124 #endif -
libcfa/src/concurrency/kernel/startup.cfa
re660761 r3e2b9c9 91 91 //----------------------------------------------------------------------------- 92 92 // Kernel storage 93 #warning duplicated in preemption.cfa94 #define KERNEL_STORAGE(T,X) __attribute((aligned(__alignof__(T)))) static char storage_##X[sizeof(T)]95 93 KERNEL_STORAGE(cluster, mainCluster); 96 94 KERNEL_STORAGE(processor, mainProcessor); -
libcfa/src/concurrency/kernel_private.hfa
re660761 r3e2b9c9 21 21 #include "alarm.hfa" 22 22 #include "stats.hfa" 23 24 #include "bits/random.hfa"25 26 23 27 24 //----------------------------------------------------------------------------- … … 89 86 //----------------------------------------------------------------------------- 90 87 // Utils 91 static inline uint64_t __tls_rand() {92 #if defined(__SIZEOF_INT128__)93 return __lehmer64( kernelTLS.rand_seed );94 #else95 return __xorshift64( kernelTLS.rand_seed );96 #endif97 }98 99 88 void doregister( struct cluster * cltr, struct $thread & thrd ); 100 89 void unregister( struct cluster * cltr, struct $thread & thrd ); … … 102 91 //----------------------------------------------------------------------------- 103 92 // I/O 104 void __kernel_io_startup ();105 void __kernel_io_shutdown ();106 107 static inline io_context * __get_io_context( void ) {108 cluster * cltr = active_cluster();109 /* paranoid */ verifyf( cltr, "No active cluster for io operation\n");110 assertf( cltr->io.cnt > 0, "Cluster %p has no default io contexts and no context was specified\n", cltr );111 /* paranoid */ verifyf( cltr->io.ctxs, "default io contexts for cluster %p are missing\n", cltr);112 return &cltr->io.ctxs[ __tls_rand() % cltr->io.cnt ];113 }114 115 93 void ^?{}(io_context & this, bool ); 116 94 … … 285 263 void ready_queue_shrink(struct cluster * cltr, int target); 286 264 287 //-----------------------------------------------------------------------288 // IO user data289 struct __io_user_data_t {290 int32_t result;291 $thread * thrd;292 };293 294 //-----------------------------------------------------------------------295 // Statics call at the end of each thread to register statistics296 #if !defined(__CFA_NO_STATISTICS__)297 static inline struct __stats_t * __tls_stats() {298 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled );299 /* paranoid */ verify( kernelTLS.this_stats );300 return kernelTLS.this_stats;301 }302 303 #define __STATS__(in_kernel, ...) { \304 if( !(in_kernel) ) disable_interrupts(); \305 with( *__tls_stats() ) { \306 __VA_ARGS__ \307 } \308 if( !(in_kernel) ) enable_interrupts( __cfaabi_dbg_ctx ); \309 }310 #else311 #define __STATS__(in_kernel, ...)312 #endif313 265 314 266 // Local Variables: // -
libcfa/src/concurrency/preemption.cfa
re660761 r3e2b9c9 61 61 #error unknown hardware architecture 62 62 #endif 63 64 #warning duplicated in startup.cfa65 #define KERNEL_STORAGE(T,X) __attribute((aligned(__alignof__(T)))) static char storage_##X[sizeof(T)]66 63 67 64 KERNEL_STORAGE(event_kernel_t, event_kernel); // private storage for event kernel -
libcfa/src/concurrency/thread.hfa
re660761 r3e2b9c9 84 84 85 85 //----------------------------------------------------------------------------- 86 // Thread getters87 static inline struct $thread * active_thread () { return TL_GET( this_thread ); }88 89 //-----------------------------------------------------------------------------90 86 // Scheduler API 91 87 … … 106 102 bool force_yield( enum __Preemption_Reason ); 107 103 108 static inline void yield() {109 force_yield(__MANUAL_PREEMPTION);110 }111 112 // Yield: yield N times113 static inline void yield( unsigned times ) {114 for( times ) {115 yield();116 }117 }118 119 104 //---------- 120 105 // sleep: force thread to block and be rescheduled after Duration duration
Note: See TracChangeset
for help on using the changeset viewer.