Changeset 7f6a7c9 for libcfa/src/concurrency
- Timestamp:
- Sep 21, 2022, 11:02:15 AM (3 years ago)
- Branches:
- ADT, ast-experimental, master, pthread-emulation
- Children:
- 95dab9e
- Parents:
- 428adbc (diff), 0bd46fd (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the(diff)links above to see all the changes relative to each parent. - Location:
- libcfa/src/concurrency
- Files:
-
- 14 edited
-
io.cfa (modified) (20 diffs)
-
io/call.cfa.in (modified) (2 diffs)
-
io/setup.cfa (modified) (13 diffs)
-
io/types.hfa (modified) (5 diffs)
-
iofwd.hfa (modified) (2 diffs)
-
kernel.cfa (modified) (5 diffs)
-
kernel.hfa (modified) (4 diffs)
-
kernel/cluster.cfa (modified) (7 diffs)
-
kernel/cluster.hfa (modified) (2 diffs)
-
kernel/fwd.hfa (modified) (14 diffs)
-
kernel/private.hfa (modified) (9 diffs)
-
kernel/startup.cfa (modified) (6 diffs)
-
monitor.hfa (modified) (2 diffs)
-
preemption.cfa (modified) (8 diffs)
Legend:
- Unmodified
- Added
- Removed
-
libcfa/src/concurrency/io.cfa
r428adbc r7f6a7c9 83 83 }; 84 84 85 static $io_context * __ioarbiter_allocate( $io_arbiter& this, __u32 idxs[], __u32 want );86 static void __ioarbiter_submit( $io_context* , __u32 idxs[], __u32 have, bool lazy );87 static void __ioarbiter_flush ( $io_context& );88 static inline void __ioarbiter_notify( $io_context& ctx );85 static io_context$ * __ioarbiter_allocate( io_arbiter$ & this, __u32 idxs[], __u32 want ); 86 static void __ioarbiter_submit( io_context$ * , __u32 idxs[], __u32 have, bool lazy ); 87 static void __ioarbiter_flush ( io_context$ & ); 88 static inline void __ioarbiter_notify( io_context$ & ctx ); 89 89 //============================================================================================= 90 90 // I/O Polling 91 91 //============================================================================================= 92 static inline unsigned __flush( struct $io_context& );93 static inline __u32 __release_sqes( struct $io_context& );92 static inline unsigned __flush( struct io_context$ & ); 93 static inline __u32 __release_sqes( struct io_context$ & ); 94 94 extern void __kernel_unpark( thread$ * thrd, unpark_hint ); 95 95 96 static void ioring_syscsll( struct $io_context& ctx, unsigned int min_comp, unsigned int flags ) {96 static void ioring_syscsll( struct io_context$ & ctx, unsigned int min_comp, unsigned int flags ) { 97 97 __STATS__( true, io.calls.flush++; ) 98 98 int ret; … … 132 132 } 133 133 134 static bool try_acquire( $io_context* ctx ) __attribute__((nonnull(1))) {134 static bool try_acquire( io_context$ * ctx ) __attribute__((nonnull(1))) { 135 135 /* paranoid */ verify( ! __preemption_enabled() ); 136 136 /* paranoid */ verify( ready_schedule_islocked() ); … … 153 153 } 154 154 155 static bool __cfa_do_drain( $io_context* ctx, cluster * cltr ) __attribute__((nonnull(1, 2))) {155 static bool __cfa_do_drain( io_context$ * ctx, cluster * cltr ) __attribute__((nonnull(1, 2))) { 156 156 /* paranoid */ verify( ! __preemption_enabled() ); 157 157 /* paranoid */ verify( ready_schedule_islocked() ); … … 213 213 214 214 cluster * const cltr = proc->cltr; 215 $io_context* const ctx = proc->io.ctx;215 io_context$ * const ctx = proc->io.ctx; 216 216 /* paranoid */ verify( cltr ); 217 217 /* paranoid */ verify( ctx ); … … 278 278 /* paranoid */ verify( proc->io.ctx ); 279 279 280 $io_context& ctx = *proc->io.ctx;280 io_context$ & ctx = *proc->io.ctx; 281 281 282 282 __ioarbiter_flush( ctx ); … … 312 312 // Allocation 313 313 // for user's convenience fill the sqes from the indexes 314 static inline void __fill(struct io_uring_sqe * out_sqes[], __u32 want, __u32 idxs[], struct $io_context* ctx) {314 static inline void __fill(struct io_uring_sqe * out_sqes[], __u32 want, __u32 idxs[], struct io_context$ * ctx) { 315 315 struct io_uring_sqe * sqes = ctx->sq.sqes; 316 316 for(i; want) { … … 322 322 // Try to directly allocate from the a given context 323 323 // Not thread-safe 324 static inline bool __alloc(struct $io_context* ctx, __u32 idxs[], __u32 want) {324 static inline bool __alloc(struct io_context$ * ctx, __u32 idxs[], __u32 want) { 325 325 __sub_ring_t & sq = ctx->sq; 326 326 const __u32 mask = *sq.mask; … … 349 349 // for convenience, return both the index and the pointer to the sqe 350 350 // sqe == &sqes[idx] 351 struct $io_context* cfa_io_allocate(struct io_uring_sqe * sqes[], __u32 idxs[], __u32 want) libcfa_public {351 struct io_context$ * cfa_io_allocate(struct io_uring_sqe * sqes[], __u32 idxs[], __u32 want) libcfa_public { 352 352 // __cfadbg_print_safe(io, "Kernel I/O : attempting to allocate %u\n", want); 353 353 354 354 disable_interrupts(); 355 355 processor * proc = __cfaabi_tls.this_processor; 356 $io_context* ctx = proc->io.ctx;356 io_context$ * ctx = proc->io.ctx; 357 357 /* paranoid */ verify( __cfaabi_tls.this_processor ); 358 358 /* paranoid */ verify( ctx ); … … 378 378 enable_interrupts(); 379 379 380 $io_arbiter* ioarb = proc->cltr->io.arbiter;380 io_arbiter$ * ioarb = proc->cltr->io.arbiter; 381 381 /* paranoid */ verify( ioarb ); 382 382 383 383 // __cfadbg_print_safe(io, "Kernel I/O : falling back on arbiter for allocation\n"); 384 384 385 struct $io_context* ret = __ioarbiter_allocate(*ioarb, idxs, want);385 struct io_context$ * ret = __ioarbiter_allocate(*ioarb, idxs, want); 386 386 387 387 // __cfadbg_print_safe(io, "Kernel I/O : slow allocation completed from ring %d\n", ret->fd); … … 393 393 //============================================================================================= 394 394 // submission 395 static inline void __submit_only( struct $io_context* ctx, __u32 idxs[], __u32 have) {395 static inline void __submit_only( struct io_context$ * ctx, __u32 idxs[], __u32 have) { 396 396 // We can proceed to the fast path 397 397 // Get the right objects … … 414 414 } 415 415 416 static inline void __submit( struct $io_context* ctx, __u32 idxs[], __u32 have, bool lazy) {416 static inline void __submit( struct io_context$ * ctx, __u32 idxs[], __u32 have, bool lazy) { 417 417 __sub_ring_t & sq = ctx->sq; 418 418 __submit_only(ctx, idxs, have); … … 428 428 } 429 429 430 void cfa_io_submit( struct $io_context* inctx, __u32 idxs[], __u32 have, bool lazy ) __attribute__((nonnull (1))) libcfa_public {430 void cfa_io_submit( struct io_context$ * inctx, __u32 idxs[], __u32 have, bool lazy ) __attribute__((nonnull (1))) libcfa_public { 431 431 // __cfadbg_print_safe(io, "Kernel I/O : attempting to submit %u (%s)\n", have, lazy ? "lazy" : "eager"); 432 432 … … 434 434 __STATS__( true, if(!lazy) io.submit.eagr += 1; ) 435 435 processor * proc = __cfaabi_tls.this_processor; 436 $io_context* ctx = proc->io.ctx;436 io_context$ * ctx = proc->io.ctx; 437 437 /* paranoid */ verify( __cfaabi_tls.this_processor ); 438 438 /* paranoid */ verify( ctx ); … … 465 465 // by io_uring 466 466 // This cannot be done by multiple threads 467 static __u32 __release_sqes( struct $io_context& ctx ) {467 static __u32 __release_sqes( struct io_context$ & ctx ) { 468 468 const __u32 mask = *ctx.sq.mask; 469 469 … … 538 538 } 539 539 540 static $io_context * __ioarbiter_allocate( $io_arbiter& this, __u32 idxs[], __u32 want ) {540 static io_context$ * __ioarbiter_allocate( io_arbiter$ & this, __u32 idxs[], __u32 want ) { 541 541 // __cfadbg_print_safe(io, "Kernel I/O : arbiter allocating\n"); 542 542 … … 557 557 } 558 558 559 static void __ioarbiter_notify( $io_arbiter & this, $io_context* ctx ) {559 static void __ioarbiter_notify( io_arbiter$ & this, io_context$ * ctx ) { 560 560 /* paranoid */ verify( !empty(this.pending.queue) ); 561 561 … … 587 587 } 588 588 589 static void __ioarbiter_notify( $io_context& ctx ) {589 static void __ioarbiter_notify( io_context$ & ctx ) { 590 590 if(!empty( ctx.arbiter->pending )) { 591 591 __ioarbiter_notify( *ctx.arbiter, &ctx ); … … 594 594 595 595 // Simply append to the pending 596 static void __ioarbiter_submit( $io_context* ctx, __u32 idxs[], __u32 have, bool lazy ) {596 static void __ioarbiter_submit( io_context$ * ctx, __u32 idxs[], __u32 have, bool lazy ) { 597 597 __cfadbg_print_safe(io, "Kernel I/O : submitting %u from the arbiter to context %u\n", have, ctx->fd); 598 598 … … 618 618 } 619 619 620 static void __ioarbiter_flush( $io_context& ctx ) {620 static void __ioarbiter_flush( io_context$ & ctx ) { 621 621 if(!empty( ctx.ext_sq )) { 622 622 __STATS__( false, io.flush.external += 1; ) … … 642 642 #if defined(CFA_WITH_IO_URING_IDLE) 643 643 bool __kernel_read(processor * proc, io_future_t & future, iovec & iov, int fd) { 644 $io_context* ctx = proc->io.ctx;644 io_context$ * ctx = proc->io.ctx; 645 645 /* paranoid */ verify( ! __preemption_enabled() ); 646 646 /* paranoid */ verify( proc == __cfaabi_tls.this_processor ); -
libcfa/src/concurrency/io/call.cfa.in
r428adbc r7f6a7c9 75 75 ; 76 76 77 extern struct $io_context* cfa_io_allocate(struct io_uring_sqe * out_sqes[], __u32 out_idxs[], __u32 want) __attribute__((nonnull (1,2)));78 extern void cfa_io_submit( struct $io_context* in_ctx, __u32 in_idxs[], __u32 have, bool lazy ) __attribute__((nonnull (1,2)));77 extern struct io_context$ * cfa_io_allocate(struct io_uring_sqe * out_sqes[], __u32 out_idxs[], __u32 want) __attribute__((nonnull (1,2))); 78 extern void cfa_io_submit( struct io_context$ * in_ctx, __u32 in_idxs[], __u32 have, bool lazy ) __attribute__((nonnull (1,2))); 79 79 #endif 80 80 … … 200 200 __u32 idx; 201 201 struct io_uring_sqe * sqe; 202 struct $io_context * ctx = cfa_io_allocate( &sqe, &idx, 1 ); 203 202 struct io_context$ * ctx = cfa_io_allocate( &sqe, &idx, 1 ); 203 204 memset(sqe, 0, sizeof(*sqe)); 204 205 sqe->opcode = IORING_OP_{op}; 206 sqe->flags = sflags; 205 207 sqe->user_data = (uintptr_t)&future; 206 sqe->flags = sflags; 207 sqe->ioprio = 0; 208 sqe->fd = 0; 209 sqe->off = 0; 210 sqe->addr = 0; 211 sqe->len = 0; 212 sqe->fsync_flags = 0; 213 sqe->__pad2[0] = 0; 214 sqe->__pad2[1] = 0; 215 sqe->__pad2[2] = 0;{body} 208 {body} 216 209 217 210 asm volatile("": : :"memory"); -
libcfa/src/concurrency/io/setup.cfa
r428adbc r7f6a7c9 28 28 void ?{}(io_context_params & this) libcfa_public {} 29 29 30 void ?{}( $io_context& this, struct cluster & cl) {}31 void ^?{}( $io_context& this) {}30 void ?{}(io_context$ & this, struct cluster & cl) {} 31 void ^?{}(io_context$ & this) {} 32 32 33 33 void __cfa_io_start( processor * proc ) {} … … 37 37 void __cfa_io_stop ( processor * proc ) {} 38 38 39 $io_arbiter* create(void) { return 0p; }40 void destroy( $io_arbiter*) {}39 io_arbiter$ * create(void) { return 0p; } 40 void destroy(io_arbiter$ *) {} 41 41 42 42 #else … … 105 105 106 106 107 static void __io_uring_setup ( $io_context& this, const io_context_params & params_in, int procfd );108 static void __io_uring_teardown( $io_context& this );109 static void __epoll_register( $io_context& ctx);110 static void __epoll_unregister( $io_context& ctx);111 void __ioarbiter_register( $io_arbiter & mutex, $io_context& ctx );112 void __ioarbiter_unregister( $io_arbiter & mutex, $io_context& ctx );113 114 void ?{}( $io_context& this, processor * proc, struct cluster & cl) {107 static void __io_uring_setup ( io_context$ & this, const io_context_params & params_in, int procfd ); 108 static void __io_uring_teardown( io_context$ & this ); 109 static void __epoll_register(io_context$ & ctx); 110 static void __epoll_unregister(io_context$ & ctx); 111 void __ioarbiter_register( io_arbiter$ & mutex, io_context$ & ctx ); 112 void __ioarbiter_unregister( io_arbiter$ & mutex, io_context$ & ctx ); 113 114 void ?{}(io_context$ & this, processor * proc, struct cluster & cl) { 115 115 /* paranoid */ verify( cl.io.arbiter ); 116 116 this.proc = proc; … … 122 122 } 123 123 124 void ^?{}( $io_context& this) {124 void ^?{}(io_context$ & this) { 125 125 __cfadbg_print_safe(io_core, "Kernel I/O : tearing down io_context %u\n", this.fd); 126 126 … … 129 129 } 130 130 131 static void __io_uring_setup( $io_context& this, const io_context_params & params_in, int procfd ) {131 static void __io_uring_setup( io_context$ & this, const io_context_params & params_in, int procfd ) { 132 132 // Step 1 : call to setup 133 133 struct io_uring_params params; … … 228 228 229 229 #if !defined(CFA_WITH_IO_URING_IDLE) 230 { 230 231 // Step 4 : eventfd 231 232 __cfadbg_print_safe(io_core, "Kernel I/O : registering %d for completion with ring %d\n", procfd, fd); … … 237 238 238 239 __cfadbg_print_safe(io_core, "Kernel I/O : registered %d for completion with ring %d\n", procfd, fd); 239 #endif 240 240 } 241 #endif 242 243 // TODO: implement a proper version of this. 244 // I have not found a better maximum that works in general but users should be able to configure it 245 // the same way they configure other I/O options 241 246 // #if defined(CFA_HAVE_IORING_REGISTER_IOWQ_MAX_WORKERS) 247 // { 242 248 // // Step 5 : max worker count 243 249 // __cfadbg_print_safe(io_core, "Kernel I/O : lmiting max workers for ring %d\n", fd); … … 252 258 253 259 // __cfadbg_print_safe(io_core, "Kernel I/O : lmited max workers for ring %d\n", fd); 260 // } 254 261 // #endif 255 262 … … 270 277 } 271 278 272 static void __io_uring_teardown( $io_context& this ) {279 static void __io_uring_teardown( io_context$ & this ) { 273 280 // Shutdown the io rings 274 281 struct __sub_ring_t & sq = this.sq; … … 313 320 // I/O Context Sleep 314 321 //============================================================================================= 315 // static inline void __epoll_ctl( $io_context& ctx, int op, const char * error) {322 // static inline void __epoll_ctl(io_context$ & ctx, int op, const char * error) { 316 323 // struct epoll_event ev; 317 324 // ev.events = EPOLLIN | EPOLLONESHOT; … … 323 330 // } 324 331 325 // static void __epoll_register( $io_context& ctx) {332 // static void __epoll_register(io_context$ & ctx) { 326 333 // __epoll_ctl(ctx, EPOLL_CTL_ADD, "ADD"); 327 334 // } 328 335 329 // static void __epoll_unregister( $io_context& ctx) {336 // static void __epoll_unregister(io_context$ & ctx) { 330 337 // // Read the current epoch so we know when to stop 331 338 // size_t curr = __atomic_load_n(&iopoll.epoch, __ATOMIC_SEQ_CST); … … 346 353 // } 347 354 348 // void __ioctx_prepare_block( $io_context& ctx) {355 // void __ioctx_prepare_block(io_context$ & ctx) { 349 356 // __cfadbg_print_safe(io_core, "Kernel I/O - epoll : Re-arming io poller %d (%p)\n", ctx.fd, &ctx); 350 357 // __epoll_ctl(ctx, EPOLL_CTL_MOD, "REARM"); … … 355 362 // I/O Context Misc Setup 356 363 //============================================================================================= 357 void ?{}( $io_arbiter& this ) {364 void ?{}( io_arbiter$ & this ) { 358 365 this.pending.empty = true; 359 366 } 360 367 361 void ^?{}( $io_arbiter& mutex this ) {}362 363 $io_arbiter* create(void) {368 void ^?{}( io_arbiter$ & mutex this ) {} 369 370 io_arbiter$ * create(void) { 364 371 return new(); 365 372 } 366 void destroy( $io_arbiter* arbiter) {373 void destroy(io_arbiter$ * arbiter) { 367 374 delete(arbiter); 368 375 } -
libcfa/src/concurrency/io/types.hfa
r428adbc r7f6a7c9 33 33 34 34 struct processor; 35 monitor $io_arbiter;35 monitor io_arbiter$; 36 36 37 37 //----------------------------------------------------------------------- … … 125 125 126 126 127 struct __attribute__((aligned(64))) $io_context{128 $io_arbiter* arbiter;127 struct __attribute__((aligned(64))) io_context$ { 128 io_arbiter$ * arbiter; 129 129 processor * proc; 130 130 … … 137 137 }; 138 138 139 static inline unsigned long long ts( $io_context*& this) {139 static inline unsigned long long ts(io_context$ *& this) { 140 140 const __u32 head = *this->cq.head; 141 141 const __u32 tail = *this->cq.tail; … … 150 150 __u32 * idxs; 151 151 __u32 want; 152 $io_context* ctx;152 io_context$ * ctx; 153 153 }; 154 154 155 monitor __attribute__((aligned(64))) $io_arbiter{155 monitor __attribute__((aligned(64))) io_arbiter$ { 156 156 __outstanding_io_queue pending; 157 157 }; … … 186 186 #endif 187 187 188 // void __ioctx_prepare_block( $io_context& ctx);188 // void __ioctx_prepare_block(io_context$ & ctx); 189 189 #endif -
libcfa/src/concurrency/iofwd.hfa
r428adbc r7f6a7c9 49 49 50 50 struct cluster; 51 struct $io_context;51 struct io_context$; 52 52 53 53 struct iovec; … … 82 82 //---------- 83 83 // underlying calls 84 extern struct $io_context* cfa_io_allocate(struct io_uring_sqe * out_sqes[], __u32 out_idxs[], __u32 want) __attribute__((nonnull (1,2)));85 extern void cfa_io_submit( struct $io_context* in_ctx, __u32 in_idxs[], __u32 have, bool lazy ) __attribute__((nonnull (1,2)));84 extern struct io_context$ * cfa_io_allocate(struct io_uring_sqe * out_sqes[], __u32 out_idxs[], __u32 want) __attribute__((nonnull (1,2))); 85 extern void cfa_io_submit( struct io_context$ * in_ctx, __u32 in_idxs[], __u32 have, bool lazy ) __attribute__((nonnull (1,2))); 86 86 87 87 //---------- -
libcfa/src/concurrency/kernel.cfa
r428adbc r7f6a7c9 305 305 RUNNING: while(true) { 306 306 thrd_dst->preempted = __NO_PREEMPTION; 307 thrd_dst->state = Active;308 307 309 308 // Update global state 310 309 kernelTLS().this_thread = thrd_dst; 310 311 // Update the state after setting this_thread 312 // so that the debugger can find all active threads 313 // in tls storage 314 thrd_dst->state = Active; 311 315 312 316 /* paranoid */ verify( ! __preemption_enabled() ); … … 335 339 /* paranoid */ verify( ! __preemption_enabled() ); 336 340 337 // Reset global state338 kernelTLS().this_thread = 0p;339 340 341 // We just finished running a thread, there are a few things that could have happened. 341 342 // 1 - Regular case : the thread has blocked and now one has scheduled it yet. … … 346 347 347 348 if(unlikely(thrd_dst->preempted != __NO_PREEMPTION)) { 349 // Reset the this_thread now that we know 350 // the state isn't active anymore 351 kernelTLS().this_thread = 0p; 352 348 353 // The thread was preempted, reschedule it and reset the flag 349 354 schedule_thread$( thrd_dst, UNPARK_LOCAL ); … … 352 357 353 358 if(unlikely(thrd_dst->state == Halting)) { 359 // Reset the this_thread now that we know 360 // the state isn't active anymore 361 kernelTLS().this_thread = 0p; 362 354 363 // The thread has halted, it should never be scheduled/run again 355 364 // finish the thread … … 360 369 /* paranoid */ verify( thrd_dst->state == Active ); 361 370 thrd_dst->state = Blocked; 371 372 // Reset the this_thread now that we know 373 // the state isn't active anymore 374 kernelTLS().this_thread = 0p; 362 375 363 376 // set state of processor coroutine to active and the thread to inactive -
libcfa/src/concurrency/kernel.hfa
r428adbc r7f6a7c9 35 35 // I/O 36 36 struct cluster; 37 struct $io_context;38 struct $io_arbiter;37 struct io_context$; 38 struct io_arbiter$; 39 39 40 40 struct io_context_params { … … 113 113 114 114 struct { 115 $io_context* ctx;115 io_context$ * ctx; 116 116 unsigned target; 117 117 volatile bool pending; … … 230 230 struct { 231 231 // Array of $io_ 232 $io_context** data;232 io_context$ ** data; 233 233 234 234 // Time since subqueues were processed … … 267 267 268 268 struct { 269 $io_arbiter* arbiter;269 io_arbiter$ * arbiter; 270 270 io_context_params params; 271 271 } io; -
libcfa/src/concurrency/kernel/cluster.cfa
r428adbc r7f6a7c9 93 93 //======================================================================= 94 94 void ?{}(__scheduler_RWLock_t & this) { 95 this. max = __max_processors();96 this. alloc = 0;97 this. ready = 0;98 this. data = alloc(this.max);99 this. write_lock = false;100 101 /*paranoid*/ verify(__atomic_is_lock_free(sizeof(this. alloc), &this.alloc));102 /*paranoid*/ verify(__atomic_is_lock_free(sizeof(this. ready), &this.ready));95 this.lock.max = __max_processors(); 96 this.lock.alloc = 0; 97 this.lock.ready = 0; 98 this.lock.data = alloc(this.lock.max); 99 this.lock.write_lock = false; 100 101 /*paranoid*/ verify(__atomic_is_lock_free(sizeof(this.lock.alloc), &this.lock.alloc)); 102 /*paranoid*/ verify(__atomic_is_lock_free(sizeof(this.lock.ready), &this.lock.ready)); 103 103 104 104 } 105 105 void ^?{}(__scheduler_RWLock_t & this) { 106 free(this. data);106 free(this.lock.data); 107 107 } 108 108 … … 110 110 //======================================================================= 111 111 // Lock-Free registering/unregistering of threads 112 unsigned register_proc_id( void ) with( *__scheduler_lock) {112 unsigned register_proc_id( void ) with(__scheduler_lock.lock) { 113 113 __kernel_rseq_register(); 114 114 … … 132 132 } 133 133 134 if(max <= alloc) abort("Trying to create more than %ud processors", __scheduler_lock ->max);134 if(max <= alloc) abort("Trying to create more than %ud processors", __scheduler_lock.lock.max); 135 135 136 136 // Step - 2 : F&A to get a new spot in the array. 137 137 uint_fast32_t n = __atomic_fetch_add(&alloc, 1, __ATOMIC_SEQ_CST); 138 if(max <= n) abort("Trying to create more than %ud processors", __scheduler_lock ->max);138 if(max <= n) abort("Trying to create more than %ud processors", __scheduler_lock.lock.max); 139 139 140 140 // Step - 3 : Mark space as used and then publish it. … … 154 154 } 155 155 156 void unregister_proc_id( unsigned id ) with( *__scheduler_lock) {156 void unregister_proc_id( unsigned id ) with(__scheduler_lock.lock) { 157 157 /* paranoid */ verify(id < ready); 158 158 /* paranoid */ verify(id == kernelTLS().sched_id); … … 169 169 // Writer side : acquire when changing the ready queue, e.g. adding more 170 170 // queues or removing them. 171 uint_fast32_t ready_mutate_lock( void ) with( *__scheduler_lock) {171 uint_fast32_t ready_mutate_lock( void ) with(__scheduler_lock.lock) { 172 172 /* paranoid */ verify( ! __preemption_enabled() ); 173 173 … … 196 196 } 197 197 198 void ready_mutate_unlock( uint_fast32_t last_s ) with( *__scheduler_lock) {198 void ready_mutate_unlock( uint_fast32_t last_s ) with(__scheduler_lock.lock) { 199 199 /* paranoid */ verify( ! __preemption_enabled() ); 200 200 … … 278 278 279 279 #if defined(CFA_HAVE_LINUX_IO_URING_H) 280 static void assign_io( $io_context** data, size_t count, dlist(processor) & list) {280 static void assign_io(io_context$ ** data, size_t count, dlist(processor) & list) { 281 281 processor * it = &list`first; 282 282 while(it) { -
libcfa/src/concurrency/kernel/cluster.hfa
r428adbc r7f6a7c9 24 24 // Calc moving average based on existing average, before and current time. 25 25 static inline unsigned long long moving_average(unsigned long long currtsc, unsigned long long instsc, unsigned long long old_avg) { 26 /* paranoid */ verifyf( currtsc < 45000000000000000, "Suspiciously large current time: %'llu (%llx)\n", currtsc, currtsc );27 /* paranoid */ verifyf( instsc < 45000000000000000, "Suspiciously large insert time: %'llu (%llx)\n", instsc, instsc );28 26 /* paranoid */ verifyf( old_avg < 15000000000000, "Suspiciously large previous average: %'llu (%llx)\n", old_avg, old_avg ); 29 27 … … 65 63 } 66 64 } 67 return (max + 2 * max) / 2;65 return 8 * max; 68 66 } 69 67 -
libcfa/src/concurrency/kernel/fwd.hfa
r428adbc r7f6a7c9 35 35 extern "C" { 36 36 extern "Cforall" { 37 extern __attribute__((aligned(64))) thread_localstruct KernelThreadData {37 extern __attribute__((aligned(64))) __thread struct KernelThreadData { 38 38 struct thread$ * volatile this_thread; 39 39 struct processor * volatile this_processor; … … 179 179 // Similar to a binary semaphore with a 'one shot' semantic 180 180 // is expected to be discarded after each party call their side 181 enum(struct thread$ *) { oneshot_ARMED = 0p, oneshot_FULFILLED = 1p }; 181 182 struct oneshot { 182 183 // Internal state : 183 // 0p : is initial state (wait will block)184 // 1p : fulfilled (wait won't block)184 // armed : initial state, wait will block 185 // fulfilled : wait won't block 185 186 // any thread : a thread is currently waiting 186 187 struct thread$ * volatile ptr; … … 189 190 static inline { 190 191 void ?{}(oneshot & this) { 191 this.ptr = 0p;192 this.ptr = oneshot_ARMED; 192 193 } 193 194 … … 199 200 for() { 200 201 struct thread$ * expected = this.ptr; 201 if(expected == 1p) return false;202 if(expected == oneshot_FULFILLED) return false; 202 203 if(__atomic_compare_exchange_n(&this.ptr, &expected, active_thread(), false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) { 203 204 park(); 204 /* paranoid */ verify( this.ptr == 1p);205 /* paranoid */ verify( this.ptr == oneshot_FULFILLED ); 205 206 return true; 206 207 } … … 211 212 // return true if a thread was unparked 212 213 thread$ * post(oneshot & this, bool do_unpark = true) { 213 struct thread$ * got = __atomic_exchange_n( &this.ptr, 1p, __ATOMIC_SEQ_CST);214 if( got == 0p || got == 1p) return 0p;214 struct thread$ * got = __atomic_exchange_n( &this.ptr, oneshot_FULFILLED, __ATOMIC_SEQ_CST); 215 if( got == oneshot_ARMED || got == oneshot_FULFILLED ) return 0p; 215 216 if(do_unpark) unpark( got ); 216 217 return got; … … 223 224 // thread on "any of" [a given set of] futures. 224 225 // does not support multiple threads waiting on the same future 226 enum(struct oneshot *) { future_ARMED = 0p, future_FULFILLED = 1p, future_PROGRESS = 2p, future_ABANDONED = 3p }; 225 227 struct future_t { 226 228 // Internal state : 227 // 0p : is initial state (wait will block)228 // 1p : fulfilled (wait won't block)229 // 2p : in progress ()230 // 3p : abandoned, server should delete229 // armed : initial state, wait will block 230 // fulfilled : result is ready, wait won't block 231 // progress : someone else is in the process of fulfilling this 232 // abandoned : client no longer cares, server should delete 231 233 // any oneshot : a context has been setup to wait, a thread could wait on it 232 234 struct oneshot * volatile ptr; … … 235 237 static inline { 236 238 void ?{}(future_t & this) { 237 this.ptr = 0p;239 this.ptr = future_ARMED; 238 240 } 239 241 … … 242 244 void reset(future_t & this) { 243 245 // needs to be in 0p or 1p 244 __atomic_exchange_n( &this.ptr, 0p, __ATOMIC_SEQ_CST);246 __atomic_exchange_n( &this.ptr, future_ARMED, __ATOMIC_SEQ_CST); 245 247 } 246 248 247 249 // check if the future is available 248 250 bool available( future_t & this ) { 249 while( this.ptr == 2p) Pause();250 return this.ptr == 1p;251 while( this.ptr == future_PROGRESS ) Pause(); 252 return this.ptr == future_FULFILLED; 251 253 } 252 254 … … 254 256 // intented to be use by wait, wait_any, waitfor, etc. rather than used directly 255 257 bool setup( future_t & this, oneshot & wait_ctx ) { 256 /* paranoid */ verify( wait_ctx.ptr == 0p || wait_ctx.ptr == 1p);258 /* paranoid */ verify( wait_ctx.ptr == oneshot_ARMED || wait_ctx.ptr == oneshot_FULFILLED ); 257 259 // The future needs to set the wait context 258 260 for() { 259 261 struct oneshot * expected = this.ptr; 260 262 // Is the future already fulfilled? 261 if(expected == 1p) return false; // Yes, just return false (didn't block)263 if(expected == future_FULFILLED) return false; // Yes, just return false (didn't block) 262 264 263 265 // The future is not fulfilled, try to setup the wait context … … 277 279 278 280 // attempt to remove the context so it doesn't get consumed. 279 if(__atomic_compare_exchange_n( &this.ptr, &expected, 0p, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) {281 if(__atomic_compare_exchange_n( &this.ptr, &expected, future_ARMED, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) { 280 282 // we still have the original context, then no one else saw it 281 283 return false; 282 284 } 283 285 284 // expected == 0p: future was never actually setup, just return285 if( expected == 0p) return false;286 287 // expected == 1p: the future is ready and the context was fully consumed286 // expected == ARMED: future was never actually setup, just return 287 if( expected == future_ARMED ) return false; 288 289 // expected == FULFILLED: the future is ready and the context was fully consumed 288 290 // the server won't use the pointer again 289 291 // It is safe to delete (which could happen after the return) 290 if( expected == 1p) return true;291 292 // expected == 2p: the future is ready but the context hasn't fully been consumed292 if( expected == future_FULFILLED ) return true; 293 294 // expected == PROGRESS: the future is ready but the context hasn't fully been consumed 293 295 // spin until it is safe to move on 294 if( expected == 2p) {295 while( this.ptr != 1p) Pause();296 /* paranoid */ verify( this.ptr == 1p);296 if( expected == future_PROGRESS ) { 297 while( this.ptr != future_FULFILLED ) Pause(); 298 /* paranoid */ verify( this.ptr == future_FULFILLED ); 297 299 return true; 298 300 } … … 305 307 // Mark the future as abandoned, meaning it will be deleted by the server 306 308 bool abandon( future_t & this ) { 307 /* paranoid */ verify( this.ptr != 3p);309 /* paranoid */ verify( this.ptr != future_ABANDONED ); 308 310 309 311 // Mark the future as abandonned 310 struct oneshot * got = __atomic_exchange_n( &this.ptr, 3p, __ATOMIC_SEQ_CST);312 struct oneshot * got = __atomic_exchange_n( &this.ptr, future_ABANDONED, __ATOMIC_SEQ_CST); 311 313 312 314 // If the future isn't already fulfilled, let the server delete it 313 if( got == 0p) return false;314 315 // got == 2p: the future is ready but the context hasn't fully been consumed315 if( got == future_ARMED ) return false; 316 317 // got == PROGRESS: the future is ready but the context hasn't fully been consumed 316 318 // spin until it is safe to move on 317 if( got == 2p) {318 while( this.ptr != 1p) Pause();319 got = 1p;319 if( got == future_PROGRESS ) { 320 while( this.ptr != future_FULFILLED ) Pause(); 321 got = future_FULFILLED; 320 322 } 321 323 322 324 // The future is completed delete it now 323 /* paranoid */ verify( this.ptr != 1p);325 /* paranoid */ verify( this.ptr != future_FULFILLED ); 324 326 free( &this ); 325 327 return true; … … 336 338 #pragma GCC diagnostic ignored "-Wfree-nonheap-object" 337 339 #endif 338 if( expected == 3p) { free( &this ); return 0p; }340 if( expected == future_ABANDONED ) { free( &this ); return 0p; } 339 341 #if defined(__GNUC__) && __GNUC__ >= 7 340 342 #pragma GCC diagnostic pop 341 343 #endif 342 344 343 /* paranoid */ verify( expected != 1p); // Future is already fulfilled, should not happen344 /* paranoid */ verify( expected != 2p); // Future is bein fulfilled by someone else, this is even less supported then the previous case.345 /* paranoid */ verify( expected != future_FULFILLED ); // Future is already fulfilled, should not happen 346 /* paranoid */ verify( expected != future_PROGRESS ); // Future is bein fulfilled by someone else, this is even less supported then the previous case. 345 347 346 348 // If there is a wait context, we need to consume it and mark it as consumed after 347 349 // If there is no context then we can skip the in progress phase 348 struct oneshot * want = expected == 0p ? 1p : 2p;350 struct oneshot * want = expected == future_ARMED ? future_FULFILLED : future_PROGRESS; 349 351 if(__atomic_compare_exchange_n(&this.ptr, &expected, want, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) { 350 if( expected == 0p) { return 0p; }352 if( expected == future_ARMED ) { return 0p; } 351 353 thread$ * ret = post( *expected, do_unpark ); 352 __atomic_store_n( &this.ptr, 1p, __ATOMIC_SEQ_CST);354 __atomic_store_n( &this.ptr, future_FULFILLED, __ATOMIC_SEQ_CST); 353 355 return ret; 354 356 } … … 366 368 367 369 // Wait for the future to tru 368 while( this.ptr == 2p) Pause();370 while( this.ptr == future_PROGRESS ) Pause(); 369 371 // Make sure the state makes sense 370 372 // Should be fulfilled, could be in progress but it's out of date if so … … 372 374 // and the oneshot should not be needed any more 373 375 __attribute__((unused)) struct oneshot * was = this.ptr; 374 /* paranoid */ verifyf( was == 1p, "Expected this.ptr to be 1p, was %p\n", was );376 /* paranoid */ verifyf( was == future_FULFILLED, "Expected this.ptr to be 1p, was %p\n", was ); 375 377 376 378 // Mark the future as fulfilled, to be consistent -
libcfa/src/concurrency/kernel/private.hfa
r428adbc r7f6a7c9 88 88 #elif defined(CFA_HAVE_LINUX_RSEQ_H) 89 89 extern "Cforall" { 90 extern __attribute__((aligned(64))) thread_localvolatile struct rseq __cfaabi_rseq;90 extern __attribute__((aligned(64))) __thread volatile struct rseq __cfaabi_rseq; 91 91 } 92 92 #else … … 139 139 //----------------------------------------------------------------------------- 140 140 // I/O 141 $io_arbiter* create(void);142 void destroy( $io_arbiter*);141 io_arbiter$ * create(void); 142 void destroy(io_arbiter$ *); 143 143 144 144 //======================================================================= … … 161 161 // Blocking acquire 162 162 static inline void __atomic_acquire(volatile bool * ll) { 163 /* paranoid */ verify( ! __preemption_enabled() ); 164 /* paranoid */ verify(ll); 165 163 166 while( __builtin_expect(__atomic_exchange_n(ll, (bool)true, __ATOMIC_SEQ_CST), false) ) { 164 167 while(__atomic_load_n(ll, (int)__ATOMIC_RELAXED)) … … 166 169 } 167 170 /* paranoid */ verify(*ll); 171 /* paranoid */ verify( ! __preemption_enabled() ); 168 172 } 169 173 170 174 // Non-Blocking acquire 171 175 static inline bool __atomic_try_acquire(volatile bool * ll) { 176 /* paranoid */ verify( ! __preemption_enabled() ); 177 /* paranoid */ verify(ll); 178 172 179 return !__atomic_exchange_n(ll, (bool)true, __ATOMIC_SEQ_CST); 173 180 } … … 175 182 // Release 176 183 static inline void __atomic_unlock(volatile bool * ll) { 184 /* paranoid */ verify( ! __preemption_enabled() ); 185 /* paranoid */ verify(ll); 177 186 /* paranoid */ verify(*ll); 178 187 __atomic_store_n(ll, (bool)false, __ATOMIC_RELEASE); … … 184 193 // have been hard-coded to for the ready-queue for 185 194 // simplicity and performance 186 struct __scheduler_RWLock_t { 187 // total cachelines allocated 188 unsigned int max; 189 190 // cachelines currently in use 191 volatile unsigned int alloc; 192 193 // cachelines ready to itereate over 194 // (!= to alloc when thread is in second half of doregister) 195 volatile unsigned int ready; 196 197 // writer lock 198 volatile bool write_lock; 199 200 // data pointer 201 volatile bool * volatile * data; 195 union __attribute__((aligned(64))) __scheduler_RWLock_t { 196 struct { 197 __attribute__((aligned(64))) char padding; 198 199 // total cachelines allocated 200 __attribute__((aligned(64))) unsigned int max; 201 202 // cachelines currently in use 203 volatile unsigned int alloc; 204 205 // cachelines ready to itereate over 206 // (!= to alloc when thread is in second half of doregister) 207 volatile unsigned int ready; 208 209 // writer lock 210 volatile bool write_lock; 211 212 // data pointer 213 volatile bool * volatile * data; 214 } lock; 215 char pad[192]; 202 216 }; 203 217 … … 205 219 void ^?{}(__scheduler_RWLock_t & this); 206 220 207 extern __scheduler_RWLock_t *__scheduler_lock;221 extern __scheduler_RWLock_t __scheduler_lock; 208 222 209 223 //----------------------------------------------------------------------- 210 224 // Reader side : acquire when using the ready queue to schedule but not 211 225 // creating/destroying queues 212 static inline void ready_schedule_lock(void) with( *__scheduler_lock) {226 static inline void ready_schedule_lock(void) with(__scheduler_lock.lock) { 213 227 /* paranoid */ verify( ! __preemption_enabled() ); 214 228 /* paranoid */ verify( ! kernelTLS().in_sched_lock ); … … 235 249 } 236 250 237 static inline void ready_schedule_unlock(void) with( *__scheduler_lock) {251 static inline void ready_schedule_unlock(void) with(__scheduler_lock.lock) { 238 252 /* paranoid */ verify( ! __preemption_enabled() ); 239 253 /* paranoid */ verify( data[kernelTLS().sched_id] == &kernelTLS().sched_lock ); … … 256 270 257 271 static inline bool ready_mutate_islocked() { 258 return __scheduler_lock ->write_lock;272 return __scheduler_lock.lock.write_lock; 259 273 } 260 274 #endif -
libcfa/src/concurrency/kernel/startup.cfa
r428adbc r7f6a7c9 113 113 KERNEL_STORAGE(thread$, mainThread); 114 114 KERNEL_STORAGE(__stack_t, mainThreadCtx); 115 KERNEL_STORAGE(__scheduler_RWLock_t, __scheduler_lock);115 // KERNEL_STORAGE(__scheduler_RWLock_t, __scheduler_lock); 116 116 KERNEL_STORAGE(eventfd_t, mainIdleEventFd); 117 117 KERNEL_STORAGE(io_future_t, mainIdleFuture); … … 123 123 processor * mainProcessor; 124 124 thread$ * mainThread; 125 __scheduler_RWLock_t * __scheduler_lock;126 125 127 126 extern "C" { … … 134 133 //----------------------------------------------------------------------------- 135 134 // Global state 136 thread_localstruct KernelThreadData __cfaabi_tls __attribute__ ((tls_model ( "initial-exec" ))) @= {135 __thread struct KernelThreadData __cfaabi_tls __attribute__ ((tls_model ( "initial-exec" ))) @= { 137 136 NULL, // cannot use 0p 138 137 NULL, … … 148 147 }; 149 148 149 __scheduler_RWLock_t __scheduler_lock @= { 0 }; 150 150 151 #if defined(CFA_HAVE_LINUX_LIBRSEQ) 151 152 // No data needed 152 153 #elif defined(CFA_HAVE_LINUX_RSEQ_H) 153 154 extern "Cforall" { 154 __attribute__((aligned(64))) thread_localvolatile struct rseq __cfaabi_rseq @= {155 __attribute__((aligned(64))) __thread volatile struct rseq __cfaabi_rseq @= { 155 156 .cpu_id : RSEQ_CPU_ID_UNINITIALIZED, 156 157 }; … … 198 199 199 200 // Initialize the global scheduler lock 200 __scheduler_lock = (__scheduler_RWLock_t*)&storage___scheduler_lock;201 ( *__scheduler_lock){};201 // __scheduler_lock = (__scheduler_RWLock_t*)&storage___scheduler_lock; 202 (__scheduler_lock){}; 202 203 203 204 // Initialize the main cluster … … 345 346 ^(*mainCluster){}; 346 347 347 ^( *__scheduler_lock){};348 ^(__scheduler_lock){}; 348 349 349 350 ^(__cfa_dbg_global_clusters.list){}; -
libcfa/src/concurrency/monitor.hfa
r428adbc r7f6a7c9 60 60 void ^?{}( monitor_dtor_guard_t & this ); 61 61 62 /* 62 63 static inline forall( T & | sized(T) | { void ^?{}( T & mutex ); } ) 63 64 void delete( T * th ) { … … 65 66 free( th ); 66 67 } 68 */ 67 69 68 70 static inline forall( T & | sized(T) | { void ^?{}( T & mutex ); } ) -
libcfa/src/concurrency/preemption.cfa
r428adbc r7f6a7c9 238 238 //---------- 239 239 // special case for preemption since used often 240 __attribute__((optimize("no-reorder-blocks"))) bool __preemption_enabled() libcfa_ public {240 __attribute__((optimize("no-reorder-blocks"))) bool __preemption_enabled() libcfa_nopreempt libcfa_public { 241 241 // create a assembler label before 242 242 // marked as clobber all to avoid movement … … 272 272 } 273 273 274 extern "C" { 275 __attribute__((visibility("hidden"))) extern void * const __start_cfatext_nopreempt; 276 __attribute__((visibility("hidden"))) extern void * const __stop_cfatext_nopreempt; 277 278 extern const __cfa_nopreempt_region __libcfa_nopreempt; 279 __attribute__((visibility("protected"))) const __cfa_nopreempt_region __libcfathrd_nopreempt @= { 280 (void * const)&__start_cfatext_nopreempt, 281 (void * const)&__stop_cfatext_nopreempt 282 }; 283 } 284 285 static inline bool __cfaabi_in( void * const ip, const struct __cfa_nopreempt_region & const region ) { 286 return ip >= region.start && ip <= region.stop; 287 } 288 274 289 275 290 //---------- 276 291 // Get data from the TLS block 277 292 // struct asm_region __cfaasm_get; 278 uintptr_t __cfatls_get( unsigned long int offset ) __attribute__((__noinline__, visibility("default"))); //no inline to avoid problems293 uintptr_t __cfatls_get( unsigned long int offset ) libcfa_nopreempt libcfa_public; //no inline to avoid problems 279 294 uintptr_t __cfatls_get( unsigned long int offset ) { 280 295 // create a assembler label before … … 295 310 extern "C" { 296 311 // Disable interrupts by incrementing the counter 297 __attribute__((__noinline__, visibility("default"))) void disable_interrupts()libcfa_public {312 void disable_interrupts() libcfa_nopreempt libcfa_public { 298 313 // create a assembler label before 299 314 // marked as clobber all to avoid movement … … 326 341 // Enable interrupts by decrementing the counter 327 342 // If counter reaches 0, execute any pending __cfactx_switch 328 void enable_interrupts( bool poll ) libcfa_ public {343 void enable_interrupts( bool poll ) libcfa_nopreempt libcfa_public { 329 344 // Cache the processor now since interrupts can start happening after the atomic store 330 345 processor * proc = __cfaabi_tls.this_processor; … … 358 373 } 359 374 } 375 376 // Check whether or not there is pending preemption 377 // force_yield( __POLL_PREEMPTION ) if appropriate 378 // return true if the thread was in an interruptable state 379 // i.e. on a real processor and not in the kernel 380 // (can return true even if no preemption was pending) 381 bool poll_interrupts() libcfa_public { 382 // Cache the processor now since interrupts can start happening after the atomic store 383 processor * proc = publicTLS_get( this_processor ); 384 if ( ! proc ) return false; 385 if ( ! __preemption_enabled() ) return false; 386 387 with( __cfaabi_tls.preemption_state ){ 388 // Signal the compiler that a fence is needed but only for signal handlers 389 __atomic_signal_fence(__ATOMIC_RELEASE); 390 if( proc->pending_preemption ) { 391 proc->pending_preemption = false; 392 force_yield( __POLL_PREEMPTION ); 393 } 394 } 395 396 return true; 397 } 360 398 } 361 399 … … 463 501 464 502 //----------------------------------------------------------------------------- 465 // Some assembly required466 #if defined( __i386 )467 #ifdef __PIC__468 #define RELOC_PRELUDE( label ) \469 "calll .Lcfaasm_prelude_" #label "$pb\n\t" \470 ".Lcfaasm_prelude_" #label "$pb:\n\t" \471 "popl %%eax\n\t" \472 ".Lcfaasm_prelude_" #label "_end:\n\t" \473 "addl $_GLOBAL_OFFSET_TABLE_+(.Lcfaasm_prelude_" #label "_end-.Lcfaasm_prelude_" #label "$pb), %%eax\n\t"474 #define RELOC_PREFIX ""475 #define RELOC_SUFFIX "@GOT(%%eax)"476 #else477 #define RELOC_PREFIX "$"478 #define RELOC_SUFFIX ""479 #endif480 #define __cfaasm_label( label ) struct asm_region label = \481 ({ \482 struct asm_region region; \483 asm( \484 RELOC_PRELUDE( label ) \485 "movl " RELOC_PREFIX "__cfaasm_" #label "_before" RELOC_SUFFIX ", %[vb]\n\t" \486 "movl " RELOC_PREFIX "__cfaasm_" #label "_after" RELOC_SUFFIX ", %[va]\n\t" \487 : [vb]"=r"(region.before), [va]"=r"(region.after) \488 ); \489 region; \490 });491 #elif defined( __x86_64 )492 #ifdef __PIC__493 #define RELOC_PREFIX ""494 #define RELOC_SUFFIX "@GOTPCREL(%%rip)"495 #else496 #define RELOC_PREFIX "$"497 #define RELOC_SUFFIX ""498 #endif499 #define __cfaasm_label( label ) struct asm_region label = \500 ({ \501 struct asm_region region; \502 asm( \503 "movq " RELOC_PREFIX "__cfaasm_" #label "_before" RELOC_SUFFIX ", %[vb]\n\t" \504 "movq " RELOC_PREFIX "__cfaasm_" #label "_after" RELOC_SUFFIX ", %[va]\n\t" \505 : [vb]"=r"(region.before), [va]"=r"(region.after) \506 ); \507 region; \508 });509 #elif defined( __aarch64__ )510 #ifdef __PIC__511 // Note that this works only for gcc512 #define __cfaasm_label( label ) struct asm_region label = \513 ({ \514 struct asm_region region; \515 asm( \516 "adrp %[vb], _GLOBAL_OFFSET_TABLE_" "\n\t" \517 "ldr %[vb], [%[vb], #:gotpage_lo15:__cfaasm_" #label "_before]" "\n\t" \518 "adrp %[va], _GLOBAL_OFFSET_TABLE_" "\n\t" \519 "ldr %[va], [%[va], #:gotpage_lo15:__cfaasm_" #label "_after]" "\n\t" \520 : [vb]"=r"(region.before), [va]"=r"(region.after) \521 ); \522 region; \523 });524 #else525 #error this is not the right thing to do526 /*527 #define __cfaasm_label( label ) struct asm_region label = \528 ({ \529 struct asm_region region; \530 asm( \531 "adrp %[vb], __cfaasm_" #label "_before" "\n\t" \532 "add %[vb], %[vb], :lo12:__cfaasm_" #label "_before" "\n\t" \533 "adrp %[va], :got:__cfaasm_" #label "_after" "\n\t" \534 "add %[va], %[va], :lo12:__cfaasm_" #label "_after" "\n\t" \535 : [vb]"=r"(region.before), [va]"=r"(region.after) \536 ); \537 region; \538 });539 */540 #endif541 #else542 #error unknown hardware architecture543 #endif544 545 503 // KERNEL ONLY 546 504 // Check if a __cfactx_switch signal handler shoud defer … … 548 506 // If false : preemption is unsafe and marked as pending 549 507 static inline bool preemption_ready( void * ip ) { 550 // Get all the region for which it is not safe to preempt551 __cfaasm_label( get );552 __cfaasm_label( check );553 __cfaasm_label( dsable );554 // __cfaasm_label( debug );555 556 508 // Check if preemption is safe 557 509 bool ready = true; 558 if( __cfaasm_in( ip, get ) ) { ready = false; goto EXIT; }; 559 if( __cfaasm_in( ip, check ) ) { ready = false; goto EXIT; }; 560 if( __cfaasm_in( ip, dsable ) ) { ready = false; goto EXIT; }; 561 // if( __cfaasm_in( ip, debug ) ) { ready = false; goto EXIT; }; 510 if( __cfaabi_in( ip, __libcfa_nopreempt ) ) { ready = false; goto EXIT; }; 511 if( __cfaabi_in( ip, __libcfathrd_nopreempt ) ) { ready = false; goto EXIT; }; 512 562 513 if( !__cfaabi_tls.preemption_state.enabled) { ready = false; goto EXIT; }; 563 514 if( __cfaabi_tls.preemption_state.in_progress ) { ready = false; goto EXIT; }; … … 643 594 // Kernel Signal Handlers 644 595 //============================================================================================= 645 __cfaabi_dbg_debug_do( static thread_localvoid * last_interrupt = 0; )596 __cfaabi_dbg_debug_do( static __thread void * last_interrupt = 0; ) 646 597 647 598 // Context switch signal handler
Note:
See TracChangeset
for help on using the changeset viewer.
