Index: libcfa/src/concurrency/io.cfa =================================================================== --- libcfa/src/concurrency/io.cfa (revision c4c8571a1aaa3100966db2d603b3a32857faebdd) +++ libcfa/src/concurrency/io.cfa (revision d93ea1d08135fda8a40ae6a4a8a6ae80da0af885) @@ -83,16 +83,16 @@ }; - static $io_context * __ioarbiter_allocate( $io_arbiter & this, __u32 idxs[], __u32 want ); - static void __ioarbiter_submit( $io_context * , __u32 idxs[], __u32 have, bool lazy ); - static void __ioarbiter_flush ( $io_context & ); - static inline void __ioarbiter_notify( $io_context & ctx ); + static io_context$ * __ioarbiter_allocate( io_arbiter$ & this, __u32 idxs[], __u32 want ); + static void __ioarbiter_submit( io_context$ * , __u32 idxs[], __u32 have, bool lazy ); + static void __ioarbiter_flush ( io_context$ & ); + static inline void __ioarbiter_notify( io_context$ & ctx ); //============================================================================================= // I/O Polling //============================================================================================= - static inline unsigned __flush( struct $io_context & ); - static inline __u32 __release_sqes( struct $io_context & ); + static inline unsigned __flush( struct io_context$ & ); + static inline __u32 __release_sqes( struct io_context$ & ); extern void __kernel_unpark( thread$ * thrd, unpark_hint ); - static void ioring_syscsll( struct $io_context & ctx, unsigned int min_comp, unsigned int flags ) { + static void ioring_syscsll( struct io_context$ & ctx, unsigned int min_comp, unsigned int flags ) { __STATS__( true, io.calls.flush++; ) int ret; @@ -132,5 +132,5 @@ } - static bool try_acquire( $io_context * ctx ) __attribute__((nonnull(1))) { + static bool try_acquire( io_context$ * ctx ) __attribute__((nonnull(1))) { /* paranoid */ verify( ! __preemption_enabled() ); /* paranoid */ verify( ready_schedule_islocked() ); @@ -153,5 +153,5 @@ } - static bool __cfa_do_drain( $io_context * ctx, cluster * cltr ) __attribute__((nonnull(1, 2))) { + static bool __cfa_do_drain( io_context$ * ctx, cluster * cltr ) __attribute__((nonnull(1, 2))) { /* paranoid */ verify( ! __preemption_enabled() ); /* paranoid */ verify( ready_schedule_islocked() ); @@ -213,5 +213,5 @@ cluster * const cltr = proc->cltr; - $io_context * const ctx = proc->io.ctx; + io_context$ * const ctx = proc->io.ctx; /* paranoid */ verify( cltr ); /* paranoid */ verify( ctx ); @@ -278,5 +278,5 @@ /* paranoid */ verify( proc->io.ctx ); - $io_context & ctx = *proc->io.ctx; + io_context$ & ctx = *proc->io.ctx; __ioarbiter_flush( ctx ); @@ -312,5 +312,5 @@ // Allocation // for user's convenience fill the sqes from the indexes - static inline void __fill(struct io_uring_sqe * out_sqes[], __u32 want, __u32 idxs[], struct $io_context * ctx) { + static inline void __fill(struct io_uring_sqe * out_sqes[], __u32 want, __u32 idxs[], struct io_context$ * ctx) { struct io_uring_sqe * sqes = ctx->sq.sqes; for(i; want) { @@ -322,5 +322,5 @@ // Try to directly allocate from the a given context // Not thread-safe - static inline bool __alloc(struct $io_context * ctx, __u32 idxs[], __u32 want) { + static inline bool __alloc(struct io_context$ * ctx, __u32 idxs[], __u32 want) { __sub_ring_t & sq = ctx->sq; const __u32 mask = *sq.mask; @@ -349,10 +349,10 @@ // for convenience, return both the index and the pointer to the sqe // sqe == &sqes[idx] - struct $io_context * cfa_io_allocate(struct io_uring_sqe * sqes[], __u32 idxs[], __u32 want) libcfa_public { + struct io_context$ * cfa_io_allocate(struct io_uring_sqe * sqes[], __u32 idxs[], __u32 want) libcfa_public { // __cfadbg_print_safe(io, "Kernel I/O : attempting to allocate %u\n", want); disable_interrupts(); processor * proc = __cfaabi_tls.this_processor; - $io_context * ctx = proc->io.ctx; + io_context$ * ctx = proc->io.ctx; /* paranoid */ verify( __cfaabi_tls.this_processor ); /* paranoid */ verify( ctx ); @@ -378,10 +378,10 @@ enable_interrupts(); - $io_arbiter * ioarb = proc->cltr->io.arbiter; + io_arbiter$ * ioarb = proc->cltr->io.arbiter; /* paranoid */ verify( ioarb ); // __cfadbg_print_safe(io, "Kernel I/O : falling back on arbiter for allocation\n"); - struct $io_context * ret = __ioarbiter_allocate(*ioarb, idxs, want); + struct io_context$ * ret = __ioarbiter_allocate(*ioarb, idxs, want); // __cfadbg_print_safe(io, "Kernel I/O : slow allocation completed from ring %d\n", ret->fd); @@ -393,5 +393,5 @@ //============================================================================================= // submission - static inline void __submit_only( struct $io_context * ctx, __u32 idxs[], __u32 have) { + static inline void __submit_only( struct io_context$ * ctx, __u32 idxs[], __u32 have) { // We can proceed to the fast path // Get the right objects @@ -414,5 +414,5 @@ } - static inline void __submit( struct $io_context * ctx, __u32 idxs[], __u32 have, bool lazy) { + static inline void __submit( struct io_context$ * ctx, __u32 idxs[], __u32 have, bool lazy) { __sub_ring_t & sq = ctx->sq; __submit_only(ctx, idxs, have); @@ -428,5 +428,5 @@ } - void cfa_io_submit( struct $io_context * inctx, __u32 idxs[], __u32 have, bool lazy ) __attribute__((nonnull (1))) libcfa_public { + void cfa_io_submit( struct io_context$ * inctx, __u32 idxs[], __u32 have, bool lazy ) __attribute__((nonnull (1))) libcfa_public { // __cfadbg_print_safe(io, "Kernel I/O : attempting to submit %u (%s)\n", have, lazy ? "lazy" : "eager"); @@ -434,5 +434,5 @@ __STATS__( true, if(!lazy) io.submit.eagr += 1; ) processor * proc = __cfaabi_tls.this_processor; - $io_context * ctx = proc->io.ctx; + io_context$ * ctx = proc->io.ctx; /* paranoid */ verify( __cfaabi_tls.this_processor ); /* paranoid */ verify( ctx ); @@ -465,5 +465,5 @@ // by io_uring // This cannot be done by multiple threads - static __u32 __release_sqes( struct $io_context & ctx ) { + static __u32 __release_sqes( struct io_context$ & ctx ) { const __u32 mask = *ctx.sq.mask; @@ -538,5 +538,5 @@ } - static $io_context * __ioarbiter_allocate( $io_arbiter & this, __u32 idxs[], __u32 want ) { + static io_context$ * __ioarbiter_allocate( io_arbiter$ & this, __u32 idxs[], __u32 want ) { // __cfadbg_print_safe(io, "Kernel I/O : arbiter allocating\n"); @@ -557,5 +557,5 @@ } - static void __ioarbiter_notify( $io_arbiter & this, $io_context * ctx ) { + static void __ioarbiter_notify( io_arbiter$ & this, io_context$ * ctx ) { /* paranoid */ verify( !empty(this.pending.queue) ); @@ -587,5 +587,5 @@ } - static void __ioarbiter_notify( $io_context & ctx ) { + static void __ioarbiter_notify( io_context$ & ctx ) { if(!empty( ctx.arbiter->pending )) { __ioarbiter_notify( *ctx.arbiter, &ctx ); @@ -594,5 +594,5 @@ // Simply append to the pending - static void __ioarbiter_submit( $io_context * ctx, __u32 idxs[], __u32 have, bool lazy ) { + static void __ioarbiter_submit( io_context$ * ctx, __u32 idxs[], __u32 have, bool lazy ) { __cfadbg_print_safe(io, "Kernel I/O : submitting %u from the arbiter to context %u\n", have, ctx->fd); @@ -618,5 +618,5 @@ } - static void __ioarbiter_flush( $io_context & ctx ) { + static void __ioarbiter_flush( io_context$ & ctx ) { if(!empty( ctx.ext_sq )) { __STATS__( false, io.flush.external += 1; ) @@ -642,5 +642,5 @@ #if defined(CFA_WITH_IO_URING_IDLE) bool __kernel_read(processor * proc, io_future_t & future, iovec & iov, int fd) { - $io_context * ctx = proc->io.ctx; + io_context$ * ctx = proc->io.ctx; /* paranoid */ verify( ! __preemption_enabled() ); /* paranoid */ verify( proc == __cfaabi_tls.this_processor ); Index: libcfa/src/concurrency/io/call.cfa.in =================================================================== --- libcfa/src/concurrency/io/call.cfa.in (revision c4c8571a1aaa3100966db2d603b3a32857faebdd) +++ libcfa/src/concurrency/io/call.cfa.in (revision d93ea1d08135fda8a40ae6a4a8a6ae80da0af885) @@ -75,6 +75,6 @@ ; - extern struct $io_context * cfa_io_allocate(struct io_uring_sqe * out_sqes[], __u32 out_idxs[], __u32 want) __attribute__((nonnull (1,2))); - extern void cfa_io_submit( struct $io_context * in_ctx, __u32 in_idxs[], __u32 have, bool lazy ) __attribute__((nonnull (1,2))); + extern struct io_context$ * cfa_io_allocate(struct io_uring_sqe * out_sqes[], __u32 out_idxs[], __u32 want) __attribute__((nonnull (1,2))); + extern void cfa_io_submit( struct io_context$ * in_ctx, __u32 in_idxs[], __u32 have, bool lazy ) __attribute__((nonnull (1,2))); #endif @@ -200,5 +200,5 @@ __u32 idx; struct io_uring_sqe * sqe; - struct $io_context * ctx = cfa_io_allocate( &sqe, &idx, 1 ); + struct io_context$ * ctx = cfa_io_allocate( &sqe, &idx, 1 ); sqe->opcode = IORING_OP_{op}; Index: libcfa/src/concurrency/io/setup.cfa =================================================================== --- libcfa/src/concurrency/io/setup.cfa (revision c4c8571a1aaa3100966db2d603b3a32857faebdd) +++ libcfa/src/concurrency/io/setup.cfa (revision d93ea1d08135fda8a40ae6a4a8a6ae80da0af885) @@ -28,6 +28,6 @@ void ?{}(io_context_params & this) libcfa_public {} - void ?{}($io_context & this, struct cluster & cl) {} - void ^?{}($io_context & this) {} + void ?{}(io_context$ & this, struct cluster & cl) {} + void ^?{}(io_context$ & this) {} void __cfa_io_start( processor * proc ) {} @@ -37,6 +37,6 @@ void __cfa_io_stop ( processor * proc ) {} - $io_arbiter * create(void) { return 0p; } - void destroy($io_arbiter *) {} + io_arbiter$ * create(void) { return 0p; } + void destroy(io_arbiter$ *) {} #else @@ -105,12 +105,12 @@ - static void __io_uring_setup ( $io_context & this, const io_context_params & params_in, int procfd ); - static void __io_uring_teardown( $io_context & this ); - static void __epoll_register($io_context & ctx); - static void __epoll_unregister($io_context & ctx); - void __ioarbiter_register( $io_arbiter & mutex, $io_context & ctx ); - void __ioarbiter_unregister( $io_arbiter & mutex, $io_context & ctx ); - - void ?{}($io_context & this, processor * proc, struct cluster & cl) { + static void __io_uring_setup ( io_context$ & this, const io_context_params & params_in, int procfd ); + static void __io_uring_teardown( io_context$ & this ); + static void __epoll_register(io_context$ & ctx); + static void __epoll_unregister(io_context$ & ctx); + void __ioarbiter_register( io_arbiter$ & mutex, io_context$ & ctx ); + void __ioarbiter_unregister( io_arbiter$ & mutex, io_context$ & ctx ); + + void ?{}(io_context$ & this, processor * proc, struct cluster & cl) { /* paranoid */ verify( cl.io.arbiter ); this.proc = proc; @@ -122,5 +122,5 @@ } - void ^?{}($io_context & this) { + void ^?{}(io_context$ & this) { __cfadbg_print_safe(io_core, "Kernel I/O : tearing down io_context %u\n", this.fd); @@ -129,5 +129,5 @@ } - static void __io_uring_setup( $io_context & this, const io_context_params & params_in, int procfd ) { + static void __io_uring_setup( io_context$ & this, const io_context_params & params_in, int procfd ) { // Step 1 : call to setup struct io_uring_params params; @@ -270,5 +270,5 @@ } - static void __io_uring_teardown( $io_context & this ) { + static void __io_uring_teardown( io_context$ & this ) { // Shutdown the io rings struct __sub_ring_t & sq = this.sq; @@ -313,5 +313,5 @@ // I/O Context Sleep //============================================================================================= - // static inline void __epoll_ctl($io_context & ctx, int op, const char * error) { + // static inline void __epoll_ctl(io_context$ & ctx, int op, const char * error) { // struct epoll_event ev; // ev.events = EPOLLIN | EPOLLONESHOT; @@ -323,9 +323,9 @@ // } - // static void __epoll_register($io_context & ctx) { + // static void __epoll_register(io_context$ & ctx) { // __epoll_ctl(ctx, EPOLL_CTL_ADD, "ADD"); // } - // static void __epoll_unregister($io_context & ctx) { + // static void __epoll_unregister(io_context$ & ctx) { // // Read the current epoch so we know when to stop // size_t curr = __atomic_load_n(&iopoll.epoch, __ATOMIC_SEQ_CST); @@ -346,5 +346,5 @@ // } - // void __ioctx_prepare_block($io_context & ctx) { + // void __ioctx_prepare_block(io_context$ & ctx) { // __cfadbg_print_safe(io_core, "Kernel I/O - epoll : Re-arming io poller %d (%p)\n", ctx.fd, &ctx); // __epoll_ctl(ctx, EPOLL_CTL_MOD, "REARM"); @@ -355,14 +355,14 @@ // I/O Context Misc Setup //============================================================================================= - void ?{}( $io_arbiter & this ) { + void ?{}( io_arbiter$ & this ) { this.pending.empty = true; } - void ^?{}( $io_arbiter & mutex this ) {} - - $io_arbiter * create(void) { + void ^?{}( io_arbiter$ & mutex this ) {} + + io_arbiter$ * create(void) { return new(); } - void destroy($io_arbiter * arbiter) { + void destroy(io_arbiter$ * arbiter) { delete(arbiter); } Index: libcfa/src/concurrency/io/types.hfa =================================================================== --- libcfa/src/concurrency/io/types.hfa (revision c4c8571a1aaa3100966db2d603b3a32857faebdd) +++ libcfa/src/concurrency/io/types.hfa (revision d93ea1d08135fda8a40ae6a4a8a6ae80da0af885) @@ -33,5 +33,5 @@ struct processor; - monitor $io_arbiter; + monitor io_arbiter$; //----------------------------------------------------------------------- @@ -125,6 +125,6 @@ - struct __attribute__((aligned(64))) $io_context { - $io_arbiter * arbiter; + struct __attribute__((aligned(64))) io_context$ { + io_arbiter$ * arbiter; processor * proc; @@ -137,5 +137,5 @@ }; - static inline unsigned long long ts($io_context *& this) { + static inline unsigned long long ts(io_context$ *& this) { const __u32 head = *this->cq.head; const __u32 tail = *this->cq.tail; @@ -150,8 +150,8 @@ __u32 * idxs; __u32 want; - $io_context * ctx; + io_context$ * ctx; }; - monitor __attribute__((aligned(64))) $io_arbiter { + monitor __attribute__((aligned(64))) io_arbiter$ { __outstanding_io_queue pending; }; @@ -186,4 +186,4 @@ #endif - // void __ioctx_prepare_block($io_context & ctx); + // void __ioctx_prepare_block(io_context$ & ctx); #endif Index: libcfa/src/concurrency/iofwd.hfa =================================================================== --- libcfa/src/concurrency/iofwd.hfa (revision c4c8571a1aaa3100966db2d603b3a32857faebdd) +++ libcfa/src/concurrency/iofwd.hfa (revision d93ea1d08135fda8a40ae6a4a8a6ae80da0af885) @@ -49,5 +49,5 @@ struct cluster; -struct $io_context; +struct io_context$; struct iovec; @@ -82,6 +82,6 @@ //---------- // underlying calls -extern struct $io_context * cfa_io_allocate(struct io_uring_sqe * out_sqes[], __u32 out_idxs[], __u32 want) __attribute__((nonnull (1,2))); -extern void cfa_io_submit( struct $io_context * in_ctx, __u32 in_idxs[], __u32 have, bool lazy ) __attribute__((nonnull (1,2))); +extern struct io_context$ * cfa_io_allocate(struct io_uring_sqe * out_sqes[], __u32 out_idxs[], __u32 want) __attribute__((nonnull (1,2))); +extern void cfa_io_submit( struct io_context$ * in_ctx, __u32 in_idxs[], __u32 have, bool lazy ) __attribute__((nonnull (1,2))); //---------- Index: libcfa/src/concurrency/kernel.cfa =================================================================== --- libcfa/src/concurrency/kernel.cfa (revision c4c8571a1aaa3100966db2d603b3a32857faebdd) +++ libcfa/src/concurrency/kernel.cfa (revision d93ea1d08135fda8a40ae6a4a8a6ae80da0af885) @@ -305,8 +305,12 @@ RUNNING: while(true) { thrd_dst->preempted = __NO_PREEMPTION; - thrd_dst->state = Active; // Update global state kernelTLS().this_thread = thrd_dst; + + // Update the state after setting this_thread + // so that the debugger can find all active threads + // in tls storage + thrd_dst->state = Active; /* paranoid */ verify( ! __preemption_enabled() ); @@ -335,7 +339,4 @@ /* paranoid */ verify( ! __preemption_enabled() ); - // Reset global state - kernelTLS().this_thread = 0p; - // We just finished running a thread, there are a few things that could have happened. // 1 - Regular case : the thread has blocked and now one has scheduled it yet. @@ -346,4 +347,8 @@ if(unlikely(thrd_dst->preempted != __NO_PREEMPTION)) { + // Reset the this_thread now that we know + // the state isn't active anymore + kernelTLS().this_thread = 0p; + // The thread was preempted, reschedule it and reset the flag schedule_thread$( thrd_dst, UNPARK_LOCAL ); @@ -352,4 +357,8 @@ if(unlikely(thrd_dst->state == Halting)) { + // Reset the this_thread now that we know + // the state isn't active anymore + kernelTLS().this_thread = 0p; + // The thread has halted, it should never be scheduled/run again // finish the thread @@ -360,4 +369,8 @@ /* paranoid */ verify( thrd_dst->state == Active ); thrd_dst->state = Blocked; + + // Reset the this_thread now that we know + // the state isn't active anymore + kernelTLS().this_thread = 0p; // set state of processor coroutine to active and the thread to inactive Index: libcfa/src/concurrency/kernel.hfa =================================================================== --- libcfa/src/concurrency/kernel.hfa (revision c4c8571a1aaa3100966db2d603b3a32857faebdd) +++ libcfa/src/concurrency/kernel.hfa (revision d93ea1d08135fda8a40ae6a4a8a6ae80da0af885) @@ -35,6 +35,6 @@ // I/O struct cluster; -struct $io_context; -struct $io_arbiter; +struct io_context$; +struct io_arbiter$; struct io_context_params { @@ -113,5 +113,5 @@ struct { - $io_context * ctx; + io_context$ * ctx; unsigned target; volatile bool pending; @@ -230,5 +230,5 @@ struct { // Array of $io_ - $io_context ** data; + io_context$ ** data; // Time since subqueues were processed @@ -267,5 +267,5 @@ struct { - $io_arbiter * arbiter; + io_arbiter$ * arbiter; io_context_params params; } io; Index: libcfa/src/concurrency/kernel/cluster.cfa =================================================================== --- libcfa/src/concurrency/kernel/cluster.cfa (revision c4c8571a1aaa3100966db2d603b3a32857faebdd) +++ libcfa/src/concurrency/kernel/cluster.cfa (revision d93ea1d08135fda8a40ae6a4a8a6ae80da0af885) @@ -278,5 +278,5 @@ #if defined(CFA_HAVE_LINUX_IO_URING_H) - static void assign_io($io_context ** data, size_t count, dlist(processor) & list) { + static void assign_io(io_context$ ** data, size_t count, dlist(processor) & list) { processor * it = &list`first; while(it) { Index: libcfa/src/concurrency/kernel/private.hfa =================================================================== --- libcfa/src/concurrency/kernel/private.hfa (revision c4c8571a1aaa3100966db2d603b3a32857faebdd) +++ libcfa/src/concurrency/kernel/private.hfa (revision d93ea1d08135fda8a40ae6a4a8a6ae80da0af885) @@ -139,6 +139,6 @@ //----------------------------------------------------------------------------- // I/O -$io_arbiter * create(void); -void destroy($io_arbiter *); +io_arbiter$ * create(void); +void destroy(io_arbiter$ *); //=======================================================================