Index: libcfa/src/concurrency/io.cfa =================================================================== --- libcfa/src/concurrency/io.cfa (revision 7ce88736e34bf185c4d55149f3117c28e91f9106) +++ libcfa/src/concurrency/io.cfa (revision 571f2200e6396d38351875d3cf8aad90ea414f68) @@ -241,8 +241,8 @@ else { const unsigned target = proc->io.target; - /* paranoid */ verify( io.tscs[target].tv != ULLONG_MAX ); + /* paranoid */ verify( io.tscs[target].t.tv != ULLONG_MAX ); HELP: if(target < ctxs_count) { const unsigned long long cutoff = calc_cutoff(ctsc, ctx->cq.id, ctxs_count, io.data, io.tscs, __shard_factor.io); - const unsigned long long age = moving_average(ctsc, io.tscs[target].tv, io.tscs[target].ma); + const unsigned long long age = moving_average(ctsc, io.tscs[target].t.tv, io.tscs[target].t.ma); __cfadbg_print_safe(io, "Kernel I/O: Help attempt on %u from %u, age %'llu vs cutoff %'llu, %s\n", target, ctx->cq.id, age, cutoff, age > cutoff ? "yes" : "no"); if(age <= cutoff) break HELP; Index: libcfa/src/concurrency/io/setup.cfa =================================================================== --- libcfa/src/concurrency/io/setup.cfa (revision 7ce88736e34bf185c4d55149f3117c28e91f9106) +++ libcfa/src/concurrency/io/setup.cfa (revision 571f2200e6396d38351875d3cf8aad90ea414f68) @@ -359,5 +359,5 @@ } - void ^?{}( $io_arbiter & this ) {} + void ^?{}( $io_arbiter & mutex this ) {} $io_arbiter * create(void) { Index: libcfa/src/concurrency/io/types.hfa =================================================================== --- libcfa/src/concurrency/io/types.hfa (revision 7ce88736e34bf185c4d55149f3117c28e91f9106) +++ libcfa/src/concurrency/io/types.hfa (revision 571f2200e6396d38351875d3cf8aad90ea414f68) @@ -125,5 +125,5 @@ - struct __attribute__((aligned(128))) $io_context { + struct __attribute__((aligned(64))) $io_context { $io_arbiter * arbiter; processor * proc; @@ -153,5 +153,5 @@ }; - struct __attribute__((aligned(128))) $io_arbiter { + monitor __attribute__((aligned(64))) $io_arbiter { __outstanding_io_queue pending; }; Index: libcfa/src/concurrency/kernel.hfa =================================================================== --- libcfa/src/concurrency/kernel.hfa (revision 7ce88736e34bf185c4d55149f3117c28e91f9106) +++ libcfa/src/concurrency/kernel.hfa (revision 571f2200e6396d38351875d3cf8aad90ea414f68) @@ -83,5 +83,5 @@ // Wrapper around kernel threads -struct __attribute__((aligned(128))) processor { +struct __attribute__((aligned(64))) processor { // Cluster from which to get threads struct cluster * cltr; @@ -171,15 +171,18 @@ // Intrusives lanes which are used by the ready queue -struct __attribute__((aligned(128))) __intrusive_lane_t; +union __attribute__((aligned(64))) __intrusive_lane_t; void ?{}(__intrusive_lane_t & this); void ^?{}(__intrusive_lane_t & this); // Aligned timestamps which are used by the ready queue and io subsystem -struct __attribute__((aligned(128))) __timestamp_t { - volatile unsigned long long tv; - volatile unsigned long long ma; -}; - -static inline void ?{}(__timestamp_t & this) { this.tv = 0; this.ma = 0; } +union __attribute__((aligned(64))) __timestamp_t { + struct { + volatile unsigned long long tv; + volatile unsigned long long ma; + } t; + char __padding[192]; +}; + +static inline void ?{}(__timestamp_t & this) { this.t.tv = 0; this.t.ma = 0; } static inline void ^?{}(__timestamp_t &) {} @@ -212,5 +215,5 @@ //----------------------------------------------------------------------------- // Cluster -struct __attribute__((aligned(128))) cluster { +struct __attribute__((aligned(64))) cluster { struct { struct { Index: libcfa/src/concurrency/kernel/cluster.cfa =================================================================== --- libcfa/src/concurrency/kernel/cluster.cfa (revision 7ce88736e34bf185c4d55149f3117c28e91f9106) +++ libcfa/src/concurrency/kernel/cluster.cfa (revision 571f2200e6396d38351875d3cf8aad90ea414f68) @@ -229,14 +229,14 @@ for( idx ; lanes_count ) { __intrusive_lane_t & sl = readyQ.data[idx]; - assert(!readyQ.data[idx].lock); + assert(!readyQ.data[idx].l.lock); if(is_empty(sl)) { - assert( sl.anchor.next == 0p ); - assert( sl.anchor.ts == MAX ); - assert( mock_head(sl) == sl.prev ); + assert( sl.l.anchor.next == 0p ); + assert( sl.l.anchor.ts == MAX ); + assert( mock_head(sl) == sl.l.prev ); } else { - assert( sl.anchor.next != 0p ); - assert( sl.anchor.ts != MAX ); - assert( mock_head(sl) != sl.prev ); + assert( sl.l.anchor.next != 0p ); + assert( sl.l.anchor.ts != MAX ); + assert( mock_head(sl) != sl.l.prev ); } } @@ -249,6 +249,6 @@ static inline void fix(__intrusive_lane_t & ll) { if(is_empty(ll)) { - verify(ll.anchor.next == 0p); - ll.prev = mock_head(ll); + verify(ll.l.anchor.next == 0p); + ll.l.prev = mock_head(ll); } } @@ -299,6 +299,6 @@ tscs = alloc(count, tscs`realloc); for(i; count) { - tscs[i].tv = rdtscl(); - tscs[i].ma = 0; + tscs[i].t.tv = rdtscl(); + tscs[i].t.ma = 0; } } @@ -400,5 +400,5 @@ for( idx; ncount ~ ocount) { // Lock is not strictly needed but makes checking invariants much easier - __attribute__((unused)) bool locked = __atomic_try_acquire(&readyQ.data[idx].lock); + __attribute__((unused)) bool locked = __atomic_try_acquire(&readyQ.data[idx].l.lock); verify(locked); @@ -418,5 +418,5 @@ // Unlock the lane - __atomic_unlock(&readyQ.data[idx].lock); + __atomic_unlock(&readyQ.data[idx].l.lock); // TODO print the queue statistics here @@ -467,34 +467,38 @@ } +#define nested_offsetof(type, field) ((off_t)(&(((type*)0)-> field))) + // Ctor void ?{}( __intrusive_lane_t & this ) { - this.lock = false; - this.prev = mock_head(this); - this.anchor.next = 0p; - this.anchor.ts = MAX; + this.l.lock = false; + this.l.prev = mock_head(this); + this.l.anchor.next = 0p; + this.l.anchor.ts = MAX; #if !defined(__CFA_NO_STATISTICS__) - this.cnt = 0; + this.l.cnt = 0; #endif // We add a boat-load of assertions here because the anchor code is very fragile - /* paranoid */ _Static_assert( offsetof( thread$, link ) == offsetof(__intrusive_lane_t, anchor) ); - /* paranoid */ verify( offsetof( thread$, link ) == offsetof(__intrusive_lane_t, anchor) ); - /* paranoid */ verify( ((uintptr_t)( mock_head(this) ) + offsetof( thread$, link )) == (uintptr_t)(&this.anchor) ); - /* paranoid */ verify( &mock_head(this)->link.next == &this.anchor.next ); - /* paranoid */ verify( &mock_head(this)->link.ts == &this.anchor.ts ); + /* paranoid */ _Static_assert( offsetof( thread$, link ) == nested_offsetof(__intrusive_lane_t, l.anchor) ); + /* paranoid */ verify( offsetof( thread$, link ) == nested_offsetof(__intrusive_lane_t, l.anchor) ); + /* paranoid */ verify( ((uintptr_t)( mock_head(this) ) + offsetof( thread$, link )) == (uintptr_t)(&this.l.anchor) ); + /* paranoid */ verify( &mock_head(this)->link.next == &this.l.anchor.next ); + /* paranoid */ verify( &mock_head(this)->link.ts == &this.l.anchor.ts ); /* paranoid */ verify( mock_head(this)->link.next == 0p ); /* paranoid */ verify( mock_head(this)->link.ts == MAX ); - /* paranoid */ verify( mock_head(this) == this.prev ); - /* paranoid */ verify( __alignof__(__intrusive_lane_t) == 128 ); - /* paranoid */ verify( __alignof__(this) == 128 ); - /* paranoid */ verifyf( ((intptr_t)(&this) % 128) == 0, "Expected address to be aligned %p %% 128 == %zd", &this, ((intptr_t)(&this) % 128) ); -} + /* paranoid */ verify( mock_head(this) == this.l.prev ); + /* paranoid */ verify( __alignof__(__intrusive_lane_t) == 64 ); + /* paranoid */ verify( __alignof__(this) == 64 ); + /* paranoid */ verifyf( ((intptr_t)(&this) % 64) == 0, "Expected address to be aligned %p %% 64 == %zd", &this, ((intptr_t)(&this) % 64) ); +} + +#undef nested_offsetof // Dtor is trivial void ^?{}( __intrusive_lane_t & this ) { // Make sure the list is empty - /* paranoid */ verify( this.anchor.next == 0p ); - /* paranoid */ verify( this.anchor.ts == MAX ); - /* paranoid */ verify( mock_head(this) == this.prev ); + /* paranoid */ verify( this.l.anchor.next == 0p ); + /* paranoid */ verify( this.l.anchor.ts == MAX ); + /* paranoid */ verify( mock_head(this) == this.l.prev ); } Index: libcfa/src/concurrency/kernel/cluster.hfa =================================================================== --- libcfa/src/concurrency/kernel/cluster.hfa (revision 7ce88736e34bf185c4d55149f3117c28e91f9106) +++ libcfa/src/concurrency/kernel/cluster.hfa (revision 571f2200e6396d38351875d3cf8aad90ea414f68) @@ -39,7 +39,7 @@ if (ts_next == ULLONG_MAX) return; unsigned long long now = rdtscl(); - unsigned long long pma = __atomic_load_n(&tscs[ idx ].ma, __ATOMIC_RELAXED); - __atomic_store_n(&tscs[ idx ].tv, ts_next, __ATOMIC_RELAXED); - __atomic_store_n(&tscs[ idx ].ma, moving_average(now, ts_prev, pma), __ATOMIC_RELAXED); + unsigned long long pma = __atomic_load_n(&tscs[ idx ].t.ma, __ATOMIC_RELAXED); + __atomic_store_n(&tscs[ idx ].t.tv, ts_next, __ATOMIC_RELAXED); + __atomic_store_n(&tscs[ idx ].t.ma, moving_average(now, ts_prev, pma), __ATOMIC_RELAXED); } @@ -61,5 +61,5 @@ if(ptsc != ULLONG_MAX) { /* paranoid */ verify( start + i < count ); - unsigned long long tsc = moving_average(ctsc, ptsc, tscs[start + i].ma); + unsigned long long tsc = moving_average(ctsc, ptsc, tscs[start + i].t.ma); if(tsc > max) max = tsc; } Index: libcfa/src/concurrency/kernel/fwd.hfa =================================================================== --- libcfa/src/concurrency/kernel/fwd.hfa (revision 7ce88736e34bf185c4d55149f3117c28e91f9106) +++ libcfa/src/concurrency/kernel/fwd.hfa (revision 571f2200e6396d38351875d3cf8aad90ea414f68) @@ -35,5 +35,5 @@ extern "C" { extern "Cforall" { - extern __attribute__((aligned(128))) thread_local struct KernelThreadData { + extern __attribute__((aligned(64))) thread_local struct KernelThreadData { struct thread$ * volatile this_thread; struct processor * volatile this_processor; Index: libcfa/src/concurrency/kernel/private.hfa =================================================================== --- libcfa/src/concurrency/kernel/private.hfa (revision 7ce88736e34bf185c4d55149f3117c28e91f9106) +++ libcfa/src/concurrency/kernel/private.hfa (revision 571f2200e6396d38351875d3cf8aad90ea414f68) @@ -88,5 +88,5 @@ #elif defined(CFA_HAVE_LINUX_RSEQ_H) extern "Cforall" { - extern __attribute__((aligned(128))) thread_local volatile struct rseq __cfaabi_rseq; + extern __attribute__((aligned(64))) thread_local volatile struct rseq __cfaabi_rseq; } #else Index: libcfa/src/concurrency/kernel/startup.cfa =================================================================== --- libcfa/src/concurrency/kernel/startup.cfa (revision 7ce88736e34bf185c4d55149f3117c28e91f9106) +++ libcfa/src/concurrency/kernel/startup.cfa (revision 571f2200e6396d38351875d3cf8aad90ea414f68) @@ -152,5 +152,5 @@ #elif defined(CFA_HAVE_LINUX_RSEQ_H) extern "Cforall" { - __attribute__((aligned(128))) thread_local volatile struct rseq __cfaabi_rseq @= { + __attribute__((aligned(64))) thread_local volatile struct rseq __cfaabi_rseq @= { .cpu_id : RSEQ_CPU_ID_UNINITIALIZED, }; Index: libcfa/src/concurrency/ready_queue.cfa =================================================================== --- libcfa/src/concurrency/ready_queue.cfa (revision 7ce88736e34bf185c4d55149f3117c28e91f9106) +++ libcfa/src/concurrency/ready_queue.cfa (revision 571f2200e6396d38351875d3cf8aad90ea414f68) @@ -81,9 +81,9 @@ /* paranoid */ verify( i < lanes_count ); // If we can't lock it retry - } while( !__atomic_try_acquire( &readyQ.data[i].lock ) ); + } while( !__atomic_try_acquire( &readyQ.data[i].l.lock ) ); } else { do { i = __tls_rand() % lanes_count; - } while( !__atomic_try_acquire( &readyQ.data[i].lock ) ); + } while( !__atomic_try_acquire( &readyQ.data[i].l.lock ) ); } } else { @@ -93,5 +93,5 @@ /* paranoid */ verify( i < lanes_count ); // If we can't lock it retry - } while( !__atomic_try_acquire( &readyQ.data[i].lock ) ); + } while( !__atomic_try_acquire( &readyQ.data[i].l.lock ) ); } @@ -100,5 +100,5 @@ // Unlock and return - __atomic_unlock( &readyQ.data[i].lock ); + __atomic_unlock( &readyQ.data[i].l.lock ); #if !defined(__CFA_NO_STATISTICS__) @@ -136,9 +136,9 @@ else { const unsigned target = proc->rdq.target; - __cfadbg_print_safe(ready_queue, "Kernel : %u considering helping %u, tcsc %llu\n", this, target, readyQ.tscs[target].tv); - /* paranoid */ verify( readyQ.tscs[target].tv != ULLONG_MAX ); + __cfadbg_print_safe(ready_queue, "Kernel : %u considering helping %u, tcsc %llu\n", this, target, readyQ.tscs[target].t.tv); + /* paranoid */ verify( readyQ.tscs[target].t.tv != ULLONG_MAX ); if(target < lanes_count) { const unsigned long long cutoff = calc_cutoff(ctsc, proc->rdq.id, lanes_count, cltr->sched.readyQ.data, cltr->sched.readyQ.tscs, __shard_factor.readyq); - const unsigned long long age = moving_average(ctsc, readyQ.tscs[target].tv, readyQ.tscs[target].ma); + const unsigned long long age = moving_average(ctsc, readyQ.tscs[target].t.tv, readyQ.tscs[target].t.ma); __cfadbg_print_safe(ready_queue, "Kernel : Help attempt on %u from %u, age %'llu vs cutoff %'llu, %s\n", target, this, age, cutoff, age > cutoff ? "yes" : "no"); if(age > cutoff) { @@ -188,5 +188,5 @@ // If we can't get the lock retry - if( !__atomic_try_acquire(&lane.lock) ) { + if( !__atomic_try_acquire(&lane.l.lock) ) { return 0p; } @@ -194,5 +194,5 @@ // If list is empty, unlock and retry if( is_empty(lane) ) { - __atomic_unlock(&lane.lock); + __atomic_unlock(&lane.l.lock); return 0p; } @@ -206,8 +206,8 @@ /* paranoid */ verify(thrd); /* paranoid */ verify(ts_next); - /* paranoid */ verify(lane.lock); + /* paranoid */ verify(lane.l.lock); // Unlock and return - __atomic_unlock(&lane.lock); + __atomic_unlock(&lane.l.lock); // Update statistics Index: libcfa/src/concurrency/ready_subqueue.hfa =================================================================== --- libcfa/src/concurrency/ready_subqueue.hfa (revision 7ce88736e34bf185c4d55149f3117c28e91f9106) +++ libcfa/src/concurrency/ready_subqueue.hfa (revision 571f2200e6396d38351875d3cf8aad90ea414f68) @@ -6,15 +6,18 @@ // Intrusives lanes which are used by the relaxed ready queue -struct __attribute__((aligned(128))) __intrusive_lane_t { - struct thread$ * prev; +union __attribute__((aligned(64))) __intrusive_lane_t { + struct { + struct thread$ * prev; - // spin lock protecting the queue - volatile bool lock; + // spin lock protecting the queue + volatile bool lock; - __thread_desc_link anchor; + __thread_desc_link anchor; - #if !defined(__CFA_NO_STATISTICS__) - unsigned cnt; - #endif + #if !defined(__CFA_NO_STATISTICS__) + unsigned cnt; + #endif + } l; + char __padding[192]; }; @@ -22,5 +25,5 @@ static inline thread$ * mock_head(const __intrusive_lane_t & this) { thread$ * rhead = (thread$ *)( - (uintptr_t)( &this.anchor ) - __builtin_offsetof( thread$, link ) + (uintptr_t)( &this.l.anchor ) - __builtin_offsetof( thread$, link ) ); return rhead; @@ -30,27 +33,27 @@ // returns true of lane was empty before push, false otherwise static inline void push( __intrusive_lane_t & this, thread$ * node ) { - /* paranoid */ verify( this.lock ); + /* paranoid */ verify( this.l.lock ); /* paranoid */ verify( node->link.next == 0p ); /* paranoid */ verify( __atomic_load_n(&node->link.ts, __ATOMIC_RELAXED) == MAX ); - /* paranoid */ verify( this.prev->link.next == 0p ); - /* paranoid */ verify( __atomic_load_n(&this.prev->link.ts, __ATOMIC_RELAXED) == MAX ); - if( this.anchor.next == 0p ) { - /* paranoid */ verify( this.anchor.next == 0p ); - /* paranoid */ verify( __atomic_load_n(&this.anchor.ts, __ATOMIC_RELAXED) == MAX ); - /* paranoid */ verify( __atomic_load_n(&this.anchor.ts, __ATOMIC_RELAXED) != 0 ); - /* paranoid */ verify( this.prev == mock_head( this ) ); + /* paranoid */ verify( this.l.prev->link.next == 0p ); + /* paranoid */ verify( __atomic_load_n(&this.l.prev->link.ts, __ATOMIC_RELAXED) == MAX ); + if( this.l.anchor.next == 0p ) { + /* paranoid */ verify( this.l.anchor.next == 0p ); + /* paranoid */ verify( __atomic_load_n(&this.l.anchor.ts, __ATOMIC_RELAXED) == MAX ); + /* paranoid */ verify( __atomic_load_n(&this.l.anchor.ts, __ATOMIC_RELAXED) != 0 ); + /* paranoid */ verify( this.l.prev == mock_head( this ) ); } else { - /* paranoid */ verify( this.anchor.next != 0p ); - /* paranoid */ verify( __atomic_load_n(&this.anchor.ts, __ATOMIC_RELAXED) != MAX ); - /* paranoid */ verify( __atomic_load_n(&this.anchor.ts, __ATOMIC_RELAXED) != 0 ); - /* paranoid */ verify( this.prev != mock_head( this ) ); + /* paranoid */ verify( this.l.anchor.next != 0p ); + /* paranoid */ verify( __atomic_load_n(&this.l.anchor.ts, __ATOMIC_RELAXED) != MAX ); + /* paranoid */ verify( __atomic_load_n(&this.l.anchor.ts, __ATOMIC_RELAXED) != 0 ); + /* paranoid */ verify( this.l.prev != mock_head( this ) ); } // Get the relevant nodes locally - this.prev->link.next = node; - __atomic_store_n(&this.prev->link.ts, rdtscl(), __ATOMIC_RELAXED); - this.prev = node; + this.l.prev->link.next = node; + __atomic_store_n(&this.l.prev->link.ts, rdtscl(), __ATOMIC_RELAXED); + this.l.prev = node; #if !defined(__CFA_NO_STATISTICS__) - this.cnt++; + this.l.cnt++; #endif } @@ -60,24 +63,24 @@ // returns true of lane was empty before push, false otherwise static inline [* thread$, unsigned long long] pop( __intrusive_lane_t & this ) { - /* paranoid */ verify( this.lock ); - /* paranoid */ verify( this.anchor.next != 0p ); - /* paranoid */ verify( __atomic_load_n(&this.anchor.ts, __ATOMIC_RELAXED) != MAX ); - /* paranoid */ verify( __atomic_load_n(&this.anchor.ts, __ATOMIC_RELAXED) != 0 ); + /* paranoid */ verify( this.l.lock ); + /* paranoid */ verify( this.l.anchor.next != 0p ); + /* paranoid */ verify( __atomic_load_n(&this.l.anchor.ts, __ATOMIC_RELAXED) != MAX ); + /* paranoid */ verify( __atomic_load_n(&this.l.anchor.ts, __ATOMIC_RELAXED) != 0 ); // Get the relevant nodes locally - thread$ * node = this.anchor.next; - this.anchor.next = node->link.next; - __atomic_store_n(&this.anchor.ts, __atomic_load_n(&node->link.ts, __ATOMIC_RELAXED), __ATOMIC_RELAXED); - bool is_empty = this.anchor.next == 0p; + thread$ * node = this.l.anchor.next; + this.l.anchor.next = node->link.next; + __atomic_store_n(&this.l.anchor.ts, __atomic_load_n(&node->link.ts, __ATOMIC_RELAXED), __ATOMIC_RELAXED); + bool is_empty = this.l.anchor.next == 0p; node->link.next = 0p; __atomic_store_n(&node->link.ts, ULLONG_MAX, __ATOMIC_RELAXED); #if !defined(__CFA_NO_STATISTICS__) - this.cnt--; + this.l.cnt--; #endif // Update head time stamp - if(is_empty) this.prev = mock_head( this ); + if(is_empty) this.l.prev = mock_head( this ); - unsigned long long ats = __atomic_load_n(&this.anchor.ts, __ATOMIC_RELAXED); + unsigned long long ats = __atomic_load_n(&this.l.anchor.ts, __ATOMIC_RELAXED); /* paranoid */ verify( node->link.next == 0p ); /* paranoid */ verify( __atomic_load_n(&node->link.ts , __ATOMIC_RELAXED) == MAX ); @@ -90,5 +93,5 @@ // Check whether or not list is empty static inline bool is_empty(__intrusive_lane_t & this) { - return this.anchor.next == 0p; + return this.l.anchor.next == 0p; } @@ -96,6 +99,6 @@ static inline unsigned long long ts(__intrusive_lane_t & this) { // Cannot verify 'emptiness' here since it may not be locked - /* paranoid */ verify(this.anchor.ts != 0); - /* paranoid */ static_assert(__atomic_always_lock_free(sizeof(this.anchor.ts), &this.anchor.ts)); - return __atomic_load_n(&this.anchor.ts, __ATOMIC_RELAXED); + /* paranoid */ verify(this.l.anchor.ts != 0); + /* paranoid */ static_assert(__atomic_always_lock_free(sizeof(this.l.anchor.ts), &this.l.anchor.ts)); + return __atomic_load_n(&this.l.anchor.ts, __ATOMIC_RELAXED); } Index: libcfa/src/concurrency/stats.hfa =================================================================== --- libcfa/src/concurrency/stats.hfa (revision 7ce88736e34bf185c4d55149f3117c28e91f9106) +++ libcfa/src/concurrency/stats.hfa (revision 571f2200e6396d38351875d3cf8aad90ea414f68) @@ -132,5 +132,5 @@ #endif - struct __attribute__((aligned(128))) __stats_t { + struct __attribute__((aligned(64))) __stats_t { __stats_readyQ_t ready; #if defined(CFA_HAVE_LINUX_IO_URING_H)