Index: libcfa/src/common.hfa =================================================================== --- libcfa/src/common.hfa (revision a5db488e6c0a055c012112dbeeb6ab737d0b9097) +++ libcfa/src/common.hfa (revision 7f543563b892c550d18d3201d22a70056e4acb5e) @@ -10,6 +10,6 @@ // Created On : Wed Jul 11 17:54:36 2018 // Last Modified By : Peter A. Buhr -// Last Modified On : Sat Aug 15 08:51:29 2020 -// Update Count : 14 +// Last Modified On : Wed May 5 14:02:04 2021 +// Update Count : 18 // @@ -67,15 +67,23 @@ static inline { - char min( char t1, char t2 ) { return t1 < t2 ? t1 : t2; } // optimization - intptr_t min( intptr_t t1, intptr_t t2 ) { return t1 < t2 ? t1 : t2; } // optimization - uintptr_t min( uintptr_t t1, uintptr_t t2 ) { return t1 < t2 ? t1 : t2; } // optimization + char min( char v1, char v2 ) { return v1 < v2 ? v1 : v2; } // optimization + int min( int v1, int v2 ) { return v1 < v2 ? v1 : v2; } + unsigned int min( unsigned int v1, unsigned int v2 ) { return v1 < v2 ? v1 : v2; } + long int min( long int v1, long int v2 ) { return v1 < v2 ? v1 : v2; } + unsigned long int min( unsigned long int v1, unsigned int v2 ) { return v1 < v2 ? v1 : v2; } + long long int min( long long int v1, long long int v2 ) { return v1 < v2 ? v1 : v2; } + unsigned long long int min( unsigned long long int v1, unsigned int v2 ) { return v1 < v2 ? v1 : v2; } forall( T | { int ? t2 ? t1 : t2; } // optimization - intptr_t max( intptr_t t1, intptr_t t2 ) { return t1 > t2 ? t1 : t2; } // optimization - uintptr_t max( uintptr_t t1, uintptr_t t2 ) { return t1 > t2 ? t1 : t2; } // optimization + char max( char v1, char v2 ) { return v1 > v2 ? v1 : v2; } // optimization + int max( int v1, int v2 ) { return v1 > v2 ? v1 : v2; } + unsigned int max( unsigned int v1, unsigned int v2 ) { return v1 > v2 ? v1 : v2; } + long int max( long int v1, long int v2 ) { return v1 > v2 ? v1 : v2; } + unsigned long int max( unsigned long int v1, unsigned long int v2 ) { return v1 > v2 ? v1 : v2; } + long long int max( long long int v1, long long int v2 ) { return v1 > v2 ? v1 : v2; } + unsigned long long int max( unsigned long long int v1, unsigned long long int v2 ) { return v1 > v2 ? v1 : v2; } forall( T | { int ?>?( T, T ); } ) - T max( T t1, T t2 ) { return t1 > t2 ? t1 : t2; } + T max( T v1, T v2 ) { return v1 > v2 ? v1 : v2; } forall( T | { T min( T, T ); T max( T, T ); } ) Index: libcfa/src/concurrency/ready_queue.cfa =================================================================== --- libcfa/src/concurrency/ready_queue.cfa (revision a5db488e6c0a055c012112dbeeb6ab737d0b9097) +++ libcfa/src/concurrency/ready_queue.cfa (revision 7f543563b892c550d18d3201d22a70056e4acb5e) @@ -398,13 +398,13 @@ if(proc->rdq.target == -1u) { - proc->rdq.target = __tls_rand() % lanes.count; - unsigned it1 = proc->rdq.itr; - unsigned it2 = proc->rdq.itr + 1; - unsigned idx1 = proc->rdq.id + (it1 % READYQ_SHARD_FACTOR); - unsigned idx2 = proc->rdq.id + (it2 % READYQ_SHARD_FACTOR); + _Static_assert(READYQ_SHARD_FACTOR == 2); + unsigned idx1 = proc->rdq.id + 0; + unsigned idx2 = proc->rdq.id + 1; unsigned long long tsc1 = ts(lanes.data[idx1]); unsigned long long tsc2 = ts(lanes.data[idx2]); - proc->rdq.cutoff = min(tsc1, tsc2); - if(proc->rdq.cutoff == 0) proc->rdq.cutoff = -1ull; + proc->rdq.target = __tls_rand() % lanes.count; + + // WARNING: std::min is polymorphic and therefore causes 500% slowdown instead of the expected 2% + proc->rdq.cutoff = tsc1 < tsc2 ? tsc1 : tsc2; } else { @@ -418,5 +418,5 @@ for(READYQ_SHARD_FACTOR) { - unsigned i = proc->rdq.id + (--proc->rdq.itr % READYQ_SHARD_FACTOR); + unsigned i = proc->rdq.id + (proc->rdq.itr++ % READYQ_SHARD_FACTOR); if($thread * t = try_pop(cltr, i __STATS(, __tls_stats()->ready.pop.local))) return t; } @@ -469,5 +469,6 @@ // Actually pop the list struct $thread * thrd; - thrd = pop(lane); + unsigned long long tsv; + [thrd, tsv] = pop(lane); /* paranoid */ verify(thrd); @@ -481,5 +482,5 @@ #if defined(USE_WORK_STEALING) - lanes.tscs[w].tv = thrd->link.ts; + lanes.tscs[w].tv = tsv; #endif @@ -663,5 +664,6 @@ while(!is_empty(lanes.data[idx])) { struct $thread * thrd; - thrd = pop(lanes.data[idx]); + unsigned long long _; + [thrd, _] = pop(lanes.data[idx]); push(cltr, thrd); Index: libcfa/src/concurrency/ready_subqueue.hfa =================================================================== --- libcfa/src/concurrency/ready_subqueue.hfa (revision a5db488e6c0a055c012112dbeeb6ab737d0b9097) +++ libcfa/src/concurrency/ready_subqueue.hfa (revision 7f543563b892c550d18d3201d22a70056e4acb5e) @@ -53,5 +53,5 @@ // Push a thread onto this lane // returns true of lane was empty before push, false otherwise -void push( __intrusive_lane_t & this, $thread * node ) { +static inline void push( __intrusive_lane_t & this, $thread * node ) { /* paranoid */ verify( node->link.next == 0p ); /* paranoid */ verify( node->link.ts == 0 ); @@ -77,9 +77,10 @@ // returns popped // returns true of lane was empty before push, false otherwise -$thread * pop( __intrusive_lane_t & this ) { +static inline [* $thread, unsigned long long] pop( __intrusive_lane_t & this ) { /* paranoid */ verify( this.anchor.next != 0p ); /* paranoid */ verify( this.anchor.ts != 0 ); // Get the relevant nodes locally + unsigned long long ts = this.anchor.ts; $thread * node = this.anchor.next; this.anchor.next = node->link.next; @@ -94,5 +95,5 @@ /* paranoid */ verify( node->link.next == 0p ); /* paranoid */ verify( node->link.ts == 0 ); - return node; + return [node, ts]; } Index: libcfa/src/heap.cfa =================================================================== --- libcfa/src/heap.cfa (revision a5db488e6c0a055c012112dbeeb6ab737d0b9097) +++ libcfa/src/heap.cfa (revision 7f543563b892c550d18d3201d22a70056e4acb5e) @@ -10,6 +10,6 @@ // Created On : Tue Dec 19 21:58:35 2017 // Last Modified By : Peter A. Buhr -// Last Modified On : Tue Apr 20 21:20:48 2021 -// Update Count : 1033 +// Last Modified On : Wed May 5 13:11:28 2021 +// Update Count : 1035 // @@ -28,5 +28,5 @@ #include "bits/locks.hfa" // __spinlock_t #include "startup.hfa" // STARTUP_PRIORITY_MEMORY -#include "math.hfa" // ceiling +#include "math.hfa" // min #include "bitmanip.hfa" // is_pow2, ceiling2