Changes in libcfa/src/concurrency/ready_queue.cfa [d2fadeb:5f6a172]

File:

• libcfa/src/concurrency/ready_queue.cfa (modified) (15 diffs)

libcfa/src/concurrency/ready_queue.cfa

@@ -35 +35 @@
 static const size_t cache_line_size = 64;
 
-#if !defined(__CFA_NO_STATISTICS__)
-        #define __STATS(...) __VA_ARGS__
-#else
-        #define __STATS(...)
-#endif
-
 // No overriden function, no environment variable, no define
 // fall back to a magic number
@@ -58 +52 @@
 #endif
 
-static inline struct $thread * try_pop(struct cluster * cltr, unsigned w __STATS(, __stats_readyQ_pop_t & stats));
-static inline struct $thread * try_pop(struct cluster * cltr, unsigned i, unsigned j __STATS(, __stats_readyQ_pop_t & stats));
+static inline [unsigned, bool] idx_from_r(unsigned r, unsigned preferred);
+static inline struct $thread * try_pop(struct cluster * cltr, unsigned w);
+static inline struct $thread * try_pop(struct cluster * cltr, unsigned i, unsigned j);
 static inline struct $thread * search(struct cluster * cltr);
-static inline [unsigned, bool] idx_from_r(unsigned r, unsigned preferred);
 
 
@@ -275 +269 @@
 
                         #if !defined(__CFA_NO_STATISTICS__)
-                                if(unlikely(external)) __atomic_fetch_add(&cltr->stats->ready.push.extrn.attempt, 1, __ATOMIC_RELAXED);
-                                else if(local) __tls_stats()->ready.push.local.attempt++;
-                                else __tls_stats()->ready.push.share.attempt++;
+                                if(external) {
+                                        if(local) __atomic_fetch_add(&cltr->stats->ready.pick.ext.local, 1, __ATOMIC_RELAXED);
+                                        __atomic_fetch_add(&cltr->stats->ready.pick.ext.attempt, 1, __ATOMIC_RELAXED);
+                                }
+                                else {
+                                        if(local) __tls_stats()->ready.pick.push.local++;
+                                        __tls_stats()->ready.pick.push.attempt++;
+                                }
                         #endif
 
@@ -303 +302 @@
                 // Update statistics
                 #if !defined(__CFA_NO_STATISTICS__)
-                        if(unlikely(external)) __atomic_fetch_add(&cltr->stats->ready.push.extrn.success, 1, __ATOMIC_RELAXED);
-                        else if(local) __tls_stats()->ready.push.local.success++;
-                        else __tls_stats()->ready.push.share.success++;
+                        if(external) {
+                                if(local) __atomic_fetch_add(&cltr->stats->ready.pick.ext.lsuccess, 1, __ATOMIC_RELAXED);
+                                __atomic_fetch_add(&cltr->stats->ready.pick.ext.success, 1, __ATOMIC_RELAXED);
+                        }
+                        else {
+                                if(local) __tls_stats()->ready.pick.push.lsuccess++;
+                                __tls_stats()->ready.pick.push.success++;
+                        }
                 #endif
         }
@@ -330 +334 @@
                         [j, localj] = idx_from_r(rj, preferred);
 
+                        #if !defined(__CFA_NO_STATISTICS__)
+                                if(locali && localj) {
+                                        __tls_stats()->ready.pick.pop.local++;
+                                }
+                        #endif
+
                         i %= count;
                         j %= count;
 
                         // try popping from the 2 picked lists
-                        struct $thread * thrd = try_pop(cltr, i, j __STATS(, *(locali || localj ? &__tls_stats()->ready.pop.local : &__tls_stats()->ready.pop.help)));
+                        struct $thread * thrd = try_pop(cltr, i, j);
                         if(thrd) {
+                                #if !defined(__CFA_NO_STATISTICS__)
+                                        if( locali || localj ) __tls_stats()->ready.pick.pop.lsuccess++;
+                                #endif
                                 return thrd;
                         }
@@ -361 +374 @@
                 unsigned i;
                 do {
-                        #if !defined(__CFA_NO_STATISTICS__)
-                                if(unlikely(external)) __atomic_fetch_add(&cltr->stats->ready.push.extrn.attempt, 1, __ATOMIC_RELAXED);
-                                else __tls_stats()->ready.push.local.attempt++;
-                        #endif
-
                         if(unlikely(external)) {
                                 i = __tls_rand() % lanes.count;
@@ -390 +398 @@
                         // Unlock and return
                         __atomic_unlock( &lanes.data[i].lock );
-                #endif
-
-                #if !defined(__CFA_NO_STATISTICS__)
-                        if(unlikely(external)) __atomic_fetch_add(&cltr->stats->ready.push.extrn.success, 1, __ATOMIC_RELAXED);
-                        else __tls_stats()->ready.push.local.success++;
                 #endif
 
@@ -419 +422 @@
                 }
                 else if(lanes.tscs[proc->rdq.target].tv < proc->rdq.cutoff) {
-                        $thread * t = try_pop(cltr, proc->rdq.target __STATS(, __tls_stats()->ready.pop.help));
+                        $thread * t = try_pop(cltr, proc->rdq.target);
                         proc->rdq.target = -1u;
                         if(t) return t;
@@ -426 +429 @@
                 for(READYQ_SHARD_FACTOR) {
                         unsigned i = proc->rdq.id + (--proc->rdq.itr % READYQ_SHARD_FACTOR);
-                        if($thread * t = try_pop(cltr, i __STATS(, __tls_stats()->ready.pop.local))) return t;
+                        if($thread * t = try_pop(cltr, i)) return t;
                 }
                 return 0p;
@@ -434 +437 @@
                 for(25) {
                         unsigned i = __tls_rand() % lanes.count;
-                        $thread * t = try_pop(cltr, i __STATS(, __tls_stats()->ready.pop.steal));
+                        $thread * t = try_pop(cltr, i);
                         if(t) return t;
                 }
@@ -450 +453 @@
 //-----------------------------------------------------------------------
 // try to pop from a lane given by index w
-static inline struct $thread * try_pop(struct cluster * cltr, unsigned w __STATS(, __stats_readyQ_pop_t & stats)) with (cltr->ready_queue) {
-        __STATS( stats.attempt++; )
-
+static inline struct $thread * try_pop(struct cluster * cltr, unsigned w) with (cltr->ready_queue) {
         // Get relevant elements locally
         __intrusive_lane_t & lane = lanes.data[w];
 
         // If list looks empty retry
-        if( is_empty(lane) ) {
-                __STATS( stats.espec++; )
-                return 0p;
-        }
+        if( is_empty(lane) ) return 0p;
 
         // If we can't get the lock retry
-        if( !__atomic_try_acquire(&lane.lock) ) {
-                __STATS( stats.elock++; )
-                return 0p;
-        }
+        if( !__atomic_try_acquire(&lane.lock) ) return 0p;
 
         // If list is empty, unlock and retry
         if( is_empty(lane) ) {
                 __atomic_unlock(&lane.lock);
-                __STATS( stats.eempty++; )
                 return 0p;
         }
@@ -486 +480 @@
 
         // Update statistics
-        __STATS( stats.success++; )
+        #if !defined(__CFA_NO_STATISTICS__)
+                __tls_stats()->ready.pick.pop.success++;
+        #endif
 
         #if defined(USE_WORK_STEALING)
@@ -505 +501 @@
         for(i; count) {
                 unsigned idx = (offset + i) % count;
-                struct $thread * thrd = try_pop(cltr, idx __STATS(, __tls_stats()->ready.pop.search));
+                struct $thread * thrd = try_pop(cltr, idx);
                 if(thrd) {
                         return thrd;
@@ -543 +539 @@
 //-----------------------------------------------------------------------
 // Given 2 indexes, pick the list with the oldest push an try to pop from it
-static inline struct $thread * try_pop(struct cluster * cltr, unsigned i, unsigned j __STATS(, __stats_readyQ_pop_t & stats)) with (cltr->ready_queue) {
+static inline struct $thread * try_pop(struct cluster * cltr, unsigned i, unsigned j) with (cltr->ready_queue) {
+        #if !defined(__CFA_NO_STATISTICS__)
+                __tls_stats()->ready.pick.pop.attempt++;
+        #endif
+
         // Pick the bet list
         int w = i;
@@ -550 +550 @@
         }
 
-        return try_pop(cltr, w __STATS(, stats));
+        return try_pop(cltr, w);
 }