Changeset d2fadeb for libcfa/src/concurrency/ready_queue.cfa

Timestamp:

Apr 19, 2021, 5:07:16 PM (2 years ago)

Author:

Thierry Delisle <tdelisle@…>

Branches:

arm-eh, enum, forall-pointer-decay, jacob/cs343-translation, master, new-ast-unique-expr, pthread-emulation, qualifiedEnum

Children:

59f3f61, 665edf40, ddd473f

Parents:

4aa495f

Message:

Changed stats to make sense with relaxed fifo and work stealing

File:

• libcfa/src/concurrency/ready_queue.cfa (modified) (15 diffs)

libcfa/src/concurrency/ready_queue.cfa

@@ -35 +35 @@
 static const size_t cache_line_size = 64;
 
+#if !defined(__CFA_NO_STATISTICS__)
+        #define __STATS(...) __VA_ARGS__
+#else
+        #define __STATS(...)
+#endif
+
 // No overriden function, no environment variable, no define
 // fall back to a magic number
@@ -52 +58 @@
 #endif
 
+static inline struct $thread * try_pop(struct cluster * cltr, unsigned w __STATS(, __stats_readyQ_pop_t & stats));
+static inline struct $thread * try_pop(struct cluster * cltr, unsigned i, unsigned j __STATS(, __stats_readyQ_pop_t & stats));
+static inline struct $thread * search(struct cluster * cltr);
 static inline [unsigned, bool] idx_from_r(unsigned r, unsigned preferred);
-static inline struct $thread * try_pop(struct cluster * cltr, unsigned w);
-static inline struct $thread * try_pop(struct cluster * cltr, unsigned i, unsigned j);
-static inline struct $thread * search(struct cluster * cltr);
 
 
@@ -269 +275 @@
 
                         #if !defined(__CFA_NO_STATISTICS__)
-                                if(external) {
-                                        if(local) __atomic_fetch_add(&cltr->stats->ready.pick.ext.local, 1, __ATOMIC_RELAXED);
-                                        __atomic_fetch_add(&cltr->stats->ready.pick.ext.attempt, 1, __ATOMIC_RELAXED);
-                                }
-                                else {
-                                        if(local) __tls_stats()->ready.pick.push.local++;
-                                        __tls_stats()->ready.pick.push.attempt++;
-                                }
+                                if(unlikely(external)) __atomic_fetch_add(&cltr->stats->ready.push.extrn.attempt, 1, __ATOMIC_RELAXED);
+                                else if(local) __tls_stats()->ready.push.local.attempt++;
+                                else __tls_stats()->ready.push.share.attempt++;
                         #endif
 
@@ -302 +303 @@
                 // Update statistics
                 #if !defined(__CFA_NO_STATISTICS__)
-                        if(external) {
-                                if(local) __atomic_fetch_add(&cltr->stats->ready.pick.ext.lsuccess, 1, __ATOMIC_RELAXED);
-                                __atomic_fetch_add(&cltr->stats->ready.pick.ext.success, 1, __ATOMIC_RELAXED);
-                        }
-                        else {
-                                if(local) __tls_stats()->ready.pick.push.lsuccess++;
-                                __tls_stats()->ready.pick.push.success++;
-                        }
+                        if(unlikely(external)) __atomic_fetch_add(&cltr->stats->ready.push.extrn.success, 1, __ATOMIC_RELAXED);
+                        else if(local) __tls_stats()->ready.push.local.success++;
+                        else __tls_stats()->ready.push.share.success++;
                 #endif
         }
@@ -334 +330 @@
                         [j, localj] = idx_from_r(rj, preferred);
 
-                        #if !defined(__CFA_NO_STATISTICS__)
-                                if(locali && localj) {
-                                        __tls_stats()->ready.pick.pop.local++;
-                                }
-                        #endif
-
                         i %= count;
                         j %= count;
 
                         // try popping from the 2 picked lists
-                        struct $thread * thrd = try_pop(cltr, i, j);
+                        struct $thread * thrd = try_pop(cltr, i, j __STATS(, *(locali || localj ? &__tls_stats()->ready.pop.local : &__tls_stats()->ready.pop.help)));
                         if(thrd) {
-                                #if !defined(__CFA_NO_STATISTICS__)
-                                        if( locali || localj ) __tls_stats()->ready.pick.pop.lsuccess++;
-                                #endif
                                 return thrd;
                         }
@@ -374 +361 @@
                 unsigned i;
                 do {
+                        #if !defined(__CFA_NO_STATISTICS__)
+                                if(unlikely(external)) __atomic_fetch_add(&cltr->stats->ready.push.extrn.attempt, 1, __ATOMIC_RELAXED);
+                                else __tls_stats()->ready.push.local.attempt++;
+                        #endif
+
                         if(unlikely(external)) {
                                 i = __tls_rand() % lanes.count;
@@ -398 +390 @@
                         // Unlock and return
                         __atomic_unlock( &lanes.data[i].lock );
+                #endif
+
+                #if !defined(__CFA_NO_STATISTICS__)
+                        if(unlikely(external)) __atomic_fetch_add(&cltr->stats->ready.push.extrn.success, 1, __ATOMIC_RELAXED);
+                        else __tls_stats()->ready.push.local.success++;
                 #endif
 
@@ -422 +419 @@
                 }
                 else if(lanes.tscs[proc->rdq.target].tv < proc->rdq.cutoff) {
-                        $thread * t = try_pop(cltr, proc->rdq.target);
+                        $thread * t = try_pop(cltr, proc->rdq.target __STATS(, __tls_stats()->ready.pop.help));
                         proc->rdq.target = -1u;
                         if(t) return t;
@@ -429 +426 @@
                 for(READYQ_SHARD_FACTOR) {
                         unsigned i = proc->rdq.id + (--proc->rdq.itr % READYQ_SHARD_FACTOR);
-                        if($thread * t = try_pop(cltr, i)) return t;
+                        if($thread * t = try_pop(cltr, i __STATS(, __tls_stats()->ready.pop.local))) return t;
                 }
                 return 0p;
@@ -437 +434 @@
                 for(25) {
                         unsigned i = __tls_rand() % lanes.count;
-                        $thread * t = try_pop(cltr, i);
+                        $thread * t = try_pop(cltr, i __STATS(, __tls_stats()->ready.pop.steal));
                         if(t) return t;
                 }
@@ -453 +450 @@
 //-----------------------------------------------------------------------
 // try to pop from a lane given by index w
-static inline struct $thread * try_pop(struct cluster * cltr, unsigned w) with (cltr->ready_queue) {
+static inline struct $thread * try_pop(struct cluster * cltr, unsigned w __STATS(, __stats_readyQ_pop_t & stats)) with (cltr->ready_queue) {
+        __STATS( stats.attempt++; )
+
         // Get relevant elements locally
         __intrusive_lane_t & lane = lanes.data[w];
 
         // If list looks empty retry
-        if( is_empty(lane) ) return 0p;
+        if( is_empty(lane) ) {
+                __STATS( stats.espec++; )
+                return 0p;
+        }
 
         // If we can't get the lock retry
-        if( !__atomic_try_acquire(&lane.lock) ) return 0p;
+        if( !__atomic_try_acquire(&lane.lock) ) {
+                __STATS( stats.elock++; )
+                return 0p;
+        }
 
         // If list is empty, unlock and retry
         if( is_empty(lane) ) {
                 __atomic_unlock(&lane.lock);
+                __STATS( stats.eempty++; )
                 return 0p;
         }
@@ -480 +486 @@
 
         // Update statistics
-        #if !defined(__CFA_NO_STATISTICS__)
-                __tls_stats()->ready.pick.pop.success++;
-        #endif
+        __STATS( stats.success++; )
 
         #if defined(USE_WORK_STEALING)
@@ -501 +505 @@
         for(i; count) {
                 unsigned idx = (offset + i) % count;
-                struct $thread * thrd = try_pop(cltr, idx);
+                struct $thread * thrd = try_pop(cltr, idx __STATS(, __tls_stats()->ready.pop.search));
                 if(thrd) {
                         return thrd;
@@ -539 +543 @@
 //-----------------------------------------------------------------------
 // Given 2 indexes, pick the list with the oldest push an try to pop from it
-static inline struct $thread * try_pop(struct cluster * cltr, unsigned i, unsigned j) with (cltr->ready_queue) {
-        #if !defined(__CFA_NO_STATISTICS__)
-                __tls_stats()->ready.pick.pop.attempt++;
-        #endif
-
+static inline struct $thread * try_pop(struct cluster * cltr, unsigned i, unsigned j __STATS(, __stats_readyQ_pop_t & stats)) with (cltr->ready_queue) {
         // Pick the bet list
         int w = i;
@@ -550 +550 @@
         }
 
-        return try_pop(cltr, w);
+        return try_pop(cltr, w __STATS(, stats));
 }