Changes in libcfa/src/concurrency/ready_queue.cfa [39fc03e:1eb239e4]

File:

• libcfa/src/concurrency/ready_queue.cfa (modified) (14 diffs)

libcfa/src/concurrency/ready_queue.cfa

@@ -17 +17 @@
 // #define __CFA_DEBUG_PRINT_READY_QUEUE__
 
+// #define USE_SNZI
+
 #include "bits/defs.hfa"
 #include "kernel_private.hfa"
@@ -192 +194 @@
 void ^?{}(__ready_queue_t & this) with (this) {
         verify( 1 == lanes.count );
-        verify( !query( snzi ) );
+        #ifdef USE_SNZI
+                verify( !query( snzi ) );
+        #endif
         free(lanes.data);
 }
@@ -198 +202 @@
 //-----------------------------------------------------------------------
 __attribute__((hot)) bool query(struct cluster * cltr) {
-        return query(cltr->ready_queue.snzi);
+        #ifdef USE_SNZI
+                return query(cltr->ready_queue.snzi);
+        #endif
+        return true;
 }
 
@@ -262 +269 @@
         bool lane_first = push(lanes.data[i], thrd);
 
-        // If this lane used to be empty we need to do more
-        if(lane_first) {
-                // Check if the entire queue used to be empty
-                first = !query(snzi);
-
-                // Update the snzi
-                arrive( snzi, i );
-        }
+        #ifdef USE_SNZI
+                // If this lane used to be empty we need to do more
+                if(lane_first) {
+                        // Check if the entire queue used to be empty
+                        first = !query(snzi);
+
+                        // Update the snzi
+                        arrive( snzi, i );
+                }
+        #endif
 
         // Unlock and return
@@ -294 +303 @@
 __attribute__((hot)) $thread * pop(struct cluster * cltr) with (cltr->ready_queue) {
         /* paranoid */ verify( lanes.count > 0 );
+        unsigned count = __atomic_load_n( &lanes.count, __ATOMIC_RELAXED );
         #if defined(BIAS)
                 // Don't bother trying locally too much
@@ -300 +310 @@
 
         // As long as the list is not empty, try finding a lane that isn't empty and pop from it
-        while( query(snzi) ) {
+        #ifdef USE_SNZI
+                while( query(snzi) ) {
+        #else
+                for(25) {
+        #endif
                 // Pick two lists at random
                 unsigned i,j;
@@ -336 +350 @@
                 #endif
 
-                i %= __atomic_load_n( &lanes.count, __ATOMIC_RELAXED );
-                j %= __atomic_load_n( &lanes.count, __ATOMIC_RELAXED );
+                i %= count;
+                j %= count;
 
                 // try popping from the 2 picked lists
@@ -353 +367 @@
 }
 
+__attribute__((hot)) struct $thread * pop_slow(struct cluster * cltr) with (cltr->ready_queue) {
+        /* paranoid */ verify( lanes.count > 0 );
+        unsigned count = __atomic_load_n( &lanes.count, __ATOMIC_RELAXED );
+        unsigned offset = __tls_rand();
+        for(i; count) {
+                unsigned idx = (offset + i) % count;
+                struct $thread * thrd = try_pop(cltr, idx);
+                if(thrd) {
+                        return thrd;
+                }
+        }
+
+        // All lanes where empty return 0p
+        return 0p;
+}
+
+
 //-----------------------------------------------------------------------
 // Given 2 indexes, pick the list with the oldest push an try to pop from it
@@ -394 +425 @@
         /* paranoid */ verify(lane.lock);
 
-        // If this was the last element in the lane
-        if(emptied) {
-                depart( snzi, w );
-        }
+        #ifdef USE_SNZI
+                // If this was the last element in the lane
+                if(emptied) {
+                        depart( snzi, w );
+                }
+        #endif
 
         // Unlock and return
@@ -430 +463 @@
 
                                 removed = true;
-                                if(emptied) {
-                                        depart( snzi, i );
-                                }
+                                #ifdef USE_SNZI
+                                        if(emptied) {
+                                                depart( snzi, i );
+                                        }
+                                #endif
                         }
                 __atomic_unlock(&lane.lock);
@@ -494 +529 @@
         // grow the ready queue
         with( cltr->ready_queue ) {
-                ^(snzi){};
+                #ifdef USE_SNZI
+                        ^(snzi){};
+                #endif
 
                 // Find new count
@@ -516 +553 @@
                 lanes.count = ncount;
 
-                // Re-create the snzi
-                snzi{ log2( lanes.count / 8 ) };
-                for( idx; (size_t)lanes.count ) {
-                        if( !is_empty(lanes.data[idx]) ) {
-                                arrive(snzi, idx);
-                        }
-                }
+                #ifdef USE_SNZI
+                        // Re-create the snzi
+                        snzi{ log2( lanes.count / 8 ) };
+                        for( idx; (size_t)lanes.count ) {
+                                if( !is_empty(lanes.data[idx]) ) {
+                                        arrive(snzi, idx);
+                                }
+                        }
+                #endif
         }
 
@@ -542 +581 @@
 
         with( cltr->ready_queue ) {
-                ^(snzi){};
+                #ifdef USE_SNZI
+                        ^(snzi){};
+                #endif
 
                 // Remember old count
@@ -596 +637 @@
                 }
 
-                // Re-create the snzi
-                snzi{ log2( lanes.count / 8 ) };
-                for( idx; (size_t)lanes.count ) {
-                        if( !is_empty(lanes.data[idx]) ) {
-                                arrive(snzi, idx);
-                        }
-                }
+                #ifdef USE_SNZI
+                        // Re-create the snzi
+                        snzi{ log2( lanes.count / 8 ) };
+                        for( idx; (size_t)lanes.count ) {
+                                if( !is_empty(lanes.data[idx]) ) {
+                                        arrive(snzi, idx);
+                                }
+                        }
+                #endif
         }