Changes in libcfa/src/concurrency/ready_queue.cfa [c993b15:d3ba775]

File:

• libcfa/src/concurrency/ready_queue.cfa (modified) (9 diffs)

libcfa/src/concurrency/ready_queue.cfa

@@ -17 +17 @@
 // #define __CFA_DEBUG_PRINT_READY_QUEUE__
 
-// #define USE_MPSC
 
 #define USE_RELAXED_FIFO
@@ -256 +255 @@
                 /* paranoid */ verify(external || kernelTLS().this_processor->rdq.id < lanes.count );
 
-                // write timestamp
-                thrd->link.ts = rdtscl();
-
                 bool local;
                 int preferred = external ? -1 : kernelTLS().this_processor->rdq.id;
@@ -277 +273 @@
                         #endif
 
-                #if defined(USE_MPSC)
-                        // mpsc always succeeds
-                } while( false );
-                #else
                         // If we can't lock it retry
                 } while( !__atomic_try_acquire( &lanes.data[i].lock ) );
-                #endif
 
                 // Actually push it
                 push(lanes.data[i], thrd);
 
-                #if !defined(USE_MPSC)
                         // Unlock and return
                         __atomic_unlock( &lanes.data[i].lock );
-                #endif
 
                 // Mark the current index in the tls rng instance as having an item
@@ -350 +339 @@
                 __cfadbg_print_safe(ready_queue, "Kernel : Pushing %p on cluster %p\n", thrd, cltr);
 
+                // #define USE_PREFERRED
+                #if !defined(USE_PREFERRED)
                 const bool external = (!kernelTLS().this_processor) || (cltr != kernelTLS().this_processor->cltr);
                 /* paranoid */ verify(external || kernelTLS().this_processor->rdq.id < lanes.count );
-
-                // write timestamp
-                thrd->link.ts = rdtscl();
+                #else
+                        unsigned preferred = thrd->preferred;
+                        const bool external = (!kernelTLS().this_processor) || preferred == -1u || thrd->curr_cluster != cltr;
+                        /* paranoid */ verifyf(external || preferred < lanes.count, "Invalid preferred queue %u for %u lanes", preferred, lanes.count );
+
+                        unsigned r = preferred % READYQ_SHARD_FACTOR;
+                        const unsigned start = preferred - r;
+                #endif
 
                 // Try to pick a lane and lock it
@@ -368 +364 @@
                         }
                         else {
+                                #if !defined(USE_PREFERRED)
                                 processor * proc = kernelTLS().this_processor;
                                 unsigned r = proc->rdq.its++;
                                 i =  proc->rdq.id + (r % READYQ_SHARD_FACTOR);
+                #else
+                                        i = start + (r++ % READYQ_SHARD_FACTOR);
+                                #endif
                         }
-
-
-                #if defined(USE_MPSC)
-                        // mpsc always succeeds
-                } while( false );
-                #else
                         // If we can't lock it retry
                 } while( !__atomic_try_acquire( &lanes.data[i].lock ) );
-                #endif
 
                 // Actually push it
                 push(lanes.data[i], thrd);
 
-                #if !defined(USE_MPSC)
                         // Unlock and return
                         __atomic_unlock( &lanes.data[i].lock );
-                #endif
 
                 #if !defined(__CFA_NO_STATISTICS__)
@@ -492 +483 @@
                 lanes.tscs[w].tv = thrd->link.ts;
         #endif
+
+        thrd->preferred = w;
 
         // return the popped thread
@@ -519 +512 @@
 // Check that all the intrusive queues in the data structure are still consistent
 static void check( __ready_queue_t & q ) with (q) {
-        #if defined(__CFA_WITH_VERIFY__) && !defined(USE_MPSC)
+        #if defined(__CFA_WITH_VERIFY__)
                 {
                         for( idx ; lanes.count ) {
@@ -525 +518 @@
                                 assert(!lanes.data[idx].lock);
 
-                                assert(head(sl)->link.prev == 0p );
-                                assert(head(sl)->link.next->link.prev == head(sl) );
-                                assert(tail(sl)->link.next == 0p );
-                                assert(tail(sl)->link.prev->link.next == tail(sl) );
-
-                                if(is_empty(sl)) {
-                                        assert(tail(sl)->link.prev == head(sl));
-                                        assert(head(sl)->link.next == tail(sl));
-                                } else {
-                                        assert(tail(sl)->link.prev != head(sl));
-                                        assert(head(sl)->link.next != tail(sl));
-                                }
+                                        if(is_empty(sl)) {
+                                                assert( sl.anchor.next == 0p );
+                                                assert( sl.anchor.ts   == 0  );
+                                                assert( mock_head(sl)  == sl.prev );
+                                        } else {
+                                                assert( sl.anchor.next != 0p );
+                                                assert( sl.anchor.ts   != 0  );
+                                                assert( mock_head(sl)  != sl.prev );
+                                        }
                         }
                 }
@@ -557 +547 @@
 // fixes the list so that the pointers back to anchors aren't left dangling
 static inline void fix(__intrusive_lane_t & ll) {
-        #if !defined(USE_MPSC)
-                // if the list is not empty then follow he pointer and fix its reverse
-                if(!is_empty(ll)) {
-                        head(ll)->link.next->link.prev = head(ll);
-                        tail(ll)->link.prev->link.next = tail(ll);
-                }
-                // Otherwise just reset the list
-                else {
-                        verify(tail(ll)->link.next == 0p);
-                        tail(ll)->link.prev = head(ll);
-                        head(ll)->link.next = tail(ll);
-                        verify(head(ll)->link.prev == 0p);
-                }
-        #endif
+                        if(is_empty(ll)) {
+                                verify(ll.anchor.next == 0p);
+                                ll.prev = mock_head(ll);
+                        }
 }