Changes in libcfa/src/concurrency/ready_queue.cfa [d3ba775:c993b15]

File:

• libcfa/src/concurrency/ready_queue.cfa (modified) (9 diffs)

libcfa/src/concurrency/ready_queue.cfa

@@ -17 +17 @@
 // #define __CFA_DEBUG_PRINT_READY_QUEUE__
 
+// #define USE_MPSC
 
 #define USE_RELAXED_FIFO
@@ -255 +256 @@
                 /* paranoid */ verify(external || kernelTLS().this_processor->rdq.id < lanes.count );
 
+                // write timestamp
+                thrd->link.ts = rdtscl();
+
                 bool local;
                 int preferred = external ? -1 : kernelTLS().this_processor->rdq.id;
@@ -273 +277 @@
                         #endif
 
+                #if defined(USE_MPSC)
+                        // mpsc always succeeds
+                } while( false );
+                #else
                         // If we can't lock it retry
                 } while( !__atomic_try_acquire( &lanes.data[i].lock ) );
+                #endif
 
                 // Actually push it
                 push(lanes.data[i], thrd);
 
+                #if !defined(USE_MPSC)
                         // Unlock and return
                         __atomic_unlock( &lanes.data[i].lock );
+                #endif
 
                 // Mark the current index in the tls rng instance as having an item
@@ -339 +350 @@
                 __cfadbg_print_safe(ready_queue, "Kernel : Pushing %p on cluster %p\n", thrd, cltr);
 
-                // #define USE_PREFERRED
-                #if !defined(USE_PREFERRED)
                 const bool external = (!kernelTLS().this_processor) || (cltr != kernelTLS().this_processor->cltr);
                 /* paranoid */ verify(external || kernelTLS().this_processor->rdq.id < lanes.count );
-                #else
-                        unsigned preferred = thrd->preferred;
-                        const bool external = (!kernelTLS().this_processor) || preferred == -1u || thrd->curr_cluster != cltr;
-                        /* paranoid */ verifyf(external || preferred < lanes.count, "Invalid preferred queue %u for %u lanes", preferred, lanes.count );
-
-                        unsigned r = preferred % READYQ_SHARD_FACTOR;
-                        const unsigned start = preferred - r;
-                #endif
+
+                // write timestamp
+                thrd->link.ts = rdtscl();
 
                 // Try to pick a lane and lock it
@@ -364 +368 @@
                         }
                         else {
-                                #if !defined(USE_PREFERRED)
                                 processor * proc = kernelTLS().this_processor;
                                 unsigned r = proc->rdq.its++;
                                 i =  proc->rdq.id + (r % READYQ_SHARD_FACTOR);
+                        }
+
+
+                #if defined(USE_MPSC)
+                        // mpsc always succeeds
+                } while( false );
                 #else
-                                        i = start + (r++ % READYQ_SHARD_FACTOR);
-                                #endif
-                        }
                         // If we can't lock it retry
                 } while( !__atomic_try_acquire( &lanes.data[i].lock ) );
+                #endif
 
                 // Actually push it
                 push(lanes.data[i], thrd);
 
+                #if !defined(USE_MPSC)
                         // Unlock and return
                         __atomic_unlock( &lanes.data[i].lock );
+                #endif
 
                 #if !defined(__CFA_NO_STATISTICS__)
@@ -483 +492 @@
                 lanes.tscs[w].tv = thrd->link.ts;
         #endif
-
-        thrd->preferred = w;
 
         // return the popped thread
@@ -512 +519 @@
 // Check that all the intrusive queues in the data structure are still consistent
 static void check( __ready_queue_t & q ) with (q) {
-        #if defined(__CFA_WITH_VERIFY__)
+        #if defined(__CFA_WITH_VERIFY__) && !defined(USE_MPSC)
                 {
                         for( idx ; lanes.count ) {
@@ -518 +525 @@
                                 assert(!lanes.data[idx].lock);
 
-                                        if(is_empty(sl)) {
-                                                assert( sl.anchor.next == 0p );
-                                                assert( sl.anchor.ts   == 0  );
-                                                assert( mock_head(sl)  == sl.prev );
-                                        } else {
-                                                assert( sl.anchor.next != 0p );
-                                                assert( sl.anchor.ts   != 0  );
-                                                assert( mock_head(sl)  != sl.prev );
-                                        }
+                                assert(head(sl)->link.prev == 0p );
+                                assert(head(sl)->link.next->link.prev == head(sl) );
+                                assert(tail(sl)->link.next == 0p );
+                                assert(tail(sl)->link.prev->link.next == tail(sl) );
+
+                                if(is_empty(sl)) {
+                                        assert(tail(sl)->link.prev == head(sl));
+                                        assert(head(sl)->link.next == tail(sl));
+                                } else {
+                                        assert(tail(sl)->link.prev != head(sl));
+                                        assert(head(sl)->link.next != tail(sl));
+                                }
                         }
                 }
@@ -547 +557 @@
 // fixes the list so that the pointers back to anchors aren't left dangling
 static inline void fix(__intrusive_lane_t & ll) {
-                        if(is_empty(ll)) {
-                                verify(ll.anchor.next == 0p);
-                                ll.prev = mock_head(ll);
-                        }
+        #if !defined(USE_MPSC)
+                // if the list is not empty then follow he pointer and fix its reverse
+                if(!is_empty(ll)) {
+                        head(ll)->link.next->link.prev = head(ll);
+                        tail(ll)->link.prev->link.next = tail(ll);
+                }
+                // Otherwise just reset the list
+                else {
+                        verify(tail(ll)->link.next == 0p);
+                        tail(ll)->link.prev = head(ll);
+                        head(ll)->link.next = tail(ll);
+                        verify(head(ll)->link.prev == 0p);
+                }
+        #endif
 }