- File:
-
- 1 edited
-
libcfa/src/concurrency/ready_subqueue.hfa (modified) (6 diffs)
Legend:
- Unmodified
- Added
- Removed
-
libcfa/src/concurrency/ready_subqueue.hfa
r2af1943 rb0be3713 6 6 7 7 // Intrusives lanes which are used by the relaxed ready queue 8 union __attribute__((aligned(64))) __intrusive_lane_t { 9 struct { 10 struct thread$ * prev; 8 struct __attribute__((aligned(128))) __intrusive_lane_t { 9 struct thread$ * prev; 11 10 12 13 11 // spin lock protecting the queue 12 volatile bool lock; 14 13 15 14 __thread_desc_link anchor; 16 15 17 #if !defined(__CFA_NO_STATISTICS__) 18 unsigned cnt; 19 #endif 20 } l; 21 char __padding[192]; 16 #if !defined(__CFA_NO_STATISTICS__) 17 unsigned cnt; 18 #endif 22 19 }; 23 20 … … 25 22 static inline thread$ * mock_head(const __intrusive_lane_t & this) { 26 23 thread$ * rhead = (thread$ *)( 27 (uintptr_t)( &this. l.anchor ) - __builtin_offsetof( thread$, link )24 (uintptr_t)( &this.anchor ) - __builtin_offsetof( thread$, link ) 28 25 ); 29 26 return rhead; … … 33 30 // returns true of lane was empty before push, false otherwise 34 31 static inline void push( __intrusive_lane_t & this, thread$ * node ) { 35 /* paranoid */ verify( this.l .lock );32 /* paranoid */ verify( this.lock ); 36 33 /* paranoid */ verify( node->link.next == 0p ); 37 34 /* paranoid */ verify( __atomic_load_n(&node->link.ts, __ATOMIC_RELAXED) == MAX ); 38 /* paranoid */ verify( this. l.prev->link.next == 0p );39 /* paranoid */ verify( __atomic_load_n(&this. l.prev->link.ts, __ATOMIC_RELAXED) == MAX );40 if( this. l.anchor.next == 0p ) {41 /* paranoid */ verify( this. l.anchor.next == 0p );42 /* paranoid */ verify( __atomic_load_n(&this. l.anchor.ts, __ATOMIC_RELAXED) == MAX );43 /* paranoid */ verify( __atomic_load_n(&this. l.anchor.ts, __ATOMIC_RELAXED) != 0 );44 /* paranoid */ verify( this. l.prev == mock_head( this ) );35 /* paranoid */ verify( this.prev->link.next == 0p ); 36 /* paranoid */ verify( __atomic_load_n(&this.prev->link.ts, __ATOMIC_RELAXED) == MAX ); 37 if( this.anchor.next == 0p ) { 38 /* paranoid */ verify( this.anchor.next == 0p ); 39 /* paranoid */ verify( __atomic_load_n(&this.anchor.ts, __ATOMIC_RELAXED) == MAX ); 40 /* paranoid */ verify( __atomic_load_n(&this.anchor.ts, __ATOMIC_RELAXED) != 0 ); 41 /* paranoid */ verify( this.prev == mock_head( this ) ); 45 42 } else { 46 /* paranoid */ verify( this. l.anchor.next != 0p );47 /* paranoid */ verify( __atomic_load_n(&this. l.anchor.ts, __ATOMIC_RELAXED) != MAX );48 /* paranoid */ verify( __atomic_load_n(&this. l.anchor.ts, __ATOMIC_RELAXED) != 0 );49 /* paranoid */ verify( this. l.prev != mock_head( this ) );43 /* paranoid */ verify( this.anchor.next != 0p ); 44 /* paranoid */ verify( __atomic_load_n(&this.anchor.ts, __ATOMIC_RELAXED) != MAX ); 45 /* paranoid */ verify( __atomic_load_n(&this.anchor.ts, __ATOMIC_RELAXED) != 0 ); 46 /* paranoid */ verify( this.prev != mock_head( this ) ); 50 47 } 51 48 52 49 // Get the relevant nodes locally 53 this. l.prev->link.next = node;54 __atomic_store_n(&this. l.prev->link.ts, rdtscl(), __ATOMIC_RELAXED);55 this. l.prev = node;50 this.prev->link.next = node; 51 __atomic_store_n(&this.prev->link.ts, rdtscl(), __ATOMIC_RELAXED); 52 this.prev = node; 56 53 #if !defined(__CFA_NO_STATISTICS__) 57 this. l.cnt++;54 this.cnt++; 58 55 #endif 59 56 } … … 63 60 // returns true of lane was empty before push, false otherwise 64 61 static inline [* thread$, unsigned long long] pop( __intrusive_lane_t & this ) { 65 /* paranoid */ verify( this.l .lock );66 /* paranoid */ verify( this. l.anchor.next != 0p );67 /* paranoid */ verify( __atomic_load_n(&this. l.anchor.ts, __ATOMIC_RELAXED) != MAX );68 /* paranoid */ verify( __atomic_load_n(&this. l.anchor.ts, __ATOMIC_RELAXED) != 0 );62 /* paranoid */ verify( this.lock ); 63 /* paranoid */ verify( this.anchor.next != 0p ); 64 /* paranoid */ verify( __atomic_load_n(&this.anchor.ts, __ATOMIC_RELAXED) != MAX ); 65 /* paranoid */ verify( __atomic_load_n(&this.anchor.ts, __ATOMIC_RELAXED) != 0 ); 69 66 70 67 // Get the relevant nodes locally 71 thread$ * node = this. l.anchor.next;72 this. l.anchor.next = node->link.next;73 __atomic_store_n(&this. l.anchor.ts, __atomic_load_n(&node->link.ts, __ATOMIC_RELAXED), __ATOMIC_RELAXED);74 bool is_empty = this. l.anchor.next == 0p;68 thread$ * node = this.anchor.next; 69 this.anchor.next = node->link.next; 70 __atomic_store_n(&this.anchor.ts, __atomic_load_n(&node->link.ts, __ATOMIC_RELAXED), __ATOMIC_RELAXED); 71 bool is_empty = this.anchor.next == 0p; 75 72 node->link.next = 0p; 76 73 __atomic_store_n(&node->link.ts, ULLONG_MAX, __ATOMIC_RELAXED); 77 74 #if !defined(__CFA_NO_STATISTICS__) 78 this. l.cnt--;75 this.cnt--; 79 76 #endif 80 77 81 78 // Update head time stamp 82 if(is_empty) this. l.prev = mock_head( this );79 if(is_empty) this.prev = mock_head( this ); 83 80 84 unsigned long long ats = __atomic_load_n(&this. l.anchor.ts, __ATOMIC_RELAXED);81 unsigned long long ats = __atomic_load_n(&this.anchor.ts, __ATOMIC_RELAXED); 85 82 /* paranoid */ verify( node->link.next == 0p ); 86 83 /* paranoid */ verify( __atomic_load_n(&node->link.ts , __ATOMIC_RELAXED) == MAX ); … … 93 90 // Check whether or not list is empty 94 91 static inline bool is_empty(__intrusive_lane_t & this) { 95 return this. l.anchor.next == 0p;92 return this.anchor.next == 0p; 96 93 } 97 94 … … 99 96 static inline unsigned long long ts(__intrusive_lane_t & this) { 100 97 // Cannot verify 'emptiness' here since it may not be locked 101 /* paranoid */ verify(this. l.anchor.ts != 0);102 /* paranoid */ static_assert(__atomic_always_lock_free(sizeof(this. l.anchor.ts), &this.l.anchor.ts));103 return __atomic_load_n(&this. l.anchor.ts, __ATOMIC_RELAXED);98 /* paranoid */ verify(this.anchor.ts != 0); 99 /* paranoid */ static_assert(__atomic_always_lock_free(sizeof(this.anchor.ts), &this.anchor.ts)); 100 return __atomic_load_n(&this.anchor.ts, __ATOMIC_RELAXED); 104 101 }
Note:
See TracChangeset
for help on using the changeset viewer.
