source: libcfa/src/concurrency/ready_subqueue.hfa @ b0be3713

pthread-emulationqualifiedEnum
Last change on this file since b0be3713 was b0be3713, checked in by Thierry Delisle <tdelisle@…>, 5 months ago

Missed an atomic store on the TS handling.

  • Property mode set to 100644
File size: 3.7 KB
Line 
1#pragma once
2
3#define __CFA_NO_SCHED_STATS__
4
5#include "limits.hfa"
6
7// Intrusives lanes which are used by the relaxed ready queue
8struct __attribute__((aligned(128))) __intrusive_lane_t {
9        struct thread$ * prev;
10
11        // spin lock protecting the queue
12        volatile bool lock;
13
14        __thread_desc_link anchor;
15
16        #if !defined(__CFA_NO_STATISTICS__)
17                unsigned cnt;
18        #endif
19};
20
21// Get the head pointer (one before the first element) from the anchor
22static inline thread$ * mock_head(const __intrusive_lane_t & this) {
23        thread$ * rhead = (thread$ *)(
24                (uintptr_t)( &this.anchor ) - __builtin_offsetof( thread$, link )
25        );
26        return rhead;
27}
28
29// Push a thread onto this lane
30// returns true of lane was empty before push, false otherwise
31static inline void push( __intrusive_lane_t & this, thread$ * node ) {
32        /* paranoid */ verify( this.lock );
33        /* paranoid */ verify( node->link.next == 0p );
34        /* paranoid */ verify( __atomic_load_n(&node->link.ts, __ATOMIC_RELAXED) == MAX  );
35        /* paranoid */ verify( this.prev->link.next == 0p );
36        /* paranoid */ verify( __atomic_load_n(&this.prev->link.ts, __ATOMIC_RELAXED)   == MAX  );
37        if( this.anchor.next == 0p ) {
38                /* paranoid */ verify( this.anchor.next == 0p );
39                /* paranoid */ verify( __atomic_load_n(&this.anchor.ts, __ATOMIC_RELAXED) == MAX );
40                /* paranoid */ verify( __atomic_load_n(&this.anchor.ts, __ATOMIC_RELAXED) != 0  );
41                /* paranoid */ verify( this.prev == mock_head( this ) );
42        } else {
43                /* paranoid */ verify( this.anchor.next != 0p );
44                /* paranoid */ verify( __atomic_load_n(&this.anchor.ts, __ATOMIC_RELAXED) != MAX );
45                /* paranoid */ verify( __atomic_load_n(&this.anchor.ts, __ATOMIC_RELAXED) != 0  );
46                /* paranoid */ verify( this.prev != mock_head( this ) );
47        }
48
49        // Get the relevant nodes locally
50        this.prev->link.next = node;
51        __atomic_store_n(&this.prev->link.ts, rdtscl(), __ATOMIC_RELAXED);
52        this.prev = node;
53        #if !defined(__CFA_NO_STATISTICS__)
54                this.cnt++;
55        #endif
56}
57
58// Pop a thread from this lane (must be non-empty)
59// returns popped
60// returns true of lane was empty before push, false otherwise
61static inline [* thread$, unsigned long long] pop( __intrusive_lane_t & this ) {
62        /* paranoid */ verify( this.lock );
63        /* paranoid */ verify( this.anchor.next != 0p );
64        /* paranoid */ verify( __atomic_load_n(&this.anchor.ts, __ATOMIC_RELAXED) != MAX );
65        /* paranoid */ verify( __atomic_load_n(&this.anchor.ts, __ATOMIC_RELAXED) != 0   );
66
67        // Get the relevant nodes locally
68        thread$ * node = this.anchor.next;
69        this.anchor.next = node->link.next;
70        __atomic_store_n(&this.anchor.ts, __atomic_load_n(&node->link.ts, __ATOMIC_RELAXED), __ATOMIC_RELAXED);
71        bool is_empty = this.anchor.next == 0p;
72        node->link.next = 0p;
73        __atomic_store_n(&node->link.ts, ULLONG_MAX, __ATOMIC_RELAXED);
74        #if !defined(__CFA_NO_STATISTICS__)
75                this.cnt--;
76        #endif
77
78        // Update head time stamp
79        if(is_empty) this.prev = mock_head( this );
80
81        unsigned long long ats = __atomic_load_n(&this.anchor.ts, __ATOMIC_RELAXED);
82        /* paranoid */ verify( node->link.next == 0p );
83        /* paranoid */ verify( __atomic_load_n(&node->link.ts , __ATOMIC_RELAXED) == MAX );
84        /* paranoid */ verify( __atomic_load_n(&node->link.ts , __ATOMIC_RELAXED) != 0   );
85        /* paranoid */ verify( ats != 0 );
86        /* paranoid */ verify( (ats == MAX) == is_empty );
87        return [node, ats];
88}
89
90// Check whether or not list is empty
91static inline bool is_empty(__intrusive_lane_t & this) {
92        return this.anchor.next == 0p;
93}
94
95// Return the timestamp
96static inline unsigned long long ts(__intrusive_lane_t & this) {
97        // Cannot verify 'emptiness' here since it may not be locked
98        /* paranoid */ verify(this.anchor.ts != 0);
99        /* paranoid */ static_assert(__atomic_always_lock_free(sizeof(this.anchor.ts), &this.anchor.ts));
100        return __atomic_load_n(&this.anchor.ts, __ATOMIC_RELAXED);
101}
Note: See TracBrowser for help on using the repository browser.