Changeset c42b8a1


Ignore:
Timestamp:
Mar 11, 2022, 1:31:58 PM (3 years ago)
Author:
Thierry Delisle <tdelisle@…>
Branches:
ADT, ast-experimental, enum, master, pthread-emulation, qualifiedEnum
Children:
884f3f67
Parents:
3c4bf05
Message:

Major cleanup and moved cluster growth to new file

Location:
libcfa/src
Files:
1 added
5 edited

Legend:

Unmodified
Added
Removed

  • libcfa/src/Makefile.am

    r3c4bf05 rc42b8a1  
    134134        concurrency/io/call.cfa \
    135135        concurrency/iofwd.hfa \
     136        concurrency/kernel/cluster.cfa \
    136137        concurrency/kernel_private.hfa \
    137138        concurrency/kernel/startup.cfa \

  • libcfa/src/concurrency/kernel.hfa

    r3c4bf05 rc42b8a1  
    205205void  ?{}(__ready_queue_t & this);
    206206void ^?{}(__ready_queue_t & this);
    207 #if !defined(__CFA_NO_STATISTICS__)
    208         unsigned cnt(const __ready_queue_t & this, unsigned idx);
    209 #endif
    210207
    211208// Idle Sleep

  • libcfa/src/concurrency/kernel_private.hfa

    r3c4bf05 rc42b8a1  
    365365void ready_queue_shrink(struct cluster * cltr);
    366366
     367//-----------------------------------------------------------------------
     368// Calc moving average based on existing average, before and current time.
     369static inline unsigned long long moving_average(unsigned long long currtsc, unsigned long long instsc, unsigned long long old_avg) {
     370        /* paranoid */ verifyf( currtsc < 45000000000000000, "Suspiciously large current time: %'llu (%llx)\n", currtsc, currtsc );
     371        /* paranoid */ verifyf( instsc  < 45000000000000000, "Suspiciously large insert time: %'llu (%llx)\n", instsc, instsc );
     372        /* paranoid */ verifyf( old_avg < 15000000000000, "Suspiciously large previous average: %'llu (%llx)\n", old_avg, old_avg );
     373
     374        const unsigned long long new_val = currtsc > instsc ? currtsc - instsc : 0;
     375        const unsigned long long total_weight = 16;
     376        const unsigned long long new_weight   = 4;
     377        const unsigned long long old_weight = total_weight - new_weight;
     378        const unsigned long long ret = ((new_weight * new_val) + (old_weight * old_avg)) / total_weight;
     379        return ret;
     380}
     381
     382static const unsigned __readyq_shard_factor = 2;
    367383
    368384// Local Variables: //

  • libcfa/src/concurrency/ready_queue.cfa

    r3c4bf05 rc42b8a1  
    2626#include "kernel_private.hfa"
    2727
    28 #include "stdlib.hfa"
    2928#include "limits.hfa"
    30 #include "math.hfa"
    31 
    32 #include <errno.h>
    33 #include <unistd.h>
    34 
    35 extern "C" {
    36         #include <sys/syscall.h>  // __NR_xxx
    37 }
     29
     30// #include <errno.h>
     31// #include <unistd.h>
    3832
    3933#include "ready_subqueue.hfa"
    … …  
    4741#endif
    4842
    49 // No overriden function, no environment variable, no define
    50 // fall back to a magic number
    51 #ifndef __CFA_MAX_PROCESSORS__
    52         #define __CFA_MAX_PROCESSORS__ 1024
    53 #endif
    54 
    55 #define READYQ_SHARD_FACTOR 2
    56 #define SEQUENTIAL_SHARD 2
    57 
    5843static inline struct thread$ * try_pop(struct cluster * cltr, unsigned w __STATS(, __stats_readyQ_pop_t & stats));
    5944static inline struct thread$ * try_pop(struct cluster * cltr, unsigned i, unsigned j __STATS(, __stats_readyQ_pop_t & stats));
    6045static inline struct thread$ * search(struct cluster * cltr);
    6146
    62 
    63 // returns the maximum number of processors the RWLock support
    64 __attribute__((weak)) unsigned __max_processors() {
    65         const char * max_cores_s = getenv("CFA_MAX_PROCESSORS");
    66         if(!max_cores_s) {
    67                 __cfadbg_print_nolock(ready_queue, "No CFA_MAX_PROCESSORS in ENV\n");
    68                 return __CFA_MAX_PROCESSORS__;
    69         }
    70 
    71         char * endptr = 0p;
    72         long int max_cores_l = strtol(max_cores_s, &endptr, 10);
    73         if(max_cores_l < 1 || max_cores_l > 65535) {
    74                 __cfadbg_print_nolock(ready_queue, "CFA_MAX_PROCESSORS out of range : %ld\n", max_cores_l);
    75                 return __CFA_MAX_PROCESSORS__;
    76         }
    77         if('\0' != *endptr) {
    78                 __cfadbg_print_nolock(ready_queue, "CFA_MAX_PROCESSORS not a decimal number : %s\n", max_cores_s);
    79                 return __CFA_MAX_PROCESSORS__;
    80         }
    81 
    82         return max_cores_l;
    83 }
    84 
    85 #if   defined(CFA_HAVE_LINUX_LIBRSEQ)
    86         // No forward declaration needed
    87         #define __kernel_rseq_register rseq_register_current_thread
    88         #define __kernel_rseq_unregister rseq_unregister_current_thread
    89 #elif defined(CFA_HAVE_LINUX_RSEQ_H)
    90         static void __kernel_raw_rseq_register  (void);
    91         static void __kernel_raw_rseq_unregister(void);
    92 
    93         #define __kernel_rseq_register __kernel_raw_rseq_register
    94         #define __kernel_rseq_unregister __kernel_raw_rseq_unregister
    95 #else
    96         // No forward declaration needed
    97         // No initialization needed
    98         static inline void noop(void) {}
    99 
    100         #define __kernel_rseq_register noop
    101         #define __kernel_rseq_unregister noop
    102 #endif
    103 
    104 //=======================================================================
    105 // Cluster wide reader-writer lock
    106 //=======================================================================
    107 void  ?{}(__scheduler_RWLock_t & this) {
    108         this.max   = __max_processors();
    109         this.alloc = 0;
    110         this.ready = 0;
    111         this.data  = alloc(this.max);
    112         this.write_lock  = false;
    113 
    114         /*paranoid*/ verify(__atomic_is_lock_free(sizeof(this.alloc), &this.alloc));
    115         /*paranoid*/ verify(__atomic_is_lock_free(sizeof(this.ready), &this.ready));
    116 
    117 }
    118 void ^?{}(__scheduler_RWLock_t & this) {
    119         free(this.data);
    120 }
    121 
    122 
    123 //=======================================================================
    124 // Lock-Free registering/unregistering of threads
    125 unsigned register_proc_id( void ) with(*__scheduler_lock) {
    126         __kernel_rseq_register();
    127 
    128         bool * handle = (bool *)&kernelTLS().sched_lock;
    129 
    130         // Step - 1 : check if there is already space in the data
    131         uint_fast32_t s = ready;
    132 
    133         // Check among all the ready
    134         for(uint_fast32_t i = 0; i < s; i++) {
    135                 bool * volatile * cell = (bool * volatile *)&data[i]; // Cforall is bugged and the double volatiles causes problems
    136                 /* paranoid */ verify( handle != *cell );
    137 
    138                 bool * null = 0p; // Re-write every loop since compare thrashes it
    139                 if( __atomic_load_n(cell, (int)__ATOMIC_RELAXED) == null
    140                         && __atomic_compare_exchange_n( cell, &null, handle, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) {
    141                         /* paranoid */ verify(i < ready);
    142                         /* paranoid */ verify( (kernelTLS().sched_id = i, true) );
    143                         return i;
    144                 }
    145         }
    146 
    147         if(max <= alloc) abort("Trying to create more than %ud processors", __scheduler_lock->max);
    148 
    149         // Step - 2 : F&A to get a new spot in the array.
    150         uint_fast32_t n = __atomic_fetch_add(&alloc, 1, __ATOMIC_SEQ_CST);
    151         if(max <= n) abort("Trying to create more than %ud processors", __scheduler_lock->max);
    152 
    153         // Step - 3 : Mark space as used and then publish it.
    154         data[n] = handle;
    155         while() {
    156                 unsigned copy = n;
    157                 if( __atomic_load_n(&ready, __ATOMIC_RELAXED) == n
    158                         && __atomic_compare_exchange_n(&ready, &copy, n + 1, true, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST))
    159                         break;
    160                 Pause();
    161         }
    162 
    163         // Return new spot.
    164         /* paranoid */ verify(n < ready);
    165         /* paranoid */ verify( (kernelTLS().sched_id = n, true) );
    166         return n;
    167 }
    168 
    169 void unregister_proc_id( unsigned id ) with(*__scheduler_lock) {
    170         /* paranoid */ verify(id < ready);
    171         /* paranoid */ verify(id == kernelTLS().sched_id);
    172         /* paranoid */ verify(data[id] == &kernelTLS().sched_lock);
    173 
    174         bool * volatile * cell = (bool * volatile *)&data[id]; // Cforall is bugged and the double volatiles causes problems
    175 
    176         __atomic_store_n(cell, 0p, __ATOMIC_RELEASE);
    177 
    178         __kernel_rseq_unregister();
    179 }
    180 
    181 //-----------------------------------------------------------------------
    182 // Writer side : acquire when changing the ready queue, e.g. adding more
    183 //  queues or removing them.
    184 uint_fast32_t ready_mutate_lock( void ) with(*__scheduler_lock) {
    185         /* paranoid */ verify( ! __preemption_enabled() );
    186 
    187         // Step 1 : lock global lock
    188         // It is needed to avoid processors that register mid Critical-Section
    189         //   to simply lock their own lock and enter.
    190         __atomic_acquire( &write_lock );
    191 
    192         // Make sure we won't deadlock ourself
    193         // Checking before acquiring the writer lock isn't safe
    194         // because someone else could have locked us.
    195         /* paranoid */ verify( ! kernelTLS().sched_lock );
    196 
    197         // Step 2 : lock per-proc lock
    198         // Processors that are currently being registered aren't counted
    199         //   but can't be in read_lock or in the critical section.
    200         // All other processors are counted
    201         uint_fast32_t s = ready;
    202         for(uint_fast32_t i = 0; i < s; i++) {
    203                 volatile bool * llock = data[i];
    204                 if(llock) __atomic_acquire( llock );
    205         }
    206 
    207         /* paranoid */ verify( ! __preemption_enabled() );
    208         return s;
    209 }
    210 
    211 void ready_mutate_unlock( uint_fast32_t last_s ) with(*__scheduler_lock) {
    212         /* paranoid */ verify( ! __preemption_enabled() );
    213 
    214         // Step 1 : release local locks
    215         // This must be done while the global lock is held to avoid
    216         //   threads that where created mid critical section
    217         //   to race to lock their local locks and have the writer
    218         //   immidiately unlock them
    219         // Alternative solution : return s in write_lock and pass it to write_unlock
    220         for(uint_fast32_t i = 0; i < last_s; i++) {
    221                 volatile bool * llock = data[i];
    222                 if(llock) __atomic_store_n(llock, (bool)false, __ATOMIC_RELEASE);
    223         }
    224 
    225         // Step 2 : release global lock
    226         /*paranoid*/ assert(true == write_lock);
    227         __atomic_store_n(&write_lock, (bool)false, __ATOMIC_RELEASE);
    228 
    229         /* paranoid */ verify( ! __preemption_enabled() );
    230 }
    231 
    23247//=======================================================================
    23348// Cforall Ready Queue used for scheduling
    23449//=======================================================================
    235 unsigned long long moving_average(unsigned long long currtsc, unsigned long long instsc, unsigned long long old_avg) {
    236         /* paranoid */ verifyf( currtsc < 45000000000000000, "Suspiciously large current time: %'llu (%llx)\n", currtsc, currtsc );
    237         /* paranoid */ verifyf( instsc  < 45000000000000000, "Suspiciously large insert time: %'llu (%llx)\n", instsc, instsc );
    238         /* paranoid */ verifyf( old_avg < 15000000000000, "Suspiciously large previous average: %'llu (%llx)\n", old_avg, old_avg );
    239 
    240         const unsigned long long new_val = currtsc > instsc ? currtsc - instsc : 0;
    241         const unsigned long long total_weight = 16;
    242         const unsigned long long new_weight   = 4;
    243         const unsigned long long old_weight = total_weight - new_weight;
    244         const unsigned long long ret = ((new_weight * new_val) + (old_weight * old_avg)) / total_weight;
    245         return ret;
    246 }
    247 
    24850void ?{}(__ready_queue_t & this) with (this) {
    24951        lanes.data   = 0p;
    … …  
    27173                // Figure out where thread was last time and make sure it's valid
    27274                /* paranoid */ verify(thrd->preferred >= 0);
    273                 if(thrd->preferred * READYQ_SHARD_FACTOR < lanes.count) {
    274                         /* paranoid */ verify(thrd->preferred * READYQ_SHARD_FACTOR < lanes.count);
    275                         unsigned start = thrd->preferred * READYQ_SHARD_FACTOR;
     75                if(thrd->preferred * __readyq_shard_factor < lanes.count) {
     76                        /* paranoid */ verify(thrd->preferred * __readyq_shard_factor < lanes.count);
     77                        unsigned start = thrd->preferred * __readyq_shard_factor;
    27678                        do {
    27779                                unsigned r = __tls_rand();
    278                                 i = start + (r % READYQ_SHARD_FACTOR);
     80                                i = start + (r % __readyq_shard_factor);
    27981                                /* paranoid */ verify( i < lanes.count );
    28082                                // If we can't lock it retry
    … …  
    28890                do {
    28991                        unsigned r = proc->rdq.its++;
    290                         i = proc->rdq.id + (r % READYQ_SHARD_FACTOR);
     92                        i = proc->rdq.id + (r % __readyq_shard_factor);
    29193                        /* paranoid */ verify( i < lanes.count );
    29294                        // If we can't lock it retry
    … …  
    309111        unsigned start = proc->rdq.id;
    310112        unsigned long long max = 0;
    311         for(i; READYQ_SHARD_FACTOR) {
     113        for(i; __readyq_shard_factor) {
    312114                unsigned long long ptsc = ts(rdq.lanes.data[start + i]);
    313115                if(ptsc != -1ull) {
    … …  
    338140        // Super important: don't write the same value over and over again
    339141        // We want to maximise our chances that his particular values stays in cache
    340         if(lanes.caches[this / READYQ_SHARD_FACTOR].id != this_cache)
    341                 __atomic_store_n(&lanes.caches[this / READYQ_SHARD_FACTOR].id, this_cache, __ATOMIC_RELAXED);
     142        if(lanes.caches[this / __readyq_shard_factor].id != this_cache)
     143                __atomic_store_n(&lanes.caches[this / __readyq_shard_factor].id, this_cache, __ATOMIC_RELAXED);
    342144
    343145        const unsigned long long ctsc = rdtscl();
    … …  
    348150                unsigned other  = (chaos >> 8) % (lanes.count);
    349151
    350                 if(ext < 3 || __atomic_load_n(&lanes.caches[other / READYQ_SHARD_FACTOR].id, __ATOMIC_RELAXED) == this_cache) {
     152                if(ext < 3 || __atomic_load_n(&lanes.caches[other / __readyq_shard_factor].id, __ATOMIC_RELAXED) == this_cache) {
    351153                        proc->rdq.target = other;
    352154                }
    … …  
    368170        }
    369171
    370         for(READYQ_SHARD_FACTOR) {
    371                 unsigned i = this + (proc->rdq.itr++ % READYQ_SHARD_FACTOR);
     172        for(__readyq_shard_factor) {
     173                unsigned i = this + (proc->rdq.itr++ % __readyq_shard_factor);
    372174                if(thread$ * t = try_pop(cltr, i __STATS(, __tls_stats()->ready.pop.local))) return t;
    373175        }
    … …  
    439241        }
    440242
    441         thrd->preferred = w / READYQ_SHARD_FACTOR;
     243        thrd->preferred = w / __readyq_shard_factor;
    442244
    443245        // return the popped thread
    … …  
    476278
    477279//-----------------------------------------------------------------------
    478 // Check that all the intrusive queues in the data structure are still consistent
    479 static void check( __ready_queue_t & q ) with (q) {
    480         #if defined(__CFA_WITH_VERIFY__)
    481                 {
    482                         for( idx ; lanes.count ) {
    483                                 __intrusive_lane_t & sl = lanes.data[idx];
    484                                 assert(!lanes.data[idx].lock);
    485 
    486                                         if(is_empty(sl)) {
    487                                                 assert( sl.anchor.next == 0p );
    488                                                 assert( sl.anchor.ts   == -1llu );
    489                                                 assert( mock_head(sl)  == sl.prev );
    490                                         } else {
    491                                                 assert( sl.anchor.next != 0p );
    492                                                 assert( sl.anchor.ts   != -1llu );
    493                                                 assert( mock_head(sl)  != sl.prev );
    494                                         }
    495                         }
    496                 }
    497         #endif
    498 }
    499 
    500 //-----------------------------------------------------------------------
    501280// Given 2 indexes, pick the list with the oldest push an try to pop from it
    502281static inline struct thread$ * try_pop(struct cluster * cltr, unsigned i, unsigned j __STATS(, __stats_readyQ_pop_t & stats)) with (cltr->ready_queue) {
    … …  
    509288        return try_pop(cltr, w __STATS(, stats));
    510289}
    511 
    512 // Call this function of the intrusive list was moved using memcpy
    513 // fixes the list so that the pointers back to anchors aren't left dangling
    514 static inline void fix(__intrusive_lane_t & ll) {
    515                         if(is_empty(ll)) {
    516                                 verify(ll.anchor.next == 0p);
    517                                 ll.prev = mock_head(ll);
    518                         }
    519 }
    520 
    521 static void assign_list(unsigned & value, dlist(processor) & list, unsigned count) {
    522         processor * it = &list`first;
    523         for(unsigned i = 0; i < count; i++) {
    524                 /* paranoid */ verifyf( it, "Unexpected null iterator, at index %u of %u\n", i, count);
    525                 it->rdq.id = value;
    526                 it->rdq.target = MAX;
    527                 value += READYQ_SHARD_FACTOR;
    528                 it = &(*it)`next;
    529         }
    530 }
    531 
    532 static void reassign_cltr_id(struct cluster * cltr) {
    533         unsigned preferred = 0;
    534         assign_list(preferred, cltr->procs.actives, cltr->procs.total - cltr->procs.idle);
    535         assign_list(preferred, cltr->procs.idles  , cltr->procs.idle );
    536 }
    537 
    538 static void fix_times( struct cluster * cltr ) with( cltr->ready_queue ) {
    539         lanes.tscs = alloc(lanes.count, lanes.tscs`realloc);
    540         for(i; lanes.count) {
    541                 lanes.tscs[i].tv = rdtscl();
    542                 lanes.tscs[i].ma = 0;
    543         }
    544 }
    545 
    546 // Grow the ready queue
    547 void ready_queue_grow(struct cluster * cltr) {
    548         size_t ncount;
    549         int target = cltr->procs.total;
    550 
    551         /* paranoid */ verify( ready_mutate_islocked() );
    552         __cfadbg_print_safe(ready_queue, "Kernel : Growing ready queue\n");
    553 
    554         // Make sure that everything is consistent
    555         /* paranoid */ check( cltr->ready_queue );
    556 
    557         // grow the ready queue
    558         with( cltr->ready_queue ) {
    559                 // Find new count
    560                 // Make sure we always have atleast 1 list
    561                 if(target >= 2) {
    562                         ncount = target * READYQ_SHARD_FACTOR;
    563                 } else {
    564                         ncount = SEQUENTIAL_SHARD;
    565                 }
    566 
    567                 // Allocate new array (uses realloc and memcpies the data)
    568                 lanes.data = alloc( ncount, lanes.data`realloc );
    569 
    570                 // Fix the moved data
    571                 for( idx; (size_t)lanes.count ) {
    572                         fix(lanes.data[idx]);
    573                 }
    574 
    575                 // Construct new data
    576                 for( idx; (size_t)lanes.count ~ ncount) {
    577                         (lanes.data[idx]){};
    578                 }
    579 
    580                 // Update original
    581                 lanes.count = ncount;
    582 
    583                 lanes.caches = alloc( target, lanes.caches`realloc );
    584         }
    585 
    586         fix_times(cltr);
    587 
    588         reassign_cltr_id(cltr);
    589 
    590         // Make sure that everything is consistent
    591         /* paranoid */ check( cltr->ready_queue );
    592 
    593         __cfadbg_print_safe(ready_queue, "Kernel : Growing ready queue done\n");
    594 
    595         /* paranoid */ verify( ready_mutate_islocked() );
    596 }
    597 
    598 // Shrink the ready queue
    599 void ready_queue_shrink(struct cluster * cltr) {
    600         /* paranoid */ verify( ready_mutate_islocked() );
    601         __cfadbg_print_safe(ready_queue, "Kernel : Shrinking ready queue\n");
    602 
    603         // Make sure that everything is consistent
    604         /* paranoid */ check( cltr->ready_queue );
    605 
    606         int target = cltr->procs.total;
    607 
    608         with( cltr->ready_queue ) {
    609                 // Remember old count
    610                 size_t ocount = lanes.count;
    611 
    612                 // Find new count
    613                 // Make sure we always have atleast 1 list
    614                 lanes.count = target >= 2 ? target * READYQ_SHARD_FACTOR: SEQUENTIAL_SHARD;
    615                 /* paranoid */ verify( ocount >= lanes.count );
    616                 /* paranoid */ verify( lanes.count == target * READYQ_SHARD_FACTOR || target < 2 );
    617 
    618                 // for printing count the number of displaced threads
    619                 #if defined(__CFA_DEBUG_PRINT__) || defined(__CFA_DEBUG_PRINT_READY_QUEUE__)
    620                         __attribute__((unused)) size_t displaced = 0;
    621                 #endif
    622 
    623                 // redistribute old data
    624                 for( idx; (size_t)lanes.count ~ ocount) {
    625                         // Lock is not strictly needed but makes checking invariants much easier
    626                         __attribute__((unused)) bool locked = __atomic_try_acquire(&lanes.data[idx].lock);
    627                         verify(locked);
    628 
    629                         // As long as we can pop from this lane to push the threads somewhere else in the queue
    630                         while(!is_empty(lanes.data[idx])) {
    631                                 struct thread$ * thrd;
    632                                 unsigned long long _;
    633                                 [thrd, _] = pop(lanes.data[idx]);
    634 
    635                                 push(cltr, thrd, true);
    636 
    637                                 // for printing count the number of displaced threads
    638                                 #if defined(__CFA_DEBUG_PRINT__) || defined(__CFA_DEBUG_PRINT_READY_QUEUE__)
    639                                         displaced++;
    640                                 #endif
    641                         }
    642 
    643                         // Unlock the lane
    644                         __atomic_unlock(&lanes.data[idx].lock);
    645 
    646                         // TODO print the queue statistics here
    647 
    648                         ^(lanes.data[idx]){};
    649                 }
    650 
    651                 __cfadbg_print_safe(ready_queue, "Kernel : Shrinking ready queue displaced %zu threads\n", displaced);
    652 
    653                 // Allocate new array (uses realloc and memcpies the data)
    654                 lanes.data = alloc( lanes.count, lanes.data`realloc );
    655 
    656                 // Fix the moved data
    657                 for( idx; (size_t)lanes.count ) {
    658                         fix(lanes.data[idx]);
    659                 }
    660 
    661                 lanes.caches = alloc( target, lanes.caches`realloc );
    662         }
    663 
    664         fix_times(cltr);
    665 
    666 
    667         reassign_cltr_id(cltr);
    668 
    669         // Make sure that everything is consistent
    670         /* paranoid */ check( cltr->ready_queue );
    671 
    672         __cfadbg_print_safe(ready_queue, "Kernel : Shrinking ready queue done\n");
    673         /* paranoid */ verify( ready_mutate_islocked() );
    674 }
    675 
    676 #if !defined(__CFA_NO_STATISTICS__)
    677         unsigned cnt(const __ready_queue_t & this, unsigned idx) {
    678                 /* paranoid */ verify(this.lanes.count > idx);
    679                 return this.lanes.data[idx].cnt;
    680         }
    681 #endif
    682 
    683 
    684 #if   defined(CFA_HAVE_LINUX_LIBRSEQ)
    685         // No definition needed
    686 #elif defined(CFA_HAVE_LINUX_RSEQ_H)
    687 
    688         #if defined( __x86_64 ) || defined( __i386 )
    689                 #define RSEQ_SIG        0x53053053
    690         #elif defined( __ARM_ARCH )
    691                 #ifdef __ARMEB__
    692                 #define RSEQ_SIG    0xf3def5e7      /* udf    #24035    ; 0x5de3 (ARMv6+) */
    693                 #else
    694                 #define RSEQ_SIG    0xe7f5def3      /* udf    #24035    ; 0x5de3 */
    695                 #endif
    696         #endif
    697 
    698         extern void __disable_interrupts_hard();
    699         extern void __enable_interrupts_hard();
    700 
    701         static void __kernel_raw_rseq_register  (void) {
    702                 /* paranoid */ verify( __cfaabi_rseq.cpu_id == RSEQ_CPU_ID_UNINITIALIZED );
    703 
    704                 // int ret = syscall(__NR_rseq, &__cfaabi_rseq, sizeof(struct rseq), 0, (sigset_t *)0p, _NSIG / 8);
    705                 int ret = syscall(__NR_rseq, &__cfaabi_rseq, sizeof(struct rseq), 0, RSEQ_SIG);
    706                 if(ret != 0) {
    707                         int e = errno;
    708                         switch(e) {
    709                         case EINVAL: abort("KERNEL ERROR: rseq register invalid argument");
    710                         case ENOSYS: abort("KERNEL ERROR: rseq register no supported");
    711                         case EFAULT: abort("KERNEL ERROR: rseq register with invalid argument");
    712                         case EBUSY : abort("KERNEL ERROR: rseq register already registered");
    713                         case EPERM : abort("KERNEL ERROR: rseq register sig  argument  on unregistration does not match the signature received on registration");
    714                         default: abort("KERNEL ERROR: rseq register unexpected return %d", e);
    715                         }
    716                 }
    717         }
    718 
    719         static void __kernel_raw_rseq_unregister(void) {
    720                 /* paranoid */ verify( __cfaabi_rseq.cpu_id >= 0 );
    721 
    722                 // int ret = syscall(__NR_rseq, &__cfaabi_rseq, sizeof(struct rseq), RSEQ_FLAG_UNREGISTER, (sigset_t *)0p, _NSIG / 8);
    723                 int ret = syscall(__NR_rseq, &__cfaabi_rseq, sizeof(struct rseq), RSEQ_FLAG_UNREGISTER, RSEQ_SIG);
    724                 if(ret != 0) {
    725                         int e = errno;
    726                         switch(e) {
    727                         case EINVAL: abort("KERNEL ERROR: rseq unregister invalid argument");
    728                         case ENOSYS: abort("KERNEL ERROR: rseq unregister no supported");
    729                         case EFAULT: abort("KERNEL ERROR: rseq unregister with invalid argument");
    730                         case EBUSY : abort("KERNEL ERROR: rseq unregister already registered");
    731                         case EPERM : abort("KERNEL ERROR: rseq unregister sig  argument  on unregistration does not match the signature received on registration");
    732                         default: abort("KERNEL ERROR: rseq unregisteunexpected return %d", e);
    733                         }
    734                 }
    735         }
    736 #else
    737         // No definition needed
    738 #endif

  • libcfa/src/concurrency/ready_subqueue.hfa

    r3c4bf05 rc42b8a1  
    2525        );
    2626        return rhead;
    27 }
    28 
    29 // Ctor
    30 void ?{}( __intrusive_lane_t & this ) {
    31         this.lock = false;
    32         this.prev = mock_head(this);
    33         this.anchor.next = 0p;
    34         this.anchor.ts   = -1llu;
    35         #if !defined(__CFA_NO_STATISTICS__)
    36                 this.cnt  = 0;
    37         #endif
    38 
    39         // We add a boat-load of assertions here because the anchor code is very fragile
    40         /* paranoid */ _Static_assert( offsetof( thread$, link ) == offsetof(__intrusive_lane_t, anchor) );
    41         /* paranoid */ verify( offsetof( thread$, link ) == offsetof(__intrusive_lane_t, anchor) );
    42         /* paranoid */ verify( ((uintptr_t)( mock_head(this) ) + offsetof( thread$, link )) == (uintptr_t)(&this.anchor) );
    43         /* paranoid */ verify( &mock_head(this)->link.next == &this.anchor.next );
    44         /* paranoid */ verify( &mock_head(this)->link.ts   == &this.anchor.ts   );
    45         /* paranoid */ verify( mock_head(this)->link.next == 0p );
    46         /* paranoid */ verify( mock_head(this)->link.ts   == -1llu  );
    47         /* paranoid */ verify( mock_head(this) == this.prev );
    48         /* paranoid */ verify( __alignof__(__intrusive_lane_t) == 128 );
    49         /* paranoid */ verify( __alignof__(this) == 128 );
    50         /* paranoid */ verifyf( ((intptr_t)(&this) % 128) == 0, "Expected address to be aligned %p %% 128 == %zd", &this, ((intptr_t)(&this) % 128) );
    51 }
    52 
    53 // Dtor is trivial
    54 void ^?{}( __intrusive_lane_t & this ) {
    55         // Make sure the list is empty
    56         /* paranoid */ verify( this.anchor.next == 0p );
    57         /* paranoid */ verify( this.anchor.ts   == -1llu );
    58         /* paranoid */ verify( mock_head(this)  == this.prev );
    5927}
    6028
Note: See TracChangeset for help on using the changeset viewer.