Ignore:
Timestamp:
Oct 19, 2022, 4:43:26 PM (2 years ago)
Author:
Thierry Delisle <tdelisle@…>
Branches:
ADT, ast-experimental, master
Children:
1a45263
Parents:
9cd5bd2 (diff), 135143b (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the (diff) links above to see all the changes relative to each parent.
Message:

Merge branch 'master' into pthread-emulation

Location:
libcfa/src/concurrency/kernel
Files:
5 edited

Legend:

Unmodified
Added
Removed
  • libcfa/src/concurrency/kernel/cluster.cfa

    r9cd5bd2 rdf6cc9d  
    221221static const unsigned __readyq_single_shard = 2;
    222222
     223void  ?{}(__timestamp_t & this) { this.t.tv = 0; this.t.ma = 0; }
     224void ^?{}(__timestamp_t &) {}
     225
    223226//-----------------------------------------------------------------------
    224227// Check that all the intrusive queues in the data structure are still consistent
     
    254257}
    255258
    256 static void assign_list(unsigned & valrq, unsigned & valio, dlist(processor) & list, unsigned count) {
    257         processor * it = &list`first;
     259static void assign_list(unsigned & valrq, unsigned & valio, dlist(struct processor) & list, unsigned count) {
     260        struct processor * it = &list`first;
    258261        for(unsigned i = 0; i < count; i++) {
    259262                /* paranoid */ verifyf( it, "Unexpected null iterator, at index %u of %u\n", i, count);
     
    278281
    279282#if defined(CFA_HAVE_LINUX_IO_URING_H)
    280         static void assign_io(io_context$ ** data, size_t count, dlist(processor) & list) {
    281                 processor * it = &list`first;
     283        static void assign_io(io_context$ ** data, size_t count, dlist(struct processor) & list) {
     284                struct processor * it = &list`first;
    282285                while(it) {
    283286                        /* paranoid */ verifyf( it, "Unexpected null iterator\n");
  • libcfa/src/concurrency/kernel/cluster.hfa

    r9cd5bd2 rdf6cc9d  
    1818#include "device/cpu.hfa"
    1919#include "kernel/private.hfa"
     20#include "math.hfa"
    2021
    2122#include <limits.h>
     23#include <inttypes.h>
     24
     25#include "clock.hfa"
     26
     27#if   defined(READYQ_USE_LINEAR_AVG)
     28
     29// no conversion needed in this case
     30static inline __readyQ_avg_t __to_readyQ_avg(unsigned long long intsc) { return intsc; }
     31
     32// warn normally all ints
     33#define warn_large_before warnf( !strict || old_avg < 33_000_000_000, "Suspiciously large previous average: %'llu (%llx), %'" PRId64 "ms \n", old_avg, old_avg, program()`ms )
     34#define warn_large_after warnf( !strict || ret < 33_000_000_000, "Suspiciously large new average after %'" PRId64 "ms cputime: %'llu (%llx) from %'llu-%'llu (%'llu, %'llu) and %'llu\n", program()`ms, ret, ret, currtsc, intsc, new_val, new_val / 1000000, old_avg )
     35
     36// 8X linear factor is just 8 * x
     37#define AVG_FACTOR( x ) (8 * (x))
     38
     39#elif defined(READYQ_USE_LOGDBL_AVG)
     40
     41// convert to log2 scale but using double
     42static inline __readyQ_avg_t __to_readyQ_avg(unsigned long long intsc) { if(unlikely(0 == intsc)) return 0.0; else return log2(intsc); }
     43
     44#define warn_large_before warnf( !strict || old_avg < 35.0, "Suspiciously large previous average: %'lf, %'" PRId64 "ms \n", old_avg, program()`ms )
     45#define warn_large_after warnf( !strict || ret < 35.3, "Suspiciously large new average after %'" PRId64 "ms cputime: %'lf from %'llu-%'llu (%'llu, %'llu) and %'lf\n", program()`ms, ret, currtsc, intsc, new_val, new_val / 1000000, old_avg ); \
     46verify(ret >= 0)
     47
     48// 8X factor in logscale is log2(8X) = log2(8) + log2(X) = 3 + log2(X)
     49#define AVG_FACTOR( x ) (3.0 + (x))
     50
     51// we need to overload the __atomic_load_n because they don't support double
     52static inline double __atomic_load_n(volatile double * ptr, int mem) {
     53        volatile uint64_t * uptr = (volatile uint64_t *)ptr;
     54        _Static_assert(sizeof(*uptr) == sizeof(*ptr));
     55        uint64_t ret = 0;
     56        ret = __atomic_load_n(uptr, mem);
     57        uint64_t *rp = &ret;
     58        double ret = *(volatile double *)rp;
     59        /* paranoid */ verify( ret == 0 || ret > 3e-100 );
     60        return ret;
     61}
     62
     63// we need to overload the __atomic_store_n because they don't support double
     64static inline void __atomic_store_n(volatile double * ptr, double val, int mem) {
     65        /* paranoid */ verify( val == 0 || val > 3e-100 );
     66        volatile uint64_t * uptr = (volatile uint64_t *)ptr;
     67        _Static_assert(sizeof(*uptr) == sizeof(*ptr));
     68        uint64_t * valp = (uint64_t *)&val;
     69        __atomic_store_n(uptr, *valp, mem);
     70}
     71
     72#elif defined(READYQ_USE_LOGDBL_AVG)
     73
     74//convert to log2 scale but with fix point u32.32 values
     75static inline __readyQ_avg_t __to_readyQ_avg(unsigned long long intsc) { return ulog2_32_32(tsc); }
     76
     77// 8X factor, +3 in logscale (see above) is + 0x3.00000000
     78#define AVG_FACTOR( x ) (0x3_00000000ull + (x))
     79
     80#else
     81#error must pick a scheme for averaging
     82#endif
    2283
    2384//-----------------------------------------------------------------------
    2485// Calc moving average based on existing average, before and current time.
    25 static inline unsigned long long moving_average(unsigned long long currtsc, unsigned long long instsc, unsigned long long old_avg) {
    26         /* paranoid */ verifyf( old_avg < 15000000000000, "Suspiciously large previous average: %'llu (%llx)\n", old_avg, old_avg );
     86static inline __readyQ_avg_t moving_average(unsigned long long currtsc, unsigned long long intsc, __readyQ_avg_t old_avg, bool strict) {
     87        (void)strict; // disable the warning around the fact this is unused in release.
     88        /* paranoid */ warn_large_before;
    2789
    28         const unsigned long long new_val = currtsc > instsc ? currtsc - instsc : 0;
    29         const unsigned long long total_weight = 16;
    30         const unsigned long long new_weight   = 4;
    31         const unsigned long long old_weight = total_weight - new_weight;
    32         const unsigned long long ret = ((new_weight * new_val) + (old_weight * old_avg)) / total_weight;
     90        const unsigned long long new_val = currtsc > intsc ? currtsc - intsc : 0;
     91        const __readyQ_avg_t total_weight = 16;
     92        const __readyQ_avg_t new_weight   = 12;
     93        const __readyQ_avg_t old_weight = total_weight - new_weight;
     94        const __readyQ_avg_t ret = ((new_weight * __to_readyQ_avg(new_val)) + (old_weight * old_avg)) / total_weight;
     95
     96        /* paranoid */ warn_large_after;
    3397        return ret;
    3498}
    3599
    36 static inline void touch_tsc(__timestamp_t * tscs, size_t idx, unsigned long long ts_prev, unsigned long long ts_next) {
     100static inline void touch_tsc(__timestamp_t * tscs, size_t idx, unsigned long long ts_prev, unsigned long long ts_next, bool strict) {
    37101        if (ts_next == ULLONG_MAX) return;
    38102        unsigned long long now = rdtscl();
    39         unsigned long long pma = __atomic_load_n(&tscs[ idx ].t.ma, __ATOMIC_RELAXED);
     103        __readyQ_avg_t pma = __atomic_load_n(&tscs[ idx ].t.ma, __ATOMIC_RELAXED);
    40104        __atomic_store_n(&tscs[ idx ].t.tv, ts_next, __ATOMIC_RELAXED);
    41         __atomic_store_n(&tscs[ idx ].t.ma, moving_average(now, ts_prev, pma), __ATOMIC_RELAXED);
     105        __atomic_store_n(&tscs[ idx ].t.ma, moving_average(now, ts_prev, pma, strict), __ATOMIC_RELAXED);
    42106}
    43107
     
    45109// Calc age a timestamp should be before needing help.
    46110forall(Data_t * | { unsigned long long ts(Data_t & this); })
    47 static inline unsigned long long calc_cutoff(
     111static inline __readyQ_avg_t calc_cutoff(
    48112        const unsigned long long ctsc,
    49113        unsigned procid,
     
    51115        Data_t * data,
    52116        __timestamp_t * tscs,
    53         const unsigned shard_factor
     117        const unsigned shard_factor,
     118        bool strict
    54119) {
    55120        unsigned start = procid;
    56         unsigned long long max = 0;
     121        __readyQ_avg_t max = 0;
    57122        for(i; shard_factor) {
    58123                unsigned long long ptsc = ts(data[start + i]);
    59124                if(ptsc != ULLONG_MAX) {
    60125                        /* paranoid */ verify( start + i < count );
    61                         unsigned long long tsc = moving_average(ctsc, ptsc, tscs[start + i].t.ma);
    62                         if(tsc > max) max = tsc;
     126                        __readyQ_avg_t avg = moving_average(ctsc, ptsc, tscs[start + i].t.ma, strict);
     127                        if(avg > max) max = avg;
    63128                }
    64129        }
    65         return 8 * max;
     130        return AVG_FACTOR( max );
    66131}
    67132
  • libcfa/src/concurrency/kernel/fwd.hfa

    r9cd5bd2 rdf6cc9d  
    276276                        // intented to be use by wait, wait_any, waitfor, etc. rather than used directly
    277277                        bool retract( future_t & this, oneshot & wait_ctx ) {
    278                                 struct oneshot * expected = this.ptr;
     278                                struct oneshot * expected = &wait_ctx;
    279279
    280280                                // attempt to remove the context so it doesn't get consumed.
  • libcfa/src/concurrency/kernel/private.hfa

    r9cd5bd2 rdf6cc9d  
    5050        #endif
    5151#endif
     52// #define READYQ_USE_LINEAR_AVG
     53#define READYQ_USE_LOGDBL_AVG
     54// #define READYQ_USE_LOGINT_AVG
     55
     56#if   defined(READYQ_USE_LINEAR_AVG)
     57typedef unsigned long long __readyQ_avg_t;
     58#elif defined(READYQ_USE_LOGDBL_AVG)
     59typedef double __readyQ_avg_t;
     60#elif defined(READYQ_USE_LOGDBL_AVG)
     61typedef unsigned long long __readyQ_avg_t;
     62#else
     63#error must pick a scheme for averaging
     64#endif
    5265
    5366extern "C" {
     
    6578//-----------------------------------------------------------------------------
    6679// Scheduler
     80union __attribute__((aligned(64))) __timestamp_t {
     81        struct {
     82                volatile unsigned long long tv;
     83                volatile __readyQ_avg_t ma;
     84        } t;
     85        char __padding[192];
     86};
     87
    6788extern "C" {
    6889        void disable_interrupts() OPTIONAL_THREAD;
  • libcfa/src/concurrency/kernel/startup.cfa

    r9cd5bd2 rdf6cc9d  
    184184
    185185
     186extern void heapManagerCtor();
     187extern void heapManagerDtor();
     188
    186189//=============================================================================================
    187190// Kernel Setup logic
     
    374377        proc->local_data = &__cfaabi_tls;
    375378
     379        heapManagerCtor();                                                                      // initialize heap
     380
    376381        __cfa_io_start( proc );
    377382        register_tls( proc );
     
    425430        unregister_tls( proc );
    426431        __cfa_io_stop( proc );
     432
     433        heapManagerDtor();                                                                      // de-initialize heap
    427434
    428435        return 0p;
Note: See TracChangeset for help on using the changeset viewer.