Ignore:
File:
1 edited

Legend:

Unmodified
Added
Removed

  • libcfa/src/concurrency/ready_queue.cfa

    re84ab3d r6ba6846  
    6767#endif
    6868
    69 static inline struct thread$ * try_pop(struct cluster * cltr, unsigned w __STATS(, __stats_readyQ_pop_t & stats));
    70 static inline struct thread$ * try_pop(struct cluster * cltr, unsigned i, unsigned j __STATS(, __stats_readyQ_pop_t & stats));
    71 static inline struct thread$ * search(struct cluster * cltr);
     69static inline struct $thread * try_pop(struct cluster * cltr, unsigned w __STATS(, __stats_readyQ_pop_t & stats));
     70static inline struct $thread * try_pop(struct cluster * cltr, unsigned i, unsigned j __STATS(, __stats_readyQ_pop_t & stats));
     71static inline struct $thread * search(struct cluster * cltr);
    7272static inline [unsigned, bool] idx_from_r(unsigned r, unsigned preferred);
    7373
    … …  
    274274//-----------------------------------------------------------------------
    275275#if defined(USE_CPU_WORK_STEALING)
    276         __attribute__((hot)) void push(struct cluster * cltr, struct thread$ * thrd, bool push_local) with (cltr->ready_queue) {
     276        __attribute__((hot)) void push(struct cluster * cltr, struct $thread * thrd, bool push_local) with (cltr->ready_queue) {
    277277                __cfadbg_print_safe(ready_queue, "Kernel : Pushing %p on cluster %p\n", thrd, cltr);
    278278
    … …  
    316316
    317317        // Pop from the ready queue from a given cluster
    318         __attribute__((hot)) thread$ * pop_fast(struct cluster * cltr) with (cltr->ready_queue) {
     318        __attribute__((hot)) $thread * pop_fast(struct cluster * cltr) with (cltr->ready_queue) {
    319319                /* paranoid */ verify( lanes.count > 0 );
    320320                /* paranoid */ verify( kernelTLS().this_processor );
    … …  
    371371                                proc->rdq.target = -1u;
    372372                                if(lanes.tscs[target].tv < cutoff && ts(lanes.data[target]) < cutoff) {
    373                                         thread$ * t = try_pop(cltr, target __STATS(, __tls_stats()->ready.pop.help));
     373                                        $thread * t = try_pop(cltr, target __STATS(, __tls_stats()->ready.pop.help));
    374374                                        proc->rdq.last = target;
    375375                                        if(t) return t;
    … …  
    379379                        unsigned last = proc->rdq.last;
    380380                        if(last != -1u && lanes.tscs[last].tv < cutoff && ts(lanes.data[last]) < cutoff) {
    381                                 thread$ * t = try_pop(cltr, last __STATS(, __tls_stats()->ready.pop.help));
     381                                $thread * t = try_pop(cltr, last __STATS(, __tls_stats()->ready.pop.help));
    382382                                if(t) return t;
    383383                        }
    … …  
    389389                for(READYQ_SHARD_FACTOR) {
    390390                        unsigned i = start + (proc->rdq.itr++ % READYQ_SHARD_FACTOR);
    391                         if(thread$ * t = try_pop(cltr, i __STATS(, __tls_stats()->ready.pop.local))) return t;
     391                        if($thread * t = try_pop(cltr, i __STATS(, __tls_stats()->ready.pop.local))) return t;
    392392                }
    393393
    … …  
    396396        }
    397397
    398         __attribute__((hot)) struct thread$ * pop_slow(struct cluster * cltr) with (cltr->ready_queue) {
     398        __attribute__((hot)) struct $thread * pop_slow(struct cluster * cltr) with (cltr->ready_queue) {
    399399                processor * const proc = kernelTLS().this_processor;
    400400                unsigned last = proc->rdq.last;
    401401                if(last != -1u) {
    402                         struct thread$ * t = try_pop(cltr, last __STATS(, __tls_stats()->ready.pop.steal));
     402                        struct $thread * t = try_pop(cltr, last __STATS(, __tls_stats()->ready.pop.steal));
    403403                        if(t) return t;
    404404                        proc->rdq.last = -1u;
    … …  
    408408                return try_pop(cltr, i __STATS(, __tls_stats()->ready.pop.steal));
    409409        }
    410         __attribute__((hot)) struct thread$ * pop_search(struct cluster * cltr) {
     410        __attribute__((hot)) struct $thread * pop_search(struct cluster * cltr) {
    411411                return search(cltr);
    412412        }
    … …  
    435435        }
    436436
    437         __attribute__((hot)) void push(struct cluster * cltr, struct thread$ * thrd, bool push_local) with (cltr->ready_queue) {
     437        __attribute__((hot)) void push(struct cluster * cltr, struct $thread * thrd, bool push_local) with (cltr->ready_queue) {
    438438                __cfadbg_print_safe(ready_queue, "Kernel : Pushing %p on cluster %p\n", thrd, cltr);
    439439
    … …  
    482482
    483483        // Pop from the ready queue from a given cluster
    484         __attribute__((hot)) thread$ * pop_fast(struct cluster * cltr) with (cltr->ready_queue) {
     484        __attribute__((hot)) $thread * pop_fast(struct cluster * cltr) with (cltr->ready_queue) {
    485485                /* paranoid */ verify( lanes.count > 0 );
    486486                /* paranoid */ verify( kernelTLS().this_processor );
    … …  
    506506
    507507                        // try popping from the 2 picked lists
    508                         struct thread$ * thrd = try_pop(cltr, i, j __STATS(, *(locali || localj ? &__tls_stats()->ready.pop.local : &__tls_stats()->ready.pop.help)));
     508                        struct $thread * thrd = try_pop(cltr, i, j __STATS(, *(locali || localj ? &__tls_stats()->ready.pop.local : &__tls_stats()->ready.pop.help)));
    509509                        if(thrd) {
    510510                                return thrd;
    … …  
    516516        }
    517517
    518         __attribute__((hot)) struct thread$ * pop_slow(struct cluster * cltr) { return pop_fast(cltr); }
    519         __attribute__((hot)) struct thread$ * pop_search(struct cluster * cltr) {
     518        __attribute__((hot)) struct $thread * pop_slow(struct cluster * cltr) { return pop_fast(cltr); }
     519        __attribute__((hot)) struct $thread * pop_search(struct cluster * cltr) {
    520520                return search(cltr);
    521521        }
    522522#endif
    523523#if defined(USE_WORK_STEALING)
    524         __attribute__((hot)) void push(struct cluster * cltr, struct thread$ * thrd, bool push_local) with (cltr->ready_queue) {
     524        __attribute__((hot)) void push(struct cluster * cltr, struct $thread * thrd, bool push_local) with (cltr->ready_queue) {
    525525                __cfadbg_print_safe(ready_queue, "Kernel : Pushing %p on cluster %p\n", thrd, cltr);
    526526
    … …  
    576576
    577577        // Pop from the ready queue from a given cluster
    578         __attribute__((hot)) thread$ * pop_fast(struct cluster * cltr) with (cltr->ready_queue) {
     578        __attribute__((hot)) $thread * pop_fast(struct cluster * cltr) with (cltr->ready_queue) {
    579579                /* paranoid */ verify( lanes.count > 0 );
    580580                /* paranoid */ verify( kernelTLS().this_processor );
    … …  
    598598                        const unsigned long long cutoff = proc->rdq.cutoff > bias ? proc->rdq.cutoff - bias : proc->rdq.cutoff;
    599599                        if(lanes.tscs[target].tv < cutoff && ts(lanes.data[target]) < cutoff) {
    600                                 thread$ * t = try_pop(cltr, target __STATS(, __tls_stats()->ready.pop.help));
     600                                $thread * t = try_pop(cltr, target __STATS(, __tls_stats()->ready.pop.help));
    601601                                if(t) return t;
    602602                        }
    … …  
    605605                for(READYQ_SHARD_FACTOR) {
    606606                        unsigned i = proc->rdq.id + (proc->rdq.itr++ % READYQ_SHARD_FACTOR);
    607                         if(thread$ * t = try_pop(cltr, i __STATS(, __tls_stats()->ready.pop.local))) return t;
     607                        if($thread * t = try_pop(cltr, i __STATS(, __tls_stats()->ready.pop.local))) return t;
    608608                }
    609609                return 0p;
    610610        }
    611611
    612         __attribute__((hot)) struct thread$ * pop_slow(struct cluster * cltr) with (cltr->ready_queue) {
     612        __attribute__((hot)) struct $thread * pop_slow(struct cluster * cltr) with (cltr->ready_queue) {
    613613                unsigned i = __tls_rand() % lanes.count;
    614614                return try_pop(cltr, i __STATS(, __tls_stats()->ready.pop.steal));
    615615        }
    616616
    617         __attribute__((hot)) struct thread$ * pop_search(struct cluster * cltr) with (cltr->ready_queue) {
     617        __attribute__((hot)) struct $thread * pop_search(struct cluster * cltr) with (cltr->ready_queue) {
    618618                return search(cltr);
    619619        }
    … …  
    628628//-----------------------------------------------------------------------
    629629// try to pop from a lane given by index w
    630 static inline struct thread$ * try_pop(struct cluster * cltr, unsigned w __STATS(, __stats_readyQ_pop_t & stats)) with (cltr->ready_queue) {
     630static inline struct $thread * try_pop(struct cluster * cltr, unsigned w __STATS(, __stats_readyQ_pop_t & stats)) with (cltr->ready_queue) {
    631631        __STATS( stats.attempt++; )
    632632
    … …  
    651651
    652652        // Actually pop the list
    653         struct thread$ * thrd;
     653        struct $thread * thrd;
    654654        unsigned long long tsv;
    655655        [thrd, tsv] = pop(lane);
    … …  
    678678// try to pop from any lanes making sure you don't miss any threads push
    679679// before the start of the function
    680 static inline struct thread$ * search(struct cluster * cltr) with (cltr->ready_queue) {
     680static inline struct $thread * search(struct cluster * cltr) with (cltr->ready_queue) {
    681681        /* paranoid */ verify( lanes.count > 0 );
    682682        unsigned count = __atomic_load_n( &lanes.count, __ATOMIC_RELAXED );
    … …  
    684684        for(i; count) {
    685685                unsigned idx = (offset + i) % count;
    686                 struct thread$ * thrd = try_pop(cltr, idx __STATS(, __tls_stats()->ready.pop.search));
     686                struct $thread * thrd = try_pop(cltr, idx __STATS(, __tls_stats()->ready.pop.search));
    687687                if(thrd) {
    688688                        return thrd;
    … …  
    719719//-----------------------------------------------------------------------
    720720// Given 2 indexes, pick the list with the oldest push an try to pop from it
    721 static inline struct thread$ * try_pop(struct cluster * cltr, unsigned i, unsigned j __STATS(, __stats_readyQ_pop_t & stats)) with (cltr->ready_queue) {
     721static inline struct $thread * try_pop(struct cluster * cltr, unsigned i, unsigned j __STATS(, __stats_readyQ_pop_t & stats)) with (cltr->ready_queue) {
    722722        // Pick the bet list
    723723        int w = i;
    … …  
    854854                                // As long as we can pop from this lane to push the threads somewhere else in the queue
    855855                                while(!is_empty(lanes.data[idx])) {
    856                                         struct thread$ * thrd;
     856                                        struct $thread * thrd;
    857857                                        unsigned long long _;
    858858                                        [thrd, _] = pop(lanes.data[idx]);
Note: See TracChangeset for help on using the changeset viewer.