Changeset 741e22c

Timestamp:

Aug 16, 2022, 4:01:54 PM (2 years ago)

Author:

Thierry Delisle <tdelisle@…>

Branches:

ADT, ast-experimental, master, pthread-emulation

Children:

71cf630

Parents:

ee0176b

Message:

Fixed potential false sharing on ready-schedule mutate lock

Location:

libcfa/src/concurrency/kernel

Files:

• cluster.cfa (modified) (6 diffs)
• cluster.hfa (modified) (1 diff)
• private.hfa (modified) (4 diffs)

libcfa/src/concurrency/kernel/cluster.cfa

@@ -93 +93 @@
 //=======================================================================
 void  ?{}(__scheduler_RWLock_t & this) {
-        this.max   = __max_processors();
-        this.alloc = 0;
-        this.ready = 0;
-        this.data  = alloc(this.max);
-        this.write_lock  = false;
-
-        /*paranoid*/ verify(__atomic_is_lock_free(sizeof(this.alloc), &this.alloc));
-        /*paranoid*/ verify(__atomic_is_lock_free(sizeof(this.ready), &this.ready));
+        this.lock.max   = __max_processors();
+        this.lock.alloc = 0;
+        this.lock.ready = 0;
+        this.lock.data  = alloc(this.lock.max);
+        this.lock.write_lock  = false;
+
+        /*paranoid*/ verify(__atomic_is_lock_free(sizeof(this.lock.alloc), &this.lock.alloc));
+        /*paranoid*/ verify(__atomic_is_lock_free(sizeof(this.lock.ready), &this.lock.ready));
 
 }
 void ^?{}(__scheduler_RWLock_t & this) {
-        free(this.data);
+        free(this.lock.data);
 }
 
@@ -110 +110 @@
 //=======================================================================
 // Lock-Free registering/unregistering of threads
-unsigned register_proc_id( void ) with(*__scheduler_lock) {
+unsigned register_proc_id( void ) with(__scheduler_lock->lock) {
         __kernel_rseq_register();
 
@@ -132 +132 @@
         }
 
-        if(max <= alloc) abort("Trying to create more than %ud processors", __scheduler_lock->max);
+        if(max <= alloc) abort("Trying to create more than %ud processors", __scheduler_lock->lock.max);
 
         // Step - 2 : F&A to get a new spot in the array.
         uint_fast32_t n = __atomic_fetch_add(&alloc, 1, __ATOMIC_SEQ_CST);
-        if(max <= n) abort("Trying to create more than %ud processors", __scheduler_lock->max);
+        if(max <= n) abort("Trying to create more than %ud processors", __scheduler_lock->lock.max);
 
         // Step - 3 : Mark space as used and then publish it.
@@ -154 +154 @@
 }
 
-void unregister_proc_id( unsigned id ) with(*__scheduler_lock) {
+void unregister_proc_id( unsigned id ) with(__scheduler_lock->lock) {
         /* paranoid */ verify(id < ready);
         /* paranoid */ verify(id == kernelTLS().sched_id);
@@ -169 +169 @@
 // Writer side : acquire when changing the ready queue, e.g. adding more
 //  queues or removing them.
-uint_fast32_t ready_mutate_lock( void ) with(*__scheduler_lock) {
+uint_fast32_t ready_mutate_lock( void ) with(__scheduler_lock->lock) {
         /* paranoid */ verify( ! __preemption_enabled() );
 
@@ -196 +196 @@
 }
 
-void ready_mutate_unlock( uint_fast32_t last_s ) with(*__scheduler_lock) {
+void ready_mutate_unlock( uint_fast32_t last_s ) with(__scheduler_lock->lock) {
         /* paranoid */ verify( ! __preemption_enabled() );
 

libcfa/src/concurrency/kernel/cluster.hfa

@@ -24 +24 @@
 // Calc moving average based on existing average, before and current time.
 static inline unsigned long long moving_average(unsigned long long currtsc, unsigned long long instsc, unsigned long long old_avg) {
-        /* paranoid */ verifyf( currtsc < 45000000000000000, "Suspiciously large current time: %'llu (%llx)\n", currtsc, currtsc );
-        /* paranoid */ verifyf( instsc  < 45000000000000000, "Suspiciously large insert time: %'llu (%llx)\n", instsc, instsc );
         /* paranoid */ verifyf( old_avg < 15000000000000, "Suspiciously large previous average: %'llu (%llx)\n", old_avg, old_avg );
 

libcfa/src/concurrency/kernel/private.hfa

@@ -184 +184 @@
 // have been hard-coded to for the ready-queue for
 // simplicity and performance
-struct __scheduler_RWLock_t {
-        // total cachelines allocated
-        unsigned int max;
-
-        // cachelines currently in use
-        volatile unsigned int alloc;
-
-        // cachelines ready to itereate over
-        // (!= to alloc when thread is in second half of doregister)
-        volatile unsigned int ready;
-
-        // writer lock
-        volatile bool write_lock;
-
-        // data pointer
-        volatile bool * volatile * data;
+union __attribute__((aligned(64))) __scheduler_RWLock_t {
+        struct {
+                // total cachelines allocated
+                unsigned int max;
+
+                // cachelines currently in use
+                volatile unsigned int alloc;
+
+                // cachelines ready to itereate over
+                // (!= to alloc when thread is in second half of doregister)
+                volatile unsigned int ready;
+
+                // writer lock
+                volatile bool write_lock;
+
+                // data pointer
+                volatile bool * volatile * data;
+        } lock;
+        char pad[192];
 };
 
@@ -210 +213 @@
 // Reader side : acquire when using the ready queue to schedule but not
 //  creating/destroying queues
-static inline void ready_schedule_lock(void) with(*__scheduler_lock) {
+static inline void ready_schedule_lock(void) with(__scheduler_lock->lock) {
         /* paranoid */ verify( ! __preemption_enabled() );
         /* paranoid */ verify( ! kernelTLS().in_sched_lock );
@@ -235 +238 @@
 }
 
-static inline void ready_schedule_unlock(void) with(*__scheduler_lock) {
+static inline void ready_schedule_unlock(void) with(__scheduler_lock->lock) {
         /* paranoid */ verify( ! __preemption_enabled() );
         /* paranoid */ verify( data[kernelTLS().sched_id] == &kernelTLS().sched_lock );
@@ -256 +259 @@
 
         static inline bool ready_mutate_islocked() {
-                return __scheduler_lock->write_lock;
+                return __scheduler_lock->lock.write_lock;
         }
 #endif