[ea7d2b0] | 1 | // |
---|
| 2 | // Cforall Version 1.0.0 Copyright (C) 2016 University of Waterloo |
---|
| 3 | // |
---|
| 4 | // The contents of this file are covered under the licence agreement in the |
---|
| 5 | // file "LICENCE" distributed with Cforall. |
---|
| 6 | // |
---|
[73abe95] | 7 | // bits/locks.hfa -- Fast internal locks. |
---|
[ea7d2b0] | 8 | // |
---|
| 9 | // Author : Thierry Delisle |
---|
| 10 | // Created On : Tue Oct 31 15:14:38 2017 |
---|
[b158d8f] | 11 | // Last Modified By : Peter A. Buhr |
---|
[fd9b524] | 12 | // Last Modified On : Wed Aug 12 14:18:07 2020 |
---|
| 13 | // Update Count : 13 |
---|
[ea7d2b0] | 14 | // |
---|
| 15 | |
---|
| 16 | #pragma once |
---|
| 17 | |
---|
[73abe95] | 18 | #include "bits/debug.hfa" |
---|
| 19 | #include "bits/defs.hfa" |
---|
[ea8b2f7] | 20 | #include <assert.h> |
---|
| 21 | |
---|
| 22 | #ifdef __cforall |
---|
| 23 | extern "C" { |
---|
| 24 | #include <pthread.h> |
---|
| 25 | } |
---|
| 26 | #endif |
---|
[ea7d2b0] | 27 | |
---|
| 28 | // pause to prevent excess processor bus usage |
---|
[fd9b524] | 29 | #if defined( __i386 ) || defined( __x86_64 ) |
---|
[ea7d2b0] | 30 | #define Pause() __asm__ __volatile__ ( "pause" : : : ) |
---|
[b158d8f] | 31 | #elif defined( __ARM_ARCH ) |
---|
[fd9b524] | 32 | #define Pause() __asm__ __volatile__ ( "YIELD" : : : ) |
---|
[ea7d2b0] | 33 | #else |
---|
| 34 | #error unsupported architecture |
---|
| 35 | #endif |
---|
| 36 | |
---|
| 37 | struct __spinlock_t { |
---|
[13073be] | 38 | // Wrap in struct to prevent false sharing with debug info |
---|
[3aeee3c] | 39 | volatile bool lock; |
---|
[ea7d2b0] | 40 | #ifdef __CFA_DEBUG__ |
---|
[13073be] | 41 | // previous function to acquire the lock |
---|
[ea7d2b0] | 42 | const char * prev_name; |
---|
[13073be] | 43 | // previous thread to acquire the lock |
---|
[ea7d2b0] | 44 | void* prev_thrd; |
---|
| 45 | #endif |
---|
[3aeee3c] | 46 | }; |
---|
[ea7d2b0] | 47 | |
---|
[0cf5b79] | 48 | #ifdef __cforall |
---|
[dbe9b08] | 49 | extern "C" { |
---|
[2026bb6] | 50 | extern void disable_interrupts() OPTIONAL_THREAD; |
---|
| 51 | extern void enable_interrupts_noPoll() OPTIONAL_THREAD; |
---|
[1997b4e] | 52 | |
---|
| 53 | #ifdef __CFA_DEBUG__ |
---|
[ae66348] | 54 | void __cfaabi_dbg_record_lock(__spinlock_t & this, const char prev_name[]); |
---|
[1997b4e] | 55 | #else |
---|
[ae66348] | 56 | #define __cfaabi_dbg_record_lock(x, y) |
---|
[1997b4e] | 57 | #endif |
---|
[dbe9b08] | 58 | } |
---|
| 59 | |
---|
[ea7d2b0] | 60 | static inline void ?{}( __spinlock_t & this ) { |
---|
| 61 | this.lock = 0; |
---|
| 62 | } |
---|
| 63 | |
---|
| 64 | // Lock the spinlock, return false if already acquired |
---|
[93c2e0a] | 65 | static inline bool try_lock ( __spinlock_t & this __cfaabi_dbg_ctx_param2 ) { |
---|
[3381ed7] | 66 | disable_interrupts(); |
---|
[93c2e0a] | 67 | bool result = (this.lock == 0) && (__atomic_test_and_set( &this.lock, __ATOMIC_ACQUIRE ) == 0); |
---|
[dbe9b08] | 68 | if( result ) { |
---|
[ae66348] | 69 | __cfaabi_dbg_record_lock( this, caller ); |
---|
[3381ed7] | 70 | } else { |
---|
| 71 | enable_interrupts_noPoll(); |
---|
[dbe9b08] | 72 | } |
---|
[ea7d2b0] | 73 | return result; |
---|
| 74 | } |
---|
| 75 | |
---|
| 76 | // Lock the spinlock, spin if already acquired |
---|
[36982fc] | 77 | static inline void lock( __spinlock_t & this __cfaabi_dbg_ctx_param2 ) { |
---|
[ea7d2b0] | 78 | #ifndef NOEXPBACK |
---|
| 79 | enum { SPIN_START = 4, SPIN_END = 64 * 1024, }; |
---|
| 80 | unsigned int spin = SPIN_START; |
---|
| 81 | #endif |
---|
| 82 | |
---|
[3381ed7] | 83 | disable_interrupts(); |
---|
[ea7d2b0] | 84 | for ( unsigned int i = 1;; i += 1 ) { |
---|
[13073be] | 85 | if ( (this.lock == 0) && (__atomic_test_and_set( &this.lock, __ATOMIC_ACQUIRE ) == 0) ) break; |
---|
[ea7d2b0] | 86 | #ifndef NOEXPBACK |
---|
| 87 | // exponential spin |
---|
| 88 | for ( volatile unsigned int s = 0; s < spin; s += 1 ) Pause(); |
---|
| 89 | |
---|
| 90 | // slowly increase by powers of 2 |
---|
| 91 | if ( i % 64 == 0 ) spin += spin; |
---|
| 92 | |
---|
| 93 | // prevent overflow |
---|
| 94 | if ( spin > SPIN_END ) spin = SPIN_START; |
---|
| 95 | #else |
---|
| 96 | Pause(); |
---|
| 97 | #endif |
---|
| 98 | } |
---|
[ae66348] | 99 | __cfaabi_dbg_record_lock( this, caller ); |
---|
[ea7d2b0] | 100 | } |
---|
| 101 | |
---|
| 102 | static inline void unlock( __spinlock_t & this ) { |
---|
[13073be] | 103 | __atomic_clear( &this.lock, __ATOMIC_RELEASE ); |
---|
[3381ed7] | 104 | enable_interrupts_noPoll(); |
---|
[ea7d2b0] | 105 | } |
---|
[ea8b2f7] | 106 | |
---|
| 107 | |
---|
| 108 | #ifdef __CFA_WITH_VERIFY__ |
---|
| 109 | extern bool __cfaabi_dbg_in_kernel(); |
---|
| 110 | #endif |
---|
| 111 | |
---|
[c66f6cb] | 112 | extern "C" { |
---|
| 113 | char * strerror(int); |
---|
| 114 | } |
---|
[6ec07e5] | 115 | #define CHECKED(x) { int err = x; if( err != 0 ) abort("KERNEL ERROR: Operation \"" #x "\" return error %d - %s\n", err, strerror(err)); } |
---|
[c66f6cb] | 116 | |
---|
[ea8b2f7] | 117 | struct __bin_sem_t { |
---|
[85b1deb] | 118 | pthread_mutex_t lock; |
---|
| 119 | pthread_cond_t cond; |
---|
[4069faad] | 120 | int val; |
---|
[ea8b2f7] | 121 | }; |
---|
| 122 | |
---|
| 123 | static inline void ?{}(__bin_sem_t & this) with( this ) { |
---|
[c66f6cb] | 124 | // Create the mutex with error checking |
---|
| 125 | pthread_mutexattr_t mattr; |
---|
| 126 | pthread_mutexattr_init( &mattr ); |
---|
| 127 | pthread_mutexattr_settype( &mattr, PTHREAD_MUTEX_ERRORCHECK_NP); |
---|
| 128 | pthread_mutex_init(&lock, &mattr); |
---|
| 129 | |
---|
[b81fd95] | 130 | pthread_cond_init (&cond, (const pthread_condattr_t *)0p); // workaround trac#208: cast should not be required |
---|
[4069faad] | 131 | val = 0; |
---|
[ea8b2f7] | 132 | } |
---|
| 133 | |
---|
| 134 | static inline void ^?{}(__bin_sem_t & this) with( this ) { |
---|
[c66f6cb] | 135 | CHECKED( pthread_mutex_destroy(&lock) ); |
---|
| 136 | CHECKED( pthread_cond_destroy (&cond) ); |
---|
[ea8b2f7] | 137 | } |
---|
| 138 | |
---|
| 139 | static inline void wait(__bin_sem_t & this) with( this ) { |
---|
| 140 | verify(__cfaabi_dbg_in_kernel()); |
---|
[c66f6cb] | 141 | CHECKED( pthread_mutex_lock(&lock) ); |
---|
[4069faad] | 142 | while(val < 1) { |
---|
[85b1deb] | 143 | pthread_cond_wait(&cond, &lock); |
---|
| 144 | } |
---|
[4069faad] | 145 | val -= 1; |
---|
[c66f6cb] | 146 | CHECKED( pthread_mutex_unlock(&lock) ); |
---|
[ea8b2f7] | 147 | } |
---|
| 148 | |
---|
[92e7631] | 149 | static inline bool post(__bin_sem_t & this) with( this ) { |
---|
[4069faad] | 150 | bool needs_signal = false; |
---|
| 151 | |
---|
[c66f6cb] | 152 | CHECKED( pthread_mutex_lock(&lock) ); |
---|
[4069faad] | 153 | if(val < 1) { |
---|
| 154 | val += 1; |
---|
| 155 | pthread_cond_signal(&cond); |
---|
| 156 | needs_signal = true; |
---|
| 157 | } |
---|
[c66f6cb] | 158 | CHECKED( pthread_mutex_unlock(&lock) ); |
---|
[85b1deb] | 159 | |
---|
[92e7631] | 160 | return needs_signal; |
---|
[85b1deb] | 161 | } |
---|
[c66f6cb] | 162 | |
---|
| 163 | #undef CHECKED |
---|
[e0f93e0] | 164 | |
---|
| 165 | struct $thread; |
---|
| 166 | extern void park( __cfaabi_dbg_ctx_param ); |
---|
| 167 | extern void unpark( struct $thread * this __cfaabi_dbg_ctx_param2 ); |
---|
| 168 | static inline struct $thread * active_thread (); |
---|
| 169 | |
---|
| 170 | // Semaphore which only supports a single thread |
---|
| 171 | struct single_sem { |
---|
| 172 | struct $thread * volatile ptr; |
---|
| 173 | }; |
---|
| 174 | |
---|
| 175 | static inline { |
---|
| 176 | void ?{}(single_sem & this) { |
---|
| 177 | this.ptr = 0p; |
---|
| 178 | } |
---|
| 179 | |
---|
| 180 | void ^?{}(single_sem & this) {} |
---|
| 181 | |
---|
| 182 | bool wait(single_sem & this) { |
---|
| 183 | for() { |
---|
| 184 | struct $thread * expected = this.ptr; |
---|
| 185 | if(expected == 1p) { |
---|
| 186 | if(__atomic_compare_exchange_n(&this.ptr, &expected, 0p, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) { |
---|
| 187 | return false; |
---|
| 188 | } |
---|
| 189 | } |
---|
| 190 | else { |
---|
| 191 | /* paranoid */ verify( expected == 0p ); |
---|
| 192 | if(__atomic_compare_exchange_n(&this.ptr, &expected, active_thread(), false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) { |
---|
| 193 | park( __cfaabi_dbg_ctx ); |
---|
| 194 | return true; |
---|
| 195 | } |
---|
| 196 | } |
---|
| 197 | |
---|
| 198 | } |
---|
| 199 | } |
---|
| 200 | |
---|
| 201 | bool post(single_sem & this) { |
---|
| 202 | for() { |
---|
| 203 | struct $thread * expected = this.ptr; |
---|
| 204 | if(expected == 1p) return false; |
---|
| 205 | if(expected == 0p) { |
---|
| 206 | if(__atomic_compare_exchange_n(&this.ptr, &expected, 1p, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) { |
---|
| 207 | return false; |
---|
| 208 | } |
---|
| 209 | } |
---|
| 210 | else { |
---|
| 211 | if(__atomic_compare_exchange_n(&this.ptr, &expected, 0p, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) { |
---|
| 212 | unpark( expected __cfaabi_dbg_ctx2 ); |
---|
| 213 | return true; |
---|
| 214 | } |
---|
| 215 | } |
---|
| 216 | } |
---|
| 217 | } |
---|
| 218 | } |
---|
[b353a49] | 219 | |
---|
| 220 | // Semaphore which only supports a single thread and one post |
---|
| 221 | // Semaphore which only supports a single thread |
---|
| 222 | struct oneshot { |
---|
| 223 | struct $thread * volatile ptr; |
---|
| 224 | }; |
---|
| 225 | |
---|
| 226 | static inline { |
---|
| 227 | void ?{}(oneshot & this) { |
---|
| 228 | this.ptr = 0p; |
---|
| 229 | } |
---|
| 230 | |
---|
| 231 | void ^?{}(oneshot & this) {} |
---|
| 232 | |
---|
| 233 | bool wait(oneshot & this) { |
---|
| 234 | for() { |
---|
| 235 | struct $thread * expected = this.ptr; |
---|
| 236 | if(expected == 1p) return false; |
---|
| 237 | /* paranoid */ verify( expected == 0p ); |
---|
| 238 | if(__atomic_compare_exchange_n(&this.ptr, &expected, active_thread(), false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) { |
---|
| 239 | park( __cfaabi_dbg_ctx ); |
---|
| 240 | /* paranoid */ verify( this.ptr == 1p ); |
---|
| 241 | return true; |
---|
| 242 | } |
---|
| 243 | } |
---|
| 244 | } |
---|
| 245 | |
---|
| 246 | bool post(oneshot & this) { |
---|
| 247 | struct $thread * got = __atomic_exchange_n( &this.ptr, 1p, __ATOMIC_SEQ_CST); |
---|
| 248 | if( got == 0p ) return false; |
---|
| 249 | unpark( got __cfaabi_dbg_ctx2 ); |
---|
| 250 | return true; |
---|
| 251 | } |
---|
| 252 | } |
---|
[b158d8f] | 253 | #endif |
---|