1 | //
|
---|
2 | // Cforall Version 1.0.0 Copyright (C) 2016 University of Waterloo
|
---|
3 | //
|
---|
4 | // The contents of this file are covered under the licence agreement in the
|
---|
5 | // file "LICENCE" distributed with Cforall.
|
---|
6 | //
|
---|
7 | // bits/locks.h -- Fast internal locks.
|
---|
8 | //
|
---|
9 | // Author : Thierry Delisle
|
---|
10 | // Created On : Tue Oct 31 15:14:38 2017
|
---|
11 | // Last Modified By : Peter A. Buhr
|
---|
12 | // Last Modified On : Fri Mar 30 18:18:13 2018
|
---|
13 | // Update Count : 9
|
---|
14 | //
|
---|
15 |
|
---|
16 | #pragma once
|
---|
17 |
|
---|
18 | #include "bits/debug.h"
|
---|
19 | #include "bits/defs.h"
|
---|
20 |
|
---|
21 | // pause to prevent excess processor bus usage
|
---|
22 | #if defined( __sparc )
|
---|
23 | #define Pause() __asm__ __volatile__ ( "rd %ccr,%g0" )
|
---|
24 | #elif defined( __i386 ) || defined( __x86_64 )
|
---|
25 | #define Pause() __asm__ __volatile__ ( "pause" : : : )
|
---|
26 | #elif defined( __ARM_ARCH )
|
---|
27 | #define Pause() __asm__ __volatile__ ( "nop" : : : )
|
---|
28 | #else
|
---|
29 | #error unsupported architecture
|
---|
30 | #endif
|
---|
31 |
|
---|
32 | #if defined( __i386 ) || defined( __x86_64 ) || defined( __ARM_ARCH )
|
---|
33 | // Intel recommendation
|
---|
34 | #define __ALIGN__ __attribute__(( aligned (128) ))
|
---|
35 | #elif defined( __sparc )
|
---|
36 | #define __ALIGN__ CALIGN
|
---|
37 | #else
|
---|
38 | #error unsupported architecture
|
---|
39 | #endif
|
---|
40 |
|
---|
41 | struct __spinlock_t {
|
---|
42 | // Wrap in struct to prevent false sharing with debug info
|
---|
43 | struct {
|
---|
44 | // Align lock on 128-bit boundary
|
---|
45 | __ALIGN__ volatile _Bool lock;
|
---|
46 | };
|
---|
47 | #ifdef __CFA_DEBUG__
|
---|
48 | // previous function to acquire the lock
|
---|
49 | const char * prev_name;
|
---|
50 | // previous thread to acquire the lock
|
---|
51 | void* prev_thrd;
|
---|
52 | #endif
|
---|
53 | } __ALIGN__;
|
---|
54 |
|
---|
55 | #ifdef __cforall
|
---|
56 | extern "C" {
|
---|
57 | extern void disable_interrupts();
|
---|
58 | extern void enable_interrupts_noPoll();
|
---|
59 | }
|
---|
60 |
|
---|
61 | extern void yield( unsigned int );
|
---|
62 |
|
---|
63 | static inline void ?{}( __spinlock_t & this ) {
|
---|
64 | this.lock = 0;
|
---|
65 | }
|
---|
66 |
|
---|
67 |
|
---|
68 | #ifdef __CFA_DEBUG__
|
---|
69 | void __cfaabi_dbg_record(__spinlock_t & this, const char * prev_name);
|
---|
70 | #else
|
---|
71 | #define __cfaabi_dbg_record(x, y)
|
---|
72 | #endif
|
---|
73 |
|
---|
74 | // Lock the spinlock, return false if already acquired
|
---|
75 | static inline _Bool try_lock ( __spinlock_t & this __cfaabi_dbg_ctx_param2 ) {
|
---|
76 | _Bool result = (this.lock == 0) && (__atomic_test_and_set( &this.lock, __ATOMIC_ACQUIRE ) == 0);
|
---|
77 | if( result ) {
|
---|
78 | disable_interrupts();
|
---|
79 | __cfaabi_dbg_record( this, caller );
|
---|
80 | }
|
---|
81 | return result;
|
---|
82 | }
|
---|
83 |
|
---|
84 | // Lock the spinlock, spin if already acquired
|
---|
85 | static inline void lock( __spinlock_t & this __cfaabi_dbg_ctx_param2 ) {
|
---|
86 | #ifndef NOEXPBACK
|
---|
87 | enum { SPIN_START = 4, SPIN_END = 64 * 1024, };
|
---|
88 | unsigned int spin = SPIN_START;
|
---|
89 | #endif
|
---|
90 |
|
---|
91 | for ( unsigned int i = 1;; i += 1 ) {
|
---|
92 | if ( (this.lock == 0) && (__atomic_test_and_set( &this.lock, __ATOMIC_ACQUIRE ) == 0) ) break;
|
---|
93 | #ifndef NOEXPBACK
|
---|
94 | // exponential spin
|
---|
95 | for ( volatile unsigned int s = 0; s < spin; s += 1 ) Pause();
|
---|
96 |
|
---|
97 | // slowly increase by powers of 2
|
---|
98 | if ( i % 64 == 0 ) spin += spin;
|
---|
99 |
|
---|
100 | // prevent overflow
|
---|
101 | if ( spin > SPIN_END ) spin = SPIN_START;
|
---|
102 | #else
|
---|
103 | Pause();
|
---|
104 | #endif
|
---|
105 | }
|
---|
106 | disable_interrupts();
|
---|
107 | __cfaabi_dbg_record( this, caller );
|
---|
108 | }
|
---|
109 |
|
---|
110 | static inline void unlock( __spinlock_t & this ) {
|
---|
111 | enable_interrupts_noPoll();
|
---|
112 | __atomic_clear( &this.lock, __ATOMIC_RELEASE );
|
---|
113 | }
|
---|
114 | #endif
|
---|