source: src/libcfa/bits/locks.h @ ea8b2f7

//
// Cforall Version 1.0.0 Copyright (C) 2016 University of Waterloo
//
// The contents of this file are covered under the licence agreement in the
// file "LICENCE" distributed with Cforall.
//
// bits/locks.h -- Fast internal locks.
//
// Author           : Thierry Delisle
// Created On       : Tue Oct 31 15:14:38 2017
// Last Modified By : Peter A. Buhr
// Last Modified On : Fri Mar 30 18:18:13 2018
// Update Count     : 9
//

#pragma once

#include "bits/debug.h"
#include "bits/defs.h"
#include <assert.h>

#ifdef __cforall
        extern "C" {
                #include <pthread.h>
        }
#endif

// pause to prevent excess processor bus usage
#if defined( __sparc )
        #define Pause() __asm__ __volatile__ ( "rd %ccr,%g0" )
#elif defined( __i386 ) || defined( __x86_64 )
        #define Pause() __asm__ __volatile__ ( "pause" : : : )
#elif defined( __ARM_ARCH )
        #define Pause() __asm__ __volatile__ ( "nop" : : : )
#else
        #error unsupported architecture
#endif

#if defined( __i386 ) || defined( __x86_64 ) || defined( __ARM_ARCH )
        // Intel recommendation
        #define __ALIGN__ __attribute__(( aligned (128) ))
#elif defined( __sparc )
        #define __ALIGN__ CALIGN
#else
        #error unsupported architecture
#endif

struct __spinlock_t {
        // Wrap in struct to prevent false sharing with debug info
        struct {
                // Align lock on 128-bit boundary
                __ALIGN__ volatile _Bool lock;
        };
        #ifdef __CFA_DEBUG__
                // previous function to acquire the lock
                const char * prev_name;
                // previous thread to acquire the lock
                void* prev_thrd;
        #endif
} __ALIGN__;

#ifdef __cforall
        extern "C" {
                extern void disable_interrupts();
                extern void enable_interrupts_noPoll();
        }

        extern void yield( unsigned int );

        static inline void ?{}( __spinlock_t & this ) {
                this.lock = 0;
        }


        #ifdef __CFA_DEBUG__
                void __cfaabi_dbg_record(__spinlock_t & this, const char * prev_name);
        #else
                #define __cfaabi_dbg_record(x, y)
        #endif

        // Lock the spinlock, return false if already acquired
        static inline _Bool try_lock  ( __spinlock_t & this __cfaabi_dbg_ctx_param2 ) {
                _Bool result = (this.lock == 0) && (__atomic_test_and_set( &this.lock, __ATOMIC_ACQUIRE ) == 0);
                if( result ) {
                        disable_interrupts();
                        __cfaabi_dbg_record( this, caller );
                }
                return result;
        }

        // Lock the spinlock, spin if already acquired
        static inline void lock( __spinlock_t & this __cfaabi_dbg_ctx_param2 ) {
                #ifndef NOEXPBACK
                        enum { SPIN_START = 4, SPIN_END = 64 * 1024, };
                        unsigned int spin = SPIN_START;
                #endif

                for ( unsigned int i = 1;; i += 1 ) {
                        if ( (this.lock == 0) && (__atomic_test_and_set( &this.lock, __ATOMIC_ACQUIRE ) == 0) ) break;
                        #ifndef NOEXPBACK
                                // exponential spin
                                for ( volatile unsigned int s = 0; s < spin; s += 1 ) Pause();

                                // slowly increase by powers of 2
                                if ( i % 64 == 0 ) spin += spin;

                                // prevent overflow
                                if ( spin > SPIN_END ) spin = SPIN_START;
                        #else
                                Pause();
                        #endif
                }
                disable_interrupts();
                __cfaabi_dbg_record( this, caller );
        }

        static inline void unlock( __spinlock_t & this ) {
                enable_interrupts_noPoll();
                __atomic_clear( &this.lock, __ATOMIC_RELEASE );
        }


        #ifdef __CFA_WITH_VERIFY__
                extern bool __cfaabi_dbg_in_kernel();
        #endif

        struct __bin_sem_t {
                int_fast8_t     counter;
                pthread_mutex_t lock;
                pthread_cond_t  cond;
        };

        static inline void ?{}(__bin_sem_t & this) with( this ) {
                counter = 0;
                pthread_mutex_init(&lock, NULL);
                pthread_cond_init (&cond, NULL);
        }

        static inline void ^?{}(__bin_sem_t & this) with( this ) {
                pthread_mutex_destroy(&lock);
                pthread_cond_destroy (&cond);
        }

        static inline void wait(__bin_sem_t & this) with( this ) {
                verify(__cfaabi_dbg_in_kernel());
                pthread_mutex_lock(&lock);
                if(counter != 0) {   // this must be a loop, not if!
                        pthread_cond_wait(&cond, &lock);
                }
                counter = 1;
                pthread_mutex_unlock(&lock);
        }

        static inline void post(__bin_sem_t & this) with( this ) {
                verify(__cfaabi_dbg_in_kernel());
                pthread_mutex_lock(&lock);
                bool needs_signal = counter == 0;
                counter = 1;
                pthread_mutex_unlock(&lock);
                if (!needs_signal)
                        pthread_cond_signal(&cond);
                }
#endif