| [34d194c] | 1 | //
|
|---|
| 2 | // Cforall Version 1.0.0 Copyright (C) 2022 University of Waterloo
|
|---|
| 3 | //
|
|---|
| 4 | // The contents of this file are covered under the licence agreement in the
|
|---|
| 5 | // file "LICENCE" distributed with Cforall.
|
|---|
| 6 | //
|
|---|
| 7 | // atomic.hfa -- simpler macros to access atomic instructions
|
|---|
| 8 | //
|
|---|
| 9 | // Author : Peter A. Buhr
|
|---|
| 10 | // Created On : Thu May 25 15:22:46 2023
|
|---|
| 11 | // Last Modified By : Peter A. Buhr
|
|---|
| [0fd9c68] | 12 | // Last Modified On : Mon Jul 6 16:29:38 2026
|
|---|
| 13 | // Update Count : 86
|
|---|
| [34d194c] | 14 | //
|
|---|
| 15 |
|
|---|
| 16 | #pragma once
|
|---|
| 17 |
|
|---|
| 18 | static inline __attribute__((always_inline)) unsigned long long int rdtsc( void ) { // read processor's time-stamp counter
|
|---|
| 19 | #if defined( __x86_64__ )
|
|---|
| 20 | uint64_t rax, rdx, ecx;
|
|---|
| 21 | __asm__ __volatile__ ( "rdtscp; " : "=a" (rax), "=d" (rdx), "=c" (ecx) :: );
|
|---|
| 22 | return (rdx << 32) + rax;
|
|---|
| 23 | #elif defined( __i386__ )
|
|---|
| 24 | uint64_t count;
|
|---|
| 25 | asm volatile ( "rdtsc" : "=A" (count) );
|
|---|
| 26 | return count;
|
|---|
| 27 | #elif defined( __aarch64__ )
|
|---|
| 28 | // https://github.com/google/benchmark/blob/v1.1.0/src/cycleclock.h#L116
|
|---|
| 29 | uint64_t count;
|
|---|
| 30 | asm volatile ( "mrs %0, cntvct_el0" : "=r" (count) );
|
|---|
| 31 | return count;
|
|---|
| 32 | #else
|
|---|
| 33 | #error unsupported architecture
|
|---|
| 34 | #endif
|
|---|
| 35 | } // rdtsc
|
|---|
| 36 |
|
|---|
| 37 | static inline __attribute__((always_inline)) void Fence( void ) { // fence to prevent code movement
|
|---|
| 38 | #if defined( __x86_64 )
|
|---|
| 39 | // __asm__ __volatile__ ( "mfence" )
|
|---|
| 40 | __asm__ __volatile__ ( "lock; addq $0,128(%%rsp);" ::: "cc" );
|
|---|
| 41 | #elif defined(__i386)
|
|---|
| 42 | __asm__ __volatile__ ( "lock; addl $0,128(%%esp);" ::: "cc" );
|
|---|
| 43 | #elif defined( __ARM_ARCH )
|
|---|
| 44 | __asm__ __volatile__ ( "DMB ISH" ::: );
|
|---|
| 45 | #else
|
|---|
| 46 | #error unsupported architecture
|
|---|
| 47 | #endif
|
|---|
| 48 | } // Fence
|
|---|
| 49 |
|
|---|
| 50 | static inline __attribute__((always_inline)) void Pause( void ) { // pause to prevent excess processor bus usage
|
|---|
| 51 | #if defined( __i386 ) || defined( __x86_64 )
|
|---|
| 52 | __asm__ __volatile__ ( "lfence" );
|
|---|
| 53 | #elif defined( __aarch64__ )
|
|---|
| 54 | __asm__ __volatile__ ( "DMB ISH" ::: );
|
|---|
| 55 | #else
|
|---|
| 56 | #error unsupported architecture
|
|---|
| 57 | #endif
|
|---|
| 58 | } // Pause
|
|---|
| 59 |
|
|---|
| 60 | // Short Names and reference parameters.
|
|---|
| 61 |
|
|---|
| [0fd9c68] | 62 | // Cforall polymorphism does not work because it takes the address of the builtin atomic and C does not generate a
|
|---|
| 63 | // wrapper function for builtins.
|
|---|
| 64 |
|
|---|
| [34d194c] | 65 | #define AtomicLdm( lock, memorder ) __atomic_load_n( (&(lock)), memorder )
|
|---|
| 66 | #define AtomicLd( lock ) AtomicLdm( lock, __ATOMIC_ACQUIRE )
|
|---|
| 67 | #define AtomicStrm( lock, value, memorder ) __atomic_store_n( (&(lock)), value, memorder )
|
|---|
| 68 | #define AtomicStr( lock, value ) AtomicStrm( lock, value, __ATOMIC_RELEASE )
|
|---|
| 69 | #define AtomicClrm( lock, memorder ) __atomic_clear( (&(lock)), memorder )
|
|---|
| 70 | #define AtomicClr( lock ) AtomicClrm( lock, __ATOMIC_RELEASE )
|
|---|
| 71 | #define AtomicTasm( lock, memorder ) __atomic_test_and_set( (&(lock)), memorder )
|
|---|
| 72 | #define AtomicTas( lock ) AtomicTasm( lock, __ATOMIC_ACQUIRE )
|
|---|
| 73 | #define AtomicCasm( change, comp, assn, smemorder, fmemorder ) ({TYPEOF(comp) __temp = (comp); __atomic_compare_exchange_n( (&(change)), (&(__temp)), (assn), false, smemorder, fmemorder ); })
|
|---|
| 74 | #define AtomicCas( change, comp, assn ) AtomicCasm( change, comp, assn, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST )
|
|---|
| 75 | #define AtomicCaswm( change, comp, assn, smemorder, fmemorder ) ({TYPEOF(comp) __temp = (comp); __atomic_compare_exchange_n( (&(change)), (&(__temp)), (assn), true, smemorder, fmemorder ); })
|
|---|
| 76 | #define AtomicCasw( change, comp, assn ) AtomicCaswm( change, comp, assn, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST )
|
|---|
| 77 | #define AtomicCasvm( change, comp, assn, smemorder, fmemorder ) __atomic_compare_exchange_n( (&(change)), (&(comp)), (assn), false, smemorder, fmemorder )
|
|---|
| 78 | #define AtomicCasv( change, comp, assn ) AtomicCasvm( change, comp, assn, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST )
|
|---|
| 79 | #define AtomicCasvwm( change, comp, assn, smemorder, fmemorder ) __atomic_compare_exchange_n( (&(change)), (&(comp)), (assn), true, smemorder, fmemorder )
|
|---|
| 80 | #define AtomicCasvw( change, comp, assn ) AtomicCasvwm( change, comp, assn, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST )
|
|---|
| 81 | #define AtomicFasm( change, assn, memorder ) __atomic_exchange_n( (&(change)), (assn), memorder )
|
|---|
| 82 | #define AtomicFas( change, assn ) AtomicFasm( change, assn, __ATOMIC_SEQ_CST )
|
|---|
| 83 | #define AtomicFaim( change, Inc, memorder ) __atomic_fetch_add( (&(change)), (Inc), memorder )
|
|---|
| 84 | #define AtomicFai( change, Inc ) AtomicFaim( change, Inc, __ATOMIC_SEQ_CST )
|
|---|
