Index: libcfa/src/Makefile.am =================================================================== --- libcfa/src/Makefile.am (revision 3859adb4d802f6f27e35aa4ed05a7c87d7ef632e) +++ libcfa/src/Makefile.am (revision 34d194c7be7287186dfb30719afe0c0c2ee07b08) @@ -11,6 +11,6 @@ ## Created On : Sun May 31 08:54:01 2015 ## Last Modified By : Peter A. Buhr -## Last Modified On : Wed Nov 1 19:03:42 2023 -## Update Count : 266 +## Last Modified On : Sat Jul 4 07:27:10 2026 +## Update Count : 267 ############################################################################### @@ -60,5 +60,5 @@ bits/queue.hfa \ bits/sequence.hfa \ - concurrency/atomic.hfa \ + bits/atomic.hfa \ concurrency/iofwd.hfa \ concurrency/barrier.hfa \ Index: libcfa/src/bits/atomic.hfa =================================================================== --- libcfa/src/bits/atomic.hfa (revision 34d194c7be7287186dfb30719afe0c0c2ee07b08) +++ libcfa/src/bits/atomic.hfa (revision 34d194c7be7287186dfb30719afe0c0c2ee07b08) @@ -0,0 +1,81 @@ +// +// Cforall Version 1.0.0 Copyright (C) 2022 University of Waterloo +// +// The contents of this file are covered under the licence agreement in the +// file "LICENCE" distributed with Cforall. +// +// atomic.hfa -- simpler macros to access atomic instructions +// +// Author : Peter A. Buhr +// Created On : Thu May 25 15:22:46 2023 +// Last Modified By : Peter A. Buhr +// Last Modified On : Sun Jul 5 23:03:28 2026 +// Update Count : 78 +// + +#pragma once + +static inline __attribute__((always_inline)) unsigned long long int rdtsc( void ) { // read processor's time-stamp counter + #if defined( __x86_64__ ) + uint64_t rax, rdx, ecx; + __asm__ __volatile__ ( "rdtscp; " : "=a" (rax), "=d" (rdx), "=c" (ecx) :: ); + return (rdx << 32) + rax; + #elif defined( __i386__ ) + uint64_t count; + asm volatile ( "rdtsc" : "=A" (count) ); + return count; + #elif defined( __aarch64__ ) + // https://github.com/google/benchmark/blob/v1.1.0/src/cycleclock.h#L116 + uint64_t count; + asm volatile ( "mrs %0, cntvct_el0" : "=r" (count) ); + return count; + #else + #error unsupported architecture + #endif +} // rdtsc + +static inline __attribute__((always_inline)) void Fence( void ) { // fence to prevent code movement + #if defined( __x86_64 ) + // __asm__ __volatile__ ( "mfence" ) + __asm__ __volatile__ ( "lock; addq $0,128(%%rsp);" ::: "cc" ); + #elif defined(__i386) + __asm__ __volatile__ ( "lock; addl $0,128(%%esp);" ::: "cc" ); + #elif defined( __ARM_ARCH ) + __asm__ __volatile__ ( "DMB ISH" ::: ); + #else + #error unsupported architecture + #endif +} // Fence + +static inline __attribute__((always_inline)) void Pause( void ) { // pause to prevent excess processor bus usage + #if defined( __i386 ) || defined( __x86_64 ) + __asm__ __volatile__ ( "lfence" ); + #elif defined( __aarch64__ ) + __asm__ __volatile__ ( "DMB ISH" ::: ); + #else + #error unsupported architecture + #endif +} // Pause + +// Short Names and reference parameters. + +#define AtomicLdm( lock, memorder ) __atomic_load_n( (&(lock)), memorder ) +#define AtomicLd( lock ) AtomicLdm( lock, __ATOMIC_ACQUIRE ) +#define AtomicStrm( lock, value, memorder ) __atomic_store_n( (&(lock)), value, memorder ) +#define AtomicStr( lock, value ) AtomicStrm( lock, value, __ATOMIC_RELEASE ) +#define AtomicClrm( lock, memorder ) __atomic_clear( (&(lock)), memorder ) +#define AtomicClr( lock ) AtomicClrm( lock, __ATOMIC_RELEASE ) +#define AtomicTasm( lock, memorder ) __atomic_test_and_set( (&(lock)), memorder ) +#define AtomicTas( lock ) AtomicTasm( lock, __ATOMIC_ACQUIRE ) +#define AtomicCasm( change, comp, assn, smemorder, fmemorder ) ({TYPEOF(comp) __temp = (comp); __atomic_compare_exchange_n( (&(change)), (&(__temp)), (assn), false, smemorder, fmemorder ); }) +#define AtomicCas( change, comp, assn ) AtomicCasm( change, comp, assn, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST ) +#define AtomicCaswm( change, comp, assn, smemorder, fmemorder ) ({TYPEOF(comp) __temp = (comp); __atomic_compare_exchange_n( (&(change)), (&(__temp)), (assn), true, smemorder, fmemorder ); }) +#define AtomicCasw( change, comp, assn ) AtomicCaswm( change, comp, assn, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST ) +#define AtomicCasvm( change, comp, assn, smemorder, fmemorder ) __atomic_compare_exchange_n( (&(change)), (&(comp)), (assn), false, smemorder, fmemorder ) +#define AtomicCasv( change, comp, assn ) AtomicCasvm( change, comp, assn, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST ) +#define AtomicCasvwm( change, comp, assn, smemorder, fmemorder ) __atomic_compare_exchange_n( (&(change)), (&(comp)), (assn), true, smemorder, fmemorder ) +#define AtomicCasvw( change, comp, assn ) AtomicCasvwm( change, comp, assn, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST ) +#define AtomicFasm( change, assn, memorder ) __atomic_exchange_n( (&(change)), (assn), memorder ) +#define AtomicFas( change, assn ) AtomicFasm( change, assn, __ATOMIC_SEQ_CST ) +#define AtomicFaim( change, Inc, memorder ) __atomic_fetch_add( (&(change)), (Inc), memorder ) +#define AtomicFai( change, Inc ) AtomicFaim( change, Inc, __ATOMIC_SEQ_CST ) Index: libcfa/src/bits/defs.hfa =================================================================== --- libcfa/src/bits/defs.hfa (revision 3859adb4d802f6f27e35aa4ed05a7c87d7ef632e) +++ libcfa/src/bits/defs.hfa (revision 34d194c7be7287186dfb30719afe0c0c2ee07b08) @@ -13,6 +13,6 @@ // Created On : Thu Nov 9 13:24:10 2017 // Last Modified By : Peter A. Buhr -// Last Modified On : Mon Oct 27 22:40:05 2025 -// Update Count : 23 +// Last Modified On : Sat Jul 4 15:04:20 2026 +// Update Count : 26 // @@ -68,27 +68,3 @@ #endif -static inline long long int rdtscl(void) { - #if defined( __i386 ) || defined( __x86_64 ) - unsigned int lo, hi; - __asm__ __volatile__ ("rdtsc" : "=a"(lo), "=d"(hi)); - return ( (unsigned long long)lo)|( ((unsigned long long)hi)<<32 ); - #elif defined( __aarch64__ ) || defined( __arm__ ) - // https://github.com/google/benchmark/blob/v1.1.0/src/cycleclock.h#L116 - long long int virtual_timer_value; - asm volatile("mrs %0, cntvct_el0" : "=r"(virtual_timer_value)); - return virtual_timer_value; - #else - #error unsupported hardware architecture - #endif -} - -// pause to prevent excess processor bus usage -#if defined( __i386 ) || defined( __x86_64 ) - #define Pause() __asm__ __volatile__ ( "pause" : : : ) -#elif defined( __ARM_ARCH ) - #define Pause() __asm__ __volatile__ ( "YIELD" : : : ) -#else - #error unsupported architecture -#endif - #define CFA_IO_LAZY (1_l64u << 32_l64u) Index: libcfa/src/bits/locks.hfa =================================================================== --- libcfa/src/bits/locks.hfa (revision 3859adb4d802f6f27e35aa4ed05a7c87d7ef632e) +++ libcfa/src/bits/locks.hfa (revision 34d194c7be7287186dfb30719afe0c0c2ee07b08) @@ -13,6 +13,6 @@ // Created On : Tue Oct 31 15:14:38 2017 // Last Modified By : Peter A. Buhr -// Last Modified On : Tue Sep 20 22:09:50 2022 -// Update Count : 18 +// Last Modified On : Sat Jul 4 06:50:46 2026 +// Update Count : 21 // @@ -21,4 +21,5 @@ #include "bits/debug.hfa" #include "bits/defs.hfa" +#include "bits/atomic.hfa" #include Index: bcfa/src/concurrency/atomic.hfa =================================================================== --- libcfa/src/concurrency/atomic.hfa (revision 3859adb4d802f6f27e35aa4ed05a7c87d7ef632e) +++ (revision ) @@ -1,48 +1,0 @@ -// -// Cforall Version 1.0.0 Copyright (C) 2022 University of Waterloo -// -// The contents of this file are covered under the licence agreement in the -// file "LICENCE" distributed with Cforall. -// -// atomic.hfa -- simpler macros to access atomic instructions -// -// Author : Peter A. Buhr -// Created On : Thu May 25 15:22:46 2023 -// Last Modified By : Peter A. Buhr -// Last Modified On : Wed Jun 14 07:48:57 2023 -// Update Count : 52 -// - -#define LOAD( val ) (LOADM( val, __ATOMIC_SEQ_CST)) -#define LOADM( val, memorder ) (__atomic_load_n( &(val), memorder)) - -#define STORE( val, assn ) (STOREM( val, assn, __ATOMIC_SEQ_CST)) -#define STOREM( val, assn, memorder ) (__atomic_store_n( &(val), assn, memorder)) - -#define TAS( lock ) (TASM( lock, __ATOMIC_ACQUIRE)) -#define TASM( lock, memorder ) (__atomic_test_and_set( &(lock), memorder)) - -#define TASCLR( lock ) (TASCLRM( lock, __ATOMIC_RELEASE)) -#define TASCLRM( lock, memorder ) (__atomic_clear( &(lock), memorder)) - -#define FAS( assn, replace ) (FASM(assn, replace, __ATOMIC_SEQ_CST)) -#define FASM( assn, replace, memorder ) (__atomic_exchange_n( &(assn), (replace), memorder)) - -#define FAI( assn, Inc ) (__atomic_fetch_add( &(assn), (Inc), __ATOMIC_SEQ_CST)) -#define FAIM( assn, Inc, memorder ) (__atomic_fetch_add( &(assn), (Inc), memorder)) - -// Use __sync because __atomic with 128-bit CAA can result in calls to pthread_mutex_lock. - -// #define CAS( assn, comp, replace ) (CASM( assn, comp, replace, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) -// #define CASM( assn, comp, replace, memorder... ) ({ typeof(comp) __temp = (comp); __atomic_compare_exchange_n( &(assn), &(__temp), (replace), false, memorder ); }) -#define CAS( assn, comp, replace ) (__sync_bool_compare_and_swap( &assn, comp, replace)) -#define CASM( assn, comp, replace, memorder... ) _Static_assert( false, "memory order unsupported for CAS macro" ); - -// #define CASV( assn, comp, replace ) (__atomic_compare_exchange_n( &(assn), &(comp), (replace), false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST )) -// #define CASVM( assn, comp, replace, memorder... ) (__atomic_compare_exchange_n( &(assn), &(comp), (replace), false, memorder, memorder )) -#define CASV( assn, comp, replace ) ({ \ - typeof(comp) temp = comp; \ - typeof(comp) old = __sync_val_compare_and_swap( &(assn), (comp), (replace) ); \ - old == temp ? true : (comp = old, false); \ -}) -#define CASVM( assn, comp, replace, memorder... ) _Static_assert( false, "memory order unsupported for CASV macro" ); Index: libcfa/src/concurrency/kernel/fwd.hfa =================================================================== --- libcfa/src/concurrency/kernel/fwd.hfa (revision 3859adb4d802f6f27e35aa4ed05a7c87d7ef632e) +++ libcfa/src/concurrency/kernel/fwd.hfa (revision 34d194c7be7287186dfb30719afe0c0c2ee07b08) @@ -19,4 +19,5 @@ #include "bits/defs.hfa" #include "bits/debug.hfa" +#include "bits/atomic.hfa" // Pause #ifdef __cforall Index: libcfa/src/concurrency/locks.hfa =================================================================== --- libcfa/src/concurrency/locks.hfa (revision 3859adb4d802f6f27e35aa4ed05a7c87d7ef632e) +++ libcfa/src/concurrency/locks.hfa (revision 34d194c7be7287186dfb30719afe0c0c2ee07b08) @@ -11,6 +11,6 @@ // Created On : Thu Jan 21 19:46:50 2021 // Last Modified By : Peter A. Buhr -// Last Modified On : Sun Mar 29 22:46:39 2026 -// Update Count : 68 +// Last Modified On : Sat Jul 4 06:52:06 2026 +// Update Count : 70 // @@ -21,4 +21,5 @@ #include "bits/weakso_locks.hfa" +#include "bits/atomic.hfa" #include "collections/lockfree.hfa" #include "collections/list.hfa" Index: libcfa/src/heap.cfa =================================================================== --- libcfa/src/heap.cfa (revision 3859adb4d802f6f27e35aa4ed05a7c87d7ef632e) +++ libcfa/src/heap.cfa (revision 34d194c7be7287186dfb30719afe0c0c2ee07b08) @@ -10,6 +10,6 @@ // Created On : Tue Dec 19 21:58:35 2017 // Last Modified By : Peter A. Buhr -// Last Modified On : Thu May 15 16:47:35 2025 -// Update Count : 1647 +// Last Modified On : Sat Jul 4 16:17:06 2026 +// Update Count : 1658 // @@ -25,6 +25,5 @@ #include "heap.hfa" -#include "bits/align.hfa" // libAlign -#include "bits/defs.hfa" // likely, unlikely +#include "bits/align.hfa" // libAlign, Pause, likely, unlikely #include "concurrency/kernel/fwd.hfa" // disable_interrupts, enable_interrupts #include "startup.hfa" // STARTUP_PRIORITY_MEMORY @@ -129,14 +128,4 @@ } // Bsearchl - -// pause to prevent excess processor bus usage -#if defined( __i386 ) || defined( __x86_64 ) - #define Pause() __asm__ __volatile__ ( "pause" : : : ) -#elif defined(__ARM_ARCH) - #define Pause() __asm__ __volatile__ ( "YIELD" : : : ) -#else - #error unsupported architecture -#endif - typedef volatile uintptr_t SpinLock_t; @@ -146,5 +135,5 @@ for ( unsigned int i = 1;; i += 1 ) { - if ( slock == 0 && __atomic_test_and_set( &slock, __ATOMIC_ACQUIRE ) == 0 ) break; // Fence + if ( slock == 0 && AtomicTas( slock ) == 0 ) break; // Fence for ( volatile unsigned int s = 0; s < spin; s += 1 ) Pause(); // exponential spin spin += spin; // powers of 2 @@ -155,5 +144,5 @@ static inline __attribute__((always_inline)) void unlock( volatile SpinLock_t & slock ) { - __atomic_clear( &slock, __ATOMIC_RELEASE ); // Fence + AtomicClr( slock ); // Fence } // spin_unlock @@ -1003,5 +992,5 @@ unlock( freeHead->returnLock ); #else - block = __atomic_exchange_n( &freeHead->returnList, 0p, __ATOMIC_SEQ_CST ); + block = AtomicFas( freeHead->returnList, 0p ); #endif // RETURNSPIN @@ -1168,6 +1157,5 @@ header->kind.real.next = freeHead->returnList; // link new node to top node // CAS resets header->kind.real.next = freeHead->returnList on failure - while ( ! __atomic_compare_exchange_n( &freeHead->returnList, &header->kind.real.next, (Heap.Storage *)header, - false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST ) ); + while ( ! AtomicCasv( freeHead->returnList, header->kind.real.next, (Heap.Storage *)header ); #ifdef __STATISTICS__