Index: libcfa/src/heap.cfa =================================================================== --- libcfa/src/heap.cfa (revision b797d97858476d1dd4df1a393b90d5f3ce4251f1) +++ libcfa/src/heap.cfa (revision 8ee54963b7cbf86746000c2fe2fbfa452b4eeb7b) @@ -10,6 +10,6 @@ // Created On : Tue Dec 19 21:58:35 2017 // Last Modified By : Peter A. Buhr -// Last Modified On : Sun Oct 30 20:56:20 2022 -// Update Count : 1584 +// Last Modified On : Wed Dec 28 12:37:38 2022 +// Update Count : 1597 // @@ -17,8 +17,6 @@ #include // memset, memcpy #include // ULONG_MAX -#include // EXIT_FAILURE #include // errno, ENOMEM, EINVAL -#include // STDERR_FILENO, sbrk, sysconf -#include // memalign, malloc_usable_size +#include // STDERR_FILENO, sbrk, sysconf, write #include // mmap, munmap extern "C" { @@ -26,4 +24,5 @@ } // extern "C" +#include "heap.hfa" #include "bits/align.hfa" // libAlign #include "bits/defs.hfa" // likely, unlikely @@ -140,5 +139,5 @@ #endif -typedef volatile uintptr_t SpinLock_t CALIGN; // aligned addressable word-size +typedef volatile uintptr_t SpinLock_t; static inline __attribute__((always_inline)) void lock( volatile SpinLock_t & slock ) { @@ -147,5 +146,5 @@ for ( unsigned int i = 1;; i += 1 ) { - if ( slock == 0 && __atomic_test_and_set( &slock, __ATOMIC_SEQ_CST ) == 0 ) break; // Fence + if ( slock == 0 && __atomic_test_and_set( &slock, __ATOMIC_ACQUIRE ) == 0 ) break; // Fence for ( volatile unsigned int s = 0; s < spin; s += 1 ) Pause(); // exponential spin spin += spin; // powers of 2 @@ -156,5 +155,5 @@ static inline __attribute__((always_inline)) void unlock( volatile SpinLock_t & slock ) { - __atomic_clear( &slock, __ATOMIC_SEQ_CST ); // Fence + __atomic_clear( &slock, __ATOMIC_RELEASE ); // Fence } // spin_unlock @@ -261,6 +260,6 @@ static_assert( libAlign() >= sizeof( Storage ), "minimum alignment < sizeof( Storage )" ); - struct __attribute__(( aligned (8) )) FreeHeader { - size_t blockSize __attribute__(( aligned(8) )); // size of allocations on this list + struct CALIGN FreeHeader { + size_t blockSize CALIGN; // size of allocations on this list #ifdef OWNERSHIP #ifdef RETURNSPIN @@ -284,5 +283,5 @@ #ifdef __CFA_DEBUG__ - int64_t allocUnfreed; // running total of allocations minus frees; can be negative + ptrdiff_t allocUnfreed; // running total of allocations minus frees; can be negative #endif // __CFA_DEBUG__ @@ -369,5 +368,5 @@ // Thread-local storage is allocated lazily when the storage is accessed. static __thread size_t PAD1 CALIGN TLSMODEL __attribute__(( unused )); // protect false sharing -static __thread Heap * volatile heapManager CALIGN TLSMODEL; +static __thread Heap * heapManager CALIGN TLSMODEL; static __thread size_t PAD2 CALIGN TLSMODEL __attribute__(( unused )); // protect further false sharing @@ -443,5 +442,5 @@ // 12K ~= 120K byte superblock. Where 128-heap superblock handles a medium sized multi-processor server. size_t remaining = heapManagersStorageEnd - heapManagersStorage; // remaining free heaps in superblock - if ( ! heapManagersStorage || remaining != 0 ) { + if ( ! heapManagersStorage || remaining == 0 ) { // Each block of heaps is a multiple of the number of cores on the computer. int HeapDim = get_nprocs(); // get_nprocs_conf does not work @@ -562,5 +561,5 @@ // allocUnfreed is set to 0 when a heap is created and it accumulates any unfreed storage during its multiple thread // usages. At the end, add up each heap allocUnfreed value across all heaps to get the total unfreed storage. - int64_t allocUnfreed = 0; + ptrdiff_t allocUnfreed = 0; for ( Heap * heap = heapMaster.heapManagersList; heap; heap = heap->nextHeapManager ) { allocUnfreed += heap->allocUnfreed; @@ -572,5 +571,5 @@ char helpText[512]; __cfaabi_bits_print_buffer( STDERR_FILENO, helpText, sizeof(helpText), - "CFA warning (UNIX pid:%ld) : program terminating with %ju(0x%jx) bytes of storage allocated but not freed.\n" + "CFA warning (UNIX pid:%ld) : program terminating with %td(%#tx) bytes of storage allocated but not freed.\n" "Possible cause is unfreed storage allocated by the program or system/library routines called from the program.\n", (long int)getpid(), allocUnfreed, allocUnfreed ); // always print the UNIX pid @@ -950,5 +949,5 @@ block = freeHead->freeList; // remove node from stack if ( unlikely( block == 0p ) ) { // no free block ? - // Freelist for this size is empty, so check return list (OWNERSHIP), carve it out of the heap, if there + // Freelist for this size is empty, so check return list (OWNERSHIP), or carve it out of the heap if there // is enough left, or get some more heap storage and carve it off. #ifdef OWNERSHIP @@ -1115,4 +1114,10 @@ while ( ! __atomic_compare_exchange_n( &freeHead->returnList, &header->kind.real.next, (Heap.Storage *)header, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST ) ); + + #ifdef __STATISTICS__ + stats.return_pushes += 1; + stats.return_storage_request += rsize; + stats.return_storage_alloc += size; + #endif // __STATISTICS__ #endif // RETURNSPIN } // if @@ -1125,10 +1130,4 @@ freeHead->freeList = (Heap.Storage *)header; #endif // ! OWNERSHIP - - #ifdef __U_STATISTICS__ - stats.return_pushes += 1; - stats.return_storage_request += rsize; - stats.return_storage_alloc += size; - #endif // __U_STATISTICS__ // OK TO BE PREEMPTED HERE AS heapManager IS NO LONGER ACCESSED. @@ -1180,9 +1179,9 @@ #ifdef __STATISTICS__ -static void incCalls( intptr_t statName ) libcfa_nopreempt { +static void incCalls( size_t statName ) libcfa_nopreempt { heapManager->stats.counters[statName].calls += 1; } // incCalls -static void incZeroCalls( intptr_t statName ) libcfa_nopreempt { +static void incZeroCalls( size_t statName ) libcfa_nopreempt { heapManager->stats.counters[statName].calls_0 += 1; } // incZeroCalls @@ -1456,6 +1455,4 @@ // 0p, no operation is performed. void free( void * addr ) libcfa_public { -// verify( heapManager ); - if ( unlikely( addr == 0p ) ) { // special case #ifdef __STATISTICS__