Index: libcfa/src/heap.cfa =================================================================== --- libcfa/src/heap.cfa (revision 6dd40916453549a3eeb4345b08b5a630c3c3e45d) +++ libcfa/src/heap.cfa (revision 3e1cd17bcf7fa71f8d3d952f264d84a7aeae095d) @@ -10,6 +10,6 @@ // Created On : Tue Dec 19 21:58:35 2017 // Last Modified By : Peter A. Buhr -// Last Modified On : Wed Jan 3 21:30:54 2024 -// Update Count : 1619 +// Last Modified On : Sun Apr 7 21:54:29 2024 +// Update Count : 1644 // @@ -114,5 +114,5 @@ -// generic Bsearchl does not inline, so substitute with hand-coded binary-search. +// CFA generic Bsearchl does not inline, so substitute with hand-coded binary-search. inline __attribute__((always_inline)) static size_t Bsearchl( unsigned int key, const unsigned int vals[], size_t dim ) { @@ -190,5 +190,5 @@ unsigned int realloc_calls, realloc_0_calls; unsigned long long int realloc_storage_request, realloc_storage_alloc; - unsigned int free_calls, free_null_calls; + unsigned int free_calls, free_null_0_calls; unsigned long long int free_storage_request, free_storage_alloc; unsigned int return_pulls, return_pushes; @@ -232,17 +232,13 @@ struct Header { // header union Kind { - struct RealHeader { + struct RealHeader { // 4-byte word => 8-byte header, 8-byte word => 16-byte header union { - struct { // 4-byte word => 8-byte header, 8-byte word => 16-byte header - union { - // 2nd low-order bit => zero filled, 3rd low-order bit => mmapped - // FreeHeader * home; // allocated block points back to home locations (must overlay alignment) - void * home; // allocated block points back to home locations (must overlay alignment) - size_t blockSize; // size for munmap (must overlay alignment) - Storage * next; // freed block points to next freed block of same size - }; - size_t size; // allocation size in bytes - }; + // 2nd low-order bit => zero filled, 3rd low-order bit => mmapped + // FreeHeader * home; // allocated block points back to home locations (must overlay alignment) + void * home; // allocated block points back to home locations (must overlay alignment) + size_t blockSize; // size for munmap (must overlay alignment) + Storage * next; // freed block points to next freed block of same size }; + size_t size; // allocation size in bytes } real; // RealHeader @@ -261,5 +257,4 @@ struct CALIGN FreeHeader { - size_t blockSize CALIGN; // size of allocations on this list #ifdef OWNERSHIP #ifdef RETURNSPIN @@ -271,4 +266,5 @@ Storage * freeList; // thread free list Heap * homeManager; // heap owner (free storage to bucket, from bucket to heap) + size_t blockSize; // size of allocations on this list }; // FreeHeader @@ -369,5 +365,5 @@ static __thread size_t PAD1 CALIGN TLSMODEL __attribute__(( unused )); // protect false sharing static __thread Heap * heapManager CALIGN TLSMODEL; -static __thread bool heapManagerBootFlag CALIGN TLSMODEL = false; +static __thread bool heapManagerBootFlag CALIGN TLSMODEL = false; static __thread size_t PAD2 CALIGN TLSMODEL __attribute__(( unused )); // protect further false sharing @@ -595,6 +591,4 @@ #ifdef __STATISTICS__ -static HeapStatistics stats; // zero filled - #define prtFmt \ "\nHeap statistics: (storage request / allocation)\n" \ @@ -607,5 +601,5 @@ " resize >0 calls %'u; 0 calls %'u; storage %'llu / %'llu bytes\n" \ " realloc >0 calls %'u; 0 calls %'u; storage %'llu / %'llu bytes\n" \ - " free !null calls %'u; null calls %'u; storage %'llu / %'llu bytes\n" \ + " free !null calls %'u; null/0 calls %'u; storage %'llu / %'llu bytes\n" \ " return pulls %'u; pushes %'u; storage %'llu / %'llu bytes\n" \ " sbrk calls %'u; storage %'llu bytes\n" \ @@ -627,5 +621,5 @@ resize_calls, resize_0_calls, resize_storage_request, resize_storage_alloc, realloc_calls, realloc_0_calls, realloc_storage_request, realloc_storage_alloc, - free_calls, free_null_calls, free_storage_request, free_storage_alloc, + free_calls, free_null_0_calls, free_storage_request, free_storage_alloc, return_pulls, return_pushes, return_storage_request, return_storage_alloc, sbrk_calls, sbrk_storage, @@ -650,5 +644,5 @@ "0 count=\"%'u;\" 0 count=\"%'u;\" size=\"%'llu / %'llu\"/> bytes\n" \ "0 count=\"%'u;\" 0 count=\"%'u;\" size=\"%'llu / %'llu\"/> bytes\n" \ - " bytes\n" \ + " bytes\n" \ " bytes\n" \ " bytes\n" \ @@ -670,5 +664,5 @@ resize_calls, resize_0_calls, resize_storage_request, resize_storage_alloc, realloc_calls, realloc_0_calls, realloc_storage_request, realloc_storage_alloc, - free_calls, free_null_calls, free_storage_request, free_storage_alloc, + free_calls, free_null_0_calls, free_storage_request, free_storage_alloc, return_pulls, return_pushes, return_storage_request, return_storage_alloc, sbrk_calls, sbrk_storage, @@ -799,7 +793,8 @@ } else { fakeHeader( header, alignment ); - if ( unlikely( MmappedBit( header ) ) ) { // mmapped ? + if ( unlikely( MmappedBit( header ) ) ) { // mmapped storage ? verify( addr < heapBegin || heapEnd < addr ); size = ClearStickyBits( header->kind.real.blockSize ); // mmap size + freeHead = 0p; // prevent uninitialized warning return true; } // if @@ -904,14 +899,9 @@ -#define BOOT_HEAP_MANAGER \ - if ( unlikely( ! heapMasterBootFlag ) ) { \ - heapManagerCtor(); /* trigger for first heap */ \ - } /* if */ - #ifdef __STATISTICS__ #define STAT_NAME __counter #define STAT_PARM , unsigned int STAT_NAME #define STAT_ARG( name ) , name -#define STAT_0_CNT( counter ) stats.counters[counter].calls_0 += 1 +#define STAT_0_CNT( counter ) heapManager->stats.counters[counter].calls_0 += 1 #else #define STAT_NAME @@ -921,22 +911,36 @@ #endif // __STATISTICS__ -// Uncomment to get allocation addresses for a 0-sized allocation rather than a null pointer. -//#define __NONNULL_0_ALLOC__ -#if ! defined( __NONNULL_0_ALLOC__ ) -#define __NULL_0_ALLOC__ unlikely( size == 0 ) || /* 0 BYTE ALLOCATION RETURNS NULL POINTER */ +#define BOOT_HEAP_MANAGER \ + if ( unlikely( ! heapMasterBootFlag ) ) { \ + heapManagerCtor(); /* trigger for first heap */ \ + } /* if */ \ + verify( heapManager ); + +#define __NONNULL_0_ALLOC__ /* Uncomment to return non-null address for malloc( 0 ). */ +#ifndef __NONNULL_0_ALLOC__ +#define __NULL_0_ALLOC__( counter, ... ) /* 0 byte allocation returns null pointer */ \ + if ( unlikely( size == 0 ) ) { \ + STAT_0_CNT( counter ); \ + __VA_ARGS__; /* call routine, if specified */ \ + return 0p; \ + } /* if */ #else -#define __NULL_0_ALLOC__ +#define __NULL_0_ALLOC__( counter, ... ) #endif // __NONNULL_0_ALLOC__ + +#ifdef __DEBUG__ +#define __OVERFLOW_MALLOC__( ... ) \ + if ( unlikely( size > ULONG_MAX - sizeof(Heap.Storage) ) ) { /* error check */ \ + __VA_ARGS__; /* call routine, if specified */ \ + return 0p; \ + } /* if */ +#else +#define __OVERFLOW_MALLOC__( ... ) +#endif // __DEBUG__ #define PROLOG( counter, ... ) \ BOOT_HEAP_MANAGER; \ - if ( \ - __NULL_0_ALLOC__ \ - unlikely( size > ULONG_MAX - sizeof(Heap.Storage) ) ) { /* error check */ \ - STAT_0_CNT( counter ); \ - __VA_ARGS__; \ - return 0p; \ - } /* if */ - + __NULL_0_ALLOC__( counter, __VA_ARGS__ ) \ + __OVERFLOW_MALLOC__( __VA_ARGS__ ) #define SCRUB_SIZE 1024lu @@ -948,5 +952,4 @@ PROLOG( STAT_NAME ); - verify( heapManager ); Heap.Storage * block; // pointer to new block of storage @@ -956,5 +959,10 @@ #ifdef __STATISTICS__ - stats.counters[STAT_NAME].calls += 1; + #ifdef __NONNULL_0_ALLOC__ + if ( unlikely( size == 0 ) ) // malloc( 0 ) ? + stats.counters[STAT_NAME].calls_0 += 1; + else + #endif // __NONNULL_0_ALLOC__ + stats.counters[STAT_NAME].calls += 1; stats.counters[STAT_NAME].request += size; #endif // __STATISTICS__ @@ -1078,4 +1086,8 @@ static void doFree( void * addr ) libcfa_nopreempt with( *heapManager ) { + // char buf[64]; + // int len = sprintf( buf, "doFree addr %p\n", addr ); + // write( 2, buf, len ); + verify( addr ); @@ -1091,5 +1103,12 @@ #endif // __STATISTICS__ || __CFA_DEBUG__ + // Do not move these down because heap can be null! #ifdef __STATISTICS__ + #ifdef __NONNULL_0_ALLOC__ + if ( unlikely( rsize == 0 ) ) // malloc( 0 ) ? + stats.free_null_0_calls += 1; + else + #endif // __NONNULL_0_ALLOC__ + heapManager->stats.free_calls += 1; // count free amd implicit frees from resize/realloc stats.free_storage_request += rsize; stats.free_storage_alloc += size; @@ -1117,4 +1136,5 @@ } // if } else { + assert( freeHead ); #ifdef __CFA_DEBUG__ // memset is NOT always inlined! @@ -1321,5 +1341,5 @@ // call to malloc(), alloc(), calloc() or realloc(). If the area pointed to was moved, a free(oaddr) is done. void * resize( void * oaddr, size_t size ) libcfa_public { - if ( unlikely( oaddr == 0p ) ) { // => malloc( size ) + if ( unlikely( oaddr == 0p ) ) { // => malloc( size ) return doMalloc( size STAT_ARG( RESIZE ) ); } // if @@ -1333,5 +1353,5 @@ size_t odsize = DataStorage( bsize, oaddr, header ); // data storage available in bucket - // same size, DO NOT preserve STICKY PROPERTIES. + // same size, DO NOT PRESERVE STICKY PROPERTIES. if ( oalign == libAlign() && size <= odsize && odsize <= size * 2 ) { // allow 50% wasted storage for smaller size ClearZeroFillBit( header ); // no alignment and turn off 0 fill @@ -1346,5 +1366,5 @@ } // if - // change size, DO NOT preserve STICKY PROPERTIES. + // change size, DO NOT PRESERVE STICKY PROPERTIES. doFree( oaddr ); // free previous storage @@ -1356,4 +1376,7 @@ // the old and new sizes. void * realloc( void * oaddr, size_t size ) libcfa_public { + // char buf[64]; + // int len = sprintf( buf, "realloc1 oaddr %p size %d\n", oaddr, size ); + // write( 2, buf, len ); if ( unlikely( oaddr == 0p ) ) { // => malloc( size ) return doMalloc( size STAT_ARG( REALLOC ) ); @@ -1498,8 +1521,4 @@ } // if - #ifdef __STATISTICS__ - incCalls( FREE ); - #endif // __STATISTICS__ - doFree( addr ); // handles heapManager == nullptr } // free @@ -1579,5 +1598,4 @@ #endif // __STATISTICS__ } // malloc_stats_clear - // Changes the file descriptor where malloc_stats() writes statistics. @@ -1700,5 +1718,5 @@ } // if - // change size, DO NOT preserve STICKY PROPERTIES. + // change size, DO NOT PRESERVE STICKY PROPERTIES. doFree( oaddr ); // free previous storage return memalignNoStats( nalign, size STAT_ARG( RESIZE ) ); // create new aligned area @@ -1707,4 +1725,7 @@ void * realloc( void * oaddr, size_t nalign, size_t size ) libcfa_public { + // char buf[64]; + // int len = sprintf( buf, "realloc2 oaddr %p size %d\n", oaddr, size ); + // write( 2, buf, len ); if ( unlikely( oaddr == 0p ) ) { // => malloc( size ) return memalignNoStats( nalign, size STAT_ARG( REALLOC ) ); Index: libcfa/src/heap.hfa =================================================================== --- libcfa/src/heap.hfa (revision 6dd40916453549a3eeb4345b08b5a630c3c3e45d) +++ libcfa/src/heap.hfa (revision 3e1cd17bcf7fa71f8d3d952f264d84a7aeae095d) @@ -10,6 +10,6 @@ // Created On : Tue May 26 11:23:55 2020 // Last Modified By : Peter A. Buhr -// Last Modified On : Mon Sep 11 11:18:18 2023 -// Update Count : 24 +// Last Modified On : Mon Apr 1 09:36:20 2024 +// Update Count : 25 // @@ -46,4 +46,5 @@ } // extern "C" +// New allocation operations. void * resize( void * oaddr, size_t alignment, size_t size ); void * realloc( void * oaddr, size_t alignment, size_t size ); @@ -51,5 +52,4 @@ // Local Variables: // -// mode: c // // tab-width: 4 // // End: //