Index: libcfa/src/heap.cfa =================================================================== --- libcfa/src/heap.cfa (revision 09da82dda65553a9f7ddeef8893821d2a45a2a87) +++ libcfa/src/heap.cfa (revision 709b8121c9e433e178ba45e3c4c29555a916ced2) @@ -10,6 +10,6 @@ // Created On : Tue Dec 19 21:58:35 2017 // Last Modified By : Peter A. Buhr -// Last Modified On : Wed Dec 16 12:28:25 2020 -// Update Count : 1023 +// Last Modified On : Sun Jan 10 11:20:49 2021 +// Update Count : 1031 // @@ -262,21 +262,21 @@ #ifdef __STATISTICS__ // Heap statistics counters. -static unsigned int malloc_calls; +static unsigned int malloc_zero_calls, malloc_calls; static unsigned long long int malloc_storage; -static unsigned int aalloc_calls; +static unsigned int aalloc_zero_calls, aalloc_calls; static unsigned long long int aalloc_storage; -static unsigned int calloc_calls; +static unsigned int calloc_zero_calls, calloc_calls; static unsigned long long int calloc_storage; -static unsigned int memalign_calls; +static unsigned int memalign_zero_calls, memalign_calls; static unsigned long long int memalign_storage; -static unsigned int amemalign_calls; +static unsigned int amemalign_zero_calls, amemalign_calls; static unsigned long long int amemalign_storage; -static unsigned int cmemalign_calls; +static unsigned int cmemalign_zero_calls, cmemalign_calls; static unsigned long long int cmemalign_storage; -static unsigned int resize_calls; +static unsigned int resize_zero_calls, resize_calls; static unsigned long long int resize_storage; -static unsigned int realloc_calls; +static unsigned int realloc_zero_calls, realloc_calls; static unsigned long long int realloc_storage; -static unsigned int free_calls; +static unsigned int free_zero_calls, free_calls; static unsigned long long int free_storage; static unsigned int mmap_calls; @@ -287,5 +287,5 @@ static unsigned long long int sbrk_storage; // Statistics file descriptor (changed by malloc_stats_fd). -static int stat_fd = STDERR_FILENO; // default stderr +static int stats_fd = STDERR_FILENO; // default stderr // Use "write" because streams may be shutdown when calls are made. @@ -293,29 +293,29 @@ char helpText[1024]; __cfaabi_bits_print_buffer( STDERR_FILENO, helpText, sizeof(helpText), - "\nHeap statistics:\n" - " malloc: calls %u / storage %llu\n" - " aalloc: calls %u / storage %llu\n" - " calloc: calls %u / storage %llu\n" - " memalign: calls %u / storage %llu\n" - " amemalign: calls %u / storage %llu\n" - " cmemalign: calls %u / storage %llu\n" - " resize: calls %u / storage %llu\n" - " realloc: calls %u / storage %llu\n" - " free: calls %u / storage %llu\n" - " mmap: calls %u / storage %llu\n" - " munmap: calls %u / storage %llu\n" - " sbrk: calls %u / storage %llu\n", - malloc_calls, malloc_storage, - aalloc_calls, aalloc_storage, - calloc_calls, calloc_storage, - memalign_calls, memalign_storage, - amemalign_calls, amemalign_storage, - cmemalign_calls, cmemalign_storage, - resize_calls, resize_storage, - realloc_calls, realloc_storage, - free_calls, free_storage, - mmap_calls, mmap_storage, - munmap_calls, munmap_storage, - sbrk_calls, sbrk_storage + "\nHeap statistics:\n" + " malloc 0-calls %'u; >0-calls %'u; storage %'llu bytes\n" + " aalloc 0-calls %'u; >0-calls %'u; storage %'llu bytes\n" + " calloc 0-calls %'u; >0-calls %'u; storage %'llu bytes\n" + " memalign 0-calls %'u; >0-calls %'u; storage %'llu bytes\n" + " amemalign 0-calls %'u; >0-calls %'u; storage %'llu bytes\n" + " cmemalign 0-calls %'u; >0-calls %'u; storage %'llu bytes\n" + " resize 0-calls %'u; >0-calls %'u; storage %'llu bytes\n" + " realloc 0-calls %'u; >0-calls %'u; storage %'llu bytes\n" + " free 0-calls %'u; >0-calls %'u; storage %'llu bytes\n" + " mmap calls %'u; storage %'llu bytes\n" + " munmap calls %'u; storage %'llu bytes\n" + " sbrk calls %'u; storage %'llu bytes\n", + malloc_zero_calls, malloc_calls, malloc_storage, + aalloc_zero_calls, aalloc_calls, aalloc_storage, + calloc_zero_calls, calloc_calls, calloc_storage, + memalign_zero_calls, memalign_calls, memalign_storage, + amemalign_zero_calls, amemalign_calls, amemalign_storage, + cmemalign_zero_calls, cmemalign_calls, cmemalign_storage, + resize_zero_calls, resize_calls, resize_storage, + realloc_zero_calls, realloc_calls, realloc_storage, + free_zero_calls, free_calls, free_storage, + mmap_calls, mmap_storage, + munmap_calls, munmap_storage, + sbrk_calls, sbrk_storage ); } // printStats @@ -328,26 +328,26 @@ "\n" "\n" - "\n" - "\n" - "\n" - "\n" - "\n" - "\n" - "\n" - "\n" - "\n" - "\n" - "\n" - "\n" + "0 count=\"%'u;\" size=\"%'llu\"/> bytes\n" + "0 count=\"%'u;\" size=\"%'llu\"/> bytes\n" + "0 count=\"%'u;\" size=\"%'llu\"/> bytes\n" + "0 count=\"%'u;\" size=\"%'llu\"/> bytes\n" + "0 count=\"%'u;\" size=\"%'llu\"/> bytes\n" + "0 count=\"%'u;\" size=\"%'llu\"/> bytes\n" + "0 count=\"%'u;\" size=\"%'llu\"/> bytes\n" + "0 count=\"%'u;\" size=\"%'llu\"/> bytes\n" + "0 count=\"%'u;\" size=\"%'llu\"/> bytes\n" + " bytes\n" + " bytes\n" + " bytes\n" "", - malloc_calls, malloc_storage, - aalloc_calls, aalloc_storage, - calloc_calls, calloc_storage, - memalign_calls, memalign_storage, - amemalign_calls, amemalign_storage, - cmemalign_calls, cmemalign_storage, - resize_calls, resize_storage, - realloc_calls, realloc_storage, - free_calls, free_storage, + malloc_zero_calls, malloc_calls, malloc_storage, + aalloc_zero_calls, aalloc_calls, aalloc_storage, + calloc_zero_calls, calloc_calls, calloc_storage, + memalign_zero_calls, memalign_calls, memalign_storage, + amemalign_zero_calls, amemalign_calls, amemalign_storage, + cmemalign_zero_calls, cmemalign_calls, cmemalign_storage, + resize_zero_calls, resize_calls, resize_storage, + realloc_zero_calls, realloc_calls, realloc_storage, + free_zero_calls, free_calls, free_storage, mmap_calls, mmap_storage, munmap_calls, munmap_storage, @@ -466,21 +466,21 @@ } // headers -#ifdef __CFA_DEBUG__ -#if __SIZEOF_POINTER__ == 4 -#define MASK 0xdeadbeef -#else -#define MASK 0xdeadbeefdeadbeef -#endif -#define STRIDE size_t - -static void * Memset( void * addr, STRIDE size ) { // debug only - if ( size % sizeof(STRIDE) != 0 ) abort( "Memset() : internal error, size %zd not multiple of %zd.", size, sizeof(STRIDE) ); - if ( (STRIDE)addr % sizeof(STRIDE) != 0 ) abort( "Memset() : internal error, addr %p not multiple of %zd.", addr, sizeof(STRIDE) ); - - STRIDE * end = (STRIDE *)addr + size / sizeof(STRIDE); - for ( STRIDE * p = (STRIDE *)addr; p < end; p += 1 ) *p = MASK; - return addr; -} // Memset -#endif // __CFA_DEBUG__ +// #ifdef __CFA_DEBUG__ +// #if __SIZEOF_POINTER__ == 4 +// #define MASK 0xdeadbeef +// #else +// #define MASK 0xdeadbeefdeadbeef +// #endif +// #define STRIDE size_t + +// static void * Memset( void * addr, STRIDE size ) { // debug only +// if ( size % sizeof(STRIDE) != 0 ) abort( "Memset() : internal error, size %zd not multiple of %zd.", size, sizeof(STRIDE) ); +// if ( (STRIDE)addr % sizeof(STRIDE) != 0 ) abort( "Memset() : internal error, addr %p not multiple of %zd.", addr, sizeof(STRIDE) ); + +// STRIDE * end = (STRIDE *)addr + size / sizeof(STRIDE); +// for ( STRIDE * p = (STRIDE *)addr; p < end; p += 1 ) *p = MASK; +// return addr; +// } // Memset +// #endif // __CFA_DEBUG__ @@ -498,6 +498,7 @@ unlock( extlock ); __cfaabi_bits_print_nolock( STDERR_FILENO, NO_MEMORY_MSG, size ); - _exit( EXIT_FAILURE ); - } // if + _exit( EXIT_FAILURE ); // give up + } // if + // Make storage executable for thunks. if ( mprotect( (char *)heapEnd + heapRemaining, increase, __map_prot ) ) { unlock( extlock ); @@ -770,30 +771,4 @@ -static inline void * callocNoStats( size_t dim, size_t elemSize ) { - size_t size = dim * elemSize; - if ( unlikely( size ) == 0 ) return 0p; // 0 BYTE ALLOCATION RETURNS NULL POINTER - char * addr = (char *)mallocNoStats( size ); - - HeapManager.Storage.Header * header; - HeapManager.FreeHeader * freeElem; - size_t bsize, alignment; - #ifndef __CFA_DEBUG__ - bool mapped = - #endif // __CFA_DEBUG__ - headers( "calloc", addr, header, freeElem, bsize, alignment ); - #ifndef __CFA_DEBUG__ - - // Mapped storage is zero filled, but in debug mode mapped memory is scrubbed in doMalloc, so it has to be reset to zero. - if ( ! mapped ) - #endif // __CFA_DEBUG__ - // <-------0000000000000000000000000000UUUUUUUUUUUUUUUUUUUUUUUUU> bsize (bucket size) U => undefined - // `-header`-addr `-size - memset( addr, '\0', size ); // set to zeros - - header->kind.real.blockSize |= 2; // mark as zero filled - return addr; -} // callocNoStats - - static inline void * memalignNoStats( size_t alignment, size_t size ) { if ( unlikely( size ) == 0 ) return 0p; // 0 BYTE ALLOCATION RETURNS NULL POINTER @@ -834,30 +809,4 @@ -static inline void * cmemalignNoStats( size_t alignment, size_t dim, size_t elemSize ) { - size_t size = dim * elemSize; - if ( unlikely( size ) == 0 ) return 0p; // 0 BYTE ALLOCATION RETURNS NULL POINTER - char * addr = (char *)memalignNoStats( alignment, size ); - - HeapManager.Storage.Header * header; - HeapManager.FreeHeader * freeElem; - size_t bsize; - #ifndef __CFA_DEBUG__ - bool mapped = - #endif // __CFA_DEBUG__ - headers( "cmemalign", addr, header, freeElem, bsize, alignment ); - - // Mapped storage is zero filled, but in debug mode mapped memory is scrubbed in doMalloc, so it has to be reset to zero. - #ifndef __CFA_DEBUG__ - if ( ! mapped ) - #endif // __CFA_DEBUG__ - // <-------0000000000000000000000000000UUUUUUUUUUUUUUUUUUUUUUUUU> bsize (bucket size) U => undefined - // `-header`-addr `-size - memset( addr, '\0', size ); // set to zeros - - header->kind.real.blockSize |= 2; // mark as zero filled - return addr; -} // cmemalignNoStats - - extern "C" { // Allocates size bytes and returns a pointer to the allocated memory. The contents are undefined. If size is 0, @@ -865,6 +814,10 @@ void * malloc( size_t size ) { #ifdef __STATISTICS__ - __atomic_add_fetch( &malloc_calls, 1, __ATOMIC_SEQ_CST ); - __atomic_add_fetch( &malloc_storage, size, __ATOMIC_SEQ_CST ); + if ( likely( size > 0 ) ) { + __atomic_add_fetch( &malloc_calls, 1, __ATOMIC_SEQ_CST ); + __atomic_add_fetch( &malloc_storage, size, __ATOMIC_SEQ_CST ); + } else { + __atomic_add_fetch( &malloc_zero_calls, 1, __ATOMIC_SEQ_CST ); + } // if #endif // __STATISTICS__ @@ -877,6 +830,10 @@ size_t size = dim * elemSize; #ifdef __STATISTICS__ - __atomic_add_fetch( &aalloc_calls, 1, __ATOMIC_SEQ_CST ); - __atomic_add_fetch( &aalloc_storage, size, __ATOMIC_SEQ_CST ); + if ( likely( size > 0 ) ) { + __atomic_add_fetch( &aalloc_calls, 1, __ATOMIC_SEQ_CST ); + __atomic_add_fetch( &aalloc_storage, size, __ATOMIC_SEQ_CST ); + } else { + __atomic_add_fetch( &aalloc_zero_calls, 1, __ATOMIC_SEQ_CST ); + } // if #endif // __STATISTICS__ @@ -887,4 +844,11 @@ // Same as aalloc() with memory set to zero. void * calloc( size_t dim, size_t elemSize ) { + size_t size = dim * elemSize; + if ( unlikely( size ) == 0 ) { // 0 BYTE ALLOCATION RETURNS NULL POINTER + #ifdef __STATISTICS__ + __atomic_add_fetch( &calloc_zero_calls, 1, __ATOMIC_SEQ_CST ); + #endif // __STATISTICS__ + return 0p; + } // if #ifdef __STATISTICS__ __atomic_add_fetch( &calloc_calls, 1, __ATOMIC_SEQ_CST ); @@ -892,5 +856,25 @@ #endif // __STATISTICS__ - return callocNoStats( dim, elemSize ); + char * addr = (char *)mallocNoStats( size ); + + HeapManager.Storage.Header * header; + HeapManager.FreeHeader * freeElem; + size_t bsize, alignment; + + #ifndef __CFA_DEBUG__ + bool mapped = + #endif // __CFA_DEBUG__ + headers( "calloc", addr, header, freeElem, bsize, alignment ); + + #ifndef __CFA_DEBUG__ + // Mapped storage is zero filled, but in debug mode mapped memory is scrubbed in doMalloc, so it has to be reset to zero. + if ( ! mapped ) + #endif // __CFA_DEBUG__ + // <-------0000000000000000000000000000UUUUUUUUUUUUUUUUUUUUUUUUU> bsize (bucket size) U => undefined + // `-header`-addr `-size + memset( addr, '\0', size ); // set to zeros + + header->kind.real.blockSize |= 2; // mark as zero filled + return addr; } // calloc @@ -901,10 +885,16 @@ // call to malloc(), alloc(), calloc() or realloc(). If the area pointed to was moved, a free(oaddr) is done. void * resize( void * oaddr, size_t size ) { + // If size is equal to 0, either NULL or a pointer suitable to be passed to free() is returned. + if ( unlikely( size == 0 ) ) { // special cases + #ifdef __STATISTICS__ + __atomic_add_fetch( &resize_zero_calls, 1, __ATOMIC_SEQ_CST ); + #endif // __STATISTICS__ + free( oaddr ); + return 0p; + } // if #ifdef __STATISTICS__ __atomic_add_fetch( &resize_calls, 1, __ATOMIC_SEQ_CST ); #endif // __STATISTICS__ - // If size is equal to 0, either NULL or a pointer suitable to be passed to free() is returned. - if ( unlikely( size == 0 ) ) { free( oaddr ); return 0p; } // special cases if ( unlikely( oaddr == 0p ) ) { #ifdef __STATISTICS__ @@ -918,6 +908,6 @@ size_t bsize, oalign; headers( "resize", oaddr, header, freeElem, bsize, oalign ); + size_t odsize = dataStorage( bsize, oaddr, header ); // data storage available in bucket - // same size, DO NOT preserve STICKY PROPERTIES. if ( oalign == libAlign() && size <= odsize && odsize <= size * 2 ) { // allow 50% wasted storage for smaller size @@ -940,10 +930,16 @@ // the old and new sizes. void * realloc( void * oaddr, size_t size ) { + // If size is equal to 0, either NULL or a pointer suitable to be passed to free() is returned. + if ( unlikely( size == 0 ) ) { // special cases + #ifdef __STATISTICS__ + __atomic_add_fetch( &realloc_zero_calls, 1, __ATOMIC_SEQ_CST ); + #endif // __STATISTICS__ + free( oaddr ); + return 0p; + } // if #ifdef __STATISTICS__ __atomic_add_fetch( &realloc_calls, 1, __ATOMIC_SEQ_CST ); #endif // __STATISTICS__ - // If size is equal to 0, either NULL or a pointer suitable to be passed to free() is returned. - if ( unlikely( size == 0 ) ) { free( oaddr ); return 0p; } // special cases if ( unlikely( oaddr == 0p ) ) { #ifdef __STATISTICS__ @@ -999,6 +995,10 @@ void * memalign( size_t alignment, size_t size ) { #ifdef __STATISTICS__ - __atomic_add_fetch( &memalign_calls, 1, __ATOMIC_SEQ_CST ); - __atomic_add_fetch( &memalign_storage, size, __ATOMIC_SEQ_CST ); + if ( likely( size > 0 ) ) { + __atomic_add_fetch( &memalign_calls, 1, __ATOMIC_SEQ_CST ); + __atomic_add_fetch( &memalign_storage, size, __ATOMIC_SEQ_CST ); + } else { + __atomic_add_fetch( &memalign_zero_calls, 1, __ATOMIC_SEQ_CST ); + } // if #endif // __STATISTICS__ @@ -1011,6 +1011,10 @@ size_t size = dim * elemSize; #ifdef __STATISTICS__ - __atomic_add_fetch( &cmemalign_calls, 1, __ATOMIC_SEQ_CST ); - __atomic_add_fetch( &cmemalign_storage, size, __ATOMIC_SEQ_CST ); + if ( likely( size > 0 ) ) { + __atomic_add_fetch( &cmemalign_calls, 1, __ATOMIC_SEQ_CST ); + __atomic_add_fetch( &cmemalign_storage, size, __ATOMIC_SEQ_CST ); + } else { + __atomic_add_fetch( &cmemalign_zero_calls, 1, __ATOMIC_SEQ_CST ); + } // if #endif // __STATISTICS__ @@ -1021,4 +1025,11 @@ // Same as calloc() with memory alignment. void * cmemalign( size_t alignment, size_t dim, size_t elemSize ) { + size_t size = dim * elemSize; + if ( unlikely( size ) == 0 ) { // 0 BYTE ALLOCATION RETURNS NULL POINTER + #ifdef __STATISTICS__ + __atomic_add_fetch( &cmemalign_zero_calls, 1, __ATOMIC_SEQ_CST ); + #endif // __STATISTICS__ + return 0p; + } // if #ifdef __STATISTICS__ __atomic_add_fetch( &cmemalign_calls, 1, __ATOMIC_SEQ_CST ); @@ -1026,5 +1037,25 @@ #endif // __STATISTICS__ - return cmemalignNoStats( alignment, dim, elemSize ); + char * addr = (char *)memalignNoStats( alignment, size ); + + HeapManager.Storage.Header * header; + HeapManager.FreeHeader * freeElem; + size_t bsize; + + #ifndef __CFA_DEBUG__ + bool mapped = + #endif // __CFA_DEBUG__ + headers( "cmemalign", addr, header, freeElem, bsize, alignment ); + + // Mapped storage is zero filled, but in debug mode mapped memory is scrubbed in doMalloc, so it has to be reset to zero. + #ifndef __CFA_DEBUG__ + if ( ! mapped ) + #endif // __CFA_DEBUG__ + // <-------0000000000000000000000000000UUUUUUUUUUUUUUUUUUUUUUUUU> bsize (bucket size) U => undefined + // `-header`-addr `-size + memset( addr, '\0', size ); // set to zeros + + header->kind.real.blockSize |= 2; // mark as zero filled + return addr; } // cmemalign @@ -1065,9 +1096,9 @@ // 0p, no operation is performed. void free( void * addr ) { - #ifdef __STATISTICS__ - __atomic_add_fetch( &free_calls, 1, __ATOMIC_SEQ_CST ); - #endif // __STATISTICS__ - if ( unlikely( addr == 0p ) ) { // special case + #ifdef __STATISTICS__ + __atomic_add_fetch( &free_zero_calls, 1, __ATOMIC_SEQ_CST ); + #endif // __STATISTICS__ + // #ifdef __CFA_DEBUG__ // if ( traceHeap() ) { @@ -1182,6 +1213,6 @@ int malloc_stats_fd( int fd __attribute__(( unused )) ) { #ifdef __STATISTICS__ - int temp = stat_fd; - stat_fd = fd; + int temp = stats_fd; + stats_fd = fd; return temp; #else @@ -1214,5 +1245,5 @@ // The string is printed on the file stream stream. The exported string includes information about all arenas (see // malloc). - int malloc_info( int options, FILE * stream ) { + int malloc_info( int options, FILE * stream __attribute__(( unused )) ) { if ( options != 0 ) { errno = EINVAL; return -1; } #ifdef __STATISTICS__ @@ -1243,18 +1274,21 @@ // Must have CFA linkage to overload with C linkage realloc. void * resize( void * oaddr, size_t nalign, size_t size ) { - #ifdef __STATISTICS__ - __atomic_add_fetch( &resize_calls, 1, __ATOMIC_SEQ_CST ); - #endif // __STATISTICS__ + // If size is equal to 0, either NULL or a pointer suitable to be passed to free() is returned. + if ( unlikely( size == 0 ) ) { // special cases + #ifdef __STATISTICS__ + __atomic_add_fetch( &resize_zero_calls, 1, __ATOMIC_SEQ_CST ); + #endif // __STATISTICS__ + free( oaddr ); + return 0p; + } // if if ( unlikely( nalign < libAlign() ) ) nalign = libAlign(); // reset alignment to minimum #ifdef __CFA_DEBUG__ - else - checkAlign( nalign ); // check alignment + else checkAlign( nalign ); // check alignment #endif // __CFA_DEBUG__ - // If size is equal to 0, either NULL or a pointer suitable to be passed to free() is returned. - if ( unlikely( size == 0 ) ) { free( oaddr ); return 0p; } // special cases if ( unlikely( oaddr == 0p ) ) { #ifdef __STATISTICS__ + __atomic_add_fetch( &resize_calls, 1, __ATOMIC_SEQ_CST ); __atomic_add_fetch( &resize_storage, size, __ATOMIC_SEQ_CST ); #endif // __STATISTICS__ @@ -1302,12 +1336,18 @@ void * realloc( void * oaddr, size_t nalign, size_t size ) { + // If size is equal to 0, either NULL or a pointer suitable to be passed to free() is returned. + if ( unlikely( size == 0 ) ) { // special cases + #ifdef __STATISTICS__ + __atomic_add_fetch( &realloc_zero_calls, 1, __ATOMIC_SEQ_CST ); + #endif // __STATISTICS__ + free( oaddr ); + return 0p; + } // if + if ( unlikely( nalign < libAlign() ) ) nalign = libAlign(); // reset alignment to minimum #ifdef __CFA_DEBUG__ - else - checkAlign( nalign ); // check alignment + else checkAlign( nalign ); // check alignment #endif // __CFA_DEBUG__ - // If size is equal to 0, either NULL or a pointer suitable to be passed to free() is returned. - if ( unlikely( size == 0 ) ) { free( oaddr ); return 0p; } // special cases if ( unlikely( oaddr == 0p ) ) { #ifdef __STATISTICS__