Index: libcfa/src/heap.cfa =================================================================== --- libcfa/src/heap.cfa (revision 53fd99584f3235dd92760cf4d0fd1bd49c6920d0) +++ libcfa/src/heap.cfa (revision 92aca37a5af06634217621f024ab9c2eef73d18c) @@ -5,11 +5,11 @@ // file "LICENCE" distributed with Cforall. // -// heap.c -- +// heap.cfa -- // // Author : Peter A. Buhr // Created On : Tue Dec 19 21:58:35 2017 // Last Modified By : Peter A. Buhr -// Last Modified On : Thu Aug 6 09:08:58 2020 -// Update Count : 861 +// Last Modified On : Sun Aug 9 12:23:20 2020 +// Update Count : 894 // @@ -23,5 +23,5 @@ #include // mmap, munmap -#include "bits/align.hfa" // libPow2 +#include "bits/align.hfa" // libAlign #include "bits/defs.hfa" // likely, unlikely #include "bits/locks.hfa" // __spinlock_t @@ -88,12 +88,7 @@ } // default_heap_expansion -bool default_heap_exhausted() __attribute__(( weak )) { // find and free some storage - // Returning false prints "out of heap memory" message and aborts. - return false; -} // default_heap_exhausted - #ifdef __CFA_DEBUG__ -static unsigned int allocUnfreed; // running total of allocations minus frees +static size_t allocUnfreed; // running total of allocations minus frees static void prtUnfreed() { @@ -101,5 +96,5 @@ // DO NOT USE STREAMS AS THEY MAY BE UNAVAILABLE AT THIS POINT. char helpText[512]; - int len = snprintf( helpText, sizeof(helpText), "CFA warning (UNIX pid:%ld) : program terminating with %u(0x%x) bytes of storage allocated but not freed.\n" + int len = snprintf( helpText, sizeof(helpText), "CFA warning (UNIX pid:%ld) : program terminating with %zu(0x%zx) bytes of storage allocated but not freed.\n" "Possible cause is unfreed storage allocated by the program or system/library routines called from the program.\n", (long int)getpid(), allocUnfreed, allocUnfreed ); // always print the UNIX pid @@ -292,5 +287,5 @@ static unsigned long long int sbrk_storage; // Statistics file descriptor (changed by malloc_stats_fd). -static int statfd = STDERR_FILENO; // default stderr +static int stat_fd = STDERR_FILENO; // default stderr // Use "write" because streams may be shutdown when calls are made. @@ -312,5 +307,5 @@ " sbrk: calls %u / storage %llu\n", malloc_calls, malloc_storage, - aalloc_calls, calloc_storage, + aalloc_calls, aalloc_storage, calloc_calls, calloc_storage, memalign_calls, memalign_storage, @@ -411,5 +406,5 @@ static inline void checkAlign( size_t alignment ) { - if ( alignment < libAlign() || ! libPow2( alignment ) ) { + if ( alignment < libAlign() || ! is_pow2( alignment ) ) { abort( "Alignment %zu for memory allocation is less than %d and/or not a power of 2.", alignment, libAlign() ); } // if @@ -443,5 +438,5 @@ header = headerAddr( addr ); - if ( unlikely( heapEnd < addr ) ) { // mmapped ? + if ( unlikely( heapEnd < addr ) ) { // mmapped ? fakeHeader( header, alignment ); size = header->kind.real.blockSize & -3; // mmap size @@ -471,5 +466,5 @@ } // headers -#define NO_MEMORY_MSG "no heap memory available for allocating %zd new bytes." +#define NO_MEMORY_MSG "insufficient heap memory available for allocating %zd new bytes." static inline void * extend( size_t size ) with( heapManager ) { @@ -479,14 +474,9 @@ // If the size requested is bigger than the current remaining storage, increase the size of the heap. - size_t increase = libCeiling( size > heapExpand ? size : heapExpand, libAlign() ); - Succeed: - { - if ( sbrk( increase ) != (void *)-1 ) break Succeed; // succeed ? - if ( default_heap_exhausted() ) { // try fix - if ( sbrk( increase ) != (void *)-1 ) break Succeed; // succeed ? - } // if + size_t increase = ceiling2( size > heapExpand ? size : heapExpand, libAlign() ); + if ( sbrk( increase ) == (void *)-1 ) { // failed, no memory ? unlock( extlock ); abort( NO_MEMORY_MSG, size ); // give up - } + } // if #ifdef __STATISTICS__ sbrk_calls += 1; @@ -524,15 +514,4 @@ posn = Bsearchl( (unsigned int)tsize, bucketSizes, (size_t)maxBucketsUsed ); HeapManager.FreeHeader * freeElem = &freeLists[posn]; - // #ifdef FASTLOOKUP - // if ( tsize < LookupSizes ) - // freeElem = &freeLists[lookup[tsize]]; - // else - // #endif // FASTLOOKUP - // freeElem = bsearchl( tsize, freeLists, (size_t)maxBucketsUsed ); // binary search - // HeapManager.FreeHeader * freeElem = - // #ifdef FASTLOOKUP - // tsize < LookupSizes ? &freeLists[lookup[tsize]] : - // #endif // FASTLOOKUP - // bsearchl( tsize, freeLists, (size_t)maxBucketsUsed ); // binary search verify( freeElem <= &freeLists[maxBucketsUsed] ); // subscripting error ? verify( tsize <= freeElem->blockSize ); // search failure ? @@ -566,21 +545,16 @@ } else { // large size => mmap if ( unlikely( size > ULONG_MAX - pageSize ) ) return 0p; - tsize = libCeiling( tsize, pageSize ); // must be multiple of page size + tsize = ceiling2( tsize, pageSize ); // must be multiple of page size #ifdef __STATISTICS__ __atomic_add_fetch( &mmap_calls, 1, __ATOMIC_SEQ_CST ); __atomic_add_fetch( &mmap_storage, tsize, __ATOMIC_SEQ_CST ); #endif // __STATISTICS__ - Succeed: - { - block = (HeapManager.Storage *)mmap( 0, tsize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, mmapFd, 0 ); - if ( block != (HeapManager.Storage *)MAP_FAILED ) break Succeed; // succeed ? - if ( errno == ENOMEM && default_heap_exhausted() ) { // out of memory and try again ? - block = (HeapManager.Storage *)mmap( 0, tsize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, mmapFd, 0 ); - if ( block != (HeapManager.Storage *)MAP_FAILED ) break Succeed; // succeed ? - } // if - if ( errno == ENOMEM ) abort( NO_MEMORY_MSG, tsize ); + + block = (HeapManager.Storage *)mmap( 0, tsize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, mmapFd, 0 ); + if ( block == (HeapManager.Storage *)MAP_FAILED ) { // failed ? + if ( errno == ENOMEM ) abort( NO_MEMORY_MSG, tsize ); // no memory // Do not call strerror( errno ) as it may call malloc. abort( "(HeapManager &)0x%p.doMalloc() : internal error, mmap failure, size:%zu error:%d.", &heapManager, tsize, errno ); - } + } //if #ifdef __CFA_DEBUG__ // Set new memory to garbage so subsequent uninitialized usages might fail. @@ -653,6 +627,5 @@ __atomic_add_fetch( &allocUnfreed, -size, __ATOMIC_SEQ_CST ); if ( traceHeap() ) { - enum { BufferSize = 64 }; - char helpText[BufferSize]; + char helpText[64]; int len = snprintf( helpText, sizeof(helpText), "Free( %p ) size:%zu\n", addr, size ); __cfaabi_bits_write( STDERR_FILENO, helpText, len ); // print debug/nodebug @@ -677,7 +650,5 @@ for ( HeapManager.Storage * p = freeLists[i].freeList; p != 0p; p = p->header.kind.real.next ) { #else - for ( HeapManager.Storage * p = top( freeLists[i].freeList ); p != 0p; /* p = getNext( p )->top */) { - typeof(p) temp = ( p )`next->top; // FIX ME: direct assignent fails, initialization works - p = temp; + for ( HeapManager.Storage * p = top( freeLists[i].freeList ); p != 0p; p = (p)`next->top ) { #endif // BUCKETLOCK total += size; @@ -721,5 +692,5 @@ char * end = (char *)sbrk( 0 ); - heapBegin = heapEnd = sbrk( (char *)libCeiling( (long unsigned int)end, libAlign() ) - end ); // move start of heap to multiple of alignment + heapBegin = heapEnd = sbrk( (char *)ceiling2( (long unsigned int)end, libAlign() ) - end ); // move start of heap to multiple of alignment } // HeapManager @@ -729,5 +700,5 @@ if ( traceHeapTerm() ) { printStats(); - // if ( prtfree() ) prtFree( heapManager, true ); + // prtUnfreed() called in heapAppStop() } // if #endif // __STATISTICS__ @@ -738,5 +709,5 @@ void memory_startup( void ) { #ifdef __CFA_DEBUG__ - if ( unlikely( heapBoot ) ) { // check for recursion during system boot + if ( heapBoot ) { // check for recursion during system boot // DO NOT USE STREAMS AS THEY MAY BE UNAVAILABLE AT THIS POINT. abort( "boot() : internal error, recursively invoked during system boot." ); @@ -757,5 +728,5 @@ static inline void * mallocNoStats( size_t size ) { // necessary for malloc statistics - verify( heapManager.heapBegin != 0 ); // called before memory_startup ? + verify( heapManager.heapBegin != 0p ); // called before memory_startup ? if ( unlikely( size ) == 0 ) return 0p; // 0 BYTE ALLOCATION RETURNS NULL POINTER @@ -793,5 +764,5 @@ -static inline void * memalignNoStats( size_t alignment, size_t size ) { // necessary for malloc statistics +static inline void * memalignNoStats( size_t alignment, size_t size ) { if ( unlikely( size ) == 0 ) return 0p; // 0 BYTE ALLOCATION RETURNS NULL POINTER @@ -815,5 +786,5 @@ // address in the block of the "next" alignment address - char * user = (char *)libCeiling( (uintptr_t)(addr + sizeof(HeapManager.Storage)), alignment ); + char * user = (char *)ceiling2( (uintptr_t)(addr + sizeof(HeapManager.Storage)), alignment ); // address of header from malloc @@ -843,7 +814,7 @@ #endif // __CFA_DEBUG__ headers( "cmemalign", addr, header, freeElem, bsize, alignment ); + + // Mapped storage is zero filled, but in debug mode mapped memory is scrubbed in doMalloc, so it has to be reset to zero. #ifndef __CFA_DEBUG__ - - // Mapped storage is zero filled, but in debug mode mapped memory is scrubbed in doMalloc, so it has to be reset to zero. if ( ! mapped ) #endif // __CFA_DEBUG__ @@ -857,13 +828,4 @@ -// supported mallopt options -#ifndef M_MMAP_THRESHOLD -#define M_MMAP_THRESHOLD (-1) -#endif // M_TOP_PAD -#ifndef M_TOP_PAD -#define M_TOP_PAD (-2) -#endif // M_TOP_PAD - - extern "C" { // Allocates size bytes and returns a pointer to the allocated memory. The contents are undefined. If size is 0, @@ -881,10 +843,11 @@ // Same as malloc() except size bytes is an array of dim elements each of elemSize bytes. void * aalloc( size_t dim, size_t elemSize ) { + size_t size = dim * elemSize; #ifdef __STATISTICS__ __atomic_add_fetch( &aalloc_calls, 1, __ATOMIC_SEQ_CST ); - __atomic_add_fetch( &aalloc_storage, dim * elemSize, __ATOMIC_SEQ_CST ); - #endif // __STATISTICS__ - - return mallocNoStats( dim * elemSize ); + __atomic_add_fetch( &aalloc_storage, size, __ATOMIC_SEQ_CST ); + #endif // __STATISTICS__ + + return mallocNoStats( size ); } // aalloc @@ -899,4 +862,5 @@ return callocNoStats( dim, elemSize ); } // calloc + // Change the size of the memory block pointed to by oaddr to size bytes. The contents are undefined. If oaddr is @@ -907,23 +871,31 @@ #ifdef __STATISTICS__ __atomic_add_fetch( &resize_calls, 1, __ATOMIC_SEQ_CST ); - __atomic_add_fetch( &resize_storage, size, __ATOMIC_SEQ_CST ); #endif // __STATISTICS__ // If size is equal to 0, either NULL or a pointer suitable to be passed to free() is returned. if ( unlikely( size == 0 ) ) { free( oaddr ); return 0p; } // special cases - if ( unlikely( oaddr == 0p ) ) return mallocNoStats( size ); + if ( unlikely( oaddr == 0p ) ) { + #ifdef __STATISTICS__ + __atomic_add_fetch( &resize_storage, size, __ATOMIC_SEQ_CST ); + #endif // __STATISTICS__ + return mallocNoStats( size ); + } // if HeapManager.Storage.Header * header; HeapManager.FreeHeader * freeElem; - size_t bsize, oalign = 0; + size_t bsize, oalign; headers( "resize", oaddr, header, freeElem, bsize, oalign ); size_t odsize = dataStorage( bsize, oaddr, header ); // data storage available in bucket // same size, DO NOT preserve STICKY PROPERTIES. - if ( oalign == 0 && size <= odsize && odsize <= size * 2 ) { // allow 50% wasted storage for smaller size + if ( oalign <= libAlign() && size <= odsize && odsize <= size * 2 ) { // allow 50% wasted storage for smaller size header->kind.real.blockSize &= -2; // no alignment and turn off 0 fill header->kind.real.size = size; // reset allocation size return oaddr; } // if + + #ifdef __STATISTICS__ + __atomic_add_fetch( &resize_storage, size, __ATOMIC_SEQ_CST ); + #endif // __STATISTICS__ // change size, DO NOT preserve STICKY PROPERTIES. @@ -938,14 +910,18 @@ #ifdef __STATISTICS__ __atomic_add_fetch( &realloc_calls, 1, __ATOMIC_SEQ_CST ); - __atomic_add_fetch( &realloc_storage, size, __ATOMIC_SEQ_CST ); #endif // __STATISTICS__ // If size is equal to 0, either NULL or a pointer suitable to be passed to free() is returned. if ( unlikely( size == 0 ) ) { free( oaddr ); return 0p; } // special cases - if ( unlikely( oaddr == 0p ) ) return mallocNoStats( size ); + if ( unlikely( oaddr == 0p ) ) { + #ifdef __STATISTICS__ + __atomic_add_fetch( &realloc_storage, size, __ATOMIC_SEQ_CST ); + #endif // __STATISTICS__ + return mallocNoStats( size ); + } // if HeapManager.Storage.Header * header; HeapManager.FreeHeader * freeElem; - size_t bsize, oalign = 0; + size_t bsize, oalign; headers( "realloc", oaddr, header, freeElem, bsize, oalign ); @@ -961,8 +937,12 @@ } // if + #ifdef __STATISTICS__ + __atomic_add_fetch( &realloc_storage, size, __ATOMIC_SEQ_CST ); + #endif // __STATISTICS__ + // change size and copy old content to new storage void * naddr; - if ( likely( oalign == 0 ) ) { // previous request memalign? + if ( likely( oalign <= libAlign() ) ) { // previous request not aligned ? naddr = mallocNoStats( size ); // create new area } else { @@ -997,10 +977,11 @@ // Same as aalloc() with memory alignment. void * amemalign( size_t alignment, size_t dim, size_t elemSize ) { + size_t size = dim * elemSize; #ifdef __STATISTICS__ __atomic_add_fetch( &cmemalign_calls, 1, __ATOMIC_SEQ_CST ); - __atomic_add_fetch( &cmemalign_storage, dim * elemSize, __ATOMIC_SEQ_CST ); - #endif // __STATISTICS__ - - return memalignNoStats( alignment, dim * elemSize ); + __atomic_add_fetch( &cmemalign_storage, size, __ATOMIC_SEQ_CST ); + #endif // __STATISTICS__ + + return memalignNoStats( alignment, size ); } // amemalign @@ -1028,5 +1009,5 @@ // free(3). int posix_memalign( void ** memptr, size_t alignment, size_t size ) { - if ( alignment < libAlign() || ! libPow2( alignment ) ) return EINVAL; // check alignment + if ( alignment < libAlign() || ! is_pow2( alignment ) ) return EINVAL; // check alignment * memptr = memalign( alignment, size ); return 0; @@ -1042,5 +1023,5 @@ // Same as valloc but rounds size to multiple of page size. void * pvalloc( size_t size ) { - return memalign( pageSize, libCeiling( size, pageSize ) ); + return memalign( pageSize, ceiling2( size, pageSize ) ); } // pvalloc @@ -1080,4 +1061,5 @@ } // malloc_alignment + // Set the alignment for an the allocation and return previous alignment or 0 if no alignment. size_t $malloc_alignment_set( void * addr, size_t alignment ) { @@ -1162,9 +1144,10 @@ } // malloc_stats + // Changes the file descripter where malloc_stats() writes statistics. int malloc_stats_fd( int fd __attribute__(( unused )) ) { #ifdef __STATISTICS__ - int temp = statfd; - statfd = fd; + int temp = stat_fd; + stat_fd = fd; return temp; #else @@ -1198,6 +1181,10 @@ // malloc). int malloc_info( int options, FILE * stream ) { - if ( options != 0 ) { errno = EINVAL; return -1; } + if ( options != 0 ) { errno = EINVAL; return -1; } + #ifdef __STATISTICS__ return printStatsXML( stream ); + #else + return 0; // unsupported + #endif // __STATISTICS__ } // malloc_info @@ -1214,5 +1201,5 @@ // Restores the state of all malloc internal bookkeeping variables to the values recorded in the opaque data // structure pointed to by state. - int malloc_set_state( void * ptr ) { + int malloc_set_state( void * ) { return 0; // unsupported } // malloc_set_state @@ -1224,10 +1211,14 @@ #ifdef __STATISTICS__ __atomic_add_fetch( &resize_calls, 1, __ATOMIC_SEQ_CST ); - __atomic_add_fetch( &resize_storage, size, __ATOMIC_SEQ_CST ); #endif // __STATISTICS__ // If size is equal to 0, either NULL or a pointer suitable to be passed to free() is returned. if ( unlikely( size == 0 ) ) { free( oaddr ); return 0p; } // special cases - if ( unlikely( oaddr == 0p ) ) return memalignNoStats( nalign, size ); + if ( unlikely( oaddr == 0p ) ) { + #ifdef __STATISTICS__ + __atomic_add_fetch( &resize_storage, size, __ATOMIC_SEQ_CST ); + #endif // __STATISTICS__ + return memalignNoStats( nalign, size ); + } // if if ( unlikely( nalign < libAlign() ) ) nalign = libAlign(); // reset alignment to minimum @@ -1239,5 +1230,5 @@ HeapManager.Storage.Header * header; HeapManager.FreeHeader * freeElem; - size_t bsize, oalign = 0; + size_t bsize, oalign; headers( "resize", oaddr, header, freeElem, bsize, oalign ); size_t odsize = dataStorage( bsize, oaddr, header ); // data storage available in bucket @@ -1253,4 +1244,8 @@ } // if } // if + + #ifdef __STATISTICS__ + __atomic_add_fetch( &resize_storage, size, __ATOMIC_SEQ_CST ); + #endif // __STATISTICS__ // change size, DO NOT preserve STICKY PROPERTIES. @@ -1269,5 +1264,5 @@ HeapManager.Storage.Header * header; HeapManager.FreeHeader * freeElem; - size_t bsize, oalign = 0; + size_t bsize, oalign; headers( "realloc", oaddr, header, freeElem, bsize, oalign ); Index: libcfa/src/heap.hfa =================================================================== --- libcfa/src/heap.hfa (revision 53fd99584f3235dd92760cf4d0fd1bd49c6920d0) +++ libcfa/src/heap.hfa (revision 92aca37a5af06634217621f024ab9c2eef73d18c) @@ -10,6 +10,6 @@ // Created On : Tue May 26 11:23:55 2020 // Last Modified By : Peter A. Buhr -// Last Modified On : Wed Aug 5 14:55:55 2020 -// Update Count : 14 +// Last Modified On : Sat Aug 8 17:36:48 2020 +// Update Count : 16 // @@ -18,5 +18,4 @@ size_t default_mmap_start(); // CFA extras size_t default_heap_expansion(); -bool default_heap_exhausted(); // unsafe to call printf! bool traceHeap(); @@ -31,4 +30,12 @@ bool checkFreeOn(); bool checkFreeOff(); + +// supported mallopt options +#ifndef M_MMAP_THRESHOLD +#define M_MMAP_THRESHOLD (-1) +#endif // M_TOP_PAD +#ifndef M_TOP_PAD +#define M_TOP_PAD (-2) +#endif // M_TOP_PAD extern "C" {