Changes in libcfa/src/heap.cfa [92aca37:c1f38e6c]
- File:
-
- 1 edited
-
libcfa/src/heap.cfa (modified) (38 diffs)
Legend:
- Unmodified
- Added
- Removed
-
libcfa/src/heap.cfa
r92aca37 rc1f38e6c 5 5 // file "LICENCE" distributed with Cforall. 6 6 // 7 // heap.c fa--7 // heap.c -- 8 8 // 9 9 // Author : Peter A. Buhr 10 10 // Created On : Tue Dec 19 21:58:35 2017 11 11 // Last Modified By : Peter A. Buhr 12 // Last Modified On : Sun Aug 9 12:23:20202013 // Update Count : 8 9412 // Last Modified On : Thu Aug 6 09:08:58 2020 13 // Update Count : 861 14 14 // 15 15 … … 23 23 #include <sys/mman.h> // mmap, munmap 24 24 25 #include "bits/align.hfa" // lib Align25 #include "bits/align.hfa" // libPow2 26 26 #include "bits/defs.hfa" // likely, unlikely 27 27 #include "bits/locks.hfa" // __spinlock_t … … 88 88 } // default_heap_expansion 89 89 90 bool default_heap_exhausted() __attribute__(( weak )) { // find and free some storage 91 // Returning false prints "out of heap memory" message and aborts. 92 return false; 93 } // default_heap_exhausted 94 90 95 91 96 #ifdef __CFA_DEBUG__ 92 static size_t allocUnfreed;// running total of allocations minus frees97 static unsigned int allocUnfreed; // running total of allocations minus frees 93 98 94 99 static void prtUnfreed() { … … 96 101 // DO NOT USE STREAMS AS THEY MAY BE UNAVAILABLE AT THIS POINT. 97 102 char helpText[512]; 98 int len = snprintf( helpText, sizeof(helpText), "CFA warning (UNIX pid:%ld) : program terminating with % zu(0x%zx) bytes of storage allocated but not freed.\n"103 int len = snprintf( helpText, sizeof(helpText), "CFA warning (UNIX pid:%ld) : program terminating with %u(0x%x) bytes of storage allocated but not freed.\n" 99 104 "Possible cause is unfreed storage allocated by the program or system/library routines called from the program.\n", 100 105 (long int)getpid(), allocUnfreed, allocUnfreed ); // always print the UNIX pid … … 287 292 static unsigned long long int sbrk_storage; 288 293 // Statistics file descriptor (changed by malloc_stats_fd). 289 static int stat _fd = STDERR_FILENO; // default stderr294 static int statfd = STDERR_FILENO; // default stderr 290 295 291 296 // Use "write" because streams may be shutdown when calls are made. … … 307 312 " sbrk: calls %u / storage %llu\n", 308 313 malloc_calls, malloc_storage, 309 aalloc_calls, aalloc_storage,314 aalloc_calls, calloc_storage, 310 315 calloc_calls, calloc_storage, 311 316 memalign_calls, memalign_storage, … … 406 411 407 412 static inline void checkAlign( size_t alignment ) { 408 if ( alignment < libAlign() || ! is_pow2( alignment ) ) {413 if ( alignment < libAlign() || ! libPow2( alignment ) ) { 409 414 abort( "Alignment %zu for memory allocation is less than %d and/or not a power of 2.", alignment, libAlign() ); 410 415 } // if … … 438 443 header = headerAddr( addr ); 439 444 440 445 if ( unlikely( heapEnd < addr ) ) { // mmapped ? 441 446 fakeHeader( header, alignment ); 442 447 size = header->kind.real.blockSize & -3; // mmap size … … 466 471 } // headers 467 472 468 #define NO_MEMORY_MSG " insufficientheap memory available for allocating %zd new bytes."473 #define NO_MEMORY_MSG "no heap memory available for allocating %zd new bytes." 469 474 470 475 static inline void * extend( size_t size ) with( heapManager ) { … … 474 479 // If the size requested is bigger than the current remaining storage, increase the size of the heap. 475 480 476 size_t increase = ceiling2( size > heapExpand ? size : heapExpand, libAlign() ); 477 if ( sbrk( increase ) == (void *)-1 ) { // failed, no memory ? 481 size_t increase = libCeiling( size > heapExpand ? size : heapExpand, libAlign() ); 482 Succeed: 483 { 484 if ( sbrk( increase ) != (void *)-1 ) break Succeed; // succeed ? 485 if ( default_heap_exhausted() ) { // try fix 486 if ( sbrk( increase ) != (void *)-1 ) break Succeed; // succeed ? 487 } // if 478 488 unlock( extlock ); 479 489 abort( NO_MEMORY_MSG, size ); // give up 480 } // if490 } 481 491 #ifdef __STATISTICS__ 482 492 sbrk_calls += 1; … … 514 524 posn = Bsearchl( (unsigned int)tsize, bucketSizes, (size_t)maxBucketsUsed ); 515 525 HeapManager.FreeHeader * freeElem = &freeLists[posn]; 526 // #ifdef FASTLOOKUP 527 // if ( tsize < LookupSizes ) 528 // freeElem = &freeLists[lookup[tsize]]; 529 // else 530 // #endif // FASTLOOKUP 531 // freeElem = bsearchl( tsize, freeLists, (size_t)maxBucketsUsed ); // binary search 532 // HeapManager.FreeHeader * freeElem = 533 // #ifdef FASTLOOKUP 534 // tsize < LookupSizes ? &freeLists[lookup[tsize]] : 535 // #endif // FASTLOOKUP 536 // bsearchl( tsize, freeLists, (size_t)maxBucketsUsed ); // binary search 516 537 verify( freeElem <= &freeLists[maxBucketsUsed] ); // subscripting error ? 517 538 verify( tsize <= freeElem->blockSize ); // search failure ? … … 545 566 } else { // large size => mmap 546 567 if ( unlikely( size > ULONG_MAX - pageSize ) ) return 0p; 547 tsize = ceiling2( tsize, pageSize ); // must be multiple of page size568 tsize = libCeiling( tsize, pageSize ); // must be multiple of page size 548 569 #ifdef __STATISTICS__ 549 570 __atomic_add_fetch( &mmap_calls, 1, __ATOMIC_SEQ_CST ); 550 571 __atomic_add_fetch( &mmap_storage, tsize, __ATOMIC_SEQ_CST ); 551 572 #endif // __STATISTICS__ 552 553 block = (HeapManager.Storage *)mmap( 0, tsize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, mmapFd, 0 ); 554 if ( block == (HeapManager.Storage *)MAP_FAILED ) { // failed ? 555 if ( errno == ENOMEM ) abort( NO_MEMORY_MSG, tsize ); // no memory 573 Succeed: 574 { 575 block = (HeapManager.Storage *)mmap( 0, tsize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, mmapFd, 0 ); 576 if ( block != (HeapManager.Storage *)MAP_FAILED ) break Succeed; // succeed ? 577 if ( errno == ENOMEM && default_heap_exhausted() ) { // out of memory and try again ? 578 block = (HeapManager.Storage *)mmap( 0, tsize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, mmapFd, 0 ); 579 if ( block != (HeapManager.Storage *)MAP_FAILED ) break Succeed; // succeed ? 580 } // if 581 if ( errno == ENOMEM ) abort( NO_MEMORY_MSG, tsize ); 556 582 // Do not call strerror( errno ) as it may call malloc. 557 583 abort( "(HeapManager &)0x%p.doMalloc() : internal error, mmap failure, size:%zu error:%d.", &heapManager, tsize, errno ); 558 } //if584 } 559 585 #ifdef __CFA_DEBUG__ 560 586 // Set new memory to garbage so subsequent uninitialized usages might fail. … … 627 653 __atomic_add_fetch( &allocUnfreed, -size, __ATOMIC_SEQ_CST ); 628 654 if ( traceHeap() ) { 629 char helpText[64]; 655 enum { BufferSize = 64 }; 656 char helpText[BufferSize]; 630 657 int len = snprintf( helpText, sizeof(helpText), "Free( %p ) size:%zu\n", addr, size ); 631 658 __cfaabi_bits_write( STDERR_FILENO, helpText, len ); // print debug/nodebug … … 650 677 for ( HeapManager.Storage * p = freeLists[i].freeList; p != 0p; p = p->header.kind.real.next ) { 651 678 #else 652 for ( HeapManager.Storage * p = top( freeLists[i].freeList ); p != 0p; p = (p)`next->top ) { 679 for ( HeapManager.Storage * p = top( freeLists[i].freeList ); p != 0p; /* p = getNext( p )->top */) { 680 typeof(p) temp = ( p )`next->top; // FIX ME: direct assignent fails, initialization works 681 p = temp; 653 682 #endif // BUCKETLOCK 654 683 total += size; … … 692 721 693 722 char * end = (char *)sbrk( 0 ); 694 heapBegin = heapEnd = sbrk( (char *) ceiling2( (long unsigned int)end, libAlign() ) - end ); // move start of heap to multiple of alignment723 heapBegin = heapEnd = sbrk( (char *)libCeiling( (long unsigned int)end, libAlign() ) - end ); // move start of heap to multiple of alignment 695 724 } // HeapManager 696 725 … … 700 729 if ( traceHeapTerm() ) { 701 730 printStats(); 702 // prtUnfreed() called in heapAppStop()731 // if ( prtfree() ) prtFree( heapManager, true ); 703 732 } // if 704 733 #endif // __STATISTICS__ … … 709 738 void memory_startup( void ) { 710 739 #ifdef __CFA_DEBUG__ 711 if ( heapBoot ) {// check for recursion during system boot740 if ( unlikely( heapBoot ) ) { // check for recursion during system boot 712 741 // DO NOT USE STREAMS AS THEY MAY BE UNAVAILABLE AT THIS POINT. 713 742 abort( "boot() : internal error, recursively invoked during system boot." ); … … 728 757 729 758 static inline void * mallocNoStats( size_t size ) { // necessary for malloc statistics 730 verify( heapManager.heapBegin != 0 p); // called before memory_startup ?759 verify( heapManager.heapBegin != 0 ); // called before memory_startup ? 731 760 if ( unlikely( size ) == 0 ) return 0p; // 0 BYTE ALLOCATION RETURNS NULL POINTER 732 761 … … 764 793 765 794 766 static inline void * memalignNoStats( size_t alignment, size_t size ) { 795 static inline void * memalignNoStats( size_t alignment, size_t size ) { // necessary for malloc statistics 767 796 if ( unlikely( size ) == 0 ) return 0p; // 0 BYTE ALLOCATION RETURNS NULL POINTER 768 797 … … 786 815 787 816 // address in the block of the "next" alignment address 788 char * user = (char *) ceiling2( (uintptr_t)(addr + sizeof(HeapManager.Storage)), alignment );817 char * user = (char *)libCeiling( (uintptr_t)(addr + sizeof(HeapManager.Storage)), alignment ); 789 818 790 819 // address of header from malloc … … 814 843 #endif // __CFA_DEBUG__ 815 844 headers( "cmemalign", addr, header, freeElem, bsize, alignment ); 845 #ifndef __CFA_DEBUG__ 816 846 817 847 // Mapped storage is zero filled, but in debug mode mapped memory is scrubbed in doMalloc, so it has to be reset to zero. 818 #ifndef __CFA_DEBUG__819 848 if ( ! mapped ) 820 849 #endif // __CFA_DEBUG__ … … 828 857 829 858 859 // supported mallopt options 860 #ifndef M_MMAP_THRESHOLD 861 #define M_MMAP_THRESHOLD (-1) 862 #endif // M_TOP_PAD 863 #ifndef M_TOP_PAD 864 #define M_TOP_PAD (-2) 865 #endif // M_TOP_PAD 866 867 830 868 extern "C" { 831 869 // Allocates size bytes and returns a pointer to the allocated memory. The contents are undefined. If size is 0, … … 843 881 // Same as malloc() except size bytes is an array of dim elements each of elemSize bytes. 844 882 void * aalloc( size_t dim, size_t elemSize ) { 845 size_t size = dim * elemSize;846 883 #ifdef __STATISTICS__ 847 884 __atomic_add_fetch( &aalloc_calls, 1, __ATOMIC_SEQ_CST ); 848 __atomic_add_fetch( &aalloc_storage, size, __ATOMIC_SEQ_CST );849 #endif // __STATISTICS__ 850 851 return mallocNoStats( size );885 __atomic_add_fetch( &aalloc_storage, dim * elemSize, __ATOMIC_SEQ_CST ); 886 #endif // __STATISTICS__ 887 888 return mallocNoStats( dim * elemSize ); 852 889 } // aalloc 853 890 … … 862 899 return callocNoStats( dim, elemSize ); 863 900 } // calloc 864 865 901 866 902 // Change the size of the memory block pointed to by oaddr to size bytes. The contents are undefined. If oaddr is … … 871 907 #ifdef __STATISTICS__ 872 908 __atomic_add_fetch( &resize_calls, 1, __ATOMIC_SEQ_CST ); 909 __atomic_add_fetch( &resize_storage, size, __ATOMIC_SEQ_CST ); 873 910 #endif // __STATISTICS__ 874 911 875 912 // If size is equal to 0, either NULL or a pointer suitable to be passed to free() is returned. 876 913 if ( unlikely( size == 0 ) ) { free( oaddr ); return 0p; } // special cases 877 if ( unlikely( oaddr == 0p ) ) { 878 #ifdef __STATISTICS__ 879 __atomic_add_fetch( &resize_storage, size, __ATOMIC_SEQ_CST ); 880 #endif // __STATISTICS__ 881 return mallocNoStats( size ); 882 } // if 914 if ( unlikely( oaddr == 0p ) ) return mallocNoStats( size ); 883 915 884 916 HeapManager.Storage.Header * header; 885 917 HeapManager.FreeHeader * freeElem; 886 size_t bsize, oalign ;918 size_t bsize, oalign = 0; 887 919 headers( "resize", oaddr, header, freeElem, bsize, oalign ); 888 920 889 921 size_t odsize = dataStorage( bsize, oaddr, header ); // data storage available in bucket 890 922 // same size, DO NOT preserve STICKY PROPERTIES. 891 if ( oalign <= libAlign()&& size <= odsize && odsize <= size * 2 ) { // allow 50% wasted storage for smaller size923 if ( oalign == 0 && size <= odsize && odsize <= size * 2 ) { // allow 50% wasted storage for smaller size 892 924 header->kind.real.blockSize &= -2; // no alignment and turn off 0 fill 893 925 header->kind.real.size = size; // reset allocation size 894 926 return oaddr; 895 927 } // if 896 897 #ifdef __STATISTICS__898 __atomic_add_fetch( &resize_storage, size, __ATOMIC_SEQ_CST );899 #endif // __STATISTICS__900 928 901 929 // change size, DO NOT preserve STICKY PROPERTIES. … … 910 938 #ifdef __STATISTICS__ 911 939 __atomic_add_fetch( &realloc_calls, 1, __ATOMIC_SEQ_CST ); 940 __atomic_add_fetch( &realloc_storage, size, __ATOMIC_SEQ_CST ); 912 941 #endif // __STATISTICS__ 913 942 914 943 // If size is equal to 0, either NULL or a pointer suitable to be passed to free() is returned. 915 944 if ( unlikely( size == 0 ) ) { free( oaddr ); return 0p; } // special cases 916 if ( unlikely( oaddr == 0p ) ) { 917 #ifdef __STATISTICS__ 918 __atomic_add_fetch( &realloc_storage, size, __ATOMIC_SEQ_CST ); 919 #endif // __STATISTICS__ 920 return mallocNoStats( size ); 921 } // if 945 if ( unlikely( oaddr == 0p ) ) return mallocNoStats( size ); 922 946 923 947 HeapManager.Storage.Header * header; 924 948 HeapManager.FreeHeader * freeElem; 925 size_t bsize, oalign ;949 size_t bsize, oalign = 0; 926 950 headers( "realloc", oaddr, header, freeElem, bsize, oalign ); 927 951 … … 937 961 } // if 938 962 939 #ifdef __STATISTICS__940 __atomic_add_fetch( &realloc_storage, size, __ATOMIC_SEQ_CST );941 #endif // __STATISTICS__942 943 963 // change size and copy old content to new storage 944 964 945 965 void * naddr; 946 if ( likely( oalign <= libAlign() ) ) { // previous request not aligned?966 if ( likely( oalign == 0 ) ) { // previous request memalign? 947 967 naddr = mallocNoStats( size ); // create new area 948 968 } else { … … 977 997 // Same as aalloc() with memory alignment. 978 998 void * amemalign( size_t alignment, size_t dim, size_t elemSize ) { 979 size_t size = dim * elemSize;980 999 #ifdef __STATISTICS__ 981 1000 __atomic_add_fetch( &cmemalign_calls, 1, __ATOMIC_SEQ_CST ); 982 __atomic_add_fetch( &cmemalign_storage, size, __ATOMIC_SEQ_CST );983 #endif // __STATISTICS__ 984 985 return memalignNoStats( alignment, size );1001 __atomic_add_fetch( &cmemalign_storage, dim * elemSize, __ATOMIC_SEQ_CST ); 1002 #endif // __STATISTICS__ 1003 1004 return memalignNoStats( alignment, dim * elemSize ); 986 1005 } // amemalign 987 1006 … … 1009 1028 // free(3). 1010 1029 int posix_memalign( void ** memptr, size_t alignment, size_t size ) { 1011 if ( alignment < libAlign() || ! is_pow2( alignment ) ) return EINVAL; // check alignment1030 if ( alignment < libAlign() || ! libPow2( alignment ) ) return EINVAL; // check alignment 1012 1031 * memptr = memalign( alignment, size ); 1013 1032 return 0; … … 1023 1042 // Same as valloc but rounds size to multiple of page size. 1024 1043 void * pvalloc( size_t size ) { 1025 return memalign( pageSize, ceiling2( size, pageSize ) );1044 return memalign( pageSize, libCeiling( size, pageSize ) ); 1026 1045 } // pvalloc 1027 1046 … … 1061 1080 } // malloc_alignment 1062 1081 1063 1064 1082 // Set the alignment for an the allocation and return previous alignment or 0 if no alignment. 1065 1083 size_t $malloc_alignment_set( void * addr, size_t alignment ) { … … 1144 1162 } // malloc_stats 1145 1163 1146 1147 1164 // Changes the file descripter where malloc_stats() writes statistics. 1148 1165 int malloc_stats_fd( int fd __attribute__(( unused )) ) { 1149 1166 #ifdef __STATISTICS__ 1150 int temp = stat _fd;1151 stat _fd = fd;1167 int temp = statfd; 1168 statfd = fd; 1152 1169 return temp; 1153 1170 #else … … 1181 1198 // malloc). 1182 1199 int malloc_info( int options, FILE * stream ) { 1183 if ( options != 0 ) { errno = EINVAL; return -1; } 1184 #ifdef __STATISTICS__ 1200 if ( options != 0 ) { errno = EINVAL; return -1; } 1185 1201 return printStatsXML( stream ); 1186 #else1187 return 0; // unsupported1188 #endif // __STATISTICS__1189 1202 } // malloc_info 1190 1203 … … 1201 1214 // Restores the state of all malloc internal bookkeeping variables to the values recorded in the opaque data 1202 1215 // structure pointed to by state. 1203 int malloc_set_state( void * ) {1216 int malloc_set_state( void * ptr ) { 1204 1217 return 0; // unsupported 1205 1218 } // malloc_set_state … … 1211 1224 #ifdef __STATISTICS__ 1212 1225 __atomic_add_fetch( &resize_calls, 1, __ATOMIC_SEQ_CST ); 1226 __atomic_add_fetch( &resize_storage, size, __ATOMIC_SEQ_CST ); 1213 1227 #endif // __STATISTICS__ 1214 1228 1215 1229 // If size is equal to 0, either NULL or a pointer suitable to be passed to free() is returned. 1216 1230 if ( unlikely( size == 0 ) ) { free( oaddr ); return 0p; } // special cases 1217 if ( unlikely( oaddr == 0p ) ) { 1218 #ifdef __STATISTICS__ 1219 __atomic_add_fetch( &resize_storage, size, __ATOMIC_SEQ_CST ); 1220 #endif // __STATISTICS__ 1221 return memalignNoStats( nalign, size ); 1222 } // if 1231 if ( unlikely( oaddr == 0p ) ) return memalignNoStats( nalign, size ); 1223 1232 1224 1233 if ( unlikely( nalign < libAlign() ) ) nalign = libAlign(); // reset alignment to minimum … … 1230 1239 HeapManager.Storage.Header * header; 1231 1240 HeapManager.FreeHeader * freeElem; 1232 size_t bsize, oalign ;1241 size_t bsize, oalign = 0; 1233 1242 headers( "resize", oaddr, header, freeElem, bsize, oalign ); 1234 1243 size_t odsize = dataStorage( bsize, oaddr, header ); // data storage available in bucket … … 1244 1253 } // if 1245 1254 } // if 1246 1247 #ifdef __STATISTICS__1248 __atomic_add_fetch( &resize_storage, size, __ATOMIC_SEQ_CST );1249 #endif // __STATISTICS__1250 1255 1251 1256 // change size, DO NOT preserve STICKY PROPERTIES. … … 1264 1269 HeapManager.Storage.Header * header; 1265 1270 HeapManager.FreeHeader * freeElem; 1266 size_t bsize, oalign ;1271 size_t bsize, oalign = 0; 1267 1272 headers( "realloc", oaddr, header, freeElem, bsize, oalign ); 1268 1273
Note:
See TracChangeset
for help on using the changeset viewer.
