Changeset 95eb7cf
- Timestamp:
- Nov 22, 2019, 3:10:22 PM (5 years ago)
- Branches:
- ADT, arm-eh, ast-experimental, enum, forall-pointer-decay, jacob/cs343-translation, jenkins-sandbox, master, new-ast, new-ast-unique-expr, pthread-emulation, qualifiedEnum
- Children:
- d74369b
- Parents:
- 0a25c34
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
libcfa/src/heap.cfa
r0a25c34 r95eb7cf 10 10 // Created On : Tue Dec 19 21:58:35 2017 11 11 // Last Modified By : Peter A. Buhr 12 // Last Modified On : Fri Oct 18 07:42:09201913 // Update Count : 55612 // Last Modified On : Fri Nov 22 14:16:30 2019 13 // Update Count : 626 14 14 // 15 15 … … 30 30 #include "malloc.h" 31 31 32 #define MIN(x, y) (y > x ? x : y) 32 33 33 34 static bool traceHeap = false; … … 50 51 51 52 52 static bool checkFree = false;53 54 inline bool checkFree() {55 return checkFree;56 } // checkFree57 58 bool checkFreeOn() {59 bool temp = checkFree;60 checkFree = true;53 static bool prtFree = false; 54 55 inline bool prtFree() { 56 return prtFree; 57 } // prtFree 58 59 bool prtFreeOn() { 60 bool temp = prtFree; 61 prtFree = true; 61 62 return temp; 62 } // checkFreeOn63 64 bool checkFreeOff() {65 bool temp = checkFree;66 checkFree = false;63 } // prtFreeOn 64 65 bool prtFreeOff() { 66 bool temp = prtFree; 67 prtFree = false; 67 68 return temp; 68 } // checkFreeOff69 } // prtFreeOff 69 70 70 71 … … 105 106 static unsigned int allocFree; // running total of allocations minus frees 106 107 107 static void checkUnfreed() {108 static void prtUnfreed() { 108 109 if ( allocFree != 0 ) { 109 110 // DO NOT USE STREAMS AS THEY MAY BE UNAVAILABLE AT THIS POINT. … … 112 113 // "Possible cause is unfreed storage allocated by the program or system/library routines called from the program.\n", 113 114 // (long int)getpid(), allocFree, allocFree ); // always print the UNIX pid 114 // __cfaabi_dbg_bits_write( helpText, len );115 } // if 116 } // checkUnfreed115 // __cfaabi_dbg_bits_write( STDERR_FILENO, helpText, len ); // print debug/nodebug 116 } // if 117 } // prtUnfreed 117 118 118 119 extern "C" { … … 123 124 void heapAppStop() { // called by __cfaabi_appready_startdown 124 125 fclose( stdin ); fclose( stdout ); 125 checkUnfreed();126 prtUnfreed(); 126 127 } // heapAppStop 127 128 } // extern "C" … … 134 135 static unsigned int maxBucketsUsed; // maximum number of buckets in use 135 136 136 137 // #comment TD : This defined is significantly different from the __ALIGN__ define from locks.hfa138 #define ALIGN 16139 137 140 138 #define SPINLOCK 0 … … 147 145 // Recursive definitions: HeapManager needs size of bucket array and bucket area needs sizeof HeapManager storage. 148 146 // Break recusion by hardcoding number of buckets and statically checking number is correct after bucket array defined. 149 enum { NoBucketSizes = 9 3}; // number of buckets sizes147 enum { NoBucketSizes = 91 }; // number of buckets sizes 150 148 151 149 struct HeapManager { … … 194 192 } kind; // Kind 195 193 } header; // Header 196 char pad[ ALIGN- sizeof( Header )];194 char pad[libAlign() - sizeof( Header )]; 197 195 char data[0]; // storage 198 196 }; // Storage 199 197 200 static_assert( ALIGN >= sizeof( Storage ), "ALIGN< sizeof( Storage )" );198 static_assert( libAlign() >= sizeof( Storage ), "libAlign() < sizeof( Storage )" ); 201 199 202 200 struct FreeHeader { … … 230 228 // Powers of 2 are common allocation sizes, so make powers of 2 generate the minimum required size. 231 229 static const unsigned int bucketSizes[] @= { // different bucket sizes 232 16, 32, 48, 64, 233 64 + sizeof(HeapManager.Storage), 96, 112, 128, 128 + sizeof(HeapManager.Storage), 160, 192, 224, 234 256 + sizeof(HeapManager.Storage), 320, 384, 448, 512 + sizeof(HeapManager.Storage), 640, 768, 896, 235 1_024 + sizeof(HeapManager.Storage), 1_536, 2_048 + sizeof(HeapManager.Storage), 2_560, 3_072, 3_584, 4_096 + sizeof(HeapManager.Storage), 6_144, 236 8_192 + sizeof(HeapManager.Storage), 9_216, 10_240, 11_264, 12_288, 13_312, 14_336, 15_360, 237 16_384 + sizeof(HeapManager.Storage), 18_432, 20_480, 22_528, 24_576, 26_624, 28_672, 30_720, 238 32_768 + sizeof(HeapManager.Storage), 36_864, 40_960, 45_056, 49_152, 53_248, 57_344, 61_440, 239 65_536 + sizeof(HeapManager.Storage), 73_728, 81_920, 90_112, 98_304, 106_496, 114_688, 122_880, 240 131_072 + sizeof(HeapManager.Storage), 147_456, 163_840, 180_224, 196_608, 212_992, 229_376, 245_760, 241 262_144 + sizeof(HeapManager.Storage), 294_912, 327_680, 360_448, 393_216, 425_984, 458_752, 491_520, 242 524_288 + sizeof(HeapManager.Storage), 655_360, 786_432, 917_504, 1_048_576 + sizeof(HeapManager.Storage), 1_179_648, 1_310_720, 1_441_792, 243 1_572_864, 1_703_936, 1_835_008, 1_966_080, 2_097_152 + sizeof(HeapManager.Storage), 2_621_440, 3_145_728, 3_670_016, 244 4_194_304 + sizeof(HeapManager.Storage) 230 16, 32, 48, 64 + sizeof(HeapManager.Storage), // 4 231 96, 112, 128 + sizeof(HeapManager.Storage), // 3 232 160, 192, 224, 256 + sizeof(HeapManager.Storage), // 4 233 320, 384, 448, 512 + sizeof(HeapManager.Storage), // 4 234 640, 768, 896, 1_024 + sizeof(HeapManager.Storage), // 4 235 1_536, 2_048 + sizeof(HeapManager.Storage), // 2 236 2_560, 3_072, 3_584, 4_096 + sizeof(HeapManager.Storage), // 4 237 6_144, 8_192 + sizeof(HeapManager.Storage), // 2 238 9_216, 10_240, 11_264, 12_288, 13_312, 14_336, 15_360, 16_384 + sizeof(HeapManager.Storage), // 8 239 18_432, 20_480, 22_528, 24_576, 26_624, 28_672, 30_720, 32_768 + sizeof(HeapManager.Storage), // 8 240 36_864, 40_960, 45_056, 49_152, 53_248, 57_344, 61_440, 65_536 + sizeof(HeapManager.Storage), // 8 241 73_728, 81_920, 90_112, 98_304, 106_496, 114_688, 122_880, 131_072 + sizeof(HeapManager.Storage), // 8 242 147_456, 163_840, 180_224, 196_608, 212_992, 229_376, 245_760, 262_144 + sizeof(HeapManager.Storage), // 8 243 294_912, 327_680, 360_448, 393_216, 425_984, 458_752, 491_520, 524_288 + sizeof(HeapManager.Storage), // 8 244 655_360, 786_432, 917_504, 1_048_576 + sizeof(HeapManager.Storage), // 4 245 1_179_648, 1_310_720, 1_441_792, 1_572_864, 1_703_936, 1_835_008, 1_966_080, 2_097_152 + sizeof(HeapManager.Storage), // 8 246 2_621_440, 3_145_728, 3_670_016, 4_194_304 + sizeof(HeapManager.Storage), // 4 245 247 }; 246 248 … … 251 253 static unsigned char lookup[LookupSizes]; // O(1) lookup for small sizes 252 254 #endif // FASTLOOKUP 255 253 256 static int mmapFd = -1; // fake or actual fd for anonymous file 254 255 256 257 #ifdef __CFA_DEBUG__ 257 258 static bool heapBoot = 0; // detect recursion during boot … … 259 260 static HeapManager heapManager __attribute__(( aligned (128) )) @= {}; // size of cache line to prevent false sharing 260 261 261 // #comment TD : The return type of this function should be commented262 static inline bool setMmapStart( size_t value ) {263 if ( value < pageSize || bucketSizes[NoBucketSizes - 1] < value ) return true;264 mmapStart = value; // set global265 266 // find the closest bucket size less than or equal to the mmapStart size267 maxBucketsUsed = bsearchl( (unsigned int)mmapStart, bucketSizes, NoBucketSizes ); // binary search268 assert( maxBucketsUsed < NoBucketSizes ); // subscript failure ?269 assert( mmapStart <= bucketSizes[maxBucketsUsed] ); // search failure ?270 return false;271 } // setMmapStart272 273 274 static void ?{}( HeapManager & manager ) with ( manager ) {275 pageSize = sysconf( _SC_PAGESIZE );276 277 for ( unsigned int i = 0; i < NoBucketSizes; i += 1 ) { // initialize the free lists278 freeLists[i].blockSize = bucketSizes[i];279 } // for280 281 #ifdef FASTLOOKUP282 unsigned int idx = 0;283 for ( unsigned int i = 0; i < LookupSizes; i += 1 ) {284 if ( i > bucketSizes[idx] ) idx += 1;285 lookup[i] = idx;286 } // for287 #endif // FASTLOOKUP288 289 if ( setMmapStart( default_mmap_start() ) ) {290 abort( "HeapManager : internal error, mmap start initialization failure." );291 } // if292 heapExpand = default_heap_expansion();293 294 char * End = (char *)sbrk( 0 );295 sbrk( (char *)libCeiling( (long unsigned int)End, libAlign() ) - End ); // move start of heap to multiple of alignment296 heapBegin = heapEnd = sbrk( 0 ); // get new start point297 } // HeapManager298 299 300 static void ^?{}( HeapManager & ) {301 #ifdef __STATISTICS__302 // if ( traceHeapTerm() ) {303 // printStats();304 // if ( checkfree() ) checkFree( heapManager, true );305 // } // if306 #endif // __STATISTICS__307 } // ~HeapManager308 309 310 static void memory_startup( void ) __attribute__(( constructor( STARTUP_PRIORITY_MEMORY ) ));311 void memory_startup( void ) {312 #ifdef __CFA_DEBUG__313 if ( unlikely( heapBoot ) ) { // check for recursion during system boot314 // DO NOT USE STREAMS AS THEY MAY BE UNAVAILABLE AT THIS POINT.315 abort( "boot() : internal error, recursively invoked during system boot." );316 } // if317 heapBoot = true;318 #endif // __CFA_DEBUG__319 320 //assert( heapManager.heapBegin != 0 );321 //heapManager{};322 if ( heapManager.heapBegin == 0 ) heapManager{};323 } // memory_startup324 325 static void memory_shutdown( void ) __attribute__(( destructor( STARTUP_PRIORITY_MEMORY ) ));326 void memory_shutdown( void ) {327 ^heapManager{};328 } // memory_shutdown329 330 262 331 263 #ifdef __STATISTICS__ 332 static unsigned long long int mmap_storage; // heap statistics counters 264 // Heap statistics counters. 265 static unsigned long long int mmap_storage; 333 266 static unsigned int mmap_calls; 334 267 static unsigned long long int munmap_storage; … … 348 281 static unsigned long long int realloc_storage; 349 282 static unsigned int realloc_calls; 350 351 static int statfd; // statistics file descriptor (changed by malloc_stats_fd) 352 283 // Statistics file descriptor (changed by malloc_stats_fd). 284 static int statfd = STDERR_FILENO; // default stderr 353 285 354 286 // Use "write" because streams may be shutdown when calls are made. 355 287 static void printStats() { 356 288 char helpText[512]; 357 __cfaabi_ dbg_bits_print_buffer(helpText, sizeof(helpText),289 __cfaabi_bits_print_buffer( STDERR_FILENO, helpText, sizeof(helpText), 358 290 "\nHeap statistics:\n" 359 291 " malloc: calls %u / storage %llu\n" … … 405 337 sbrk_calls, sbrk_storage 406 338 ); 407 return write( fileno( stream ), helpText, len ); // -1 => error 339 __cfaabi_bits_write( fileno( stream ), helpText, len ); // ensures all bytes written or exit 340 return len; 408 341 } // printStatsXML 409 342 #endif // __STATISTICS__ 343 410 344 411 345 // #comment TD : Is this the samething as Out-of-Memory? … … 418 352 419 353 static inline void checkAlign( size_t alignment ) { 420 if ( alignment < sizeof(void *) || ! libPow2( alignment ) ) {421 abort( "Alignment %zu for memory allocation is less than sizeof(void *) and/or not a power of 2.", alignment);354 if ( alignment < libAlign() || ! libPow2( alignment ) ) { 355 abort( "Alignment %zu for memory allocation is less than %d and/or not a power of 2.", alignment, libAlign() ); 422 356 } // if 423 357 } // checkAlign … … 431 365 432 366 367 static inline bool setMmapStart( size_t value ) { // true => mmapped, false => sbrk 368 if ( value < pageSize || bucketSizes[NoBucketSizes - 1] < value ) return true; 369 mmapStart = value; // set global 370 371 // find the closest bucket size less than or equal to the mmapStart size 372 maxBucketsUsed = bsearchl( (unsigned int)mmapStart, bucketSizes, NoBucketSizes ); // binary search 373 assert( maxBucketsUsed < NoBucketSizes ); // subscript failure ? 374 assert( mmapStart <= bucketSizes[maxBucketsUsed] ); // search failure ? 375 return false; 376 } // setMmapStart 377 378 433 379 static inline void checkHeader( bool check, const char * name, void * addr ) { 434 380 if ( unlikely( check ) ) { // bad address ? … … 439 385 } // checkHeader 440 386 441 // #comment TD : function should be commented and/or have a more evocative name 442 // this isn't either a check or a constructor which is what I would expect this function to be 443 static inline void fakeHeader( HeapManager.Storage.Header *& header, size_t & size, size_t & alignment ) { 387 388 static inline void fakeHeader( HeapManager.Storage.Header *& header, size_t & alignment ) { 444 389 if ( unlikely( (header->kind.fake.alignment & 1) == 1 ) ) { // fake header ? 445 390 size_t offset = header->kind.fake.offset; … … 452 397 } // fakeHeader 453 398 454 // #comment TD : Why is this a define 399 400 // <-------+----------------------------------------------------> bsize (bucket size) 401 // |header |addr 402 //================================================================================== 403 // | alignment 404 // <-----------------<------------+-----------------------------> bsize (bucket size) 405 // |fake-header | addr 455 406 #define headerAddr( addr ) ((HeapManager.Storage.Header *)( (char *)addr - sizeof(HeapManager.Storage) )) 456 407 457 static inline bool headers( const char * name, void * addr, HeapManager.Storage.Header *& header, HeapManager.FreeHeader *& freeElem, size_t & size, size_t & alignment ) with ( heapManager ) { 408 // <-------<<--------------------- dsize ---------------------->> bsize (bucket size) 409 // |header |addr 410 //================================================================================== 411 // | alignment 412 // <------------------------------<<---------- dsize --------->>> bsize (bucket size) 413 // |fake-header |addr 414 #define dataStorage( bsize, addr, header ) (bsize - ( (char *)addr - (char *)header )) 415 416 417 static inline bool headers( const char * name __attribute__(( unused )), void * addr, HeapManager.Storage.Header *& header, HeapManager.FreeHeader *& freeElem, size_t & size, size_t & alignment ) with ( heapManager ) { 458 418 header = headerAddr( addr ); 459 419 460 420 if ( unlikely( heapEnd < addr ) ) { // mmapped ? 461 fakeHeader( header, size,alignment );421 fakeHeader( header, alignment ); 462 422 size = header->kind.real.blockSize & -3; // mmap size 463 423 return true; … … 468 428 #endif // __CFA_DEBUG__ 469 429 470 // #comment TD : This code looks weird...471 // It's called as the first statement of both branches of the last if, with the same parameters in all cases472 473 430 // header may be safe to dereference 474 fakeHeader( header, size,alignment );431 fakeHeader( header, alignment ); 475 432 #ifdef __CFA_DEBUG__ 476 433 checkHeader( header < (HeapManager.Storage.Header *)heapBegin || (HeapManager.Storage.Header *)heapEnd < header, name, addr ); // bad address ? (offset could be + or -) … … 500 457 unlock( extlock ); 501 458 errno = ENOMEM; 502 return 0 ;459 return 0p; 503 460 } // if 504 461 #ifdef __STATISTICS__ … … 541 498 // along with the block and is a multiple of the alignment size. 542 499 543 if ( unlikely( size > ~0ul - sizeof(HeapManager.Storage) ) ) return 0 ;500 if ( unlikely( size > ~0ul - sizeof(HeapManager.Storage) ) ) return 0p; 544 501 size_t tsize = size + sizeof(HeapManager.Storage); 545 502 if ( likely( tsize < mmapStart ) ) { // small size => sbrk … … 574 531 block = freeElem->freeList.pop(); 575 532 #endif // SPINLOCK 576 if ( unlikely( block == 0 ) ) {// no free block ?533 if ( unlikely( block == 0p ) ) { // no free block ? 577 534 #if defined( SPINLOCK ) 578 535 unlock( freeElem->lock ); … … 583 540 584 541 block = (HeapManager.Storage *)extend( tsize ); // mutual exclusion on call 585 if ( unlikely( block == 0 ) ) return 0;542 if ( unlikely( block == 0p ) ) return 0p; 586 543 #if defined( SPINLOCK ) 587 544 } else { … … 593 550 block->header.kind.real.home = freeElem; // pointer back to free list of apropriate size 594 551 } else { // large size => mmap 595 if ( unlikely( size > ~0ul - pageSize ) ) return 0 ;552 if ( unlikely( size > ~0ul - pageSize ) ) return 0p; 596 553 tsize = libCeiling( tsize, pageSize ); // must be multiple of page size 597 554 #ifdef __STATISTICS__ … … 611 568 } // if 612 569 613 void * a rea= &(block->data); // adjust off header to user bytes570 void * addr = &(block->data); // adjust off header to user bytes 614 571 615 572 #ifdef __CFA_DEBUG__ 616 assert( ((uintptr_t)a rea& (libAlign() - 1)) == 0 ); // minimum alignment ?573 assert( ((uintptr_t)addr & (libAlign() - 1)) == 0 ); // minimum alignment ? 617 574 __atomic_add_fetch( &allocFree, tsize, __ATOMIC_SEQ_CST ); 618 575 if ( traceHeap() ) { 619 576 enum { BufferSize = 64 }; 620 577 char helpText[BufferSize]; 621 int len = snprintf( helpText, BufferSize, "%p = Malloc( %zu ) (allocated %zu)\n", a rea, size, tsize );622 // int len = snprintf( helpText, BufferSize, "Malloc %p %zu\n", a rea, size );623 __cfaabi_ dbg_bits_write( helpText, len );578 int len = snprintf( helpText, BufferSize, "%p = Malloc( %zu ) (allocated %zu)\n", addr, size, tsize ); 579 // int len = snprintf( helpText, BufferSize, "Malloc %p %zu\n", addr, size ); 580 __cfaabi_bits_write( STDERR_FILENO, helpText, len ); // print debug/nodebug 624 581 } // if 625 582 #endif // __CFA_DEBUG__ 626 583 627 return a rea;584 return addr; 628 585 } // doMalloc 629 586 … … 631 588 static inline void doFree( void * addr ) with ( heapManager ) { 632 589 #ifdef __CFA_DEBUG__ 633 if ( unlikely( heapManager.heapBegin == 0 ) ) {590 if ( unlikely( heapManager.heapBegin == 0p ) ) { 634 591 abort( "doFree( %p ) : internal error, called before heap is initialized.", addr ); 635 592 } // if … … 677 634 char helpText[BufferSize]; 678 635 int len = snprintf( helpText, sizeof(helpText), "Free( %p ) size:%zu\n", addr, size ); 679 __cfaabi_ dbg_bits_write( helpText, len );636 __cfaabi_bits_write( STDERR_FILENO, helpText, len ); // print debug/nodebug 680 637 } // if 681 638 #endif // __CFA_DEBUG__ … … 683 640 684 641 685 size_t checkFree( HeapManager & manager ) with ( manager ) {642 size_t prtFree( HeapManager & manager ) with ( manager ) { 686 643 size_t total = 0; 687 644 #ifdef __STATISTICS__ 688 __cfaabi_ dbg_bits_acquire();689 __cfaabi_ dbg_bits_print_nolock("\nBin lists (bin size : free blocks on list)\n" );645 __cfaabi_bits_acquire(); 646 __cfaabi_bits_print_nolock( STDERR_FILENO, "\nBin lists (bin size : free blocks on list)\n" ); 690 647 #endif // __STATISTICS__ 691 648 for ( unsigned int i = 0; i < maxBucketsUsed; i += 1 ) { … … 696 653 697 654 #if defined( SPINLOCK ) 698 for ( HeapManager.Storage * p = freeLists[i].freeList; p != 0 ; p = p->header.kind.real.next ) {655 for ( HeapManager.Storage * p = freeLists[i].freeList; p != 0p; p = p->header.kind.real.next ) { 699 656 #else 700 for ( HeapManager.Storage * p = freeLists[i].freeList.top(); p != 0 ; p = p->header.kind.real.next.top ) {657 for ( HeapManager.Storage * p = freeLists[i].freeList.top(); p != 0p; p = p->header.kind.real.next.top ) { 701 658 #endif // SPINLOCK 702 659 total += size; … … 707 664 708 665 #ifdef __STATISTICS__ 709 __cfaabi_ dbg_bits_print_nolock("%7zu, %-7u ", size, N );710 if ( (i + 1) % 8 == 0 ) __cfaabi_ dbg_bits_print_nolock("\n" );666 __cfaabi_bits_print_nolock( STDERR_FILENO, "%7zu, %-7u ", size, N ); 667 if ( (i + 1) % 8 == 0 ) __cfaabi_bits_print_nolock( STDERR_FILENO, "\n" ); 711 668 #endif // __STATISTICS__ 712 669 } // for 713 670 #ifdef __STATISTICS__ 714 __cfaabi_ dbg_bits_print_nolock("\ntotal free blocks:%zu\n", total );715 __cfaabi_ dbg_bits_release();671 __cfaabi_bits_print_nolock( STDERR_FILENO, "\ntotal free blocks:%zu\n", total ); 672 __cfaabi_bits_release(); 716 673 #endif // __STATISTICS__ 717 674 return (char *)heapEnd - (char *)heapBegin - total; 718 } // checkFree 675 } // prtFree 676 677 678 static void ?{}( HeapManager & manager ) with ( manager ) { 679 pageSize = sysconf( _SC_PAGESIZE ); 680 681 for ( unsigned int i = 0; i < NoBucketSizes; i += 1 ) { // initialize the free lists 682 freeLists[i].blockSize = bucketSizes[i]; 683 } // for 684 685 #ifdef FASTLOOKUP 686 unsigned int idx = 0; 687 for ( unsigned int i = 0; i < LookupSizes; i += 1 ) { 688 if ( i > bucketSizes[idx] ) idx += 1; 689 lookup[i] = idx; 690 } // for 691 #endif // FASTLOOKUP 692 693 if ( setMmapStart( default_mmap_start() ) ) { 694 abort( "HeapManager : internal error, mmap start initialization failure." ); 695 } // if 696 heapExpand = default_heap_expansion(); 697 698 char * End = (char *)sbrk( 0 ); 699 sbrk( (char *)libCeiling( (long unsigned int)End, libAlign() ) - End ); // move start of heap to multiple of alignment 700 heapBegin = heapEnd = sbrk( 0 ); // get new start point 701 } // HeapManager 702 703 704 static void ^?{}( HeapManager & ) { 705 #ifdef __STATISTICS__ 706 // if ( traceHeapTerm() ) { 707 // printStats(); 708 // if ( prtfree() ) prtFree( heapManager, true ); 709 // } // if 710 #endif // __STATISTICS__ 711 } // ~HeapManager 712 713 714 static void memory_startup( void ) __attribute__(( constructor( STARTUP_PRIORITY_MEMORY ) )); 715 void memory_startup( void ) { 716 #ifdef __CFA_DEBUG__ 717 if ( unlikely( heapBoot ) ) { // check for recursion during system boot 718 // DO NOT USE STREAMS AS THEY MAY BE UNAVAILABLE AT THIS POINT. 719 abort( "boot() : internal error, recursively invoked during system boot." ); 720 } // if 721 heapBoot = true; 722 #endif // __CFA_DEBUG__ 723 724 //assert( heapManager.heapBegin != 0 ); 725 //heapManager{}; 726 if ( heapManager.heapBegin == 0p ) heapManager{}; 727 } // memory_startup 728 729 static void memory_shutdown( void ) __attribute__(( destructor( STARTUP_PRIORITY_MEMORY ) )); 730 void memory_shutdown( void ) { 731 ^heapManager{}; 732 } // memory_shutdown 719 733 720 734 721 735 static inline void * mallocNoStats( size_t size ) { // necessary for malloc statistics 722 736 //assert( heapManager.heapBegin != 0 ); 723 if ( unlikely( heapManager.heapBegin == 0 ) ) heapManager{}; // called before memory_startup ?724 void * a rea= doMalloc( size );725 if ( unlikely( a rea == 0) ) errno = ENOMEM; // POSIX726 return a rea;737 if ( unlikely( heapManager.heapBegin == 0p ) ) heapManager{}; // called before memory_startup ? 738 void * addr = doMalloc( size ); 739 if ( unlikely( addr == 0p ) ) errno = ENOMEM; // POSIX 740 return addr; 727 741 } // mallocNoStats 742 743 744 static inline void * callocNoStats( size_t noOfElems, size_t elemSize ) { 745 size_t size = noOfElems * elemSize; 746 char * addr = (char *)mallocNoStats( size ); 747 if ( unlikely( addr == 0p ) ) return 0p; 748 749 HeapManager.Storage.Header * header; 750 HeapManager.FreeHeader * freeElem; 751 size_t bsize, alignment; 752 bool mapped __attribute__(( unused )) = headers( "calloc", addr, header, freeElem, bsize, alignment ); 753 #ifndef __CFA_DEBUG__ 754 // Mapped storage is zero filled, but in debug mode mapped memory is scrubbed in doMalloc, so it has to be reset to zero. 755 if ( ! mapped ) 756 #endif // __CFA_DEBUG__ 757 // Zero entire data space even when > than size => realloc without a new allocation and zero fill works. 758 // <-------00000000000000000000000000000000000000000000000000000> bsize (bucket size) 759 // `-header`-addr `-size 760 memset( addr, '\0', bsize - sizeof(HeapManager.Storage) ); // set to zeros 761 762 header->kind.real.blockSize |= 2; // mark as zero filled 763 return addr; 764 } // callocNoStats 728 765 729 766 … … 745 782 // subtract libAlign() because it is already the minimum alignment 746 783 // add sizeof(Storage) for fake header 747 // #comment TD : this is the only place that calls doMalloc without calling mallocNoStats, why ? 748 char * area = (char *)doMalloc( size + alignment - libAlign() + sizeof(HeapManager.Storage) ); 749 if ( unlikely( area == 0 ) ) return area; 784 char * addr = (char *)mallocNoStats( size + alignment - libAlign() + sizeof(HeapManager.Storage) ); 785 if ( unlikely( addr == 0p ) ) return addr; 750 786 751 787 // address in the block of the "next" alignment address 752 char * user = (char *)libCeiling( (uintptr_t)(a rea+ sizeof(HeapManager.Storage)), alignment );788 char * user = (char *)libCeiling( (uintptr_t)(addr + sizeof(HeapManager.Storage)), alignment ); 753 789 754 790 // address of header from malloc 755 HeapManager.Storage.Header * realHeader = headerAddr( a rea);791 HeapManager.Storage.Header * realHeader = headerAddr( addr ); 756 792 // address of fake header * before* the alignment location 757 793 HeapManager.Storage.Header * fakeHeader = headerAddr( user ); … … 763 799 return user; 764 800 } // memalignNoStats 801 802 803 static inline void * cmemalignNoStats( size_t alignment, size_t noOfElems, size_t elemSize ) { 804 size_t size = noOfElems * elemSize; 805 char * addr = (char *)memalignNoStats( alignment, size ); 806 if ( unlikely( addr == 0p ) ) return 0p; 807 HeapManager.Storage.Header * header; 808 HeapManager.FreeHeader * freeElem; 809 size_t bsize; 810 bool mapped __attribute__(( unused )) = headers( "cmemalign", addr, header, freeElem, bsize, alignment ); 811 #ifndef __CFA_DEBUG__ 812 // Mapped storage is zero filled, but in debug mode mapped memory is scrubbed in doMalloc, so it has to be reset to zero. 813 if ( ! mapped ) 814 #endif // __CFA_DEBUG__ 815 memset( addr, '\0', dataStorage( bsize, addr, header ) ); // set to zeros 816 header->kind.real.blockSize |= 2; // mark as zero filled 817 818 return addr; 819 } // cmemalignNoStats 765 820 766 821 … … 776 831 extern "C" { 777 832 // The malloc() function allocates size bytes and returns a pointer to the allocated memory. The memory is not 778 // initialized. If size is 0, then malloc() returns either NULL, or a unique pointer value that can later be833 // initialized. If size is 0, then malloc() returns either 0p, or a unique pointer value that can later be 779 834 // successfully passed to free(). 780 835 void * malloc( size_t size ) { … … 788 843 789 844 // The calloc() function allocates memory for an array of nmemb elements of size bytes each and returns a pointer to 790 // the allocated memory. The memory is set to zero. If nmemb or size is 0, then calloc() returns either NULL, or a845 // the allocated memory. The memory is set to zero. If nmemb or size is 0, then calloc() returns either 0p, or a 791 846 // unique pointer value that can later be successfully passed to free(). 792 847 void * calloc( size_t noOfElems, size_t elemSize ) { 793 size_t size = noOfElems * elemSize;794 848 #ifdef __STATISTICS__ 795 849 __atomic_add_fetch( &calloc_calls, 1, __ATOMIC_SEQ_CST ); 796 __atomic_add_fetch( &calloc_storage, size, __ATOMIC_SEQ_CST ); 797 #endif // __STATISTICS__ 798 799 char * area = (char *)mallocNoStats( size ); 800 if ( unlikely( area == 0 ) ) return 0; 850 __atomic_add_fetch( &calloc_storage, noOfElems * elemSize, __ATOMIC_SEQ_CST ); 851 #endif // __STATISTICS__ 852 853 return callocNoStats( noOfElems, elemSize ); 854 } // calloc 855 856 // The realloc() function changes the size of the memory block pointed to by ptr to size bytes. The contents will be 857 // unchanged in the range from the start of the region up to the minimum of the old and new sizes. If the new size 858 // is larger than the old size, the added memory will not be initialized. If ptr is 0p, then the call is 859 // equivalent to malloc(size), for all values of size; if size is equal to zero, and ptr is not 0p, then the call 860 // is equivalent to free(ptr). Unless ptr is 0p, it must have been returned by an earlier call to malloc(), 861 // calloc() or realloc(). If the area pointed to was moved, a free(ptr) is done. 862 void * realloc( void * oaddr, size_t size ) { 863 #ifdef __STATISTICS__ 864 __atomic_add_fetch( &realloc_calls, 1, __ATOMIC_SEQ_CST ); 865 #endif // __STATISTICS__ 866 867 if ( unlikely( size == 0 ) ) { free( oaddr ); return 0p; } // special cases 868 if ( unlikely( oaddr == 0p ) ) return mallocNoStats( size ); 801 869 802 870 HeapManager.Storage.Header * header; 803 871 HeapManager.FreeHeader * freeElem; 804 size_t asize, alignment; 805 bool mapped __attribute__(( unused )) = headers( "calloc", area, header, freeElem, asize, alignment ); 806 #ifndef __CFA_DEBUG__ 807 // Mapped storage is zero filled, but in debug mode mapped memory is scrubbed in doMalloc, so it has to be reset to zero. 808 if ( ! mapped ) 809 #endif // __CFA_DEBUG__ 810 memset( area, '\0', asize - sizeof(HeapManager.Storage) ); // set to zeros 811 812 header->kind.real.blockSize |= 2; // mark as zero filled 813 return area; 814 } // calloc 815 816 // #comment TD : Document this function 817 void * cmemalign( size_t alignment, size_t noOfElems, size_t elemSize ) { 818 size_t size = noOfElems * elemSize; 819 #ifdef __STATISTICS__ 820 __atomic_add_fetch( &cmemalign_calls, 1, __ATOMIC_SEQ_CST ); 821 __atomic_add_fetch( &cmemalign_storage, size, __ATOMIC_SEQ_CST ); 822 #endif // __STATISTICS__ 823 824 char * area = (char *)memalignNoStats( alignment, size ); 825 if ( unlikely( area == 0 ) ) return 0; 826 HeapManager.Storage.Header * header; 827 HeapManager.FreeHeader * freeElem; 828 size_t asize; 829 bool mapped __attribute__(( unused )) = headers( "cmemalign", area, header, freeElem, asize, alignment ); 830 #ifndef __CFA_DEBUG__ 831 // Mapped storage is zero filled, but in debug mode mapped memory is scrubbed in doMalloc, so it has to be reset to zero. 832 if ( ! mapped ) 833 #endif // __CFA_DEBUG__ 834 memset( area, '\0', asize - ( (char *)area - (char *)header ) ); // set to zeros 835 header->kind.real.blockSize |= 2; // mark as zero filled 836 837 return area; 838 } // cmemalign 839 840 // The realloc() function changes the size of the memory block pointed to by ptr to size bytes. The contents will be 841 // unchanged in the range from the start of the region up to the minimum of the old and new sizes. If the new size 842 // is larger than the old size, the added memory will not be initialized. If ptr is NULL, then the call is 843 // equivalent to malloc(size), for all values of size; if size is equal to zero, and ptr is not NULL, then the call 844 // is equivalent to free(ptr). Unless ptr is NULL, it must have been returned by an earlier call to malloc(), 845 // calloc() or realloc(). If the area pointed to was moved, a free(ptr) is done. 846 void * realloc( void * addr, size_t size ) { 847 #ifdef __STATISTICS__ 848 __atomic_add_fetch( &realloc_calls, 1, __ATOMIC_SEQ_CST ); 849 #endif // __STATISTICS__ 850 851 if ( unlikely( addr == 0 ) ) return mallocNoStats( size ); // special cases 852 if ( unlikely( size == 0 ) ) { free( addr ); return 0; } 853 854 HeapManager.Storage.Header * header; 855 HeapManager.FreeHeader * freeElem; 856 size_t asize, alignment = 0; 857 headers( "realloc", addr, header, freeElem, asize, alignment ); 858 859 size_t usize = asize - ( (char *)addr - (char *)header ); // compute the amount of user storage in the block 860 if ( usize >= size ) { // already sufficient storage 872 size_t bsize, oalign = 0; 873 headers( "realloc", oaddr, header, freeElem, bsize, oalign ); 874 875 size_t odsize = dataStorage( bsize, oaddr, header ); // data storage available in bucket 876 if ( size <= odsize && odsize <= size * 2 ) { // allow up to 50% wasted storage in smaller size 877 // Do not know size of original allocation => cannot do 0 fill for any additional space because do not know 878 // where to start filling, i.e., do not overwrite existing values in space. 879 // 861 880 // This case does not result in a new profiler entry because the previous one still exists and it must match with 862 881 // the free for this memory. Hence, this realloc does not appear in the profiler output. 863 return addr;882 return oaddr; 864 883 } // if 865 884 … … 868 887 #endif // __STATISTICS__ 869 888 870 void * area; 871 if ( unlikely( alignment != 0 ) ) { // previous request memalign? 872 area = memalign( alignment, size ); // create new aligned area 889 // change size and copy old content to new storage 890 891 void * naddr; 892 if ( unlikely( oalign != 0 ) ) { // previous request memalign? 893 if ( unlikely( header->kind.real.blockSize & 2 ) ) { // previous request zero fill 894 naddr = cmemalignNoStats( oalign, 1, size ); // create new aligned area 895 } else { 896 naddr = memalignNoStats( oalign, size ); // create new aligned area 897 } // if 873 898 } else { 874 area = mallocNoStats( size ); // create new area 899 if ( unlikely( header->kind.real.blockSize & 2 ) ) { // previous request zero fill 900 naddr = callocNoStats( 1, size ); // create new area 901 } else { 902 naddr = mallocNoStats( size ); // create new area 903 } // if 875 904 } // if 876 if ( unlikely( area == 0 ) ) return 0; 877 if ( unlikely( header->kind.real.blockSize & 2 ) ) { // previous request zero fill (calloc/cmemalign) ? 878 assert( (header->kind.real.blockSize & 1) == 0 ); 879 bool mapped __attribute__(( unused )) = headers( "realloc", area, header, freeElem, asize, alignment ); 880 #ifndef __CFA_DEBUG__ 881 // Mapped storage is zero filled, but in debug mode mapped memory is scrubbed in doMalloc, so it has to be reset to zero. 882 if ( ! mapped ) 883 #endif // __CFA_DEBUG__ 884 memset( (char *)area + usize, '\0', asize - ( (char *)area - (char *)header ) - usize ); // zero-fill back part 885 header->kind.real.blockSize |= 2; // mark new request as zero fill 886 } // if 887 memcpy( area, addr, usize ); // copy bytes 888 free( addr ); 889 return area; 905 if ( unlikely( naddr == 0p ) ) return 0p; 906 headers( "realloc", naddr, header, freeElem, bsize, oalign ); 907 size_t ndsize = dataStorage( bsize, naddr, header ); // data storage avilable in bucket 908 // To preserve prior fill, the entire bucket must be copied versus the size. 909 memcpy( naddr, oaddr, MIN( odsize, ndsize ) ); // copy bytes 910 free( oaddr ); 911 return naddr; 890 912 } // realloc 891 913 … … 898 920 #endif // __STATISTICS__ 899 921 900 void * area = memalignNoStats( alignment, size ); 901 902 return area; 922 return memalignNoStats( alignment, size ); 903 923 } // memalign 924 925 926 // The cmemalign() function is the same as calloc() with memory alignment. 927 void * cmemalign( size_t alignment, size_t noOfElems, size_t elemSize ) { 928 #ifdef __STATISTICS__ 929 __atomic_add_fetch( &cmemalign_calls, 1, __ATOMIC_SEQ_CST ); 930 __atomic_add_fetch( &cmemalign_storage, noOfElems * elemSize, __ATOMIC_SEQ_CST ); 931 #endif // __STATISTICS__ 932 933 return cmemalignNoStats( alignment, noOfElems, elemSize ); 934 } // cmemalign 904 935 905 936 // The function aligned_alloc() is the same as memalign(), except for the added restriction that size should be a … … 912 943 // The function posix_memalign() allocates size bytes and places the address of the allocated memory in *memptr. The 913 944 // address of the allocated memory will be a multiple of alignment, which must be a power of two and a multiple of 914 // sizeof(void *). If size is 0, then posix_memalign() returns either NULL, or a unique pointer value that can later945 // sizeof(void *). If size is 0, then posix_memalign() returns either 0p, or a unique pointer value that can later 915 946 // be successfully passed to free(3). 916 947 int posix_memalign( void ** memptr, size_t alignment, size_t size ) { 917 948 if ( alignment < sizeof(void *) || ! libPow2( alignment ) ) return EINVAL; // check alignment 918 949 * memptr = memalign( alignment, size ); 919 if ( unlikely( * memptr == 0 ) ) return ENOMEM;950 if ( unlikely( * memptr == 0p ) ) return ENOMEM; 920 951 return 0; 921 952 } // posix_memalign … … 930 961 // The free() function frees the memory space pointed to by ptr, which must have been returned by a previous call to 931 962 // malloc(), calloc() or realloc(). Otherwise, or if free(ptr) has already been called before, undefined behavior 932 // occurs. If ptr is NULL, no operation is performed.963 // occurs. If ptr is 0p, no operation is performed. 933 964 void free( void * addr ) { 934 965 #ifdef __STATISTICS__ … … 936 967 #endif // __STATISTICS__ 937 968 938 // #comment TD : To decrease nesting I would but the special case in the 939 // else instead, plus it reads more naturally to have the 940 // short / normal case instead 941 if ( unlikely( addr == 0 ) ) { // special case 942 #ifdef __CFA_DEBUG__ 943 if ( traceHeap() ) { 944 #define nullmsg "Free( 0x0 ) size:0\n" 945 // Do not debug print free( 0 ), as it can cause recursive entry from sprintf. 946 __cfaabi_dbg_bits_write( nullmsg, sizeof(nullmsg) - 1 ); 947 } // if 948 #endif // __CFA_DEBUG__ 969 if ( unlikely( addr == 0p ) ) { // special case 970 // #ifdef __CFA_DEBUG__ 971 // if ( traceHeap() ) { 972 // #define nullmsg "Free( 0x0 ) size:0\n" 973 // // Do not debug print free( 0 ), as it can cause recursive entry from sprintf. 974 // __cfaabi_dbg_write( nullmsg, sizeof(nullmsg) - 1 ); 975 // } // if 976 // #endif // __CFA_DEBUG__ 949 977 return; 950 978 } // exit … … 953 981 } // free 954 982 955 // The mallopt() function adjusts parameters that control the behavior of the memory-allocation functions (see956 // malloc(3)). The param argument specifies the parameter to be modified, and value specifies the new value for that957 // parameter.958 int mallopt( int option, int value ) {959 choose( option ) {960 case M_TOP_PAD:961 if ( setHeapExpand( value ) ) fallthru default;962 case M_MMAP_THRESHOLD:963 if ( setMmapStart( value ) ) fallthru default;964 default:965 // #comment TD : 1 for unsopported feels wrong966 return 1; // success, or unsupported967 } // switch968 return 0; // error969 } // mallopt970 971 // The malloc_trim() function attempts to release free memory at the top of the heap (by calling sbrk(2) with a972 // suitable argument).973 int malloc_trim( size_t ) {974 return 0; // => impossible to release memory975 } // malloc_trim976 977 // The malloc_usable_size() function returns the number of usable bytes in the block pointed to by ptr, a pointer to978 // a block of memory allocated by malloc(3) or a related function.979 size_t malloc_usable_size( void * addr ) {980 if ( unlikely( addr == 0 ) ) return 0; // null allocation has 0 size981 982 HeapManager.Storage.Header * header;983 HeapManager.FreeHeader * freeElem;984 size_t size, alignment;985 986 headers( "malloc_usable_size", addr, header, freeElem, size, alignment );987 size_t usize = size - ( (char *)addr - (char *)header ); // compute the amount of user storage in the block988 return usize;989 } // malloc_usable_size990 991 983 992 984 // The malloc_alignment() function returns the alignment of the allocation. 993 985 size_t malloc_alignment( void * addr ) { 994 if ( unlikely( addr == 0 ) ) return libAlign(); // minimum alignment986 if ( unlikely( addr == 0p ) ) return libAlign(); // minimum alignment 995 987 HeapManager.Storage.Header * header = headerAddr( addr ); 996 988 if ( (header->kind.fake.alignment & 1) == 1 ) { // fake header ? … … 1004 996 // The malloc_zero_fill() function returns true if the allocation is zero filled, i.e., initially allocated by calloc(). 1005 997 bool malloc_zero_fill( void * addr ) { 1006 if ( unlikely( addr == 0 ) ) return false; // null allocation is not zero fill998 if ( unlikely( addr == 0p ) ) return false; // null allocation is not zero fill 1007 999 HeapManager.Storage.Header * header = headerAddr( addr ); 1008 1000 if ( (header->kind.fake.alignment & 1) == 1 ) { // fake header ? … … 1013 1005 1014 1006 1007 // The malloc_usable_size() function returns the number of usable bytes in the block pointed to by ptr, a pointer to 1008 // a block of memory allocated by malloc(3) or a related function. 1009 size_t malloc_usable_size( void * addr ) { 1010 if ( unlikely( addr == 0p ) ) return 0; // null allocation has 0 size 1011 HeapManager.Storage.Header * header; 1012 HeapManager.FreeHeader * freeElem; 1013 size_t bsize, alignment; 1014 1015 headers( "malloc_usable_size", addr, header, freeElem, bsize, alignment ); 1016 return dataStorage( bsize, addr, header ); // data storage in bucket 1017 } // malloc_usable_size 1018 1019 1015 1020 // The malloc_stats() function prints (on default standard error) statistics about memory allocated by malloc(3) and 1016 1021 // related functions. … … 1018 1023 #ifdef __STATISTICS__ 1019 1024 printStats(); 1020 if ( checkFree() ) checkFree( heapManager );1025 if ( prtFree() ) prtFree( heapManager ); 1021 1026 #endif // __STATISTICS__ 1022 1027 } // malloc_stats 1023 1028 1024 1029 // The malloc_stats_fd() function changes the file descripter where malloc_stats() writes the statistics. 1025 int malloc_stats_fd( int fd ) {1030 int malloc_stats_fd( int fd __attribute__(( unused )) ) { 1026 1031 #ifdef __STATISTICS__ 1027 1032 int temp = statfd; … … 1033 1038 } // malloc_stats_fd 1034 1039 1040 1041 // The mallopt() function adjusts parameters that control the behavior of the memory-allocation functions (see 1042 // malloc(3)). The param argument specifies the parameter to be modified, and value specifies the new value for that 1043 // parameter. 1044 int mallopt( int option, int value ) { 1045 choose( option ) { 1046 case M_TOP_PAD: 1047 if ( setHeapExpand( value ) ) return 1; 1048 case M_MMAP_THRESHOLD: 1049 if ( setMmapStart( value ) ) return 1; 1050 } // switch 1051 return 0; // error, unsupported 1052 } // mallopt 1053 1054 // The malloc_trim() function attempts to release free memory at the top of the heap (by calling sbrk(2) with a 1055 // suitable argument). 1056 int malloc_trim( size_t ) { 1057 return 0; // => impossible to release memory 1058 } // malloc_trim 1059 1060 1035 1061 // The malloc_info() function exports an XML string that describes the current state of the memory-allocation 1036 1062 // implementation in the caller. The string is printed on the file stream stream. The exported string includes 1037 1063 // information about all arenas (see malloc(3)). 1038 1064 int malloc_info( int options, FILE * stream ) { 1065 if ( options != 0 ) { errno = EINVAL; return -1; } 1039 1066 return printStatsXML( stream ); 1040 1067 } // malloc_info … … 1046 1073 // structure is returned as the function result. (It is the caller's responsibility to free(3) this memory.) 1047 1074 void * malloc_get_state( void ) { 1048 return 0 ; // unsupported1075 return 0p; // unsupported 1049 1076 } // malloc_get_state 1050 1077 … … 1058 1085 1059 1086 1087 // Must have CFA linkage to overload with C linkage realloc. 1088 void * realloc( void * oaddr, size_t nalign, size_t size ) { 1089 #ifdef __STATISTICS__ 1090 __atomic_add_fetch( &realloc_calls, 1, __ATOMIC_SEQ_CST ); 1091 #endif // __STATISTICS__ 1092 1093 if ( unlikely( size == 0 ) ) { free( oaddr ); return 0p; } // special cases 1094 if ( unlikely( oaddr == 0p ) ) return mallocNoStats( size ); 1095 1096 if ( unlikely( nalign == 0 ) ) nalign = libAlign(); // reset alignment to minimum 1097 #ifdef __CFA_DEBUG__ 1098 else 1099 checkAlign( nalign ); // check alignment 1100 #endif // __CFA_DEBUG__ 1101 1102 HeapManager.Storage.Header * header; 1103 HeapManager.FreeHeader * freeElem; 1104 size_t bsize, oalign = 0; 1105 headers( "realloc", oaddr, header, freeElem, bsize, oalign ); 1106 1107 size_t odsize = dataStorage( bsize, oaddr, header ); // data storage available in bucket 1108 1109 if ( oalign != 0 && (uintptr_t)oaddr % nalign == 0 ) { // has alignment and just happens to work out 1110 headerAddr( oaddr )->kind.fake.alignment = nalign | 1; // update alignment (could be the same) 1111 return realloc( oaddr, size ); 1112 } // if 1113 1114 #ifdef __STATISTICS__ 1115 __atomic_add_fetch( &realloc_storage, size, __ATOMIC_SEQ_CST ); 1116 #endif // __STATISTICS__ 1117 1118 // change size and copy old content to new storage 1119 1120 void * naddr; 1121 if ( unlikely( header->kind.real.blockSize & 2 ) ) { // previous request zero fill 1122 naddr = cmemalignNoStats( nalign, 1, size ); // create new aligned area 1123 } else { 1124 naddr = memalignNoStats( nalign, size ); // create new aligned area 1125 } // if 1126 size_t ndsize = dataStorage( bsize, naddr, header ); // data storage avilable in bucket 1127 // To preserve prior fill, the entire bucket must be copied versus the size. 1128 memcpy( naddr, oaddr, MIN( odsize, ndsize ) ); // copy bytes 1129 free( oaddr ); 1130 return naddr; 1131 } // realloc 1132 1133 1060 1134 // Local Variables: // 1061 1135 // tab-width: 4 //
Note: See TracChangeset
for help on using the changeset viewer.