Changeset 19e5d65d
- Timestamp:
- Apr 24, 2022, 10:47:18 AM (2 years ago)
- Branches:
- ADT, ast-experimental, master, pthread-emulation, qualifiedEnum
- Children:
- 3229b4f
- Parents:
- 58e97d9
- File:
-
- 1 edited
-
libcfa/src/heap.cfa (modified) (37 diffs)
Legend:
- Unmodified
- Added
- Removed
-
libcfa/src/heap.cfa
r58e97d9 r19e5d65d 10 10 // Created On : Tue Dec 19 21:58:35 2017 11 11 // Last Modified By : Peter A. Buhr 12 // Last Modified On : Fri Apr 22 18:25:23202213 // Update Count : 11 2112 // Last Modified On : Sun Apr 24 09:58:01 2022 13 // Update Count : 1146 14 14 // 15 15 … … 240 240 static_assert( NoBucketSizes == sizeof(bucketSizes) / sizeof(bucketSizes[0] ), "size of bucket array wrong" ); 241 241 242 243 242 // The constructor for heapManager is called explicitly in memory_startup. 244 243 static Heap heapManager __attribute__(( aligned (128) )) @= {}; // size of cache line to prevent false sharing 244 245 246 //####################### Memory Allocation Routines Helpers #################### 245 247 246 248 … … 327 329 328 330 // Use "write" because streams may be shutdown when calls are made. 329 static voidprintStats() { // see malloc_stats331 static int printStats() { // see malloc_stats 330 332 char helpText[sizeof(prtFmt) + 1024]; // space for message and values 331 __cfaabi_bits_print_buffer( STDERR_FILENO, helpText, sizeof(helpText), prtFmt,333 return __cfaabi_bits_print_buffer( STDERR_FILENO, helpText, sizeof(helpText), prtFmt, 332 334 stats.malloc_calls, stats.malloc_0_calls, stats.malloc_storage_request, stats.malloc_storage_alloc, 333 335 stats.aalloc_calls, stats.aalloc_0_calls, stats.aalloc_storage_request, stats.aalloc_storage_alloc, … … 417 419 // <-----------------<------------+-----------------------------> bsize (bucket size) 418 420 // |fake-header | addr 419 #define headerAddr( addr ) ((Heap.Storage.Header *)( (char *)addr - sizeof(Heap.Storage) ))420 #define realHeader( header ) ((Heap.Storage.Header *)((char *)header - header->kind.fake.offset))421 #define HeaderAddr( addr ) ((Heap.Storage.Header *)( (char *)addr - sizeof(Heap.Storage) )) 422 #define RealHeader( header ) ((Heap.Storage.Header *)((char *)header - header->kind.fake.offset)) 421 423 422 424 // <-------<<--------------------- dsize ---------------------->> bsize (bucket size) … … 426 428 // <------------------------------<<---------- dsize --------->>> bsize (bucket size) 427 429 // |fake-header |addr 428 #define dataStorage( bsize, addr, header ) (bsize - ( (char *)addr - (char *)header ))430 #define DataStorage( bsize, addr, header ) (bsize - ( (char *)addr - (char *)header )) 429 431 430 432 431 433 static inline void checkAlign( size_t alignment ) { 432 if ( alignment < libAlign() || ! is_pow2( alignment) ) {433 abort( " Alignment %zu for memory allocation is less than %d and/or not a power of 2.", alignment, libAlign() );434 if ( unlikely( alignment < libAlign() || ! is_pow2( alignment ) ) ) { 435 abort( "**** Error **** alignment %zu for memory allocation is less than %d and/or not a power of 2.", alignment, libAlign() ); 434 436 } // if 435 437 } // checkAlign … … 438 440 static inline void checkHeader( bool check, const char name[], void * addr ) { 439 441 if ( unlikely( check ) ) { // bad address ? 440 abort( " Attempt to %s storage %p with address outside the heap.\n"442 abort( "**** Error **** attempt to %s storage %p with address outside the heap.\n" 441 443 "Possible cause is duplicate free on same block or overwriting of memory.", 442 444 name, addr ); … … 445 447 446 448 449 // Manipulate sticky bits stored in unused 3 low-order bits of an address. 450 // bit0 => alignment => fake header 451 // bit1 => zero filled (calloc) 452 // bit2 => mapped allocation versus sbrk 453 #define StickyBits( header ) (((header)->kind.real.blockSize & 0x7)) 454 #define ClearStickyBits( addr ) (typeof(addr))((uintptr_t)(addr) & ~7) 455 #define MarkAlignmentBit( align ) ((align) | 1) 456 #define AlignmentBit( header ) ((((header)->kind.fake.alignment) & 1)) 457 #define ClearAlignmentBit( header ) (((header)->kind.fake.alignment) & ~1) 458 #define ZeroFillBit( header ) ((((header)->kind.real.blockSize) & 2)) 459 #define ClearZeroFillBit( header ) ((((header)->kind.real.blockSize) &= ~2)) 460 #define MarkZeroFilledBit( header ) ((header)->kind.real.blockSize |= 2) 461 #define MmappedBit( header ) ((((header)->kind.real.blockSize) & 4)) 462 #define MarkMmappedBit( size ) ((size) | 4) 463 464 447 465 static inline void fakeHeader( Heap.Storage.Header *& header, size_t & alignment ) { 448 if ( unlikely( (header->kind.fake.alignment & 1) == 1 ) ) {// fake header ?449 alignment = header->kind.fake.alignment & -2; // removeflag from value466 if ( unlikely( AlignmentBit( header ) ) ) { // fake header ? 467 alignment = ClearAlignmentBit( header ); // clear flag from value 450 468 #ifdef __CFA_DEBUG__ 451 469 checkAlign( alignment ); // check alignment 452 470 #endif // __CFA_DEBUG__ 453 header = realHeader( header ); // backup from fake to real header471 header = RealHeader( header ); // backup from fake to real header 454 472 } else { 455 473 alignment = libAlign(); // => no fake header … … 458 476 459 477 460 static inline bool headers( const char name[] __attribute__(( unused )), void * addr, Heap.Storage.Header *& header, Heap.FreeHeader *& freeElem, 461 size_t & size, size_t & alignment ) with( heapManager ) { 462 header = headerAddr( addr ); 463 464 if ( unlikely( addr < heapBegin || heapEnd < addr ) ) { // mmapped ? 465 fakeHeader( header, alignment ); 466 size = header->kind.real.blockSize & -3; // mmap size 467 return true; 468 } // if 478 static inline bool headers( const char name[] __attribute__(( unused )), void * addr, Heap.Storage.Header *& header, 479 Heap.FreeHeader *& freeHead, size_t & size, size_t & alignment ) with( heapManager ) { 480 header = HeaderAddr( addr ); 469 481 470 482 #ifdef __CFA_DEBUG__ … … 472 484 #endif // __CFA_DEBUG__ 473 485 474 // header may be safe to dereference 475 fakeHeader( header, alignment ); 486 if ( likely( ! StickyBits( header ) ) ) { // no sticky bits ? 487 freeHead = (Heap.FreeHeader *)(header->kind.real.home); 488 alignment = libAlign(); 489 } else { 490 fakeHeader( header, alignment ); 491 if ( unlikely( MmappedBit( header ) ) ) { 492 assert( addr < heapBegin || heapEnd < addr ); 493 size = ClearStickyBits( header->kind.real.blockSize ); // mmap size 494 return true; 495 } // if 496 497 freeHead = (Heap.FreeHeader *)(ClearStickyBits( header->kind.real.home )); 498 } // if 499 size = freeHead->blockSize; 500 476 501 #ifdef __CFA_DEBUG__ 477 502 checkHeader( header < (Heap.Storage.Header *)heapBegin || (Heap.Storage.Header *)heapEnd < header, name, addr ); // bad address ? (offset could be + or -) 503 504 if ( freeHead < &freeLists[0] || &freeLists[NoBucketSizes] <= freeHead ) { 505 abort( "Attempt to %s storage %p with corrupted header.\n" 506 "Possible cause is duplicate free on same block or overwriting of header information.", 507 name, addr ); 508 } // if 478 509 #endif // __CFA_DEBUG__ 479 510 480 freeElem = (Heap.FreeHeader *)((size_t)header->kind.real.home & -3);481 #ifdef __CFA_DEBUG__482 if ( freeElem < &freeLists[0] || &freeLists[NoBucketSizes] <= freeElem ) {483 abort( "Attempt to %s storage %p with corrupted header.\n"484 "Possible cause is duplicate free on same block or overwriting of header information.",485 name, addr );486 } // if487 #endif // __CFA_DEBUG__488 size = freeElem->blockSize;489 511 return false; 490 512 } // headers … … 513 535 static inline void * extend( size_t size ) with( heapManager ) { 514 536 lock( extlock __cfaabi_dbg_ctx2 ); 537 515 538 ptrdiff_t rem = heapRemaining - size; 516 if ( rem < 0) {539 if ( unlikely( rem < 0 ) ) { 517 540 // If the size requested is bigger than the current remaining storage, increase the size of the heap. 518 541 519 size_t increase = ceiling2( size > heapExpand ? size : heapExpand, __page_size);542 size_t increase = ceiling2( size > heapExpand ? size : heapExpand, libAlign() ); 520 543 // Do not call abort or strerror( errno ) as they may call malloc. 521 544 if ( sbrk( increase ) == (void *)-1 ) { // failed, no memory ? … … 524 547 _exit( EXIT_FAILURE ); // give up 525 548 } // if 549 526 550 // Make storage executable for thunks. 527 if ( mprotect( (char *)heapEnd + heapRemaining, increase, __map_prot ) ) { 528 unlock( extlock ); 529 __cfaabi_bits_print_nolock( STDERR_FILENO, "extend() : internal error, mprotect failure, heapEnd:%p size:%zd, errno:%d.\n", heapEnd, increase, errno ); 530 _exit( EXIT_FAILURE ); 531 } // if 551 // if ( mprotect( (char *)heapEnd + heapRemaining, increase, __map_prot ) ) { 552 // unlock( extlock ); 553 // __cfaabi_bits_print_nolock( STDERR_FILENO, "extend() : internal error, mprotect failure, heapEnd:%p size:%zd, errno:%d.\n", heapEnd, increase, errno ); 554 // _exit( EXIT_FAILURE ); 555 // } // if 556 532 557 #ifdef __STATISTICS__ 533 558 sbrk_calls += 1; … … 555 580 // Look up size in the size list. Make sure the user request includes space for the header that must be allocated 556 581 // along with the block and is a multiple of the alignment size. 557 558 if ( unlikely( size > ULONG_MAX - sizeof(Heap.Storage) ) ) return 0p;559 582 size_t tsize = size + sizeof(Heap.Storage); 583 560 584 if ( likely( tsize < mmapStart ) ) { // small size => sbrk 561 585 size_t posn; … … 615 639 //Memset( block, tsize ); 616 640 #endif // __CFA_DEBUG__ 617 block->header.kind.real.blockSize = tsize;// storage size for munmap641 block->header.kind.real.blockSize = MarkMmappedBit( tsize ); // storage size for munmap 618 642 } // if 619 643 … … 825 849 826 850 // address of header from malloc 827 Heap.Storage.Header * realHeader = headerAddr( addr );828 realHeader->kind.real.size = size; // correct size to eliminate above alignment offset851 Heap.Storage.Header * RealHeader = HeaderAddr( addr ); 852 RealHeader->kind.real.size = size; // correct size to eliminate above alignment offset 829 853 // address of fake header * before* the alignment location 830 Heap.Storage.Header * fakeHeader = headerAddr( user );854 Heap.Storage.Header * fakeHeader = HeaderAddr( user ); 831 855 // SKULLDUGGERY: insert the offset to the start of the actual storage block and remember alignment 832 fakeHeader->kind.fake.offset = (char *)fakeHeader - (char *) realHeader;856 fakeHeader->kind.fake.offset = (char *)fakeHeader - (char *)RealHeader; 833 857 // SKULLDUGGERY: odd alignment imples fake header 834 fakeHeader->kind.fake.alignment = alignment | 1;858 fakeHeader->kind.fake.alignment = MarkAlignmentBit( alignment ); 835 859 836 860 return user; 837 861 } // memalignNoStats 862 863 864 //####################### Memory Allocation Routines #################### 838 865 839 866 … … 904 931 memset( addr, '\0', size ); // set to zeros 905 932 906 header->kind.real.blockSize |= 2; // mark as zero filled933 MarkZeroFilledBit( header ); // mark as zero fill 907 934 return addr; 908 935 } // calloc … … 938 965 headers( "resize", oaddr, header, freeElem, bsize, oalign ); 939 966 940 size_t odsize = dataStorage( bsize, oaddr, header ); // data storage available in bucket967 size_t odsize = DataStorage( bsize, oaddr, header ); // data storage available in bucket 941 968 // same size, DO NOT preserve STICKY PROPERTIES. 942 969 if ( oalign == libAlign() && size <= odsize && odsize <= size * 2 ) { // allow 50% wasted storage for smaller size 943 header->kind.real.blockSize &= -2;// no alignment and turn off 0 fill970 ClearZeroFillBit( header ); // no alignment and turn off 0 fill 944 971 header->kind.real.size = size; // reset allocation size 945 972 return oaddr; … … 983 1010 headers( "realloc", oaddr, header, freeElem, bsize, oalign ); 984 1011 985 size_t odsize = dataStorage( bsize, oaddr, header ); // data storage available in bucket1012 size_t odsize = DataStorage( bsize, oaddr, header ); // data storage available in bucket 986 1013 size_t osize = header->kind.real.size; // old allocation size 987 bool ozfill = (header->kind.real.blockSize & 2);// old allocation zero filled1014 bool ozfill = ZeroFillBit( header ); // old allocation zero filled 988 1015 if ( unlikely( size <= odsize ) && odsize <= size * 2 ) { // allow up to 50% wasted storage 989 1016 header->kind.real.size = size; // reset allocation size … … 1012 1039 1013 1040 if ( unlikely( ozfill ) ) { // previous request zero fill ? 1014 header->kind.real.blockSize |= 2;// mark new request as zero filled1041 MarkZeroFilledBit( header ); // mark new request as zero filled 1015 1042 if ( size > osize ) { // previous request larger ? 1016 1043 memset( (char *)naddr + osize, '\0', size - osize ); // initialize added storage … … 1019 1046 return naddr; 1020 1047 } // realloc 1048 1049 1050 // Same as realloc() except the new allocation size is large enough for an array of nelem elements of size elsize. 1051 void * reallocarray( void * oaddr, size_t dim, size_t elemSize ) { 1052 return realloc( oaddr, dim * elemSize ); 1053 } // reallocarray 1021 1054 1022 1055 … … 1085 1118 memset( addr, '\0', size ); // set to zeros 1086 1119 1087 header->kind.real.blockSize |= 2;// mark as zero filled1120 MarkZeroFilledBit( header ); // mark as zero filled 1088 1121 return addr; 1089 1122 } // cmemalign … … 1091 1124 1092 1125 // Same as memalign(), but ISO/IEC 2011 C11 Section 7.22.2 states: the value of size shall be an integral multiple 1093 1126 // of alignment. This requirement is universally ignored. 1094 1127 void * aligned_alloc( size_t alignment, size_t size ) { 1095 1128 return memalign( alignment, size ); … … 1102 1135 // free(3). 1103 1136 int posix_memalign( void ** memptr, size_t alignment, size_t size ) { 1104 if ( alignment < libAlign() || ! is_pow2( alignment) ) return EINVAL; // check alignment1105 * 1137 if ( unlikely( alignment < libAlign() || ! is_pow2( alignment ) ) ) return EINVAL; // check alignment 1138 *memptr = memalign( alignment, size ); 1106 1139 return 0; 1107 1140 } // posix_memalign … … 1117 1150 // Same as valloc but rounds size to multiple of page size. 1118 1151 void * pvalloc( size_t size ) { 1119 return memalign( __page_size, ceiling2( size, __page_size ) ); 1152 return memalign( __page_size, ceiling2( size, __page_size ) ); // round size to multiple of page size 1120 1153 } // pvalloc 1121 1154 … … 1147 1180 size_t malloc_alignment( void * addr ) { 1148 1181 if ( unlikely( addr == 0p ) ) return libAlign(); // minimum alignment 1149 Heap.Storage.Header * header = headerAddr( addr );1150 if ( (header->kind.fake.alignment & 1) == 1 ) {// fake header ?1151 return header->kind.fake.alignment & -2; // removeflag from value1182 Heap.Storage.Header * header = HeaderAddr( addr ); 1183 if ( unlikely( AlignmentBit( header ) ) ) { // fake header ? 1184 return ClearAlignmentBit( header ); // clear flag from value 1152 1185 } else { 1153 1186 return libAlign(); // minimum alignment … … 1157 1190 1158 1191 // Set the alignment for an the allocation and return previous alignment or 0 if no alignment. 1159 size_t malloc_alignment_set$( void * addr, size_t alignment ) {1160 if ( unlikely( addr == 0p ) ) return libAlign(); // minimum alignment1161 size_t ret;1162 Heap.Storage.Header * header = headerAddr( addr );1163 if ( (header->kind.fake.alignment & 1) == 1 ) { // fake header ?1164 ret = header->kind.fake.alignment & -2; // remove flag from old value1165 header->kind.fake.alignment = alignment | 1; // add flag to new value1166 } else {1167 ret = 0; // => no alignment to change1168 } // if1169 return ret;1170 } // malloc_alignment_set$1192 // size_t malloc_alignment_set$( void * addr, size_t alignment ) { 1193 // if ( unlikely( addr == 0p ) ) return libAlign(); // minimum alignment 1194 // size_t ret; 1195 // Heap.Storage.Header * header = HeaderAddr( addr ); 1196 // if ( (header->kind.fake.alignment & 1) == 1 ) { // fake header ? 1197 // ret = header->kind.fake.alignment & -2; // remove flag from old value 1198 // header->kind.fake.alignment = alignment | 1; // add flag to new value 1199 // } else { 1200 // ret = 0; // => no alignment to change 1201 // } // if 1202 // return ret; 1203 // } // malloc_alignment_set$ 1171 1204 1172 1205 … … 1174 1207 bool malloc_zero_fill( void * addr ) { 1175 1208 if ( unlikely( addr == 0p ) ) return false; // null allocation is not zero fill 1176 Heap.Storage.Header * header = headerAddr( addr );1177 if ( (header->kind.fake.alignment & 1) == 1 ) {// fake header ?1178 header = realHeader( header ); // backup from fake to real header1179 } // if 1180 return (header->kind.real.blockSize & 2) != 0;// zero filled ?1209 Heap.Storage.Header * header = HeaderAddr( addr ); 1210 if ( unlikely( AlignmentBit( header ) ) ) { // fake header ? 1211 header = RealHeader( header ); // backup from fake to real header 1212 } // if 1213 return ZeroFillBit( header ); // zero filled ? 1181 1214 } // malloc_zero_fill 1182 1215 1183 1216 // Set allocation is zero filled and return previous zero filled. 1184 bool malloc_zero_fill_set$( void * addr ) {1185 if ( unlikely( addr == 0p ) ) return false; // null allocation is not zero fill1186 Heap.Storage.Header * header = headerAddr( addr );1187 if ( (header->kind.fake.alignment & 1) == 1 ) { // fake header ?1188 header = realHeader( header ); // backup from fake to real header1189 } // if1190 bool ret = (header->kind.real.blockSize & 2) != 0; // zero filled ?1191 header->kind.real.blockSize |= 2; // mark as zero filled1192 return ret;1193 } // malloc_zero_fill_set$1194 1195 1196 // Returns original total allocation size (not bucket size) => array size is dimension * size if(T).1217 // bool malloc_zero_fill_set$( void * addr ) { 1218 // if ( unlikely( addr == 0p ) ) return false; // null allocation is not zero fill 1219 // Heap.Storage.Header * header = HeaderAddr( addr ); 1220 // if ( (header->kind.fake.alignment & 1) == 1 ) { // fake header ? 1221 // header = RealHeader( header ); // backup from fake to real header 1222 // } // if 1223 // bool ret = (header->kind.real.blockSize & 2) != 0; // zero filled ? 1224 // header->kind.real.blockSize |= 2; // mark as zero filled 1225 // return ret; 1226 // } // malloc_zero_fill_set$ 1227 1228 1229 // Returns original total allocation size (not bucket size) => array size is dimension * sizeof(T). 1197 1230 size_t malloc_size( void * addr ) { 1198 1231 if ( unlikely( addr == 0p ) ) return 0; // null allocation has zero size 1199 Heap.Storage.Header * header = headerAddr( addr );1200 if ( (header->kind.fake.alignment & 1) == 1 ) {// fake header ?1201 header = realHeader( header ); // backup from fake to real header1232 Heap.Storage.Header * header = HeaderAddr( addr ); 1233 if ( unlikely( AlignmentBit( header ) ) ) { // fake header ? 1234 header = RealHeader( header ); // backup from fake to real header 1202 1235 } // if 1203 1236 return header->kind.real.size; … … 1205 1238 1206 1239 // Set allocation size and return previous size. 1207 size_t malloc_size_set$( void * addr, size_t size ) {1208 if ( unlikely( addr == 0p ) ) return 0; // null allocation has 0 size1209 Heap.Storage.Header * header = headerAddr( addr );1210 if ( (header->kind.fake.alignment & 1) == 1 ) { // fake header ?1211 header = realHeader( header ); // backup from fake to real header1212 } // if1213 size_t ret = header->kind.real.size;1214 header->kind.real.size = size;1215 return ret;1216 } // malloc_size_set$1240 // size_t malloc_size_set$( void * addr, size_t size ) { 1241 // if ( unlikely( addr == 0p ) ) return 0; // null allocation has 0 size 1242 // Heap.Storage.Header * header = HeaderAddr( addr ); 1243 // if ( (header->kind.fake.alignment & 1) == 1 ) { // fake header ? 1244 // header = RealHeader( header ); // backup from fake to real header 1245 // } // if 1246 // size_t ret = header->kind.real.size; 1247 // header->kind.real.size = size; 1248 // return ret; 1249 // } // malloc_size_set$ 1217 1250 1218 1251 … … 1226 1259 1227 1260 headers( "malloc_usable_size", addr, header, freeElem, bsize, alignment ); 1228 return dataStorage( bsize, addr, header ); // data storage in bucket1261 return DataStorage( bsize, addr, header ); // data storage in bucket 1229 1262 } // malloc_usable_size 1230 1263 … … 1239 1272 1240 1273 1241 // Changes the file descript er where malloc_stats() writes statistics.1274 // Changes the file descriptor where malloc_stats() writes statistics. 1242 1275 int malloc_stats_fd( int fd __attribute__(( unused )) ) { 1243 1276 #ifdef __STATISTICS__ … … 1246 1279 return temp; 1247 1280 #else 1248 return -1; 1281 return -1; // unsupported 1249 1282 #endif // __STATISTICS__ 1250 1283 } // malloc_stats_fd 1284 1285 1286 // Prints an XML string that describes the current state of the memory-allocation implementation in the caller. 1287 // The string is printed on the file stream stream. The exported string includes information about all arenas (see 1288 // malloc). 1289 int malloc_info( int options, FILE * stream __attribute__(( unused )) ) { 1290 if ( options != 0 ) { errno = EINVAL; return -1; } 1291 #ifdef __STATISTICS__ 1292 return printStatsXML( stream ); 1293 #else 1294 return 0; // unsupported 1295 #endif // __STATISTICS__ 1296 } // malloc_info 1251 1297 1252 1298 … … 1254 1300 // specifies the parameter to be modified, and value specifies the new value for that parameter. 1255 1301 int mallopt( int option, int value ) { 1302 if ( value < 0 ) return 0; 1256 1303 choose( option ) { 1257 1304 case M_TOP_PAD: 1258 heapExpand = ceiling2( value, __page_size ); return 1; 1305 heapExpand = ceiling2( value, __page_size ); 1306 return 1; 1259 1307 case M_MMAP_THRESHOLD: 1260 1308 if ( setMmapStart( value ) ) return 1; 1261 break; 1262 } // switch 1309 } // choose 1263 1310 return 0; // error, unsupported 1264 1311 } // mallopt … … 1269 1316 return 0; // => impossible to release memory 1270 1317 } // malloc_trim 1271 1272 1273 // Exports an XML string that describes the current state of the memory-allocation implementation in the caller.1274 // The string is printed on the file stream stream. The exported string includes information about all arenas (see1275 // malloc).1276 int malloc_info( int options, FILE * stream __attribute__(( unused )) ) {1277 if ( options != 0 ) { errno = EINVAL; return -1; }1278 #ifdef __STATISTICS__1279 return printStatsXML( stream );1280 #else1281 return 0; // unsupported1282 #endif // __STATISTICS__1283 } // malloc_info1284 1318 1285 1319 … … 1299 1333 } // malloc_set_state 1300 1334 1335 1301 1336 // Sets the amount (bytes) to extend the heap when there is insufficent free storage to service an allocation. 1302 1337 __attribute__((weak)) size_t malloc_expansion() { return __CFA_DEFAULT_HEAP_EXPANSION__; } … … 1335 1370 1336 1371 // Attempt to reuse existing alignment. 1337 Heap.Storage.Header * header = headerAddr( oaddr );1338 bool isFakeHeader = header->kind.fake.alignment & 1;// old fake header ?1372 Heap.Storage.Header * header = HeaderAddr( oaddr ); 1373 bool isFakeHeader = AlignmentBit( header ); // old fake header ? 1339 1374 size_t oalign; 1340 if ( isFakeHeader ) { 1341 oalign = header->kind.fake.alignment & -2; // old alignment 1342 if ( (uintptr_t)oaddr % nalign == 0 // lucky match ? 1375 1376 if ( unlikely( isFakeHeader ) ) { 1377 oalign = ClearAlignmentBit( header ); // old alignment 1378 if ( unlikely( (uintptr_t)oaddr % nalign == 0 // lucky match ? 1343 1379 && ( oalign <= nalign // going down 1344 1380 || (oalign >= nalign && oalign <= 256) ) // little alignment storage wasted ? 1345 ) {1346 headerAddr( oaddr )->kind.fake.alignment = nalign | 1; // update alignment (could be the same)1381 ) ) { 1382 HeaderAddr( oaddr )->kind.fake.alignment = MarkAlignmentBit( nalign ); // update alignment (could be the same) 1347 1383 Heap.FreeHeader * freeElem; 1348 1384 size_t bsize, oalign; 1349 1385 headers( "resize", oaddr, header, freeElem, bsize, oalign ); 1350 size_t odsize = dataStorage( bsize, oaddr, header ); // data storage available in bucket1386 size_t odsize = DataStorage( bsize, oaddr, header ); // data storage available in bucket 1351 1387 1352 1388 if ( size <= odsize && odsize <= size * 2 ) { // allow 50% wasted data storage 1353 headerAddr( oaddr )->kind.fake.alignment = nalign | 1; // update alignment (could be the same) 1354 1355 header->kind.real.blockSize &= -2; // turn off 0 fill 1389 HeaderAddr( oaddr )->kind.fake.alignment = MarkAlignmentBit( nalign ); // update alignment (could be the same) 1390 ClearZeroFillBit( header ); // turn off 0 fill 1356 1391 header->kind.real.size = size; // reset allocation size 1357 1392 return oaddr; … … 1397 1432 1398 1433 // Attempt to reuse existing alignment. 1399 Heap.Storage.Header * header = headerAddr( oaddr );1400 bool isFakeHeader = header->kind.fake.alignment & 1;// old fake header ?1434 Heap.Storage.Header * header = HeaderAddr( oaddr ); 1435 bool isFakeHeader = AlignmentBit( header ); // old fake header ? 1401 1436 size_t oalign; 1402 if ( isFakeHeader) {1403 oalign = header->kind.fake.alignment & -2;// old alignment1404 if ( (uintptr_t)oaddr % nalign == 0// lucky match ?1437 if ( unlikely( isFakeHeader ) ) { 1438 oalign = ClearAlignmentBit( header ); // old alignment 1439 if ( unlikely( (uintptr_t)oaddr % nalign == 0 // lucky match ? 1405 1440 && ( oalign <= nalign // going down 1406 1441 || (oalign >= nalign && oalign <= 256) ) // little alignment storage wasted ? 1407 ) {1408 headerAddr( oaddr )->kind.fake.alignment = nalign | 1; // update alignment (could be the same)1409 return realloc( oaddr, size ); // duplicate alignment andspecial case checks1442 ) ) { 1443 HeaderAddr( oaddr )->kind.fake.alignment = MarkAlignmentBit( nalign ); // update alignment (could be the same) 1444 return realloc( oaddr, size ); // duplicate special case checks 1410 1445 } // if 1411 1446 } else if ( ! isFakeHeader // old real header (aligned on libAlign) ? 1412 && nalign == libAlign() ) // new alignment also on libAlign => no fake header needed 1413 return realloc( oaddr, size ); // duplicate alignment and special case checks 1447 && nalign == libAlign() ) { // new alignment also on libAlign => no fake header needed 1448 return realloc( oaddr, size ); // duplicate special case checks 1449 } // if 1414 1450 1415 1451 #ifdef __STATISTICS__ … … 1425 1461 1426 1462 size_t osize = header->kind.real.size; // old allocation size 1427 bool ozfill = (header->kind.real.blockSize & 2);// old allocation zero filled1463 bool ozfill = ZeroFillBit( header ); // old allocation zero filled 1428 1464 1429 1465 void * naddr = memalignNoStats( nalign, size ); // create new aligned area … … 1434 1470 1435 1471 if ( unlikely( ozfill ) ) { // previous request zero fill ? 1436 header->kind.real.blockSize |= 2;// mark new request as zero filled1472 MarkZeroFilledBit( header ); // mark new request as zero filled 1437 1473 if ( size > osize ) { // previous request larger ? 1438 1474 memset( (char *)naddr + osize, '\0', size - osize ); // initialize added storage
Note: See TracChangeset
for help on using the changeset viewer.
