Context Navigation

heap.cfa @ b6830d74

ADTaaron-thesisarm-ehast-experimentalcleanup-dtorsdeferred_resndemanglerenumforall-pointer-decayjacob/cs343-translationjenkins-sandboxnew-astnew-ast-unique-exprno_listpersistent-indexerpthread-emulationqualifiedEnum

Last change on this file since b6830d74 was b6830d74, checked in by Thierry Delisle <tdelisle@…>, 6 years ago
First pass of comment on heap.cfa
Property mode set to `100644`
File size: 38.6 KB

Rev	Line
[b6830d74]	1	// #comment TD : this file uses both spaces and tabs for indentation
	2
[73abe95]	3	//
[c4f68dc]	4	// Cforall Version 1.0.0 Copyright (C) 2017 University of Waterloo
	5	//
	6	// The contents of this file are covered under the licence agreement in the
	7	// file "LICENCE" distributed with Cforall.
[73abe95]	8	//
	9	// heap.c --
	10	//
[c4f68dc]	11	// Author : Peter A. Buhr
	12	// Created On : Tue Dec 19 21:58:35 2017
	13	// Last Modified By : Peter A. Buhr
[93c2e0a]	14	// Last Modified On : Sat Aug 11 08:22:16 2018
	15	// Update Count : 495
[73abe95]	16	//
[c4f68dc]	17
	18	#include <unistd.h> // sbrk, sysconf
	19	#include <stdbool.h> // true, false
	20	#include <stdio.h> // snprintf, fileno
	21	#include <errno.h> // errno
	22	extern "C" {
	23	#include <sys/mman.h> // mmap, munmap
	24	} // extern "C"
	25
[b6830d74]	26	// #comment TD : Many of these should be merged into math I believe
[73abe95]	27	#include "bits/align.hfa" // libPow2
	28	#include "bits/defs.hfa" // likely, unlikely
	29	#include "bits/locks.hfa" // __spinlock_t
	30	#include "startup.hfa" // STARTUP_PRIORITY_MEMORY
[58b6d1b]	31	#include "stdlib.hfa" // bsearchl
[c4f68dc]	32	#include "malloc.h"
	33
	34
	35	enum {
	36	__CFA_DEFAULT_MMAP_START__ = (512 * 1024 + 1),
	37	__CFA_DEFAULT_HEAP_EXPANSION__ = (1 * 1024 * 1024),
	38	};
	39
	40	size_t default_mmap_start() __attribute__(( weak )) {
[b6830d74]	41	return __CFA_DEFAULT_MMAP_START__;
[c4f68dc]	42	} // default_mmap_start
	43
	44	size_t default_heap_expansion() __attribute__(( weak )) {
[b6830d74]	45	return __CFA_DEFAULT_HEAP_EXPANSION__;
[c4f68dc]	46	} // default_heap_expansion
	47
	48
	49	// supported mallopt options
	50	#ifndef M_MMAP_THRESHOLD
	51	#define M_MMAP_THRESHOLD (-1)
	52	#endif // M_TOP_PAD
	53	#ifndef M_TOP_PAD
	54	#define M_TOP_PAD (-2)
	55	#endif // M_TOP_PAD
	56
	57	#define FASTLOOKUP
	58	#define __STATISTICS__
	59
	60	#define SPINLOCK 0
	61	#define LOCKFREE 1
	62	#define BUCKETLOCK SPINLOCK
	63	#if BUCKETLOCK == LOCKFREE
	64	#include <uStackLF.h>
	65	#endif // LOCKFREE
	66
[b6830d74]	67	// #comment TD : This defined is significantly different from the __ALIGN__ define from locks.hfa
[c4f68dc]	68	#define ALIGN 16
	69
	70	// enum { NoBucketSizes = 93, // number of buckets sizes
	71	// #ifdef FASTLOOKUP
	72	// LookupSizes = 65536, // number of fast lookup sizes
	73	// #endif // FASTLOOKUP
	74	// };
	75	#define NoBucketSizes 93 // number of buckets sizes
	76	#ifdef FASTLOOKUP
	77	#define LookupSizes 65536 // number of fast lookup sizes
	78	#endif // FASTLOOKUP
	79
	80
[93c2e0a]	81	static bool traceHeap = false;
[d46ed6e]	82
[93c2e0a]	83	inline bool traceHeap() {
[d46ed6e]	84	return traceHeap;
	85	} // traceHeap
	86
[93c2e0a]	87	bool traceHeapOn() {
	88	bool temp = traceHeap;
[d46ed6e]	89	traceHeap = true;
	90	return temp;
	91	} // traceHeapOn
	92
[93c2e0a]	93	bool traceHeapOff() {
	94	bool temp = traceHeap;
[d46ed6e]	95	traceHeap = false;
	96	return temp;
	97	} // traceHeapOff
	98
	99
[93c2e0a]	100	static bool checkFree = false;
[d46ed6e]	101
[93c2e0a]	102	inline bool checkFree() {
[5d4fa18]	103	return checkFree;
	104	} // checkFree
	105
[93c2e0a]	106	bool checkFreeOn() {
	107	bool temp = checkFree;
[5d4fa18]	108	checkFree = true;
	109	return temp;
	110	} // checkFreeOn
	111
[93c2e0a]	112	bool checkFreeOff() {
	113	bool temp = checkFree;
[5d4fa18]	114	checkFree = false;
	115	return temp;
	116	} // checkFreeOff
	117
	118
[93c2e0a]	119	// static bool traceHeapTerm = false;
[5d4fa18]	120
[93c2e0a]	121	// inline bool traceHeapTerm() {
[5d4fa18]	122	// return traceHeapTerm;
	123	// } // traceHeapTerm
[d46ed6e]	124
[93c2e0a]	125	// bool traceHeapTermOn() {
	126	// bool temp = traceHeapTerm;
[5d4fa18]	127	// traceHeapTerm = true;
[d46ed6e]	128	// return temp;
[5d4fa18]	129	// } // traceHeapTermOn
[d46ed6e]	130
[93c2e0a]	131	// bool traceHeapTermOff() {
	132	// bool temp = traceHeapTerm;
[5d4fa18]	133	// traceHeapTerm = false;
[d46ed6e]	134	// return temp;
[5d4fa18]	135	// } // traceHeapTermOff
[d46ed6e]	136
	137
[f0b3f51]	138	#ifdef __CFA_DEBUG__
[93c2e0a]	139	static unsigned int allocFree; // running total of allocations minus frees
[d46ed6e]	140
	141	static void checkUnfreed() {
[b6830d74]	142	if ( allocFree != 0 ) {
[d46ed6e]	143	// DO NOT USE STREAMS AS THEY MAY BE UNAVAILABLE AT THIS POINT.
	144	// char helpText[512];
[93c2e0a]	145	// int len = snprintf( helpText, sizeof(helpText), "CFA warning (UNIX pid:%ld) : program terminating with %u(0x%x) bytes of storage allocated but not freed.\n"
[d46ed6e]	146	// "Possible cause is unfreed storage allocated by the program or system/library routines called from the program.\n",
[93c2e0a]	147	// (long int)getpid(), allocFree, allocFree ); // always print the UNIX pid
[d46ed6e]	148	// __cfaabi_dbg_bits_write( helpText, len );
[b6830d74]	149	} // if
[d46ed6e]	150	} // checkUnfreed
	151
	152	extern "C" {
	153	void heapAppStart() { // called by __cfaabi_appready_startup
[93c2e0a]	154	allocFree = 0;
[d46ed6e]	155	} // heapAppStart
	156
	157	void heapAppStop() { // called by __cfaabi_appready_startdown
[93c2e0a]	158	fclose( stdin ); fclose( stdout );
[d46ed6e]	159	checkUnfreed();
	160	} // heapAppStop
	161	} // extern "C"
	162	#endif // __CFA_DEBUG__
	163
	164
[c4f68dc]	165	struct HeapManager {
	166	// struct FreeHeader; // forward declaration
	167
	168	struct Storage {
	169	struct Header { // header
	170	union Kind {
	171	struct RealHeader {
	172	union {
[b6830d74]	173	// #comment TD : this code use byte size but the comment uses bit size
	174
[c4f68dc]	175	struct { // 32-bit word => 64-bit header, 64-bit word => 128-bit header
[f0b3f51]	176	#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ && __SIZEOF_POINTER__ == 4
[c4f68dc]	177	uint32_t padding; // unused, force home/blocksize to overlay alignment in fake header
[f0b3f51]	178	#endif // __ORDER_BIG_ENDIAN__ && __U_WORDSIZE__ == 32
[c4f68dc]	179
	180	union {
	181	// FreeHeader * home; // allocated block points back to home locations (must overlay alignment)
	182	void * home; // allocated block points back to home locations (must overlay alignment)
	183	size_t blockSize; // size for munmap (must overlay alignment)
	184	#if BUCKLOCK == SPINLOCK
	185	Storage * next; // freed block points next freed block of same size
	186	#endif // SPINLOCK
	187	};
	188
[f0b3f51]	189	#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ && __SIZEOF_POINTER__ == 4
[c4f68dc]	190	uint32_t padding; // unused, force home/blocksize to overlay alignment in fake header
[f0b3f51]	191	#endif // __ORDER_LITTLE_ENDIAN__ && __U_WORDSIZE__ == 32
[c4f68dc]	192
	193	};
[b6830d74]	194
	195	// #comment TD : C++ code
[c4f68dc]	196	#if BUCKLOCK == LOCKFREE
	197	Stack<Storage>::Link next; // freed block points next freed block of same size (double-wide)
	198	#endif // LOCKFREE
	199	};
[93c2e0a]	200	} real; // RealHeader
[c4f68dc]	201	struct FakeHeader {
	202	#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
	203	uint32_t alignment; // low-order bits of home/blockSize used for tricks
[f0b3f51]	204	#endif // __ORDER_LITTLE_ENDIAN__
[c4f68dc]	205
	206	uint32_t offset;
	207
	208	#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
	209	uint32_t alignment; // low-order bits of home/blockSize used for tricks
[f0b3f51]	210	#endif // __ORDER_BIG_ENDIAN__
[93c2e0a]	211	} fake; // FakeHeader
	212	} kind; // Kind
[c4f68dc]	213	} header; // Header
	214	char pad[ALIGN - sizeof( Header )];
	215	char data[0]; // storage
	216	}; // Storage
	217
	218	static_assert( ALIGN >= sizeof( Storage ), "ALIGN < sizeof( Storage )" );
	219
	220	struct FreeHeader {
	221	#if BUCKLOCK == SPINLOCK
	222	__spinlock_t lock; // must be first field for alignment
	223	Storage * freeList;
	224	#elif BUCKLOCK == LOCKFREE
[b6830d74]	225	// #comment TD : C++ code
[c4f68dc]	226	StackLF<Storage> freeList;
	227	#else
	228	#error undefined lock type for bucket lock
	229	#endif // SPINLOCK
	230	size_t blockSize; // size of allocations on this list
	231	}; // FreeHeader
	232
	233	// must be first fields for alignment
	234	__spinlock_t extlock; // protects allocation-buffer extension
	235	FreeHeader freeLists[NoBucketSizes]; // buckets for different allocation sizes
	236
	237	void * heapBegin; // start of heap
	238	void * heapEnd; // logical end of heap
	239	size_t heapRemaining; // amount of storage not allocated in the current chunk
	240	}; // HeapManager
	241
[5d4fa18]	242	static inline size_t getKey( const HeapManager.FreeHeader & freeheader ) { return freeheader.blockSize; }
	243	// statically allocated variables => zero filled.
	244
	245
	246	static size_t pageSize; // architecture pagesize
	247	static size_t heapExpand; // sbrk advance
	248	static size_t mmapStart; // cross over point for mmap
	249	static unsigned int maxBucketsUsed; // maximum number of buckets in use
	250
[b6830d74]	251	// #comment TD : This array is not const but it feels like it should be
[5d4fa18]	252	// Powers of 2 are common allocation sizes, so make powers of 2 generate the minimum required size.
	253	static unsigned int bucketSizes[NoBucketSizes] @= { // different bucket sizes
[b6830d74]	254	16, 32, 48, 64,
	255	64 + sizeof(HeapManager.Storage), 96, 112, 128, 128 + sizeof(HeapManager.Storage), 160, 192, 224,
	256	256 + sizeof(HeapManager.Storage), 320, 384, 448, 512 + sizeof(HeapManager.Storage), 640, 768, 896,
	257	1_024 + sizeof(HeapManager.Storage), 1_536, 2_048 + sizeof(HeapManager.Storage), 2_560, 3_072, 3_584, 4_096 + sizeof(HeapManager.Storage), 6_144,
	258	8_192 + sizeof(HeapManager.Storage), 9_216, 10_240, 11_264, 12_288, 13_312, 14_336, 15_360,
	259	16_384 + sizeof(HeapManager.Storage), 18_432, 20_480, 22_528, 24_576, 26_624, 28_672, 30_720,
	260	32_768 + sizeof(HeapManager.Storage), 36_864, 40_960, 45_056, 49_152, 53_248, 57_344, 61_440,
	261	65_536 + sizeof(HeapManager.Storage), 73_728, 81_920, 90_112, 98_304, 106_496, 114_688, 122_880,
	262	131_072 + sizeof(HeapManager.Storage), 147_456, 163_840, 180_224, 196_608, 212_992, 229_376, 245_760,
	263	262_144 + sizeof(HeapManager.Storage), 294_912, 327_680, 360_448, 393_216, 425_984, 458_752, 491_520,
	264	524_288 + sizeof(HeapManager.Storage), 655_360, 786_432, 917_504, 1_048_576 + sizeof(HeapManager.Storage), 1_179_648, 1_310_720, 1_441_792,
	265	1_572_864, 1_703_936, 1_835_008, 1_966_080, 2_097_152 + sizeof(HeapManager.Storage), 2_621_440, 3_145_728, 3_670_016,
	266	4_194_304 + sizeof(HeapManager.Storage)
[5d4fa18]	267	};
	268	#ifdef FASTLOOKUP
	269	static unsigned char lookup[LookupSizes]; // O(1) lookup for small sizes
	270	#endif // FASTLOOKUP
	271	static int mmapFd = -1; // fake or actual fd for anonymous file
	272
	273
	274	#ifdef __CFA_DEBUG__
[93c2e0a]	275	static bool heapBoot = 0; // detect recursion during boot
[5d4fa18]	276	#endif // __CFA_DEBUG__
	277	static HeapManager heapManager __attribute__(( aligned (128) )) @= {}; // size of cache line to prevent false sharing
	278
[b6830d74]	279	// #comment TD : The return type of this function should be commented
[93c2e0a]	280	static inline bool setMmapStart( size_t value ) {
[b6830d74]	281	if ( value < pageSize \|\| bucketSizes[NoBucketSizes - 1] < value ) return true;
	282	mmapStart = value; // set global
	283
	284	// find the closest bucket size less than or equal to the mmapStart size
	285	maxBucketsUsed = bsearchl( (unsigned int)mmapStart, bucketSizes, NoBucketSizes ); // binary search
	286	assert( maxBucketsUsed < NoBucketSizes ); // subscript failure ?
	287	assert( mmapStart <= bucketSizes[maxBucketsUsed] ); // search failure ?
	288	return false;
[d46ed6e]	289	} // setMmapStart
	290
	291
	292	static void ?{}( HeapManager & manager ) with ( manager ) {
[b6830d74]	293	pageSize = sysconf( _SC_PAGESIZE );
[73abe95]	294
[b6830d74]	295	for ( unsigned int i = 0; i < NoBucketSizes; i += 1 ) { // initialize the free lists
[d46ed6e]	296	freeLists[i].blockSize = bucketSizes[i];
[b6830d74]	297	} // for
[d46ed6e]	298
	299	#ifdef FASTLOOKUP
[b6830d74]	300	unsigned int idx = 0;
	301	for ( unsigned int i = 0; i < LookupSizes; i += 1 ) {
[d46ed6e]	302	if ( i > bucketSizes[idx] ) idx += 1;
	303	lookup[i] = idx;
[b6830d74]	304	} // for
[d46ed6e]	305	#endif // FASTLOOKUP
	306
[b6830d74]	307	if ( setMmapStart( default_mmap_start() ) ) {
[d46ed6e]	308	abort( "HeapManager : internal error, mmap start initialization failure." );
[b6830d74]	309	} // if
	310	heapExpand = default_heap_expansion();
[d46ed6e]	311
[b6830d74]	312	char * End = (char *)sbrk( 0 );
	313	sbrk( (char *)libCeiling( (long unsigned int)End, libAlign() ) - End ); // move start of heap to multiple of alignment
	314	heapBegin = heapEnd = sbrk( 0 ); // get new start point
[d46ed6e]	315	} // HeapManager
	316
	317
	318	static void ^?{}( HeapManager & ) {
	319	#ifdef __STATISTICS__
[5d4fa18]	320	// if ( traceHeapTerm() ) {
[d46ed6e]	321	// printStats();
[5d4fa18]	322	// if ( checkfree() ) checkFree( heapManager, true );
[d46ed6e]	323	// } // if
	324	#endif // __STATISTICS__
	325	} // ~HeapManager
	326
	327
	328	static void memory_startup( void ) __attribute__(( constructor( STARTUP_PRIORITY_MEMORY ) ));
	329	void memory_startup( void ) {
	330	#ifdef __CFA_DEBUG__
	331	if ( unlikely( heapBoot ) ) { // check for recursion during system boot
	332	// DO NOT USE STREAMS AS THEY MAY BE UNAVAILABLE AT THIS POINT.
	333	abort( "boot() : internal error, recursively invoked during system boot." );
	334	} // if
	335	heapBoot = true;
	336	#endif // __CFA_DEBUG__
[c4f68dc]	337
[b6830d74]	338	// #comment TD : This assertion seems redundent with the above code
[f0b3f51]	339	assert( heapManager.heapBegin == 0 );
[d46ed6e]	340	heapManager{};
	341	} // memory_startup
[c4f68dc]	342
[d46ed6e]	343	static void memory_shutdown( void ) __attribute__(( destructor( STARTUP_PRIORITY_MEMORY ) ));
	344	void memory_shutdown( void ) {
	345	^heapManager{};
	346	} // memory_shutdown
	347
[c4f68dc]	348
	349	#ifdef __STATISTICS__
[d46ed6e]	350	static unsigned long long int mmap_storage; // heap statistics counters
[c4f68dc]	351	static unsigned int mmap_calls;
	352	static unsigned long long int munmap_storage;
	353	static unsigned int munmap_calls;
	354	static unsigned long long int sbrk_storage;
	355	static unsigned int sbrk_calls;
	356	static unsigned long long int malloc_storage;
	357	static unsigned int malloc_calls;
	358	static unsigned long long int free_storage;
	359	static unsigned int free_calls;
	360	static unsigned long long int calloc_storage;
	361	static unsigned int calloc_calls;
	362	static unsigned long long int memalign_storage;
	363	static unsigned int memalign_calls;
	364	static unsigned long long int cmemalign_storage;
	365	static unsigned int cmemalign_calls;
	366	static unsigned long long int realloc_storage;
	367	static unsigned int realloc_calls;
[d46ed6e]	368
	369	static int statfd; // statistics file descriptor (changed by malloc_stats_fd)
[c4f68dc]	370
	371
	372	// Use "write" because streams may be shutdown when calls are made.
[d46ed6e]	373	static void printStats() {
[b6830d74]	374	char helpText[512];
[93c2e0a]	375	__cfaabi_dbg_bits_print_buffer( helpText, sizeof(helpText),
[5d4fa18]	376	"\nHeap statistics:\n"
	377	" malloc: calls %u / storage %llu\n"
	378	" calloc: calls %u / storage %llu\n"
	379	" memalign: calls %u / storage %llu\n"
	380	" cmemalign: calls %u / storage %llu\n"
	381	" realloc: calls %u / storage %llu\n"
	382	" free: calls %u / storage %llu\n"
	383	" mmap: calls %u / storage %llu\n"
	384	" munmap: calls %u / storage %llu\n"
	385	" sbrk: calls %u / storage %llu\n",
	386	malloc_calls, malloc_storage,
	387	calloc_calls, calloc_storage,
	388	memalign_calls, memalign_storage,
	389	cmemalign_calls, cmemalign_storage,
	390	realloc_calls, realloc_storage,
	391	free_calls, free_storage,
	392	mmap_calls, mmap_storage,
	393	munmap_calls, munmap_storage,
	394	sbrk_calls, sbrk_storage
[c4f68dc]	395	);
[d46ed6e]	396	} // printStats
[c4f68dc]	397
[b6830d74]	398	// #comment TD : Why do we have this?
[d46ed6e]	399	static int printStatsXML( FILE * stream ) {
[b6830d74]	400	char helpText[512];
	401	int len = snprintf( helpText, sizeof(helpText),
[c4f68dc]	402	"<malloc version=\"1\">\n"
	403	"<heap nr=\"0\">\n"
	404	"<sizes>\n"
	405	"</sizes>\n"
	406	"<total type=\"malloc\" count=\"%u\" size=\"%llu\"/>\n"
	407	"<total type=\"calloc\" count=\"%u\" size=\"%llu\"/>\n"
	408	"<total type=\"memalign\" count=\"%u\" size=\"%llu\"/>\n"
	409	"<total type=\"cmemalign\" count=\"%u\" size=\"%llu\"/>\n"
	410	"<total type=\"realloc\" count=\"%u\" size=\"%llu\"/>\n"
	411	"<total type=\"free\" count=\"%u\" size=\"%llu\"/>\n"
	412	"<total type=\"mmap\" count=\"%u\" size=\"%llu\"/>\n"
	413	"<total type=\"munmap\" count=\"%u\" size=\"%llu\"/>\n"
	414	"<total type=\"sbrk\" count=\"%u\" size=\"%llu\"/>\n"
	415	"</malloc>",
	416	malloc_calls, malloc_storage,
	417	calloc_calls, calloc_storage,
	418	memalign_calls, memalign_storage,
	419	cmemalign_calls, cmemalign_storage,
	420	realloc_calls, realloc_storage,
	421	free_calls, free_storage,
	422	mmap_calls, mmap_storage,
	423	munmap_calls, munmap_storage,
	424	sbrk_calls, sbrk_storage
	425	);
[b6830d74]	426	return write( fileno( stream ), helpText, len ); // -1 => error
[d46ed6e]	427	} // printStatsXML
[c4f68dc]	428	#endif // __STATISTICS__
	429
[b6830d74]	430	// #comment TD : Is this the samething as Out-of-Memory?
[c4f68dc]	431	static inline void noMemory() {
[b6830d74]	432	abort( "Heap memory exhausted at %zu bytes.\n"
[c4f68dc]	433	"Possible cause is very large memory allocation and/or large amount of unfreed storage allocated by the program or system/library routines.",
	434	((char )(sbrk( 0 )) - (char )(heapManager.heapBegin)) );
	435	} // noMemory
	436
	437
	438	static inline void checkAlign( size_t alignment ) {
[b6830d74]	439	if ( alignment < sizeof(void *) \|\| ! libPow2( alignment ) ) {
[c4f68dc]	440	abort( "Alignment %zu for memory allocation is less than sizeof(void *) and/or not a power of 2.", alignment );
[b6830d74]	441	} // if
[c4f68dc]	442	} // checkAlign
	443
	444
[93c2e0a]	445	static inline bool setHeapExpand( size_t value ) {
[b6830d74]	446	if ( heapExpand < pageSize ) return true;
	447	heapExpand = value;
	448	return false;
[c4f68dc]	449	} // setHeapExpand
	450
	451
[93c2e0a]	452	static inline void checkHeader( bool check, const char * name, void * addr ) {
[b6830d74]	453	if ( unlikely( check ) ) { // bad address ?
[c4f68dc]	454	abort( "Attempt to %s storage %p with address outside the heap.\n"
	455	"Possible cause is duplicate free on same block or overwriting of memory.",
	456	name, addr );
[b6830d74]	457	} // if
[c4f68dc]	458	} // checkHeader
	459
[b6830d74]	460	// #comment TD : function should be commented and/or have a more evocative name
	461	// this isn't either a check or a constructor which is what I would expect this function to be
[c4f68dc]	462	static inline void fakeHeader( HeapManager.Storage.Header *& header, size_t & size, size_t & alignment ) {
[b6830d74]	463	if ( unlikely( (header->kind.fake.alignment & 1) == 1 ) ) { // fake header ?
[c4f68dc]	464	size_t offset = header->kind.fake.offset;
	465	alignment = header->kind.fake.alignment & -2; // remove flag from value
	466	#ifdef __CFA_DEBUG__
	467	checkAlign( alignment ); // check alignment
	468	#endif // __CFA_DEBUG__
	469	header = (HeapManager.Storage.Header )((char )header - offset);
[b6830d74]	470	} // if
[c4f68dc]	471	} // fakeHeader
	472
[b6830d74]	473	// #comment TD : Why is this a define
[c4f68dc]	474	#define headerAddr( addr ) ((HeapManager.Storage.Header )( (char )addr - sizeof(HeapManager.Storage) ))
	475
[93c2e0a]	476	static inline bool headers( const char * name, void * addr, HeapManager.Storage.Header & header, HeapManager.FreeHeader & freeElem, size_t & size, size_t & alignment ) with ( heapManager ) {
[b6830d74]	477	header = headerAddr( addr );
[c4f68dc]	478
[b6830d74]	479	if ( unlikely( heapEnd < addr ) ) { // mmapped ?
[c4f68dc]	480	fakeHeader( header, size, alignment );
	481	size = header->kind.real.blockSize & -3; // mmap size
	482	return true;
[b6830d74]	483	} // if
[c4f68dc]	484
	485	#ifdef __CFA_DEBUG__
[b6830d74]	486	checkHeader( addr < heapBegin \|\| header < (HeapManager.Storage.Header *)heapBegin, name, addr ); // bad low address ?
[c4f68dc]	487	#endif // __CFA_DEBUG__
[b6830d74]	488
	489	// #comment TD : This code looks weird...
	490	// It's called as the first statement of both branches of the last if, with the same parameters in all cases
	491
	492	// header may be safe to dereference
	493	fakeHeader( header, size, alignment );
[c4f68dc]	494	#ifdef __CFA_DEBUG__
[b6830d74]	495	checkHeader( header < (HeapManager.Storage.Header )heapBegin \|\| (HeapManager.Storage.Header )heapEnd < header, name, addr ); // bad address ? (offset could be + or -)
[c4f68dc]	496	#endif // __CFA_DEBUG__
	497
[b6830d74]	498	freeElem = (HeapManager.FreeHeader *)((size_t)header->kind.real.home & -3);
[c4f68dc]	499	#ifdef __CFA_DEBUG__
[b6830d74]	500	if ( freeElem < &freeLists[0] \|\| &freeLists[NoBucketSizes] <= freeElem ) {
	501	abort( "Attempt to %s storage %p with corrupted header.\n"
	502	"Possible cause is duplicate free on same block or overwriting of header information.",
	503	name, addr );
	504	} // if
[c4f68dc]	505	#endif // __CFA_DEBUG__
[b6830d74]	506	size = freeElem->blockSize;
	507	return false;
[c4f68dc]	508	} // headers
	509
	510
	511	static inline void * extend( size_t size ) with ( heapManager ) {
[b6830d74]	512	lock( extlock __cfaabi_dbg_ctx2 );
	513	ptrdiff_t rem = heapRemaining - size;
	514	if ( rem < 0 ) {
[c4f68dc]	515	// If the size requested is bigger than the current remaining storage, increase the size of the heap.
	516
	517	size_t increase = libCeiling( size > heapExpand ? size : heapExpand, libAlign() );
	518	if ( sbrk( increase ) == (void *)-1 ) {
	519	unlock( extlock );
	520	errno = ENOMEM;
	521	return 0;
	522	} // if
	523	#ifdef __STATISTICS__
	524	sbrk_calls += 1;
	525	sbrk_storage += increase;
	526	#endif // __STATISTICS__
	527	#ifdef __CFA_DEBUG__
	528	// Set new memory to garbage so subsequent uninitialized usages might fail.
	529	memset( (char *)heapEnd + heapRemaining, '\377', increase );
	530	#endif // __CFA_DEBUG__
	531	rem = heapRemaining + increase - size;
[b6830d74]	532	} // if
[c4f68dc]	533
[b6830d74]	534	HeapManager.Storage * block = (HeapManager.Storage *)heapEnd;
	535	heapRemaining = rem;
	536	heapEnd = (char *)heapEnd + size;
	537	unlock( extlock );
	538	return block;
[c4f68dc]	539	} // extend
	540
	541
	542	static inline void * doMalloc( size_t size ) with ( heapManager ) {
[b6830d74]	543	HeapManager.Storage * block;
[c4f68dc]	544
[b6830d74]	545	// Look up size in the size list. Make sure the user request includes space for the header that must be allocated
	546	// along with the block and is a multiple of the alignment size.
[c4f68dc]	547
[b6830d74]	548	size_t tsize = size + sizeof(HeapManager.Storage);
	549	if ( likely( tsize < mmapStart ) ) { // small size => sbrk
[c4f68dc]	550	HeapManager.FreeHeader * freeElem =
	551	#ifdef FASTLOOKUP
	552	tsize < LookupSizes ? &freeLists[lookup[tsize]] :
	553	#endif // FASTLOOKUP
	554	bsearchl( tsize, freeLists, (size_t)maxBucketsUsed ); // binary search
	555	assert( freeElem <= &freeLists[maxBucketsUsed] ); // subscripting error ?
	556	assert( tsize <= freeElem->blockSize ); // search failure ?
	557	tsize = freeElem->blockSize; // total space needed for request
	558
	559	// Spin until the lock is acquired for this particular size of block.
	560
	561	#if defined( SPINLOCK )
[b6830d74]	562	lock( freeElem->lock __cfaabi_dbg_ctx2 );
	563	block = freeElem->freeList; // remove node from stack
[c4f68dc]	564	#else
[b6830d74]	565	block = freeElem->freeList.pop();
[c4f68dc]	566	#endif // SPINLOCK
	567	if ( unlikely( block == 0 ) ) { // no free block ?
	568	#if defined( SPINLOCK )
	569	unlock( freeElem->lock );
	570	#endif // SPINLOCK
	571	// Freelist for that size was empty, so carve it out of the heap if there's enough left, or get some more
	572	// and then carve it off.
	573
	574	block = (HeapManager.Storage *)extend( tsize ); // mutual exclusion on call
	575	if ( unlikely( block == 0 ) ) return 0;
	576	#if defined( SPINLOCK )
	577	} else {
	578	freeElem->freeList = block->header.kind.real.next;
	579	unlock( freeElem->lock );
	580	#endif // SPINLOCK
	581	} // if
	582
	583	block->header.kind.real.home = freeElem; // pointer back to free list of apropriate size
[b6830d74]	584	} else { // large size => mmap
[c4f68dc]	585	tsize = libCeiling( tsize, pageSize ); // must be multiple of page size
	586	#ifdef __STATISTICS__
[b6830d74]	587	__atomic_add_fetch( &mmap_calls, 1, __ATOMIC_SEQ_CST );
	588	__atomic_add_fetch( &mmap_storage, tsize, __ATOMIC_SEQ_CST );
[c4f68dc]	589	#endif // __STATISTICS__
	590	block = (HeapManager.Storage *)mmap( 0, tsize, PROT_READ \| PROT_WRITE, MAP_PRIVATE \| MAP_ANONYMOUS, mmapFd, 0 );
	591	if ( block == (HeapManager.Storage *)MAP_FAILED ) {
	592	// Do not call strerror( errno ) as it may call malloc.
	593	abort( "(HeapManager &)0x%p.doMalloc() : internal error, mmap failure, size:%zu error:%d.", &heapManager, tsize, errno );
	594	} // if
	595	#ifdef __CFA_DEBUG__
	596	// Set new memory to garbage so subsequent uninitialized usages might fail.
	597	memset( block, '\377', tsize );
	598	#endif // __CFA_DEBUG__
	599	block->header.kind.real.blockSize = tsize; // storage size for munmap
[b6830d74]	600	} // if
[c4f68dc]	601
[b6830d74]	602	void * area = &(block->data); // adjust off header to user bytes
[c4f68dc]	603
	604	#ifdef __CFA_DEBUG__
[b6830d74]	605	assert( ((uintptr_t)area & (libAlign() - 1)) == 0 ); // minimum alignment ?
	606	__atomic_add_fetch( &allocFree, tsize, __ATOMIC_SEQ_CST );
	607	if ( traceHeap() ) {
	608	enum { BufferSize = 64 };
	609	char helpText[BufferSize];
	610	int len = snprintf( helpText, BufferSize, "%p = Malloc( %zu ) (allocated %zu)\n", area, size, tsize );
	611	// int len = snprintf( helpText, BufferSize, "Malloc %p %zu\n", area, size );
	612	__cfaabi_dbg_bits_write( helpText, len );
	613	} // if
[c4f68dc]	614	#endif // __CFA_DEBUG__
	615
[b6830d74]	616	return area;
[c4f68dc]	617	} // doMalloc
	618
	619
	620	static inline void doFree( void * addr ) with ( heapManager ) {
	621	#ifdef __CFA_DEBUG__
[b6830d74]	622	if ( unlikely( heapManager.heapBegin == 0 ) ) {
	623	abort( "doFree( %p ) : internal error, called before heap is initialized.", addr );
	624	} // if
[c4f68dc]	625	#endif // __CFA_DEBUG__
	626
[b6830d74]	627	HeapManager.Storage.Header * header;
	628	HeapManager.FreeHeader * freeElem;
	629	size_t size, alignment; // not used (see realloc)
[c4f68dc]	630
[b6830d74]	631	if ( headers( "free", addr, header, freeElem, size, alignment ) ) { // mmapped ?
[c4f68dc]	632	#ifdef __STATISTICS__
[b6830d74]	633	__atomic_add_fetch( &munmap_calls, 1, __ATOMIC_SEQ_CST );
	634	__atomic_add_fetch( &munmap_storage, size, __ATOMIC_SEQ_CST );
[c4f68dc]	635	#endif // __STATISTICS__
	636	if ( munmap( header, size ) == -1 ) {
	637	#ifdef __CFA_DEBUG__
	638	abort( "Attempt to deallocate storage %p not allocated or with corrupt header.\n"
	639	"Possible cause is invalid pointer.",
	640	addr );
	641	#endif // __CFA_DEBUG__
	642	} // if
[b6830d74]	643	} else {
[c4f68dc]	644	#ifdef __CFA_DEBUG__
[b6830d74]	645	// Set free memory to garbage so subsequent usages might fail.
	646	memset( ((HeapManager.Storage *)header)->data, '\377', freeElem->blockSize - sizeof( HeapManager.Storage ) );
[c4f68dc]	647	#endif // __CFA_DEBUG__
	648
	649	#ifdef __STATISTICS__
[b6830d74]	650	free_storage += size;
[c4f68dc]	651	#endif // __STATISTICS__
	652	#if defined( SPINLOCK )
[b6830d74]	653	lock( freeElem->lock __cfaabi_dbg_ctx2 ); // acquire spin lock
	654	header->kind.real.next = freeElem->freeList; // push on stack
	655	freeElem->freeList = (HeapManager.Storage *)header;
	656	unlock( freeElem->lock ); // release spin lock
[c4f68dc]	657	#else
[b6830d74]	658	freeElem->freeList.push( (HeapManager.Storage )header );
[c4f68dc]	659	#endif // SPINLOCK
[b6830d74]	660	} // if
[c4f68dc]	661
	662	#ifdef __CFA_DEBUG__
[b6830d74]	663	__atomic_add_fetch( &allocFree, -size, __ATOMIC_SEQ_CST );
	664	if ( traceHeap() ) {
	665	char helpText[64];
	666	int len = snprintf( helpText, sizeof(helpText), "Free( %p ) size:%zu\n", addr, size );
	667	__cfaabi_dbg_bits_write( helpText, len );
	668	} // if
[c4f68dc]	669	#endif // __CFA_DEBUG__
	670	} // doFree
	671
	672
[5d4fa18]	673	size_t checkFree( HeapManager & manager ) with ( manager ) {
[b6830d74]	674	size_t total = 0;
[c4f68dc]	675	#ifdef __STATISTICS__
[b6830d74]	676	__cfaabi_dbg_bits_acquire();
	677	__cfaabi_dbg_bits_print_nolock( "\nBin lists (bin size : free blocks on list)\n" );
[c4f68dc]	678	#endif // __STATISTICS__
[b6830d74]	679	for ( unsigned int i = 0; i < maxBucketsUsed; i += 1 ) {
[d46ed6e]	680	size_t size = freeLists[i].blockSize;
	681	#ifdef __STATISTICS__
	682	unsigned int N = 0;
	683	#endif // __STATISTICS__
[b6830d74]	684
[d46ed6e]	685	#if defined( SPINLOCK )
	686	for ( HeapManager.Storage * p = freeLists[i].freeList; p != 0; p = p->header.kind.real.next ) {
	687	#else
	688	for ( HeapManager.Storage * p = freeLists[i].freeList.top(); p != 0; p = p->header.kind.real.next.top ) {
	689	#endif // SPINLOCK
	690	total += size;
	691	#ifdef __STATISTICS__
	692	N += 1;
	693	#endif // __STATISTICS__
[b6830d74]	694	} // for
	695
[d46ed6e]	696	#ifdef __STATISTICS__
[b6830d74]	697	__cfaabi_dbg_bits_print_nolock( "%7zu, %-7u ", size, N );
	698	if ( (i + 1) % 8 == 0 ) __cfaabi_dbg_bits_print_nolock( "\n" );
[d46ed6e]	699	#endif // __STATISTICS__
	700	} // for
	701	#ifdef __STATISTICS__
[b6830d74]	702	__cfaabi_dbg_bits_print_nolock( "\ntotal free blocks:%zu\n", total );
	703	__cfaabi_dbg_bits_release();
[d46ed6e]	704	#endif // __STATISTICS__
	705	return (char )heapEnd - (char )heapBegin - total;
	706	} // checkFree
[c4f68dc]	707
[b6830d74]	708	// #comment TD : This is not a good name, plus this feels like it could easily be folded into doMalloc
[c4f68dc]	709	static inline void * malloc2( size_t size ) { // necessary for malloc statistics
[f0b3f51]	710	assert( heapManager.heapBegin != 0 );
[b6830d74]	711	void * area = doMalloc( size );
	712	if ( unlikely( area == 0 ) ) errno = ENOMEM; // POSIX
	713	return area;
[c4f68dc]	714	} // malloc2
	715
	716
	717	static inline void * memalign2( size_t alignment, size_t size ) { // necessary for malloc statistics
	718	#ifdef __CFA_DEBUG__
[b6830d74]	719	checkAlign( alignment ); // check alignment
[c4f68dc]	720	#endif // __CFA_DEBUG__
	721
[b6830d74]	722	// if alignment <= default alignment, do normal malloc as two headers are unnecessary
	723	if ( unlikely( alignment <= libAlign() ) ) return malloc2( size );
	724
	725	// Allocate enough storage to guarantee an address on the alignment boundary, and sufficient space before it for
	726	// administrative storage. NOTE, WHILE THERE ARE 2 HEADERS, THE FIRST ONE IS IMPLICITLY CREATED BY DOMALLOC.
	727	// .-------------v-----------------v----------------v----------,
	728	// \| Real Header \| ... padding ... \| Fake Header \| data ... \|
	729	// `-------------^-----------------^-+--------------^----------'
	730	// \|<--------------------------------' offset/align \|<-- alignment boundary
	731
	732	// subtract libAlign() because it is already the minimum alignment
	733	// add sizeof(Storage) for fake header
	734	// #comment TD : this is the only place that calls doMalloc without calling malloc2, why ?
	735	char * area = (char *)doMalloc( size + alignment - libAlign() + sizeof(HeapManager.Storage) );
	736	if ( unlikely( area == 0 ) ) return area;
	737
	738	// address in the block of the "next" alignment address
	739	char * user = (char *)libCeiling( (uintptr_t)(area + sizeof(HeapManager.Storage)), alignment );
	740
	741	// address of header from malloc
	742	HeapManager.Storage.Header * realHeader = headerAddr( area );
	743	// address of fake header * before* the alignment location
	744	HeapManager.Storage.Header * fakeHeader = headerAddr( user );
	745	// SKULLDUGGERY: insert the offset to the start of the actual storage block and remember alignment
	746	fakeHeader->kind.fake.offset = (char )fakeHeader - (char )realHeader;
	747	// SKULLDUGGERY: odd alignment imples fake header
	748	fakeHeader->kind.fake.alignment = alignment \| 1;
	749
	750	return user;
[c4f68dc]	751	} // memalign2
	752
	753
	754	extern "C" {
[b6830d74]	755	// The malloc() function allocates size bytes and returns a pointer to the
	756	// allocated memory. The memory is not initialized. If size is 0, then malloc()
	757	// returns either NULL, or a unique pointer value that can later be successfully
	758	// passed to free().
	759	void * malloc( size_t size ) {
[c4f68dc]	760	#ifdef __STATISTICS__
[b6830d74]	761	__atomic_add_fetch( &malloc_calls, 1, __ATOMIC_SEQ_CST );
	762	__atomic_add_fetch( &malloc_storage, size, __ATOMIC_SEQ_CST );
[c4f68dc]	763	#endif // __STATISTICS__
	764
	765	return malloc2( size );
[b6830d74]	766	} // malloc
[c4f68dc]	767
[b6830d74]	768	// The calloc() function allocates memory for an array of nmemb elements of
	769	// size bytes each and returns a pointer to the allocated memory. The memory
	770	// is set to zero. If nmemb or size is 0, then calloc() returns either NULL,
	771	// or a unique pointer value that can later be successfully passed to free().
	772	void * calloc( size_t noOfElems, size_t elemSize ) {
[c4f68dc]	773	size_t size = noOfElems * elemSize;
	774	#ifdef __STATISTICS__
[b6830d74]	775	__atomic_add_fetch( &calloc_calls, 1, __ATOMIC_SEQ_CST );
	776	__atomic_add_fetch( &calloc_storage, size, __ATOMIC_SEQ_CST );
[c4f68dc]	777	#endif // __STATISTICS__
	778
	779	char * area = (char *)malloc2( size );
	780	if ( unlikely( area == 0 ) ) return 0;
[b6830d74]	781
[c4f68dc]	782	HeapManager.Storage.Header * header;
	783	HeapManager.FreeHeader * freeElem;
	784	size_t asize, alignment;
[93c2e0a]	785	bool mapped __attribute__(( unused )) = headers( "calloc", area, header, freeElem, asize, alignment );
[c4f68dc]	786	#ifndef __CFA_DEBUG__
[73abe95]	787	// Mapped storage is zero filled, but in debug mode mapped memory is scrubbed in doMalloc, so it has to be reset to zero.
[c4f68dc]	788	if ( ! mapped )
	789	#endif // __CFA_DEBUG__
	790	memset( area, '\0', asize - sizeof(HeapManager.Storage) ); // set to zeros
[b6830d74]	791
[c4f68dc]	792	header->kind.real.blockSize \|= 2; // mark as zero filled
	793	return area;
[b6830d74]	794	} // calloc
[c4f68dc]	795
[b6830d74]	796	// #comment TD : Document this function
	797	void * cmemalign( size_t alignment, size_t noOfElems, size_t elemSize ) {
[c4f68dc]	798	size_t size = noOfElems * elemSize;
	799	#ifdef __STATISTICS__
[b6830d74]	800	__atomic_add_fetch( &cmemalign_calls, 1, __ATOMIC_SEQ_CST );
	801	__atomic_add_fetch( &cmemalign_storage, size, __ATOMIC_SEQ_CST );
[c4f68dc]	802	#endif // __STATISTICS__
	803
	804	char * area = (char *)memalign2( alignment, size );
	805	if ( unlikely( area == 0 ) ) return 0;
	806	HeapManager.Storage.Header * header;
	807	HeapManager.FreeHeader * freeElem;
	808	size_t asize;
[93c2e0a]	809	bool mapped __attribute__(( unused )) = headers( "cmemalign", area, header, freeElem, asize, alignment );
[c4f68dc]	810	#ifndef __CFA_DEBUG__
[73abe95]	811	// Mapped storage is zero filled, but in debug mode mapped memory is scrubbed in doMalloc, so it has to be reset to zero.
[c4f68dc]	812	if ( ! mapped )
	813	#endif // __CFA_DEBUG__
	814	memset( area, '\0', asize - ( (char )area - (char )header ) ); // set to zeros
	815	header->kind.real.blockSize \|= 2; // mark as zero filled
	816
	817	return area;
[b6830d74]	818	} // cmemalign
	819
	820	// The realloc() function changes the size of the memory block pointed to by
	821	// ptr to size bytes. The contents will be unchanged in the range from the
	822	// start of the region up to the minimum of the old and new sizes. If the new
	823	// size is larger than the old size, the added memory will not be initialized.
	824	// If ptr is NULL, then the call is equivalent to malloc(size), for all values
	825	// of size; if size is equal to zero, and ptr is not NULL, then the call is
	826	// equivalent to free(ptr). Unless ptr is NULL, it must have been returned by
	827	// an earlier call to malloc(), calloc() or realloc(). If the area pointed to
	828	// was moved, a free(ptr) is done.
	829	void * realloc( void * addr, size_t size ) {
[c4f68dc]	830	#ifdef __STATISTICS__
[b6830d74]	831	__atomic_add_fetch( &realloc_calls, 1, __ATOMIC_SEQ_CST );
[c4f68dc]	832	#endif // __STATISTICS__
	833
	834	if ( unlikely( addr == 0 ) ) return malloc2( size ); // special cases
	835	if ( unlikely( size == 0 ) ) { free( addr ); return 0; }
	836
	837	HeapManager.Storage.Header * header;
	838	HeapManager.FreeHeader * freeElem;
	839	size_t asize, alignment = 0;
	840	headers( "realloc", addr, header, freeElem, asize, alignment );
	841
	842	size_t usize = asize - ( (char )addr - (char )header ); // compute the amount of user storage in the block
	843	if ( usize >= size ) { // already sufficient storage
	844	// This case does not result in a new profiler entry because the previous one still exists and it must match with
	845	// the free for this memory. Hence, this realloc does not appear in the profiler output.
	846	return addr;
	847	} // if
	848
	849	#ifdef __STATISTICS__
[b6830d74]	850	__atomic_add_fetch( &realloc_storage, size, __ATOMIC_SEQ_CST );
[c4f68dc]	851	#endif // __STATISTICS__
	852
	853	void * area;
	854	if ( unlikely( alignment != 0 ) ) { // previous request memalign?
	855	area = memalign( alignment, size ); // create new area
	856	} else {
	857	area = malloc2( size ); // create new area
	858	} // if
	859	if ( unlikely( area == 0 ) ) return 0;
	860	if ( unlikely( header->kind.real.blockSize & 2 ) ) { // previous request zero fill (calloc/cmemalign) ?
	861	assert( (header->kind.real.blockSize & 1) == 0 );
[93c2e0a]	862	bool mapped __attribute__(( unused )) = headers( "realloc", area, header, freeElem, asize, alignment );
[c4f68dc]	863	#ifndef __CFA_DEBUG__
[73abe95]	864	// Mapped storage is zero filled, but in debug mode mapped memory is scrubbed in doMalloc, so it has to be reset to zero.
[c4f68dc]	865	if ( ! mapped )
	866	#endif // __CFA_DEBUG__
	867	memset( (char )area + usize, '\0', asize - ( (char )area - (char *)header ) - usize ); // zero-fill back part
	868	header->kind.real.blockSize \|= 2; // mark new request as zero fill
	869	} // if
	870	memcpy( area, addr, usize ); // copy bytes
	871	free( addr );
	872	return area;
[b6830d74]	873	} // realloc
[c4f68dc]	874
	875
[b6830d74]	876	// The obsolete function memalign() allocates size bytes and returns
	877	// a pointer to the allocated memory. The memory address will be a
	878	// multiple of alignment, which must be a power of two.
	879	void * memalign( size_t alignment, size_t size ) __attribute__ ((deprecated));
	880	void * memalign( size_t alignment, size_t size ) {
[c4f68dc]	881	#ifdef __STATISTICS__
	882	__atomic_add_fetch( &memalign_calls, 1, __ATOMIC_SEQ_CST );
	883	__atomic_add_fetch( &memalign_storage, size, __ATOMIC_SEQ_CST );
	884	#endif // __STATISTICS__
	885
	886	void * area = memalign2( alignment, size );
	887
	888	return area;
[b6830d74]	889	} // memalign
[c4f68dc]	890
[b6830d74]	891	// The function aligned_alloc() is the same as memalign(), except for
	892	// the added restriction that size should be a multiple of alignment.
	893	void * aligned_alloc( size_t alignment, size_t size ) {
[c4f68dc]	894	return memalign( alignment, size );
[b6830d74]	895	} // aligned_alloc
[c4f68dc]	896
	897
[b6830d74]	898	// The function posix_memalign() allocates size bytes and places the address
	899	// of the allocated memory in *memptr. The address of the allocated memory
	900	// will be a multiple of alignment, which must be a power of two and a multiple
	901	// of sizeof(void *). If size is 0, then posix_memalign() returns either NULL,
	902	// or a unique pointer value that can later be successfully passed to free(3).
	903	int posix_memalign( void ** memptr, size_t alignment, size_t size ) {
[c4f68dc]	904	if ( alignment < sizeof(void *) \|\| ! libPow2( alignment ) ) return EINVAL; // check alignment
	905	* memptr = memalign( alignment, size );
	906	if ( unlikely( * memptr == 0 ) ) return ENOMEM;
	907	return 0;
[b6830d74]	908	} // posix_memalign
[c4f68dc]	909
[b6830d74]	910	// The obsolete function valloc() allocates size bytes and returns a pointer
	911	// to the allocated memory. The memory address will be a multiple of the page size.
	912	// It is equivalent to memalign(sysconf(_SC_PAGESIZE),size).
	913	void * valloc( size_t size ) __attribute__ ((deprecated));
	914	void * valloc( size_t size ) {
[c4f68dc]	915	return memalign( pageSize, size );
[b6830d74]	916	} // valloc
[c4f68dc]	917
	918
[b6830d74]	919	// The free() function frees the memory space pointed to by ptr, which must
	920	// have been returned by a previous call to malloc(), calloc() or realloc().
	921	// Otherwise, or if free(ptr) has already been called before, undefined
	922	// behavior occurs. If ptr is NULL, no operation is performed.
	923	void free( void * addr ) {
[c4f68dc]	924	#ifdef __STATISTICS__
[b6830d74]	925	__atomic_add_fetch( &free_calls, 1, __ATOMIC_SEQ_CST );
[c4f68dc]	926	#endif // __STATISTICS__
	927
[b6830d74]	928	// #comment TD : To decrease nesting I would but the special case in the
	929	// else instead, plus it reads more naturally to have the
	930	// short / normal case instead
[c4f68dc]	931	if ( unlikely( addr == 0 ) ) { // special case
[d46ed6e]	932	#ifdef __CFA_DEBUG__
[b6830d74]	933	if ( traceHeap() ) {
	934	#define nullmsg "Free( 0x0 ) size:0\n"
	935	// Do not debug print free( 0 ), as it can cause recursive entry from sprintf.
	936	__cfaabi_dbg_bits_write( nullmsg, sizeof(nullmsg) - 1 );
	937	} // if
[d46ed6e]	938	#endif // __CFA_DEBUG__
[c4f68dc]	939	return;
	940	} // exit
	941
	942	doFree( addr );
[b6830d74]	943	} // free
[93c2e0a]	944
[b6830d74]	945	// The mallopt() function adjusts parameters that control the behavior of the
	946	// memory-allocation functions (see malloc(3)). The param argument specifies
	947	// the parameter to be modified, and value specifies the new value for that
	948	// parameter.
	949	int mallopt( int option, int value ) {
[c4f68dc]	950	choose( option ) {
[b6830d74]	951	case M_TOP_PAD:
	952	if ( setHeapExpand( value ) ) fallthru default;
	953	case M_MMAP_THRESHOLD:
	954	if ( setMmapStart( value ) ) fallthru default;
	955	default:
	956	// #comment TD : 1 for unsopported feels wrong
	957	return 1; // success, or unsupported
[c4f68dc]	958	} // switch
	959	return 0; // error
[b6830d74]	960	} // mallopt
[c4f68dc]	961
[b6830d74]	962	// The malloc_trim() function attempts to release free memory at the top
	963	// of the heap (by calling sbrk(2) with a suitable argument).
[c4f68dc]	964	int malloc_trim( size_t ) {
	965	return 0; // => impossible to release memory
	966	} // malloc_trim
	967
[b6830d74]	968	// The malloc_usable_size() function returns the number of usable bytes in the
	969	// block pointed to by ptr, a pointer to a block of memory allocated by
	970	// malloc(3) or a related function.
	971	size_t malloc_usable_size( void * addr ) {
[c4f68dc]	972	if ( unlikely( addr == 0 ) ) return 0; // null allocation has 0 size
[b6830d74]	973
[c4f68dc]	974	HeapManager.Storage.Header * header;
	975	HeapManager.FreeHeader * freeElem;
	976	size_t size, alignment;
	977
	978	headers( "malloc_usable_size", addr, header, freeElem, size, alignment );
	979	size_t usize = size - ( (char )addr - (char )header ); // compute the amount of user storage in the block
	980	return usize;
[b6830d74]	981	} // malloc_usable_size
[c4f68dc]	982
	983
[b6830d74]	984	// #comment TD : Document this function
	985	size_t malloc_alignment( void * addr ) {
[c4f68dc]	986	if ( unlikely( addr == 0 ) ) return libAlign(); // minimum alignment
	987	HeapManager.Storage.Header * header = (HeapManager.Storage.Header )( (char )addr - sizeof(HeapManager.Storage) );
	988	if ( (header->kind.fake.alignment & 1) == 1 ) { // fake header ?
	989	return header->kind.fake.alignment & -2; // remove flag from value
	990	} else {
	991	return libAlign (); // minimum alignment
	992	} // if
[b6830d74]	993	} // malloc_alignment
[c4f68dc]	994
	995
[b6830d74]	996	// #comment TD : Document this function
	997	bool malloc_zero_fill( void * addr ) {
[c4f68dc]	998	if ( unlikely( addr == 0 ) ) return false; // null allocation is not zero fill
[b6830d74]	999
[c4f68dc]	1000	HeapManager.Storage.Header * header = (HeapManager.Storage.Header )( (char )addr - sizeof(HeapManager.Storage) );
	1001	if ( (header->kind.fake.alignment & 1) == 1 ) { // fake header ?
	1002	header = (HeapManager.Storage.Header )((char )header - header->kind.fake.offset);
	1003	} // if
	1004	return (header->kind.real.blockSize & 2) != 0; // zero filled (calloc/cmemalign) ?
[b6830d74]	1005	} // malloc_zero_fill
[c4f68dc]	1006
	1007
[b6830d74]	1008	// #comment TD : Document this function
	1009	void malloc_stats( void ) {
[c4f68dc]	1010	#ifdef __STATISTICS__
[b6830d74]	1011	printStats();
	1012	if ( checkFree() ) checkFree( heapManager );
[c4f68dc]	1013	#endif // __STATISTICS__
[b6830d74]	1014	} // malloc_stats
[c4f68dc]	1015
[b6830d74]	1016	// #comment TD : Document this function
	1017	int malloc_stats_fd( int fd ) {
[c4f68dc]	1018	#ifdef __STATISTICS__
[b6830d74]	1019	int temp = statfd;
	1020	statfd = fd;
	1021	return temp;
[c4f68dc]	1022	#else
[b6830d74]	1023	return -1;
[c4f68dc]	1024	#endif // __STATISTICS__
[b6830d74]	1025	} // malloc_stats_fd
[c4f68dc]	1026
	1027
[b6830d74]	1028	// #comment TD : Document this function
[c4f68dc]	1029	int malloc_info( int options, FILE * stream ) {
[d46ed6e]	1030	return printStatsXML( stream );
[c4f68dc]	1031	} // malloc_info
	1032
	1033
[b6830d74]	1034	// #comment TD : What are these two functions for?
[c4f68dc]	1035	void * malloc_get_state( void ) {
	1036	return 0;
	1037	} // malloc_get_state
	1038
	1039	int malloc_set_state( void * ptr ) {
	1040	return 0;
	1041	} // malloc_set_state
	1042	} // extern "C"
	1043
	1044
	1045	// Local Variables: //
	1046	// tab-width: 4 //
	1047	// compile-command: "cfa -nodebug -O2 heap.c" //
	1048	// End: //

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

Original Format