Index: libcfa/src/heap.cfa
===================================================================
--- libcfa/src/heap.cfa	(revision c1f38e6cb761f0c262410a4cbc49bf127a2918c4)
+++ libcfa/src/heap.cfa	(revision 92aca37a5af06634217621f024ab9c2eef73d18c)
@@ -5,11 +5,11 @@
 // file "LICENCE" distributed with Cforall.
 //
-// heap.c --
+// heap.cfa --
 //
 // Author           : Peter A. Buhr
 // Created On       : Tue Dec 19 21:58:35 2017
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Thu Aug  6 09:08:58 2020
-// Update Count     : 861
+// Last Modified On : Sun Aug  9 12:23:20 2020
+// Update Count     : 894
 //
 
@@ -23,5 +23,5 @@
 #include <sys/mman.h>									// mmap, munmap
 
-#include "bits/align.hfa"								// libPow2
+#include "bits/align.hfa"								// libAlign
 #include "bits/defs.hfa"								// likely, unlikely
 #include "bits/locks.hfa"								// __spinlock_t
@@ -88,12 +88,7 @@
 } // default_heap_expansion
 
-bool default_heap_exhausted() __attribute__(( weak )) {	// find and free some storage
-	// Returning false prints "out of heap memory" message and aborts.
-	return false;
-} // default_heap_exhausted
-
 
 #ifdef __CFA_DEBUG__
-static unsigned int allocUnfreed;						// running total of allocations minus frees
+static size_t allocUnfreed;								// running total of allocations minus frees
 
 static void prtUnfreed() {
@@ -101,5 +96,5 @@
 		// DO NOT USE STREAMS AS THEY MAY BE UNAVAILABLE AT THIS POINT.
 		char helpText[512];
-		int len = snprintf( helpText, sizeof(helpText), "CFA warning (UNIX pid:%ld) : program terminating with %u(0x%x) bytes of storage allocated but not freed.\n"
+		int len = snprintf( helpText, sizeof(helpText), "CFA warning (UNIX pid:%ld) : program terminating with %zu(0x%zx) bytes of storage allocated but not freed.\n"
 							"Possible cause is unfreed storage allocated by the program or system/library routines called from the program.\n",
 							(long int)getpid(), allocUnfreed, allocUnfreed ); // always print the UNIX pid
@@ -292,5 +287,5 @@
 static unsigned long long int sbrk_storage;
 // Statistics file descriptor (changed by malloc_stats_fd).
-static int statfd = STDERR_FILENO;						// default stderr
+static int stat_fd = STDERR_FILENO;						// default stderr
 
 // Use "write" because streams may be shutdown when calls are made.
@@ -312,5 +307,5 @@
 									"  sbrk: calls %u / storage %llu\n",
 									malloc_calls, malloc_storage,
-									aalloc_calls, calloc_storage,
+									aalloc_calls, aalloc_storage,
 									calloc_calls, calloc_storage,
 									memalign_calls, memalign_storage,
@@ -411,5 +406,5 @@
 
 static inline void checkAlign( size_t alignment ) {
-	if ( alignment < libAlign() || ! libPow2( alignment ) ) {
+	if ( alignment < libAlign() || ! is_pow2( alignment ) ) {
 		abort( "Alignment %zu for memory allocation is less than %d and/or not a power of 2.", alignment, libAlign() );
 	} // if
@@ -443,5 +438,5 @@
 	header = headerAddr( addr );
 
-	if ( unlikely( heapEnd < addr ) ) {					// mmapped ?
+  if ( unlikely( heapEnd < addr ) ) {					// mmapped ?
 		fakeHeader( header, alignment );
 		size = header->kind.real.blockSize & -3;		// mmap size
@@ -471,5 +466,5 @@
 } // headers
 
-#define NO_MEMORY_MSG "no heap memory available for allocating %zd new bytes."
+#define NO_MEMORY_MSG "insufficient heap memory available for allocating %zd new bytes."
 
 static inline void * extend( size_t size ) with( heapManager ) {
@@ -479,14 +474,9 @@
 		// If the size requested is bigger than the current remaining storage, increase the size of the heap.
 
-		size_t increase = libCeiling( size > heapExpand ? size : heapExpand, libAlign() );
-	  Succeed:
-		{
-			if ( sbrk( increase ) != (void *)-1 ) break Succeed; // succeed ?
-			if ( default_heap_exhausted() )	{			// try fix
-				if ( sbrk( increase ) != (void *)-1 ) break Succeed; // succeed ?
-			} // if
+		size_t increase = ceiling2( size > heapExpand ? size : heapExpand, libAlign() );
+		if ( sbrk( increase ) == (void *)-1 ) {			// failed, no memory ?
 			unlock( extlock );
 			abort( NO_MEMORY_MSG, size );				// give up
-		}
+		} // if
 		#ifdef __STATISTICS__
 		sbrk_calls += 1;
@@ -524,15 +514,4 @@
 			posn = Bsearchl( (unsigned int)tsize, bucketSizes, (size_t)maxBucketsUsed );
 		HeapManager.FreeHeader * freeElem = &freeLists[posn];
-		// #ifdef FASTLOOKUP
-		// if ( tsize < LookupSizes )
-		// 	freeElem = &freeLists[lookup[tsize]];
-		// else
-		// #endif // FASTLOOKUP
-		//  	freeElem = bsearchl( tsize, freeLists, (size_t)maxBucketsUsed ); // binary search
-		// HeapManager.FreeHeader * freeElem =
-		// 	#ifdef FASTLOOKUP
-		// 	tsize < LookupSizes ? &freeLists[lookup[tsize]] :
-		// 	#endif // FASTLOOKUP
-		// 	bsearchl( tsize, freeLists, (size_t)maxBucketsUsed ); // binary search
 		verify( freeElem <= &freeLists[maxBucketsUsed] ); // subscripting error ?
 		verify( tsize <= freeElem->blockSize );			// search failure ?
@@ -566,21 +545,16 @@
 	} else {											// large size => mmap
   if ( unlikely( size > ULONG_MAX - pageSize ) ) return 0p;
-		tsize = libCeiling( tsize, pageSize );			// must be multiple of page size
+		tsize = ceiling2( tsize, pageSize );			// must be multiple of page size
 		#ifdef __STATISTICS__
 		__atomic_add_fetch( &mmap_calls, 1, __ATOMIC_SEQ_CST );
 		__atomic_add_fetch( &mmap_storage, tsize, __ATOMIC_SEQ_CST );
 		#endif // __STATISTICS__
-	  Succeed:
-		{
-			block = (HeapManager.Storage *)mmap( 0, tsize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, mmapFd, 0 );
-			if ( block != (HeapManager.Storage *)MAP_FAILED ) break Succeed; // succeed ?
-			if ( errno == ENOMEM && default_heap_exhausted() ) { // out of memory and try again ?
-				block = (HeapManager.Storage *)mmap( 0, tsize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, mmapFd, 0 );
-				if ( block != (HeapManager.Storage *)MAP_FAILED ) break Succeed; // succeed ?
-			} // if
-			if ( errno == ENOMEM ) abort( NO_MEMORY_MSG, tsize );
+
+		block = (HeapManager.Storage *)mmap( 0, tsize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, mmapFd, 0 );
+		if ( block == (HeapManager.Storage *)MAP_FAILED ) { // failed ?
+			if ( errno == ENOMEM ) abort( NO_MEMORY_MSG, tsize ); // no memory
 			// Do not call strerror( errno ) as it may call malloc.
 			abort( "(HeapManager &)0x%p.doMalloc() : internal error, mmap failure, size:%zu error:%d.", &heapManager, tsize, errno );
-		}
+		} //if
 		#ifdef __CFA_DEBUG__
 		// Set new memory to garbage so subsequent uninitialized usages might fail.
@@ -653,6 +627,5 @@
 	__atomic_add_fetch( &allocUnfreed, -size, __ATOMIC_SEQ_CST );
 	if ( traceHeap() ) {
-		enum { BufferSize = 64 };
-		char helpText[BufferSize];
+		char helpText[64];
 		int len = snprintf( helpText, sizeof(helpText), "Free( %p ) size:%zu\n", addr, size );
 		__cfaabi_bits_write( STDERR_FILENO, helpText, len ); // print debug/nodebug
@@ -677,7 +650,5 @@
 		for ( HeapManager.Storage * p = freeLists[i].freeList; p != 0p; p = p->header.kind.real.next ) {
 		#else
-		for ( HeapManager.Storage * p = top( freeLists[i].freeList ); p != 0p; /* p = getNext( p )->top */) {
-			typeof(p) temp = ( p )`next->top;			// FIX ME: direct assignent fails, initialization works
-			p = temp;
+		for ( HeapManager.Storage * p = top( freeLists[i].freeList ); p != 0p; p = (p)`next->top ) {
 		#endif // BUCKETLOCK
 			total += size;
@@ -721,5 +692,5 @@
 
 	char * end = (char *)sbrk( 0 );
-	heapBegin = heapEnd = sbrk( (char *)libCeiling( (long unsigned int)end, libAlign() ) - end ); // move start of heap to multiple of alignment
+	heapBegin = heapEnd = sbrk( (char *)ceiling2( (long unsigned int)end, libAlign() ) - end ); // move start of heap to multiple of alignment
 } // HeapManager
 
@@ -729,5 +700,5 @@
 	if ( traceHeapTerm() ) {
 		printStats();
-		// if ( prtfree() ) prtFree( heapManager, true );
+		// prtUnfreed() called in heapAppStop()
 	} // if
 	#endif // __STATISTICS__
@@ -738,5 +709,5 @@
 void memory_startup( void ) {
 	#ifdef __CFA_DEBUG__
-	if ( unlikely( heapBoot ) ) {						// check for recursion during system boot
+	if ( heapBoot ) {									// check for recursion during system boot
 		// DO NOT USE STREAMS AS THEY MAY BE UNAVAILABLE AT THIS POINT.
 		abort( "boot() : internal error, recursively invoked during system boot." );
@@ -757,5 +728,5 @@
 
 static inline void * mallocNoStats( size_t size ) {		// necessary for malloc statistics
-	verify( heapManager.heapBegin != 0 );				// called before memory_startup ?
+	verify( heapManager.heapBegin != 0p );				// called before memory_startup ?
   if ( unlikely( size ) == 0 ) return 0p;				// 0 BYTE ALLOCATION RETURNS NULL POINTER
 
@@ -793,5 +764,5 @@
 
 
-static inline void * memalignNoStats( size_t alignment, size_t size ) { // necessary for malloc statistics
+static inline void * memalignNoStats( size_t alignment, size_t size ) {
   if ( unlikely( size ) == 0 ) return 0p;				// 0 BYTE ALLOCATION RETURNS NULL POINTER
 
@@ -815,5 +786,5 @@
 
 	// address in the block of the "next" alignment address
-	char * user = (char *)libCeiling( (uintptr_t)(addr + sizeof(HeapManager.Storage)), alignment );
+	char * user = (char *)ceiling2( (uintptr_t)(addr + sizeof(HeapManager.Storage)), alignment );
 
 	// address of header from malloc
@@ -843,7 +814,7 @@
 	#endif // __CFA_DEBUG__
 		headers( "cmemalign", addr, header, freeElem, bsize, alignment );
+
+	// Mapped storage is zero filled, but in debug mode mapped memory is scrubbed in doMalloc, so it has to be reset to zero.
 	#ifndef __CFA_DEBUG__
-
-	// Mapped storage is zero filled, but in debug mode mapped memory is scrubbed in doMalloc, so it has to be reset to zero.
 	if ( ! mapped )
 	#endif // __CFA_DEBUG__
@@ -857,13 +828,4 @@
 
 
-// supported mallopt options
-#ifndef M_MMAP_THRESHOLD
-#define M_MMAP_THRESHOLD (-1)
-#endif // M_TOP_PAD
-#ifndef M_TOP_PAD
-#define M_TOP_PAD (-2)
-#endif // M_TOP_PAD
-
-
 extern "C" {
 	// Allocates size bytes and returns a pointer to the allocated memory.  The contents are undefined. If size is 0,
@@ -881,10 +843,11 @@
 	// Same as malloc() except size bytes is an array of dim elements each of elemSize bytes.
 	void * aalloc( size_t dim, size_t elemSize ) {
+		size_t size = dim * elemSize;
 		#ifdef __STATISTICS__
 		__atomic_add_fetch( &aalloc_calls, 1, __ATOMIC_SEQ_CST );
-		__atomic_add_fetch( &aalloc_storage, dim * elemSize, __ATOMIC_SEQ_CST );
-		#endif // __STATISTICS__
-
-		return mallocNoStats( dim * elemSize );
+		__atomic_add_fetch( &aalloc_storage, size, __ATOMIC_SEQ_CST );
+		#endif // __STATISTICS__
+
+		return mallocNoStats( size );
 	} // aalloc
 
@@ -899,4 +862,5 @@
 		return callocNoStats( dim, elemSize );
 	} // calloc
+
 
 	// Change the size of the memory block pointed to by oaddr to size bytes. The contents are undefined.  If oaddr is
@@ -907,23 +871,31 @@
 		#ifdef __STATISTICS__
 		__atomic_add_fetch( &resize_calls, 1, __ATOMIC_SEQ_CST );
-		__atomic_add_fetch( &resize_storage, size, __ATOMIC_SEQ_CST );
 		#endif // __STATISTICS__
 
 		// If size is equal to 0, either NULL or a pointer suitable to be passed to free() is returned.
 	  if ( unlikely( size == 0 ) ) { free( oaddr ); return 0p; } // special cases
-	  if ( unlikely( oaddr == 0p ) ) return mallocNoStats( size );
+	  if ( unlikely( oaddr == 0p ) ) {
+			#ifdef __STATISTICS__
+			__atomic_add_fetch( &resize_storage, size, __ATOMIC_SEQ_CST );
+			#endif // __STATISTICS__
+			return mallocNoStats( size );
+		} // if
 
 		HeapManager.Storage.Header * header;
 		HeapManager.FreeHeader * freeElem;
-		size_t bsize, oalign = 0;
+		size_t bsize, oalign;
 		headers( "resize", oaddr, header, freeElem, bsize, oalign );
 
 		size_t odsize = dataStorage( bsize, oaddr, header ); // data storage available in bucket
 		// same size, DO NOT preserve STICKY PROPERTIES.
-	  if ( oalign == 0 && size <= odsize && odsize <= size * 2 ) { // allow 50% wasted storage for smaller size
+		if ( oalign <= libAlign() && size <= odsize && odsize <= size * 2 ) { // allow 50% wasted storage for smaller size
 			header->kind.real.blockSize &= -2;			// no alignment and turn off 0 fill
 			header->kind.real.size = size;				// reset allocation size
 			return oaddr;
 		} // if
+
+		#ifdef __STATISTICS__
+		__atomic_add_fetch( &resize_storage, size, __ATOMIC_SEQ_CST );
+		#endif // __STATISTICS__
 
 		// change size, DO NOT preserve STICKY PROPERTIES.
@@ -938,14 +910,18 @@
 		#ifdef __STATISTICS__
 		__atomic_add_fetch( &realloc_calls, 1, __ATOMIC_SEQ_CST );
-		__atomic_add_fetch( &realloc_storage, size, __ATOMIC_SEQ_CST );
 		#endif // __STATISTICS__
 
 		// If size is equal to 0, either NULL or a pointer suitable to be passed to free() is returned.
 	  if ( unlikely( size == 0 ) ) { free( oaddr ); return 0p; } // special cases
-	  if ( unlikely( oaddr == 0p ) ) return mallocNoStats( size );
+	  if ( unlikely( oaddr == 0p ) ) {
+			#ifdef __STATISTICS__
+			__atomic_add_fetch( &realloc_storage, size, __ATOMIC_SEQ_CST );
+			#endif // __STATISTICS__
+			return mallocNoStats( size );
+		} // if
 
 		HeapManager.Storage.Header * header;
 		HeapManager.FreeHeader * freeElem;
-		size_t bsize, oalign = 0;
+		size_t bsize, oalign;
 		headers( "realloc", oaddr, header, freeElem, bsize, oalign );
 
@@ -961,8 +937,12 @@
 		} // if
 
+		#ifdef __STATISTICS__
+	  	__atomic_add_fetch( &realloc_storage, size, __ATOMIC_SEQ_CST );
+		#endif // __STATISTICS__
+
 		// change size and copy old content to new storage
 
 		void * naddr;
-		if ( likely( oalign == 0 ) ) {					// previous request memalign?
+		if ( likely( oalign <= libAlign() ) ) {			// previous request not aligned ?
 			naddr = mallocNoStats( size );				// create new area
 		} else {
@@ -997,10 +977,11 @@
 	// Same as aalloc() with memory alignment.
 	void * amemalign( size_t alignment, size_t dim, size_t elemSize ) {
+		size_t size = dim * elemSize;
 		#ifdef __STATISTICS__
 		__atomic_add_fetch( &cmemalign_calls, 1, __ATOMIC_SEQ_CST );
-		__atomic_add_fetch( &cmemalign_storage, dim * elemSize, __ATOMIC_SEQ_CST );
-		#endif // __STATISTICS__
-
-		return memalignNoStats( alignment, dim * elemSize );
+		__atomic_add_fetch( &cmemalign_storage, size, __ATOMIC_SEQ_CST );
+		#endif // __STATISTICS__
+
+		return memalignNoStats( alignment, size );
 	} // amemalign
 
@@ -1028,5 +1009,5 @@
 	// free(3).
 	int posix_memalign( void ** memptr, size_t alignment, size_t size ) {
-	  if ( alignment < libAlign() || ! libPow2( alignment ) ) return EINVAL; // check alignment
+	  if ( alignment < libAlign() || ! is_pow2( alignment ) ) return EINVAL; // check alignment
 		* memptr = memalign( alignment, size );
 		return 0;
@@ -1042,5 +1023,5 @@
 	// Same as valloc but rounds size to multiple of page size.
 	void * pvalloc( size_t size ) {
-		return memalign( pageSize, libCeiling( size, pageSize ) );
+		return memalign( pageSize, ceiling2( size, pageSize ) );
 	} // pvalloc
 
@@ -1080,4 +1061,5 @@
 	} // malloc_alignment
 
+
 	// Set the alignment for an the allocation and return previous alignment or 0 if no alignment.
 	size_t $malloc_alignment_set( void * addr, size_t alignment ) {
@@ -1162,9 +1144,10 @@
 	} // malloc_stats
 
+
 	// Changes the file descripter where malloc_stats() writes statistics.
 	int malloc_stats_fd( int fd __attribute__(( unused )) ) {
 		#ifdef __STATISTICS__
-		int temp = statfd;
-		statfd = fd;
+		int temp = stat_fd;
+		stat_fd = fd;
 		return temp;
 		#else
@@ -1198,6 +1181,10 @@
 	// malloc).
 	int malloc_info( int options, FILE * stream ) {
-		if ( options != 0 ) { errno = EINVAL; return -1; }
+	  if ( options != 0 ) { errno = EINVAL; return -1; }
+		#ifdef __STATISTICS__
 		return printStatsXML( stream );
+		#else
+		return 0;										// unsupported
+		#endif // __STATISTICS__
 	} // malloc_info
 
@@ -1214,5 +1201,5 @@
 	// Restores the state of all malloc internal bookkeeping variables to the values recorded in the opaque data
 	// structure pointed to by state.
-	int malloc_set_state( void * ptr ) {
+	int malloc_set_state( void * ) {
 		return 0;										// unsupported
 	} // malloc_set_state
@@ -1224,10 +1211,14 @@
 	#ifdef __STATISTICS__
 	__atomic_add_fetch( &resize_calls, 1, __ATOMIC_SEQ_CST );
-	__atomic_add_fetch( &resize_storage, size, __ATOMIC_SEQ_CST );
 	#endif // __STATISTICS__
 
 	// If size is equal to 0, either NULL or a pointer suitable to be passed to free() is returned.
   if ( unlikely( size == 0 ) ) { free( oaddr ); return 0p; } // special cases
-  if ( unlikely( oaddr == 0p ) ) return memalignNoStats( nalign, size );
+  if ( unlikely( oaddr == 0p ) ) {
+		#ifdef __STATISTICS__
+		__atomic_add_fetch( &resize_storage, size, __ATOMIC_SEQ_CST );
+		#endif // __STATISTICS__
+		return memalignNoStats( nalign, size );
+	} // if
 
 	if ( unlikely( nalign < libAlign() ) ) nalign = libAlign(); // reset alignment to minimum
@@ -1239,5 +1230,5 @@
 	HeapManager.Storage.Header * header;
 	HeapManager.FreeHeader * freeElem;
-	size_t bsize, oalign = 0;
+	size_t bsize, oalign;
 	headers( "resize", oaddr, header, freeElem, bsize, oalign );
 	size_t odsize = dataStorage( bsize, oaddr, header ); // data storage available in bucket
@@ -1253,4 +1244,8 @@
 		} // if
 	} // if
+
+	#ifdef __STATISTICS__
+	__atomic_add_fetch( &resize_storage, size, __ATOMIC_SEQ_CST );
+	#endif // __STATISTICS__
 
 	// change size, DO NOT preserve STICKY PROPERTIES.
@@ -1269,5 +1264,5 @@
 	HeapManager.Storage.Header * header;
 	HeapManager.FreeHeader * freeElem;
-	size_t bsize, oalign = 0;
+	size_t bsize, oalign;
 	headers( "realloc", oaddr, header, freeElem, bsize, oalign );
 
