diff options
Diffstat (limited to 'drivers/nedmalloc/nedmalloc.cpp')
| -rw-r--r-- | drivers/nedmalloc/nedmalloc.cpp | 1467 |
1 files changed, 0 insertions, 1467 deletions
diff --git a/drivers/nedmalloc/nedmalloc.cpp b/drivers/nedmalloc/nedmalloc.cpp deleted file mode 100644 index 9aac277a2a..0000000000 --- a/drivers/nedmalloc/nedmalloc.cpp +++ /dev/null @@ -1,1467 +0,0 @@ -#ifdef NEDMALLOC_ENABLED -/* Alternative malloc implementation for multiple threads without -lock contention based on dlmalloc. (C) 2005-2009 Niall Douglas - -Boost Software License - Version 1.0 - August 17th, 2003 - -Permission is hereby granted, free of charge, to any person or organization -obtaining a copy of the software and accompanying documentation covered by -this license (the "Software") to use, reproduce, display, distribute, -execute, and transmit the Software, and to prepare derivative works of the -Software, and to permit third-parties to whom the Software is furnished to -do so, all subject to the following: - -The copyright notices in the Software and this entire statement, including -the above license grant, this restriction and the following disclaimer, -must be included in all copies of the Software, in whole or in part, and -all derivative works of the Software, unless such copies or derivative -works are solely in the form of machine-executable object code generated by -a source language processor. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT -SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE -FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, -ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. -*/ - -#ifdef _MSC_VER -/* Enable full aliasing on MSVC */ -/*#pragma optimize("a", on)*/ -#pragma warning(push) -#pragma warning(disable:4100) /* unreferenced formal parameter */ -#pragma warning(disable:4127) /* conditional expression is constant */ -#pragma warning(disable:4706) /* assignment within conditional expression */ -#endif - -/*#define ENABLE_TOLERANT_NEDMALLOC 1*/ -/*#define ENABLE_FAST_HEAP_DETECTION 1*/ -/*#define NEDMALLOC_DEBUG 1*/ - -/*#define FULLSANITYCHECKS*/ -/* If link time code generation is on, don't force or prevent inlining */ -#if defined(_MSC_VER) && defined(NEDMALLOC_DLL_EXPORTS) -#define FORCEINLINE -#define NOINLINE -#endif - - -#include "nedmalloc.h" -#ifdef WIN32 - #include <malloc.h> - #include <stddef.h> -#endif -#if USE_ALLOCATOR==1 - #define MSPACES 1 - #define ONLY_MSPACES 1 -#endif -#define USE_DL_PREFIX 1 -#ifndef USE_LOCKS - #define USE_LOCKS 1 -#endif -#define FOOTERS 1 /* Need to enable footers so frees lock the right mspace */ -#ifndef NEDMALLOC_DEBUG - #if defined(DEBUG) || defined(_DEBUG) - #define NEDMALLOC_DEBUG 1 - #else - #define NEDMALLOC_DEBUG 0 - #endif -#endif -/* We need to consistently define DEBUG=0|1, _DEBUG and NDEBUG for dlmalloc */ -#undef DEBUG -#undef _DEBUG -#if NEDMALLOC_DEBUG - #define _DEBUG - #define DEBUG 1 -#else - #define DEBUG 0 -#endif -#ifdef NDEBUG /* Disable assert checking on release builds */ - #undef DEBUG - #undef _DEBUG -#endif -/* The default of 64Kb means we spend too much time kernel-side */ -#ifndef DEFAULT_GRANULARITY -#define DEFAULT_GRANULARITY (1*1024*1024) -#if DEBUG -#define DEFAULT_GRANULARITY_ALIGNED -#endif -#endif -/*#define USE_SPIN_LOCKS 0*/ - - -#include "malloc.c.h" -#ifdef NDEBUG /* Disable assert checking on release builds */ - #undef DEBUG -#elif !NEDMALLOC_DEBUG - #ifdef __GNUC__ - #warning DEBUG is defined so allocator will run with assert checking! Define NDEBUG to run at full speed. - #elif defined(_MSC_VER) - #pragma message(__FILE__ ": WARNING: DEBUG is defined so allocator will run with assert checking! Define NDEBUG to run at full speed.") - #endif -#endif - -/* The maximum concurrent threads in a pool possible */ -#ifndef MAXTHREADSINPOOL -#define MAXTHREADSINPOOL 16 -#endif -/* The maximum number of threadcaches which can be allocated */ -#ifndef THREADCACHEMAXCACHES -#define THREADCACHEMAXCACHES 256 -#endif -/* The maximum size to be allocated from the thread cache */ -#ifndef THREADCACHEMAX -#define THREADCACHEMAX 8192 -#endif -#if 0 -/* The number of cache entries for finer grained bins. This is (topbitpos(THREADCACHEMAX)-4)*2 */ -#define THREADCACHEMAXBINS ((13-4)*2) -#else -/* The number of cache entries. This is (topbitpos(THREADCACHEMAX)-4) */ -#define THREADCACHEMAXBINS (13-4) -#endif -/* Point at which the free space in a thread cache is garbage collected */ -#ifndef THREADCACHEMAXFREESPACE -#define THREADCACHEMAXFREESPACE (512*1024) -#endif - - -#ifdef WIN32 - #define TLSVAR DWORD - #define TLSALLOC(k) (*(k)=TlsAlloc(), TLS_OUT_OF_INDEXES==*(k)) - #define TLSFREE(k) (!TlsFree(k)) - #define TLSGET(k) TlsGetValue(k) - #define TLSSET(k, a) (!TlsSetValue(k, a)) - #ifdef DEBUG -static LPVOID ChkedTlsGetValue(DWORD idx) -{ - LPVOID ret=TlsGetValue(idx); - assert(S_OK==GetLastError()); - return ret; -} - #undef TLSGET - #define TLSGET(k) ChkedTlsGetValue(k) - #endif -#else - #define TLSVAR pthread_key_t - #define TLSALLOC(k) pthread_key_create(k, 0) - #define TLSFREE(k) pthread_key_delete(k) - #define TLSGET(k) pthread_getspecific(k) - #define TLSSET(k, a) pthread_setspecific(k, a) -#endif - -#if defined(__cplusplus) -#if !defined(NO_NED_NAMESPACE) -namespace nedalloc { -#else -extern "C" { -#endif -#endif - -#if USE_ALLOCATOR==0 -static void *unsupported_operation(const char *opname) THROWSPEC -{ - fprintf(stderr, "nedmalloc: The operation %s is not supported under this build configuration\n", opname); - abort(); - return 0; -} -static size_t mspacecounter=(size_t) 0xdeadbeef; -#endif -#ifndef ENABLE_FAST_HEAP_DETECTION -static void *RESTRICT leastusedaddress; -static size_t largestusedblock; -#endif - -static FORCEINLINE void *CallMalloc(void *RESTRICT mspace, size_t size, size_t alignment) THROWSPEC -{ - void *RESTRICT ret=0; - size_t _alignment=alignment; -#if USE_MAGIC_HEADERS - size_t *_ret=0; - size+=alignment+3*sizeof(size_t); - _alignment=0; -#endif -#if USE_ALLOCATOR==0 - ret=_alignment ? -#ifdef _MSC_VER - /* This is the MSVCRT equivalent */ - _aligned_malloc(size, _alignment) -#elif defined(__linux__) || defined(__FreeBSD__) || defined(__APPLE__) - /* This is the glibc/ptmalloc2/dlmalloc/BSD libc equivalent. */ - memalign(_alignment, size) -#else -#error Cannot aligned allocate with the memory allocator of an unknown system! -#endif - : malloc(size); -#elif USE_ALLOCATOR==1 - ret=_alignment ? mspace_memalign((mstate) mspace, _alignment, size) : mspace_malloc((mstate) mspace, size); -#ifndef ENABLE_FAST_HEAP_DETECTION - if(ret) - { - size_t truesize=chunksize(mem2chunk(ret)); - if(!leastusedaddress || (void *)((mstate) mspace)->least_addr<leastusedaddress) leastusedaddress=(void *)((mstate) mspace)->least_addr; - if(!largestusedblock || truesize>largestusedblock) largestusedblock=(truesize+mparams.page_size) & ~(mparams.page_size-1); - } -#endif -#endif - if(!ret) return 0; -#if USE_MAGIC_HEADERS - _ret=(size_t *) ret; - ret=(void *)(_ret+3); - if(alignment) ret=(void *)(((size_t) ret+alignment-1)&~(alignment-1)); - for(; _ret<(size_t *)ret-2; _ret++) *_ret=*(size_t *)"NEDMALOC"; - _ret[0]=(size_t) mspace; - _ret[1]=size-3*sizeof(size_t); -#endif - return ret; -} - -static FORCEINLINE void *CallCalloc(void *RESTRICT mspace, size_t size, size_t alignment) THROWSPEC -{ - void *RESTRICT ret=0; -#if USE_MAGIC_HEADERS - size_t *_ret=0; - size+=alignment+3*sizeof(size_t); -#endif -#if USE_ALLOCATOR==0 - ret=calloc(1, size); -#elif USE_ALLOCATOR==1 - ret=mspace_calloc((mstate) mspace, 1, size); -#ifndef ENABLE_FAST_HEAP_DETECTION - if(ret) - { - size_t truesize=chunksize(mem2chunk(ret)); - if(!leastusedaddress || (void *)((mstate) mspace)->least_addr<leastusedaddress) leastusedaddress=(void *)((mstate) mspace)->least_addr; - if(!largestusedblock || truesize>largestusedblock) largestusedblock=(truesize+mparams.page_size) & ~(mparams.page_size-1); - } -#endif -#endif - if(!ret) return 0; -#if USE_MAGIC_HEADERS - _ret=(size_t *) ret; - ret=(void *)(_ret+3); - if(alignment) ret=(void *)(((size_t) ret+alignment-1)&~(alignment-1)); - for(; _ret<(size_t *)ret-2; _ret++) *_ret=*(size_t *) "NEDMALOC"; - _ret[0]=(size_t) mspace; - _ret[1]=size-3*sizeof(size_t); -#endif - return ret; -} - -static FORCEINLINE void *CallRealloc(void *RESTRICT mspace, void *RESTRICT mem, int isforeign, size_t oldsize, size_t newsize) THROWSPEC -{ - void *RESTRICT ret=0; -#if USE_MAGIC_HEADERS - mstate oldmspace=0; - size_t *_ret=0, *_mem=(size_t *) mem-3; -#endif - if(isforeign) - { /* Transfer */ -#if USE_MAGIC_HEADERS - assert(_mem[0]!=*(size_t *) "NEDMALOC"); -#endif - if((ret=CallMalloc(mspace, newsize, 0))) - { -#if defined(DEBUG) - printf("*** nedmalloc frees system allocated block %p\n", mem); -#endif - memcpy(ret, mem, oldsize<newsize ? oldsize : newsize); - free(mem); - } - return ret; - } -#if USE_MAGIC_HEADERS - assert(_mem[0]==*(size_t *) "NEDMALOC"); - newsize+=3*sizeof(size_t); - oldmspace=(mstate) _mem[1]; - assert(oldsize>=_mem[2]); - for(; *_mem==*(size_t *) "NEDMALOC"; *_mem--=*(size_t *) "nedmaloc"); - mem=(void *)(++_mem); -#endif -#if USE_ALLOCATOR==0 - ret=realloc(mem, newsize); -#elif USE_ALLOCATOR==1 - ret=mspace_realloc((mstate) mspace, mem, newsize); -#ifndef ENABLE_FAST_HEAP_DETECTION - if(ret) - { - size_t truesize=chunksize(mem2chunk(ret)); - if(!largestusedblock || truesize>largestusedblock) largestusedblock=(truesize+mparams.page_size) & ~(mparams.page_size-1); - } -#endif -#endif - if(!ret) - { /* Put it back the way it was */ -#if USE_MAGIC_HEADERS - for(; *_mem==0; *_mem++=*(size_t *) "NEDMALOC"); -#endif - return 0; - } -#if USE_MAGIC_HEADERS - _ret=(size_t *) ret; - ret=(void *)(_ret+3); - for(; _ret<(size_t *)ret-2; _ret++) *_ret=*(size_t *) "NEDMALOC"; - _ret[0]=(size_t) mspace; - _ret[1]=newsize-3*sizeof(size_t); -#endif - return ret; -} - -static FORCEINLINE void CallFree(void *RESTRICT mspace, void *RESTRICT mem, int isforeign) THROWSPEC -{ -#if USE_MAGIC_HEADERS - mstate oldmspace=0; - size_t *_mem=(size_t *) mem-3, oldsize=0; -#endif - if(isforeign) - { -#if USE_MAGIC_HEADERS - assert(_mem[0]!=*(size_t *) "NEDMALOC"); -#endif -#if defined(DEBUG) - printf("*** nedmalloc frees system allocated block %p\n", mem); -#endif - free(mem); - return; - } -#if USE_MAGIC_HEADERS - assert(_mem[0]==*(size_t *) "NEDMALOC"); - oldmspace=(mstate) _mem[1]; - oldsize=_mem[2]; - for(; *_mem==*(size_t *) "NEDMALOC"; *_mem--=*(size_t *) "nedmaloc"); - mem=(void *)(++_mem); -#endif -#if USE_ALLOCATOR==0 - free(mem); -#elif USE_ALLOCATOR==1 - mspace_free((mstate) mspace, mem); -#endif -} - -static NEDMALLOCNOALIASATTR mstate nedblkmstate(void *RESTRICT mem) THROWSPEC -{ - if(mem) - { -#if USE_MAGIC_HEADERS - size_t *_mem=(size_t *) mem-3; - if(_mem[0]==*(size_t *) "NEDMALOC") - { - return (mstate) _mem[1]; - } - else return 0; -#else -#if USE_ALLOCATOR==0 - /* Fail everything */ - return 0; -#elif USE_ALLOCATOR==1 -#ifdef ENABLE_FAST_HEAP_DETECTION -#ifdef WIN32 - /* On Windows for RELEASE both x86 and x64 the NT heap precedes each block with an eight byte header - which looks like: - normal: 4 bytes of size, 4 bytes of [char < 64, char < 64, char < 64 bit 0 always set, char random ] - mmaped: 4 bytes of size 4 bytes of [zero, zero, 0xb, zero ] - - On Windows for DEBUG both x86 and x64 the preceding four bytes is always 0xfdfdfdfd (no man's land). - */ -#pragma pack(push, 1) - struct _HEAP_ENTRY - { - USHORT Size; - USHORT PreviousSize; - UCHAR Cookie; /* SegmentIndex */ - UCHAR Flags; /* always bit 0 (HEAP_ENTRY_BUSY). bit 1=(HEAP_ENTRY_EXTRA_PRESENT), bit 2=normal block (HEAP_ENTRY_FILL_PATTERN), bit 3=mmap block (HEAP_ENTRY_VIRTUAL_ALLOC). Bit 4 (HEAP_ENTRY_LAST_ENTRY) could be set */ - UCHAR UnusedBytes; - UCHAR SmallTagIndex; /* fastbin index. Always one of 0x02, 0x03, 0x04 < 0x80 */ - } *RESTRICT he=((struct _HEAP_ENTRY *) mem)-1; -#pragma pack(pop) - unsigned int header=((unsigned int *)mem)[-1], mask1=0x8080E100, result1, mask2=0xFFFFFF06, result2; - result1=header & mask1; /* Positive testing for NT heap */ - result2=header & mask2; /* Positive testing for dlmalloc */ - if(result1==0x00000100 && result2!=0x00000102) - { /* This is likely a NT heap block */ - return 0; - } -#endif -#ifdef __linux__ - /* On Linux glibc uses ptmalloc2 (really dlmalloc) just as we do, but prev_foot contains rubbish - when the preceding block is allocated because ptmalloc2 finds the local mstate by rounding the ptr - down to the nearest megabyte. It's like dlmalloc with FOOTERS disabled. */ - mchunkptr p=mem2chunk(mem); - mstate fm=get_mstate_for(p); - /* If it's a ptmalloc2 block, fm is likely to be some crazy value */ - if(!is_aligned(fm)) return 0; - if((size_t)mem-(size_t)fm>=(size_t)1<<(SIZE_T_BITSIZE-1)) return 0; - if(ok_magic(fm)) - return fm; - else - return 0; - if(1) { } -#endif - else - { - mchunkptr p=mem2chunk(mem); - mstate fm=get_mstate_for(p); - assert(ok_magic(fm)); /* If this fails, someone tried to free a block twice */ - if(ok_magic(fm)) - return fm; - } -#else -//#ifdef WIN32 -// __try -//#endif - { - /* We try to return zero here if it isn't one of our own blocks, however - the current block annotation scheme used by dlmalloc makes it impossible - to be absolutely sure of avoiding a segfault. - - mchunkptr->prev_foot = mem-(2*size_t) = mstate ^ mparams.magic for PRECEDING block; - mchunkptr->head = mem-(1*size_t) = 8 multiple size of this block with bottom three bits = FLAG_BITS - FLAG_BITS = bit 0 is CINUSE (currently in use unless is mmap), bit 1 is PINUSE (previous block currently - in use unless mmap), bit 2 is UNUSED and currently is always zero. - */ - register void *RESTRICT leastusedaddress_=leastusedaddress; /* Cache these to avoid register reloading */ - register size_t largestusedblock_=largestusedblock; - if(!is_aligned(mem)) return 0; /* Would fail very rarely as all allocators return aligned blocks */ - if(mem<leastusedaddress_) return 0; /* Simple but effective */ - { - mchunkptr p=mem2chunk(mem); - mstate fm=0; - int ismmapped=is_mmapped(p); - if((!ismmapped && !is_inuse(p)) || (p->head & FLAG4_BIT)) return 0; - /* Reduced uncertainty by 0.5^2 = 25.0% */ - /* size should never exceed largestusedblock */ - if(chunksize(p)>largestusedblock_) return 0; - /* Reduced uncertainty by a minimum of 0.5^3 = 12.5%, maximum 0.5^16 = 0.0015% */ - /* Having sanity checked prev_foot and head, check next block */ - if(!ismmapped && (!next_pinuse(p) || (next_chunk(p)->head & FLAG4_BIT))) return 0; - /* Reduced uncertainty by 0.5^5 = 3.13% or 0.5^18 = 0.00038% */ - #if 0 - /* If previous block is free, check that its next block pointer equals us */ - if(!ismmapped && !pinuse(p)) - if(next_chunk(prev_chunk(p))!=p) return 0; - /* We could start comparing prev_foot's for similarity but it starts getting slow. */ - #endif - fm = get_mstate_for(p); - if(!is_aligned(fm) || (void *)fm<leastusedaddress_) return 0; - if((size_t)mem-(size_t)fm>=(size_t)1<<(SIZE_T_BITSIZE-1)) return 0; - assert(ok_magic(fm)); /* If this fails, someone tried to free a block twice */ - if(ok_magic(fm)) - return fm; - } - } -//#ifdef WIN32 -// __except(1) { } -//#endif -#endif -#endif -#endif - } - return 0; -} -NEDMALLOCNOALIASATTR size_t nedblksize(int *RESTRICT isforeign, void *RESTRICT mem) THROWSPEC -{ - if(mem) - { - if(isforeign) *isforeign=1; -#if USE_MAGIC_HEADERS - { - size_t *_mem=(size_t *) mem-3; - if(_mem[0]==*(size_t *) "NEDMALOC") - { - mstate mspace=(mstate) _mem[1]; - size_t size=_mem[2]; - if(isforeign) *isforeign=0; - return size; - } - } -#elif USE_ALLOCATOR==1 - if(nedblkmstate(mem)) - { - mchunkptr p=mem2chunk(mem); - if(isforeign) *isforeign=0; - return chunksize(p)-overhead_for(p); - } -#ifdef DEBUG - else - { - int a=1; /* Set breakpoints here if needed */ - } -#endif -#endif -#if defined(ENABLE_TOLERANT_NEDMALLOC) || USE_ALLOCATOR==0 -#ifdef _MSC_VER - /* This is the MSVCRT equivalent */ - return _msize(mem); -#elif defined(__linux__) - /* This is the glibc/ptmalloc2/dlmalloc equivalent. */ - return malloc_usable_size(mem); -#elif defined(__FreeBSD__) || defined(__APPLE__) - /* This is the BSD libc equivalent. */ - return malloc_size(mem); -#else -#error Cannot tolerate the memory allocator of an unknown system! -#endif -#endif - } - return 0; -} - -NEDMALLOCNOALIASATTR void nedsetvalue(void *v) THROWSPEC { nedpsetvalue((nedpool *) 0, v); } -NEDMALLOCNOALIASATTR NEDMALLOCPTRATTR void * nedmalloc(size_t size) THROWSPEC { return nedpmalloc((nedpool *) 0, size); } -NEDMALLOCNOALIASATTR NEDMALLOCPTRATTR void * nedcalloc(size_t no, size_t size) THROWSPEC { return nedpcalloc((nedpool *) 0, no, size); } -NEDMALLOCNOALIASATTR NEDMALLOCPTRATTR void * nedrealloc(void *mem, size_t size) THROWSPEC { return nedprealloc((nedpool *) 0, mem, size); } -NEDMALLOCNOALIASATTR void nedfree(void *mem) THROWSPEC { nedpfree((nedpool *) 0, mem); } -NEDMALLOCNOALIASATTR NEDMALLOCPTRATTR void * nedmemalign(size_t alignment, size_t bytes) THROWSPEC { return nedpmemalign((nedpool *) 0, alignment, bytes); } -NEDMALLOCNOALIASATTR struct nedmallinfo nedmallinfo(void) THROWSPEC { return nedpmallinfo((nedpool *) 0); } -NEDMALLOCNOALIASATTR int nedmallopt(int parno, int value) THROWSPEC { return nedpmallopt((nedpool *) 0, parno, value); } -NEDMALLOCNOALIASATTR int nedmalloc_trim(size_t pad) THROWSPEC { return nedpmalloc_trim((nedpool *) 0, pad); } -void nedmalloc_stats() THROWSPEC { nedpmalloc_stats((nedpool *) 0); } -NEDMALLOCNOALIASATTR size_t nedmalloc_footprint() THROWSPEC { return nedpmalloc_footprint((nedpool *) 0); } -NEDMALLOCNOALIASATTR NEDMALLOCPTRATTR void **nedindependent_calloc(size_t elemsno, size_t elemsize, void **chunks) THROWSPEC { return nedpindependent_calloc((nedpool *) 0, elemsno, elemsize, chunks); } -NEDMALLOCNOALIASATTR NEDMALLOCPTRATTR void **nedindependent_comalloc(size_t elems, size_t *sizes, void **chunks) THROWSPEC { return nedpindependent_comalloc((nedpool *) 0, elems, sizes, chunks); } - -struct threadcacheblk_t; -typedef struct threadcacheblk_t threadcacheblk; -struct threadcacheblk_t -{ /* Keep less than 16 bytes on 32 bit systems and 32 bytes on 64 bit systems */ -#ifdef FULLSANITYCHECKS - unsigned int magic; -#endif - unsigned int lastUsed, size; - threadcacheblk *next, *prev; -}; -typedef struct threadcache_t -{ -#ifdef FULLSANITYCHECKS - unsigned int magic1; -#endif - int mymspace; /* Last mspace entry this thread used */ - long threadid; - unsigned int mallocs, frees, successes; - size_t freeInCache; /* How much free space is stored in this cache */ - threadcacheblk *bins[(THREADCACHEMAXBINS+1)*2]; -#ifdef FULLSANITYCHECKS - unsigned int magic2; -#endif -} threadcache; -struct nedpool_t -{ - MLOCK_T mutex; - void *uservalue; - int threads; /* Max entries in m to use */ - threadcache *caches[THREADCACHEMAXCACHES]; - TLSVAR mycache; /* Thread cache for this thread. 0 for unset, negative for use mspace-1 directly, otherwise is cache-1 */ - mstate m[MAXTHREADSINPOOL+1]; /* mspace entries for this pool */ -}; -static nedpool syspool; - -static FORCEINLINE NEDMALLOCNOALIASATTR unsigned int size2binidx(size_t _size) THROWSPEC -{ /* 8=1000 16=10000 20=10100 24=11000 32=100000 48=110000 4096=1000000000000 */ - unsigned int topbit, size=(unsigned int)(_size>>4); - /* 16=1 20=1 24=1 32=10 48=11 64=100 96=110 128=1000 4096=100000000 */ - -#if defined(__GNUC__) - topbit = sizeof(size)*__CHAR_BIT__ - 1 - __builtin_clz(size); -#elif defined(_MSC_VER) && _MSC_VER>=1300 - { - unsigned long bsrTopBit; - - _BitScanReverse(&bsrTopBit, size); - - topbit = bsrTopBit; - } -#else -#if 0 - union { - unsigned asInt[2]; - double asDouble; - }; - int n; - - asDouble = (double)size + 0.5; - topbit = (asInt[!FOX_BIGENDIAN] >> 20) - 1023; -#else - { - unsigned int x=size; - x = x | (x >> 1); - x = x | (x >> 2); - x = x | (x >> 4); - x = x | (x >> 8); - x = x | (x >>16); - x = ~x; - x = x - ((x >> 1) & 0x55555555); - x = (x & 0x33333333) + ((x >> 2) & 0x33333333); - x = (x + (x >> 4)) & 0x0F0F0F0F; - x = x + (x << 8); - x = x + (x << 16); - topbit=31 - (x >> 24); - } -#endif -#endif - return topbit; -} - - -#ifdef FULLSANITYCHECKS -static void tcsanitycheck(threadcacheblk **ptr) THROWSPEC -{ - assert((ptr[0] && ptr[1]) || (!ptr[0] && !ptr[1])); - if(ptr[0] && ptr[1]) - { - assert(nedblksize(ptr[0])>=sizeof(threadcacheblk)); - assert(nedblksize(ptr[1])>=sizeof(threadcacheblk)); - assert(*(unsigned int *) "NEDN"==ptr[0]->magic); - assert(*(unsigned int *) "NEDN"==ptr[1]->magic); - assert(!ptr[0]->prev); - assert(!ptr[1]->next); - if(ptr[0]==ptr[1]) - { - assert(!ptr[0]->next); - assert(!ptr[1]->prev); - } - } -} -static void tcfullsanitycheck(threadcache *tc) THROWSPEC -{ - threadcacheblk **tcbptr=tc->bins; - int n; - for(n=0; n<=THREADCACHEMAXBINS; n++, tcbptr+=2) - { - threadcacheblk *b, *ob=0; - tcsanitycheck(tcbptr); - for(b=tcbptr[0]; b; ob=b, b=b->next) - { - assert(*(unsigned int *) "NEDN"==b->magic); - assert(!ob || ob->next==b); - assert(!ob || b->prev==ob); - } - } -} -#endif - -static NOINLINE void RemoveCacheEntries(nedpool *RESTRICT p, threadcache *RESTRICT tc, unsigned int age) THROWSPEC -{ -#ifdef FULLSANITYCHECKS - tcfullsanitycheck(tc); -#endif - if(tc->freeInCache) - { - threadcacheblk **tcbptr=tc->bins; - int n; - for(n=0; n<=THREADCACHEMAXBINS; n++, tcbptr+=2) - { - threadcacheblk **tcb=tcbptr+1; /* come from oldest end of list */ - /*tcsanitycheck(tcbptr);*/ - for(; *tcb && tc->frees-(*tcb)->lastUsed>=age; ) - { - threadcacheblk *f=*tcb; - size_t blksize=f->size; /*nedblksize(f);*/ - assert(blksize<=nedblksize(0, f)); - assert(blksize); -#ifdef FULLSANITYCHECKS - assert(*(unsigned int *) "NEDN"==(*tcb)->magic); -#endif - *tcb=(*tcb)->prev; - if(*tcb) - (*tcb)->next=0; - else - *tcbptr=0; - tc->freeInCache-=blksize; - assert((long) tc->freeInCache>=0); - CallFree(0, f, 0); - /*tcsanitycheck(tcbptr);*/ - } - } - } -#ifdef FULLSANITYCHECKS - tcfullsanitycheck(tc); -#endif -} -static void DestroyCaches(nedpool *RESTRICT p) THROWSPEC -{ - if(p->caches) - { - threadcache *tc; - int n; - for(n=0; n<THREADCACHEMAXCACHES; n++) - { - if((tc=p->caches[n])) - { - tc->frees++; - RemoveCacheEntries(p, tc, 0); - assert(!tc->freeInCache); - tc->mymspace=-1; - tc->threadid=0; - CallFree(0, tc, 0); - p->caches[n]=0; - } - } - } -} - -static NOINLINE threadcache *AllocCache(nedpool *RESTRICT p) THROWSPEC -{ - threadcache *tc=0; - int n, end; - ACQUIRE_LOCK(&p->mutex); - for(n=0; n<THREADCACHEMAXCACHES && p->caches[n]; n++); - if(THREADCACHEMAXCACHES==n) - { /* List exhausted, so disable for this thread */ - RELEASE_LOCK(&p->mutex); - return 0; - } - tc=p->caches[n]=(threadcache *) CallCalloc(p->m[0], sizeof(threadcache), 0); - if(!tc) - { - RELEASE_LOCK(&p->mutex); - return 0; - } -#ifdef FULLSANITYCHECKS - tc->magic1=*(unsigned int *)"NEDMALC1"; - tc->magic2=*(unsigned int *)"NEDMALC2"; -#endif - tc->threadid=(long)(size_t)CURRENT_THREAD; - for(end=0; p->m[end]; end++); - tc->mymspace=abs(tc->threadid) % end; - RELEASE_LOCK(&p->mutex); - if(TLSSET(p->mycache, (void *)(size_t)(n+1))) abort(); - return tc; -} - -static void *threadcache_malloc(nedpool *RESTRICT p, threadcache *RESTRICT tc, size_t *RESTRICT _size) THROWSPEC -{ - void *RESTRICT ret=0; - size_t size=*_size, blksize=0; - unsigned int bestsize; - unsigned int idx=size2binidx(size); - threadcacheblk *RESTRICT blk, **RESTRICT binsptr; -#ifdef FULLSANITYCHECKS - tcfullsanitycheck(tc); -#endif - /* Calculate best fit bin size */ - bestsize=1<<(idx+4); -#if 0 - /* Finer grained bin fit */ - idx<<=1; - if(size>bestsize) - { - idx++; - bestsize+=bestsize>>1; - } - if(size>bestsize) - { - idx++; - bestsize=1<<(4+(idx>>1)); - } -#else - if(size>bestsize) - { - idx++; - bestsize<<=1; - } -#endif - assert(bestsize>=size); - if(size<bestsize) size=bestsize; - assert(size<=THREADCACHEMAX); - assert(idx<=THREADCACHEMAXBINS); - binsptr=&tc->bins[idx*2]; - /* Try to match close, but move up a bin if necessary */ - blk=*binsptr; - if(!blk || blk->size<size) - { /* Bump it up a bin */ - if(idx<THREADCACHEMAXBINS) - { - idx++; - binsptr+=2; - blk=*binsptr; - } - } - if(blk) - { - blksize=blk->size; /*nedblksize(blk);*/ - assert(nedblksize(0, blk)>=blksize); - assert(blksize>=size); - if(blk->next) - blk->next->prev=0; - *binsptr=blk->next; - if(!*binsptr) - binsptr[1]=0; -#ifdef FULLSANITYCHECKS - blk->magic=0; -#endif - assert(binsptr[0]!=blk && binsptr[1]!=blk); - assert(nedblksize(0, blk)>=sizeof(threadcacheblk) && nedblksize(0, blk)<=THREADCACHEMAX+CHUNK_OVERHEAD); - /*printf("malloc: %p, %p, %p, %lu\n", p, tc, blk, (long) _size);*/ - ret=(void *) blk; - } - ++tc->mallocs; - if(ret) - { - assert(blksize>=size); - ++tc->successes; - tc->freeInCache-=blksize; - assert((long) tc->freeInCache>=0); - } -#if defined(DEBUG) && 0 - if(!(tc->mallocs & 0xfff)) - { - printf("*** threadcache=%u, mallocs=%u (%f), free=%u (%f), freeInCache=%u\n", (unsigned int) tc->threadid, tc->mallocs, - (float) tc->successes/tc->mallocs, tc->frees, (float) tc->successes/tc->frees, (unsigned int) tc->freeInCache); - } -#endif -#ifdef FULLSANITYCHECKS - tcfullsanitycheck(tc); -#endif - *_size=size; - return ret; -} -static NOINLINE void ReleaseFreeInCache(nedpool *RESTRICT p, threadcache *RESTRICT tc, int mymspace) THROWSPEC -{ - unsigned int age=THREADCACHEMAXFREESPACE/8192; - /*ACQUIRE_LOCK(&p->m[mymspace]->mutex);*/ - while(age && tc->freeInCache>=THREADCACHEMAXFREESPACE) - { - RemoveCacheEntries(p, tc, age); - /*printf("*** Removing cache entries older than %u (%u)\n", age, (unsigned int) tc->freeInCache);*/ - age>>=1; - } - /*RELEASE_LOCK(&p->m[mymspace]->mutex);*/ -} -static void threadcache_free(nedpool *RESTRICT p, threadcache *RESTRICT tc, int mymspace, void *RESTRICT mem, size_t size) THROWSPEC -{ - unsigned int bestsize; - unsigned int idx=size2binidx(size); - threadcacheblk **RESTRICT binsptr, *RESTRICT tck=(threadcacheblk *) mem; - assert(size>=sizeof(threadcacheblk) && size<=THREADCACHEMAX+CHUNK_OVERHEAD); -#ifdef DEBUG - /* Make sure this is a valid memory block */ - assert(nedblksize(0, mem)); -#endif -#ifdef FULLSANITYCHECKS - tcfullsanitycheck(tc); -#endif - /* Calculate best fit bin size */ - bestsize=1<<(idx+4); -#if 0 - /* Finer grained bin fit */ - idx<<=1; - if(size>bestsize) - { - unsigned int biggerbestsize=bestsize+bestsize<<1; - if(size>=biggerbestsize) - { - idx++; - bestsize=biggerbestsize; - } - } -#endif - if(bestsize!=size) /* dlmalloc can round up, so we round down to preserve indexing */ - size=bestsize; - binsptr=&tc->bins[idx*2]; - assert(idx<=THREADCACHEMAXBINS); - if(tck==*binsptr) - { - fprintf(stderr, "nedmalloc: Attempt to free already freed memory block %p - aborting!\n", tck); - abort(); - } -#ifdef FULLSANITYCHECKS - tck->magic=*(unsigned int *) "NEDN"; -#endif - tck->lastUsed=++tc->frees; - tck->size=(unsigned int) size; - tck->next=*binsptr; - tck->prev=0; - if(tck->next) - tck->next->prev=tck; - else - binsptr[1]=tck; - assert(!*binsptr || (*binsptr)->size==tck->size); - *binsptr=tck; - assert(tck==tc->bins[idx*2]); - assert(tc->bins[idx*2+1]==tck || binsptr[0]->next->prev==tck); - /*printf("free: %p, %p, %p, %lu\n", p, tc, mem, (long) size);*/ - tc->freeInCache+=size; -#ifdef FULLSANITYCHECKS - tcfullsanitycheck(tc); -#endif -#if 1 - if(tc->freeInCache>=THREADCACHEMAXFREESPACE) - ReleaseFreeInCache(p, tc, mymspace); -#endif -} - - - - -static NOINLINE int InitPool(nedpool *RESTRICT p, size_t capacity, int threads) THROWSPEC -{ /* threads is -1 for system pool */ - ensure_initialization(); - ACQUIRE_MALLOC_GLOBAL_LOCK(); - if(p->threads) goto done; - if(INITIAL_LOCK(&p->mutex)) goto err; - if(TLSALLOC(&p->mycache)) goto err; -#if USE_ALLOCATOR==0 - p->m[0]=(mstate) mspacecounter++; -#elif USE_ALLOCATOR==1 - if(!(p->m[0]=(mstate) create_mspace(capacity, 1))) goto err; - p->m[0]->extp=p; -#endif - p->threads=(threads<1 || threads>MAXTHREADSINPOOL) ? MAXTHREADSINPOOL : threads; -done: - RELEASE_MALLOC_GLOBAL_LOCK(); - return 1; -err: - if(threads<0) - abort(); /* If you can't allocate for system pool, we're screwed */ - DestroyCaches(p); - if(p->m[0]) - { -#if USE_ALLOCATOR==1 - destroy_mspace(p->m[0]); -#endif - p->m[0]=0; - } - if(p->mycache) - { - if(TLSFREE(p->mycache)) abort(); - p->mycache=0; - } - RELEASE_MALLOC_GLOBAL_LOCK(); - return 0; -} -static NOINLINE mstate FindMSpace(nedpool *RESTRICT p, threadcache *RESTRICT tc, int *RESTRICT lastUsed, size_t size) THROWSPEC -{ /* Gets called when thread's last used mspace is in use. The strategy - is to run through the list of all available mspaces looking for an - unlocked one and if we fail, we create a new one so long as we don't - exceed p->threads */ - int n, end; - for(n=end=*lastUsed+1; p->m[n]; end=++n) - { - if(TRY_LOCK(&p->m[n]->mutex)) goto found; - } - for(n=0; n<*lastUsed && p->m[n]; n++) - { - if(TRY_LOCK(&p->m[n]->mutex)) goto found; - } - if(end<p->threads) - { - mstate temp; -#if USE_ALLOCATOR==0 - temp=(mstate) mspacecounter++; -#elif USE_ALLOCATOR==1 - if(!(temp=(mstate) create_mspace(size, 1))) - goto badexit; -#endif - /* Now we're ready to modify the lists, we lock */ - ACQUIRE_LOCK(&p->mutex); - while(p->m[end] && end<p->threads) - end++; - if(end>=p->threads) - { /* Drat, must destroy it now */ - RELEASE_LOCK(&p->mutex); -#if USE_ALLOCATOR==1 - destroy_mspace((mstate) temp); -#endif - goto badexit; - } - /* We really want to make sure this goes into memory now but we - have to be careful of breaking aliasing rules, so write it twice */ - *((volatile struct malloc_state **) &p->m[end])=p->m[end]=temp; - ACQUIRE_LOCK(&p->m[end]->mutex); - /*printf("Created mspace idx %d\n", end);*/ - RELEASE_LOCK(&p->mutex); - n=end; - goto found; - } - /* Let it lock on the last one it used */ -badexit: - ACQUIRE_LOCK(&p->m[*lastUsed]->mutex); - return p->m[*lastUsed]; -found: - *lastUsed=n; - if(tc) - tc->mymspace=n; - else - { - if(TLSSET(p->mycache, (void *)(size_t)(-(n+1)))) abort(); - } - return p->m[n]; -} - -typedef struct PoolList_t -{ - size_t size; /* Size of list */ - size_t length; /* Actual entries in list */ -#ifdef DEBUG - nedpool *list[1]; /* Force testing of list expansion */ -#else - nedpool *list[16]; -#endif -} PoolList; -static MLOCK_T poollistlock; -static PoolList *poollist; -NEDMALLOCPTRATTR nedpool *nedcreatepool(size_t capacity, int threads) THROWSPEC -{ - nedpool *ret=0; - if(!poollist) - { - PoolList *newpoollist=0; - if(!(newpoollist=(PoolList *) nedpcalloc(0, 1, sizeof(PoolList)+sizeof(nedpool *)))) return 0; - INITIAL_LOCK(&poollistlock); - ACQUIRE_LOCK(&poollistlock); - poollist=newpoollist; - poollist->size=sizeof(poollist->list)/sizeof(nedpool *); - } - else - ACQUIRE_LOCK(&poollistlock); - if(poollist->length==poollist->size) - { - PoolList *newpoollist=0; - size_t newsize=0; - newsize=sizeof(PoolList)+(poollist->size+1)*sizeof(nedpool *); - if(!(newpoollist=(PoolList *) nedprealloc(0, poollist, newsize))) goto badexit; - poollist=newpoollist; - memset(&poollist->list[poollist->size], 0, newsize-((size_t)&poollist->list[poollist->size]-(size_t)&poollist->list[0])); - poollist->size=((newsize-((char *)&poollist->list[0]-(char *)poollist))/sizeof(nedpool *))-1; - assert(poollist->size>poollist->length); - } - if(!(ret=(nedpool *) nedpcalloc(0, 1, sizeof(nedpool)))) goto badexit; - if(!InitPool(ret, capacity, threads)) - { - nedpfree(0, ret); - goto badexit; - } - poollist->list[poollist->length++]=ret; -badexit: - RELEASE_LOCK(&poollistlock); - return ret; -} -void neddestroypool(nedpool *p) THROWSPEC -{ - unsigned int n; - ACQUIRE_LOCK(&p->mutex); - DestroyCaches(p); - for(n=0; p->m[n]; n++) - { -#if USE_ALLOCATOR==1 - destroy_mspace(p->m[n]); -#endif - p->m[n]=0; - } - RELEASE_LOCK(&p->mutex); - if(TLSFREE(p->mycache)) abort(); - nedpfree(0, p); - ACQUIRE_LOCK(&poollistlock); - assert(poollist); - for(n=0; n<poollist->length && poollist->list[n]!=p; n++); - assert(n!=poollist->length); - memmove(&poollist->list[n], &poollist->list[n+1], (size_t)&poollist->list[poollist->length]-(size_t)&poollist->list[n]); - if(!--poollist->length) - { - assert(!poollist->list[0]); - nedpfree(0, poollist); - poollist=0; - } - RELEASE_LOCK(&poollistlock); -} -void neddestroysyspool() THROWSPEC -{ - nedpool *p=&syspool; - int n; - ACQUIRE_LOCK(&p->mutex); - DestroyCaches(p); - for(n=0; p->m[n]; n++) - { -#if USE_ALLOCATOR==1 - destroy_mspace(p->m[n]); -#endif - p->m[n]=0; - } - /* Render syspool unusable */ - for(n=0; n<THREADCACHEMAXCACHES; n++) - p->caches[n]=(threadcache *)(size_t)(sizeof(size_t)>4 ? 0xdeadbeefdeadbeefULL : 0xdeadbeefUL); - for(n=0; n<MAXTHREADSINPOOL+1; n++) - p->m[n]=(mstate)(size_t)(sizeof(size_t)>4 ? 0xdeadbeefdeadbeefULL : 0xdeadbeefUL); - if(TLSFREE(p->mycache)) abort(); - RELEASE_LOCK(&p->mutex); -} -nedpool **nedpoollist() THROWSPEC -{ - nedpool **ret=0; - if(poollist) - { - ACQUIRE_LOCK(&poollistlock); - if(!(ret=(nedpool **) nedmalloc((poollist->length+1)*sizeof(nedpool *)))) goto badexit; - memcpy(ret, poollist->list, (poollist->length+1)*sizeof(nedpool *)); -badexit: - RELEASE_LOCK(&poollistlock); - } - return ret; -} - -void nedpsetvalue(nedpool *p, void *v) THROWSPEC -{ - if(!p) { p=&syspool; if(!syspool.threads) InitPool(&syspool, 0, -1); } - p->uservalue=v; -} -void *nedgetvalue(nedpool **p, void *mem) THROWSPEC -{ - nedpool *np=0; - mstate fm=nedblkmstate(mem); - if(!fm || !fm->extp) return 0; - np=(nedpool *) fm->extp; - if(p) *p=np; - return np->uservalue; -} - -void nedtrimthreadcache(nedpool *p, int disable) THROWSPEC -{ - int mycache; - if(!p) - { - p=&syspool; - if(!syspool.threads) InitPool(&syspool, 0, -1); - } - mycache=(int)(size_t) TLSGET(p->mycache); - if(!mycache) - { /* Set to mspace 0 */ - if(disable && TLSSET(p->mycache, (void *)(size_t)-1)) abort(); - } - else if(mycache>0) - { /* Set to last used mspace */ - threadcache *tc=p->caches[mycache-1]; -#if defined(DEBUG) - printf("Threadcache utilisation: %lf%% in cache with %lf%% lost to other threads\n", - 100.0*tc->successes/tc->mallocs, 100.0*((double) tc->mallocs-tc->frees)/tc->mallocs); -#endif - if(disable && TLSSET(p->mycache, (void *)(size_t)(-tc->mymspace))) abort(); - tc->frees++; - RemoveCacheEntries(p, tc, 0); - assert(!tc->freeInCache); - if(disable) - { - tc->mymspace=-1; - tc->threadid=0; - CallFree(0, p->caches[mycache-1], 0); - p->caches[mycache-1]=0; - } - } -} -void neddisablethreadcache(nedpool *p) THROWSPEC -{ - nedtrimthreadcache(p, 1); -} - -#define GETMSPACE(m,p,tc,ms,s,action) \ - do \ - { \ - mstate m = GetMSpace((p),(tc),(ms),(s)); \ - action; \ - if(USE_ALLOCATOR==1) { RELEASE_LOCK(&m->mutex); } \ - } while (0) - -static FORCEINLINE mstate GetMSpace(nedpool *RESTRICT p, threadcache *RESTRICT tc, int mymspace, size_t size) THROWSPEC -{ /* Returns a locked and ready for use mspace */ - mstate m=p->m[mymspace]; - assert(m); -#if USE_ALLOCATOR==1 - if(!TRY_LOCK(&p->m[mymspace]->mutex)) m=FindMSpace(p, tc, &mymspace, size); - /*assert(IS_LOCKED(&p->m[mymspace]->mutex));*/ -#endif - return m; -} -static NOINLINE void GetThreadCache_cold1(nedpool *RESTRICT *RESTRICT p) THROWSPEC -{ - *p=&syspool; - if(!syspool.threads) InitPool(&syspool, 0, -1); -} -static NOINLINE void GetThreadCache_cold2(nedpool *RESTRICT *RESTRICT p, threadcache *RESTRICT *RESTRICT tc, int *RESTRICT mymspace, int mycache) THROWSPEC -{ - if(!mycache) - { /* Need to allocate a new cache */ - *tc=AllocCache(*p); - if(!*tc) - { /* Disable */ - if(TLSSET((*p)->mycache, (void *)(size_t)-1)) abort(); - *mymspace=0; - } - else - *mymspace=(*tc)->mymspace; - } - else - { /* Cache disabled, but we do have an assigned thread pool */ - *tc=0; - *mymspace=-mycache-1; - } -} -static FORCEINLINE void GetThreadCache(nedpool *RESTRICT *RESTRICT p, threadcache *RESTRICT *RESTRICT tc, int *RESTRICT mymspace, size_t *RESTRICT size) THROWSPEC -{ - int mycache; - if(size && *size<sizeof(threadcacheblk)) *size=sizeof(threadcacheblk); - if(!*p) - GetThreadCache_cold1(p); - mycache=(int)(size_t) TLSGET((*p)->mycache); - if(mycache>0) - { /* Already have a cache */ - *tc=(*p)->caches[mycache-1]; - *mymspace=(*tc)->mymspace; - } - else GetThreadCache_cold2(p, tc, mymspace, mycache); - assert(*mymspace>=0); - assert(!(*tc) || (long)(size_t)CURRENT_THREAD==(*tc)->threadid); -#ifdef FULLSANITYCHECKS - if(*tc) - { - if(*(unsigned int *)"NEDMALC1"!=(*tc)->magic1 || *(unsigned int *)"NEDMALC2"!=(*tc)->magic2) - { - abort(); - } - } -#endif -} - -NEDMALLOCPTRATTR void * nedpmalloc(nedpool *p, size_t size) THROWSPEC -{ - void *ret=0; - threadcache *tc; - int mymspace; - GetThreadCache(&p, &tc, &mymspace, &size); -#if THREADCACHEMAX - if(tc && size<=THREADCACHEMAX) - { /* Use the thread cache */ - ret=threadcache_malloc(p, tc, &size); - } -#endif - if(!ret) - { /* Use this thread's mspace */ - GETMSPACE(m, p, tc, mymspace, size, - ret=CallMalloc(m, size, 0)); - } - return ret; -} -NEDMALLOCPTRATTR void * nedpcalloc(nedpool *p, size_t no, size_t size) THROWSPEC -{ - size_t rsize=size*no; - void *ret=0; - threadcache *tc; - int mymspace; - GetThreadCache(&p, &tc, &mymspace, &rsize); -#if THREADCACHEMAX - if(tc && rsize<=THREADCACHEMAX) - { /* Use the thread cache */ - if((ret=threadcache_malloc(p, tc, &rsize))) - memset(ret, 0, rsize); - } -#endif - if(!ret) - { /* Use this thread's mspace */ - GETMSPACE(m, p, tc, mymspace, rsize, - ret=CallCalloc(m, rsize, 0)); - } - return ret; -} -NEDMALLOCPTRATTR void * nedprealloc(nedpool *p, void *mem, size_t size) THROWSPEC -{ - void *ret=0; - threadcache *tc; - int mymspace, isforeign=1; - size_t memsize; - if(!mem) return nedpmalloc(p, size); - memsize=nedblksize(&isforeign, mem); - assert(memsize); - if(!memsize) - { - fprintf(stderr, "nedmalloc: nedprealloc() called with a block not created by nedmalloc!\n"); - abort(); - } - else if(size<=memsize && memsize-size< -#ifdef DEBUG - 32 -#else - 1024 -#endif - ) /* If realloc size is within 1Kb smaller than existing, noop it */ - return mem; - GetThreadCache(&p, &tc, &mymspace, &size); -#if THREADCACHEMAX - if(tc && size && size<=THREADCACHEMAX) - { /* Use the thread cache */ - if((ret=threadcache_malloc(p, tc, &size))) - { - memcpy(ret, mem, memsize<size ? memsize : size); - if(memsize>=sizeof(threadcacheblk) && memsize<=(THREADCACHEMAX+CHUNK_OVERHEAD)) - threadcache_free(p, tc, mymspace, mem, memsize); - else - CallFree(0, mem, isforeign); - } - } -#endif - if(!ret) - { /* Reallocs always happen in the mspace they happened in, so skip - locking the preferred mspace for this thread */ - ret=CallRealloc(p->m[mymspace], mem, isforeign, memsize, size); - } - return ret; -} -void nedpfree(nedpool *p, void *mem) THROWSPEC -{ /* Frees always happen in the mspace they happened in, so skip - locking the preferred mspace for this thread */ - threadcache *tc; - int mymspace, isforeign=1; - size_t memsize; - if(!mem) - { /* If you tried this on FreeBSD you'd be sorry! */ -#ifdef DEBUG - fprintf(stderr, "nedmalloc: WARNING nedpfree() called with zero. This is not portable behaviour!\n"); -#endif - return; - } - memsize=nedblksize(&isforeign, mem); - assert(memsize); - if(!memsize) - { - fprintf(stderr, "nedmalloc: nedpfree() called with a block not created by nedmalloc!\n"); - abort(); - } - GetThreadCache(&p, &tc, &mymspace, 0); -#if THREADCACHEMAX - if(mem && tc && memsize>=sizeof(threadcacheblk) && memsize<=(THREADCACHEMAX+CHUNK_OVERHEAD)) - threadcache_free(p, tc, mymspace, mem, memsize); - else -#endif - CallFree(0, mem, isforeign); -} -NEDMALLOCPTRATTR void * nedpmemalign(nedpool *p, size_t alignment, size_t bytes) THROWSPEC -{ - void *ret; - threadcache *tc; - int mymspace; - GetThreadCache(&p, &tc, &mymspace, &bytes); - { /* Use this thread's mspace */ - GETMSPACE(m, p, tc, mymspace, bytes, - ret=CallMalloc(m, bytes, alignment)); - } - return ret; -} -struct nedmallinfo nedpmallinfo(nedpool *p) THROWSPEC -{ - int n; - struct nedmallinfo ret={0}; - if(!p) { p=&syspool; if(!syspool.threads) InitPool(&syspool, 0, -1); } - for(n=0; p->m[n]; n++) - { -#if USE_ALLOCATOR==1 && !NO_MALLINFO - struct mallinfo t=mspace_mallinfo(p->m[n]); - ret.arena+=t.arena; - ret.ordblks+=t.ordblks; - ret.hblkhd+=t.hblkhd; - ret.usmblks+=t.usmblks; - ret.uordblks+=t.uordblks; - ret.fordblks+=t.fordblks; - ret.keepcost+=t.keepcost; -#endif - } - return ret; -} -int nedpmallopt(nedpool *p, int parno, int value) THROWSPEC -{ -#if USE_ALLOCATOR==1 - return mspace_mallopt(parno, value); -#else - return 0; -#endif -} -NEDMALLOCNOALIASATTR void* nedmalloc_internals(size_t *granularity, size_t *magic) THROWSPEC -{ -#if USE_ALLOCATOR==1 - if(granularity) *granularity=mparams.granularity; - if(magic) *magic=mparams.magic; - return (void *) &syspool; -#else - if(granularity) *granularity=0; - if(magic) *magic=0; - return 0; -#endif -} -int nedpmalloc_trim(nedpool *p, size_t pad) THROWSPEC -{ - int n, ret=0; - if(!p) { p=&syspool; if(!syspool.threads) InitPool(&syspool, 0, -1); } - for(n=0; p->m[n]; n++) - { -#if USE_ALLOCATOR==1 - ret+=mspace_trim(p->m[n], pad); -#endif - } - return ret; -} -void nedpmalloc_stats(nedpool *p) THROWSPEC -{ - int n; - if(!p) { p=&syspool; if(!syspool.threads) InitPool(&syspool, 0, -1); } - for(n=0; p->m[n]; n++) - { -#if USE_ALLOCATOR==1 - mspace_malloc_stats(p->m[n]); -#endif - } -} -size_t nedpmalloc_footprint(nedpool *p) THROWSPEC -{ - size_t ret=0; - int n; - if(!p) { p=&syspool; if(!syspool.threads) InitPool(&syspool, 0, -1); } - for(n=0; p->m[n]; n++) - { -#if USE_ALLOCATOR==1 - ret+=mspace_footprint(p->m[n]); -#endif - } - return ret; -} -NEDMALLOCPTRATTR void **nedpindependent_calloc(nedpool *p, size_t elemsno, size_t elemsize, void **chunks) THROWSPEC -{ - void **ret; - threadcache *tc; - int mymspace; - GetThreadCache(&p, &tc, &mymspace, &elemsize); -#if USE_ALLOCATOR==0 - GETMSPACE(m, p, tc, mymspace, elemsno*elemsize, - ret=unsupported_operation("independent_calloc")); -#elif USE_ALLOCATOR==1 - GETMSPACE(m, p, tc, mymspace, elemsno*elemsize, - ret=mspace_independent_calloc(m, elemsno, elemsize, chunks)); -#endif - return ret; -} -NEDMALLOCPTRATTR void **nedpindependent_comalloc(nedpool *p, size_t elems, size_t *sizes, void **chunks) THROWSPEC -{ - void **ret; - threadcache *tc; - int mymspace; - size_t i, *adjustedsizes=(size_t *) alloca(elems*sizeof(size_t)); - if(!adjustedsizes) return 0; - for(i=0; i<elems; i++) - adjustedsizes[i]=sizes[i]<sizeof(threadcacheblk) ? sizeof(threadcacheblk) : sizes[i]; - GetThreadCache(&p, &tc, &mymspace, 0); -#if USE_ALLOCATOR==0 - GETMSPACE(m, p, tc, mymspace, 0, - ret=unsupported_operation("independent_comalloc")); -#elif USE_ALLOCATOR==1 - GETMSPACE(m, p, tc, mymspace, 0, - ret=mspace_independent_comalloc(m, elems, adjustedsizes, chunks)); -#endif - return ret; -} - -#if defined(__cplusplus) -} -#endif - -#ifdef _MSC_VER -#pragma warning(pop) -#endif - -#endif |