diff options
Diffstat (limited to 'thirdparty/zstd/compress')
| -rw-r--r-- | thirdparty/zstd/compress/huf_compress.c | 10 | ||||
| -rw-r--r-- | thirdparty/zstd/compress/zstd_compress.c | 1569 | ||||
| -rw-r--r-- | thirdparty/zstd/compress/zstd_opt.h | 45 | ||||
| -rw-r--r-- | thirdparty/zstd/compress/zstdmt_compress.c | 520 | ||||
| -rw-r--r-- | thirdparty/zstd/compress/zstdmt_compress.h | 62 |
5 files changed, 1530 insertions, 676 deletions
diff --git a/thirdparty/zstd/compress/huf_compress.c b/thirdparty/zstd/compress/huf_compress.c index fe11aafb8f..7af0789a9c 100644 --- a/thirdparty/zstd/compress/huf_compress.c +++ b/thirdparty/zstd/compress/huf_compress.c @@ -266,7 +266,8 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits) if (highTotal <= lowTotal) break; } } /* only triggered when no more rank 1 symbol left => find closest one (note : there is necessarily at least one !) */ - while ((nBitsToDecrease<=HUF_TABLELOG_MAX) && (rankLast[nBitsToDecrease] == noSymbol)) /* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */ + /* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */ + while ((nBitsToDecrease<=HUF_TABLELOG_MAX) && (rankLast[nBitsToDecrease] == noSymbol)) nBitsToDecrease ++; totalCost -= 1 << (nBitsToDecrease-1); if (rankLast[nBitsToDecrease-1] == noSymbol) @@ -463,12 +464,15 @@ size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, si { case 3 : HUF_encodeSymbol(&bitC, ip[n+ 2], CTable); HUF_FLUSHBITS_2(&bitC); + /* fall-through */ case 2 : HUF_encodeSymbol(&bitC, ip[n+ 1], CTable); HUF_FLUSHBITS_1(&bitC); + /* fall-through */ case 1 : HUF_encodeSymbol(&bitC, ip[n+ 0], CTable); HUF_FLUSHBITS(&bitC); - case 0 : - default: ; + /* fall-through */ + case 0 : /* fall-through */ + default: break; } for (; n>0; n-=4) { /* note : n&3==0 at this stage */ diff --git a/thirdparty/zstd/compress/zstd_compress.c b/thirdparty/zstd/compress/zstd_compress.c index c08b315dab..9300357f2d 100644 --- a/thirdparty/zstd/compress/zstd_compress.c +++ b/thirdparty/zstd/compress/zstd_compress.c @@ -9,6 +9,14 @@ /*-************************************* +* Tuning parameters +***************************************/ +#ifndef ZSTD_CLEVEL_DEFAULT +# define ZSTD_CLEVEL_DEFAULT 3 +#endif + + +/*-************************************* * Dependencies ***************************************/ #include <string.h> /* memset */ @@ -18,26 +26,7 @@ #define HUF_STATIC_LINKING_ONLY #include "huf.h" #include "zstd_internal.h" /* includes zstd.h */ - - -/*-************************************* -* Debug -***************************************/ -#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=1) -# include <assert.h> -#else -# define assert(condition) ((void)0) -#endif - -#define ZSTD_STATIC_ASSERT(c) { enum { ZSTD_static_assert = 1/(int)(!!(c)) }; } - -#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=2) -# include <stdio.h> - static unsigned g_debugLevel = ZSTD_DEBUG; -# define DEBUGLOG(l, ...) if (l<=g_debugLevel) { fprintf(stderr, __FILE__ ": "); fprintf(stderr, __VA_ARGS__); fprintf(stderr, " \n"); } -#else -# define DEBUGLOG(l, ...) {} /* disabled */ -#endif +#include "zstdmt_compress.h" /*-************************************* @@ -79,6 +68,15 @@ static void ZSTD_resetSeqStore(seqStore_t* ssPtr) /*-************************************* * Context memory management ***************************************/ +typedef enum { zcss_init=0, zcss_load, zcss_flush } ZSTD_cStreamStage; + +struct ZSTD_CDict_s { + void* dictBuffer; + const void* dictContent; + size_t dictContentSize; + ZSTD_CCtx* refContext; +}; /* typedef'd to ZSTD_CDict within "zstd.h" */ + struct ZSTD_CCtx_s { const BYTE* nextSrc; /* next block here to continue on current prefix */ const BYTE* base; /* All regular indexes relative to this position */ @@ -90,19 +88,21 @@ struct ZSTD_CCtx_s { U32 hashLog3; /* dispatch table : larger == faster, more memory */ U32 loadedDictEnd; /* index of end of dictionary */ U32 forceWindow; /* force back-references to respect limit of 1<<wLog, even for dictionary */ - U32 forceRawDict; /* Force loading dictionary in "content-only" mode (no header analysis) */ ZSTD_compressionStage_e stage; U32 rep[ZSTD_REP_NUM]; U32 repToConfirm[ZSTD_REP_NUM]; U32 dictID; - ZSTD_parameters params; + int compressionLevel; + ZSTD_parameters requestedParams; + ZSTD_parameters appliedParams; void* workSpace; size_t workSpaceSize; size_t blockSize; - U64 frameContentSize; + U64 pledgedSrcSizePlusOne; /* this way, 0 (default) == unknown */ U64 consumedSrcSize; XXH64_state_t xxhState; ZSTD_customMem customMem; + size_t staticSize; seqStore_t seqStore; /* sequences storage ptrs */ U32* hashTable; @@ -115,31 +115,92 @@ struct ZSTD_CCtx_s { FSE_CTable* matchlengthCTable; FSE_CTable* litlengthCTable; unsigned* entropyScratchSpace; + + /* streaming */ + char* inBuff; + size_t inBuffSize; + size_t inToCompress; + size_t inBuffPos; + size_t inBuffTarget; + char* outBuff; + size_t outBuffSize; + size_t outBuffContentSize; + size_t outBuffFlushedSize; + ZSTD_cStreamStage streamStage; + U32 frameEnded; + + /* Dictionary */ + ZSTD_dictMode_e dictMode; /* select restricting dictionary to "rawContent" or "fullDict" only */ + U32 dictContentByRef; + ZSTD_CDict* cdictLocal; + const ZSTD_CDict* cdict; + const void* prefix; + size_t prefixSize; + + /* Multi-threading */ + U32 nbThreads; + ZSTDMT_CCtx* mtctx; }; + ZSTD_CCtx* ZSTD_createCCtx(void) { - return ZSTD_createCCtx_advanced(defaultCustomMem); + return ZSTD_createCCtx_advanced(ZSTD_defaultCMem); } ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) { ZSTD_CCtx* cctx; - if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem; - if (!customMem.customAlloc || !customMem.customFree) return NULL; + if (!customMem.customAlloc ^ !customMem.customFree) return NULL; - cctx = (ZSTD_CCtx*) ZSTD_malloc(sizeof(ZSTD_CCtx), customMem); + cctx = (ZSTD_CCtx*) ZSTD_calloc(sizeof(ZSTD_CCtx), customMem); if (!cctx) return NULL; - memset(cctx, 0, sizeof(ZSTD_CCtx)); cctx->customMem = customMem; + cctx->compressionLevel = ZSTD_CLEVEL_DEFAULT; + ZSTD_STATIC_ASSERT(zcss_init==0); + ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN==(0ULL - 1)); + return cctx; +} + +ZSTD_CCtx* ZSTD_initStaticCCtx(void *workspace, size_t workspaceSize) +{ + ZSTD_CCtx* cctx = (ZSTD_CCtx*) workspace; + if (workspaceSize <= sizeof(ZSTD_CCtx)) return NULL; /* minimum size */ + if ((size_t)workspace & 7) return NULL; /* must be 8-aligned */ + memset(workspace, 0, workspaceSize); /* may be a bit generous, could memset be smaller ? */ + cctx->staticSize = workspaceSize; + cctx->workSpace = (void*)(cctx+1); + cctx->workSpaceSize = workspaceSize - sizeof(ZSTD_CCtx); + + /* entropy space (never moves) */ + /* note : this code should be shared with resetCCtx, rather than copy/pasted */ + { void* ptr = cctx->workSpace; + cctx->hufCTable = (HUF_CElt*)ptr; + ptr = (char*)cctx->hufCTable + hufCTable_size; + cctx->offcodeCTable = (FSE_CTable*) ptr; + ptr = (char*)ptr + offcodeCTable_size; + cctx->matchlengthCTable = (FSE_CTable*) ptr; + ptr = (char*)ptr + matchlengthCTable_size; + cctx->litlengthCTable = (FSE_CTable*) ptr; + ptr = (char*)ptr + litlengthCTable_size; + assert(((size_t)ptr & 3) == 0); /* ensure correct alignment */ + cctx->entropyScratchSpace = (unsigned*) ptr; + } + return cctx; } size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx) { if (cctx==NULL) return 0; /* support free on NULL */ + if (cctx->staticSize) return ERROR(memory_allocation); /* not compatible with static CCtx */ ZSTD_free(cctx->workSpace, cctx->customMem); + cctx->workSpace = NULL; + ZSTD_freeCDict(cctx->cdictLocal); + cctx->cdictLocal = NULL; + ZSTDMT_freeCCtx(cctx->mtctx); + cctx->mtctx = NULL; ZSTD_free(cctx, cctx->customMem); return 0; /* reserved as a potential error code in the future */ } @@ -147,46 +208,294 @@ size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx) size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx) { if (cctx==NULL) return 0; /* support sizeof on NULL */ - return sizeof(*cctx) + cctx->workSpaceSize; + DEBUGLOG(5, "sizeof(*cctx) : %u", (U32)sizeof(*cctx)); + DEBUGLOG(5, "workSpaceSize : %u", (U32)cctx->workSpaceSize); + DEBUGLOG(5, "streaming buffers : %u", (U32)(cctx->outBuffSize + cctx->inBuffSize)); + DEBUGLOG(5, "inner MTCTX : %u", (U32)ZSTDMT_sizeof_CCtx(cctx->mtctx)); + return sizeof(*cctx) + cctx->workSpaceSize + + ZSTD_sizeof_CDict(cctx->cdictLocal) + + cctx->outBuffSize + cctx->inBuffSize + + ZSTDMT_sizeof_CCtx(cctx->mtctx); } +size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs) +{ + return ZSTD_sizeof_CCtx(zcs); /* same object */ +} + +/* private API call, for dictBuilder only */ +const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) { return &(ctx->seqStore); } + +static ZSTD_parameters ZSTD_getParamsFromCCtx(const ZSTD_CCtx* cctx) { return cctx->appliedParams; } + +/* older variant; will be deprecated */ size_t ZSTD_setCCtxParameter(ZSTD_CCtx* cctx, ZSTD_CCtxParameter param, unsigned value) { switch(param) { case ZSTD_p_forceWindow : cctx->forceWindow = value>0; cctx->loadedDictEnd = 0; return 0; - case ZSTD_p_forceRawDict : cctx->forceRawDict = value>0; return 0; + ZSTD_STATIC_ASSERT(ZSTD_dm_auto==0); + ZSTD_STATIC_ASSERT(ZSTD_dm_rawContent==1); + case ZSTD_p_forceRawDict : cctx->dictMode = (ZSTD_dictMode_e)(value>0); return 0; default: return ERROR(parameter_unknown); } } -const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) /* hidden interface */ + +#define ZSTD_CLEVEL_CUSTOM 999 +static void ZSTD_cLevelToCParams(ZSTD_CCtx* cctx) { - return &(ctx->seqStore); + if (cctx->compressionLevel==ZSTD_CLEVEL_CUSTOM) return; + cctx->requestedParams.cParams = ZSTD_getCParams(cctx->compressionLevel, + cctx->pledgedSrcSizePlusOne-1, 0); + cctx->compressionLevel = ZSTD_CLEVEL_CUSTOM; } -static ZSTD_parameters ZSTD_getParamsFromCCtx(const ZSTD_CCtx* cctx) +#define CLAMPCHECK(val,min,max) { \ + if (((val)<(min)) | ((val)>(max))) { \ + return ERROR(compressionParameter_outOfBound); \ +} } + +size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value) { - return cctx->params; + if (cctx->streamStage != zcss_init) return ERROR(stage_wrong); + + switch(param) + { + case ZSTD_p_compressionLevel : + if ((int)value > ZSTD_maxCLevel()) value = ZSTD_maxCLevel(); /* cap max compression level */ + if (value == 0) return 0; /* special value : 0 means "don't change anything" */ + if (cctx->cdict) return ERROR(stage_wrong); + cctx->compressionLevel = value; + return 0; + + case ZSTD_p_windowLog : + DEBUGLOG(5, "setting ZSTD_p_windowLog = %u (cdict:%u)", + value, (cctx->cdict!=NULL)); + if (value == 0) return 0; /* special value : 0 means "don't change anything" */ + if (cctx->cdict) return ERROR(stage_wrong); + CLAMPCHECK(value, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX); + ZSTD_cLevelToCParams(cctx); + cctx->requestedParams.cParams.windowLog = value; + return 0; + + case ZSTD_p_hashLog : + if (value == 0) return 0; /* special value : 0 means "don't change anything" */ + if (cctx->cdict) return ERROR(stage_wrong); + CLAMPCHECK(value, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX); + ZSTD_cLevelToCParams(cctx); + cctx->requestedParams.cParams.hashLog = value; + return 0; + + case ZSTD_p_chainLog : + if (value == 0) return 0; /* special value : 0 means "don't change anything" */ + if (cctx->cdict) return ERROR(stage_wrong); + CLAMPCHECK(value, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX); + ZSTD_cLevelToCParams(cctx); + cctx->requestedParams.cParams.chainLog = value; + return 0; + + case ZSTD_p_searchLog : + if (value == 0) return 0; /* special value : 0 means "don't change anything" */ + if (cctx->cdict) return ERROR(stage_wrong); + CLAMPCHECK(value, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX); + ZSTD_cLevelToCParams(cctx); + cctx->requestedParams.cParams.searchLog = value; + return 0; + + case ZSTD_p_minMatch : + if (value == 0) return 0; /* special value : 0 means "don't change anything" */ + if (cctx->cdict) return ERROR(stage_wrong); + CLAMPCHECK(value, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX); + ZSTD_cLevelToCParams(cctx); + cctx->requestedParams.cParams.searchLength = value; + return 0; + + case ZSTD_p_targetLength : + if (value == 0) return 0; /* special value : 0 means "don't change anything" */ + if (cctx->cdict) return ERROR(stage_wrong); + CLAMPCHECK(value, ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX); + ZSTD_cLevelToCParams(cctx); + cctx->requestedParams.cParams.targetLength = value; + return 0; + + case ZSTD_p_compressionStrategy : + if (value == 0) return 0; /* special value : 0 means "don't change anything" */ + if (cctx->cdict) return ERROR(stage_wrong); + CLAMPCHECK(value, (unsigned)ZSTD_fast, (unsigned)ZSTD_btultra); + ZSTD_cLevelToCParams(cctx); + cctx->requestedParams.cParams.strategy = (ZSTD_strategy)value; + return 0; + + case ZSTD_p_contentSizeFlag : + DEBUGLOG(5, "set content size flag = %u", (value>0)); + /* Content size written in frame header _when known_ (default:1) */ + cctx->requestedParams.fParams.contentSizeFlag = value>0; + return 0; + + case ZSTD_p_checksumFlag : + /* A 32-bits content checksum will be calculated and written at end of frame (default:0) */ + cctx->requestedParams.fParams.checksumFlag = value>0; + return 0; + + case ZSTD_p_dictIDFlag : /* When applicable, dictionary's dictID is provided in frame header (default:1) */ + DEBUGLOG(5, "set dictIDFlag = %u", (value>0)); + cctx->requestedParams.fParams.noDictIDFlag = (value==0); + return 0; + + /* Dictionary parameters */ + case ZSTD_p_dictMode : + if (cctx->cdict) return ERROR(stage_wrong); /* must be set before loading */ + /* restrict dictionary mode, to "rawContent" or "fullDict" only */ + ZSTD_STATIC_ASSERT((U32)ZSTD_dm_fullDict > (U32)ZSTD_dm_rawContent); + if (value > (unsigned)ZSTD_dm_fullDict) + return ERROR(compressionParameter_outOfBound); + cctx->dictMode = (ZSTD_dictMode_e)value; + return 0; + + case ZSTD_p_refDictContent : + if (cctx->cdict) return ERROR(stage_wrong); /* must be set before loading */ + /* dictionary content will be referenced, instead of copied */ + cctx->dictContentByRef = value>0; + return 0; + + case ZSTD_p_forceMaxWindow : /* Force back-references to remain < windowSize, + * even when referencing into Dictionary content + * default : 0 when using a CDict, 1 when using a Prefix */ + cctx->forceWindow = value>0; + cctx->loadedDictEnd = 0; + return 0; + + case ZSTD_p_nbThreads: + if (value==0) return 0; + DEBUGLOG(5, " setting nbThreads : %u", value); +#ifndef ZSTD_MULTITHREAD + if (value > 1) return ERROR(compressionParameter_unsupported); +#endif + if ((value>1) && (cctx->nbThreads != value)) { + if (cctx->staticSize) /* MT not compatible with static alloc */ + return ERROR(compressionParameter_unsupported); + ZSTDMT_freeCCtx(cctx->mtctx); + cctx->nbThreads = 1; + cctx->mtctx = ZSTDMT_createCCtx(value); + if (cctx->mtctx == NULL) return ERROR(memory_allocation); + } + cctx->nbThreads = value; + return 0; + + case ZSTD_p_jobSize: + if (cctx->nbThreads <= 1) return ERROR(compressionParameter_unsupported); + assert(cctx->mtctx != NULL); + return ZSTDMT_setMTCtxParameter(cctx->mtctx, ZSTDMT_p_sectionSize, value); + + case ZSTD_p_overlapSizeLog: + DEBUGLOG(5, " setting overlap with nbThreads == %u", cctx->nbThreads); + if (cctx->nbThreads <= 1) return ERROR(compressionParameter_unsupported); + assert(cctx->mtctx != NULL); + return ZSTDMT_setMTCtxParameter(cctx->mtctx, ZSTDMT_p_overlapSectionLog, value); + + default: return ERROR(parameter_unknown); + } +} + +ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize) +{ + DEBUGLOG(5, " setting pledgedSrcSize to %u", (U32)pledgedSrcSize); + if (cctx->streamStage != zcss_init) return ERROR(stage_wrong); + cctx->pledgedSrcSizePlusOne = pledgedSrcSize+1; + return 0; +} + +ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize) +{ + if (cctx->streamStage != zcss_init) return ERROR(stage_wrong); + if (cctx->staticSize) return ERROR(memory_allocation); /* no malloc for static CCtx */ + DEBUGLOG(5, "load dictionary of size %u", (U32)dictSize); + ZSTD_freeCDict(cctx->cdictLocal); /* in case one already exists */ + if (dict==NULL || dictSize==0) { /* no dictionary mode */ + cctx->cdictLocal = NULL; + cctx->cdict = NULL; + } else { + ZSTD_compressionParameters const cParams = + cctx->compressionLevel == ZSTD_CLEVEL_CUSTOM ? + cctx->requestedParams.cParams : + ZSTD_getCParams(cctx->compressionLevel, 0, dictSize); + cctx->cdictLocal = ZSTD_createCDict_advanced( + dict, dictSize, + cctx->dictContentByRef, cctx->dictMode, + cParams, cctx->customMem); + cctx->cdict = cctx->cdictLocal; + if (cctx->cdictLocal == NULL) + return ERROR(memory_allocation); + } + return 0; +} + +size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) +{ + if (cctx->streamStage != zcss_init) return ERROR(stage_wrong); + cctx->cdict = cdict; + cctx->prefix = NULL; /* exclusive */ + cctx->prefixSize = 0; + return 0; +} + +size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize) +{ + if (cctx->streamStage != zcss_init) return ERROR(stage_wrong); + cctx->cdict = NULL; /* prefix discards any prior cdict */ + cctx->prefix = prefix; + cctx->prefixSize = prefixSize; + return 0; +} + +static void ZSTD_startNewCompression(ZSTD_CCtx* cctx) +{ + cctx->streamStage = zcss_init; + cctx->pledgedSrcSizePlusOne = 0; } +/*! ZSTD_CCtx_reset() : + * Also dumps dictionary */ +void ZSTD_CCtx_reset(ZSTD_CCtx* cctx) +{ + ZSTD_startNewCompression(cctx); + cctx->cdict = NULL; +} -/** ZSTD_checkParams() : - ensure param values remain within authorized range. +/** ZSTD_checkCParams() : + control CParam values remain within authorized range. @return : 0, or an error code if one value is beyond authorized range */ size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams) { -# define CLAMPCHECK(val,min,max) { if ((val<min) | (val>max)) return ERROR(compressionParameter_unsupported); } CLAMPCHECK(cParams.windowLog, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX); CLAMPCHECK(cParams.chainLog, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX); CLAMPCHECK(cParams.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX); CLAMPCHECK(cParams.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX); CLAMPCHECK(cParams.searchLength, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX); CLAMPCHECK(cParams.targetLength, ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX); - if ((U32)(cParams.strategy) > (U32)ZSTD_btopt2) return ERROR(compressionParameter_unsupported); + if ((U32)(cParams.strategy) > (U32)ZSTD_btultra) return ERROR(compressionParameter_unsupported); return 0; } +/** ZSTD_clampCParams() : + * make CParam values within valid range. + * @return : valid CParams */ +static ZSTD_compressionParameters ZSTD_clampCParams(ZSTD_compressionParameters cParams) +{ +# define CLAMP(val,min,max) { \ + if (val<min) val=min; \ + else if (val>max) val=max; \ + } + CLAMP(cParams.windowLog, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX); + CLAMP(cParams.chainLog, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX); + CLAMP(cParams.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX); + CLAMP(cParams.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX); + CLAMP(cParams.searchLength, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX); + CLAMP(cParams.targetLength, ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX); + if ((U32)(cParams.strategy) > (U32)ZSTD_btultra) cParams.strategy = ZSTD_btultra; + return cParams; +} /** ZSTD_cycleLog() : * condition for correct operation : hashLog > 1 */ @@ -196,14 +505,15 @@ static U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat) return hashLog - btScale; } -/** ZSTD_adjustCParams() : +/** ZSTD_adjustCParams_internal() : optimize `cPar` for a given input (`srcSize` and `dictSize`). mostly downsizing to reduce memory consumption and initialization. Both `srcSize` and `dictSize` are optional (use 0 if unknown), but if both are 0, no optimization can be done. Note : cPar is considered validated at this stage. Use ZSTD_checkParams() to ensure that. */ -ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize) +ZSTD_compressionParameters ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize) { + assert(ZSTD_checkCParams(cPar)==0); if (srcSize+dictSize == 0) return cPar; /* no size information available : no adjustment */ /* resize params, to use less memory when necessary */ @@ -223,10 +533,16 @@ ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, u return cPar; } +ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize) +{ + cPar = ZSTD_clampCParams(cPar); + return ZSTD_adjustCParams_internal(cPar, srcSize, dictSize); +} -size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams) + +size_t ZSTD_estimateCCtxSize_advanced(ZSTD_compressionParameters cParams) { - size_t const blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, (size_t)1 << cParams.windowLog); + size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog); U32 const divider = (cParams.searchLength==3) ? 3 : 4; size_t const maxNbSeq = blockSize / divider; size_t const tokenSpace = blockSize + 11*maxNbSeq; @@ -240,31 +556,64 @@ size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams) + entropyScratchSpace_size; size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); - size_t const optSpace = ((MaxML+1) + (MaxLL+1) + (MaxOff+1) + (1<<Litbits))*sizeof(U32) + size_t const optBudget = ((MaxML+1) + (MaxLL+1) + (MaxOff+1) + (1<<Litbits))*sizeof(U32) + (ZSTD_OPT_NUM+1)*(sizeof(ZSTD_match_t) + sizeof(ZSTD_optimal_t)); - size_t const neededSpace = entropySpace + tableSpace + tokenSpace - + (((cParams.strategy == ZSTD_btopt) || (cParams.strategy == ZSTD_btopt2)) ? optSpace : 0); + size_t const optSpace = ((cParams.strategy == ZSTD_btopt) || (cParams.strategy == ZSTD_btultra)) ? optBudget : 0; + size_t const neededSpace = entropySpace + tableSpace + tokenSpace + optSpace; + DEBUGLOG(5, "sizeof(ZSTD_CCtx) : %u", (U32)sizeof(ZSTD_CCtx)); + DEBUGLOG(5, "estimate workSpace : %u", (U32)neededSpace); return sizeof(ZSTD_CCtx) + neededSpace; } +size_t ZSTD_estimateCCtxSize(int compressionLevel) +{ + ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, 0, 0); + return ZSTD_estimateCCtxSize_advanced(cParams); +} -static U32 ZSTD_equivalentParams(ZSTD_parameters param1, ZSTD_parameters param2) +size_t ZSTD_estimateCStreamSize_advanced(ZSTD_compressionParameters cParams) { - return (param1.cParams.hashLog == param2.cParams.hashLog) - & (param1.cParams.chainLog == param2.cParams.chainLog) - & (param1.cParams.strategy == param2.cParams.strategy) - & ((param1.cParams.searchLength==3) == (param2.cParams.searchLength==3)); + size_t const CCtxSize = ZSTD_estimateCCtxSize_advanced(cParams); + size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog); + size_t const inBuffSize = ((size_t)1 << cParams.windowLog) + blockSize; + size_t const outBuffSize = ZSTD_compressBound(blockSize) + 1; + size_t const streamingSize = inBuffSize + outBuffSize; + + return CCtxSize + streamingSize; +} + +size_t ZSTD_estimateCStreamSize(int compressionLevel) { + ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, 0, 0); + return ZSTD_estimateCStreamSize_advanced(cParams); +} + + +static U32 ZSTD_equivalentParams(ZSTD_compressionParameters cParams1, + ZSTD_compressionParameters cParams2) +{ + U32 bslog1 = MIN(cParams1.windowLog, ZSTD_BLOCKSIZELOG_MAX); + U32 bslog2 = MIN(cParams2.windowLog, ZSTD_BLOCKSIZELOG_MAX); + return (bslog1 == bslog2) /* same block size */ + & (cParams1.hashLog == cParams2.hashLog) + & (cParams1.chainLog == cParams2.chainLog) + & (cParams1.strategy == cParams2.strategy) /* opt parser space */ + & ((cParams1.searchLength==3) == (cParams2.searchLength==3)); /* hashlog3 space */ } /*! ZSTD_continueCCtx() : - reuse CCtx without reset (note : requires no dictionary) */ -static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, ZSTD_parameters params, U64 frameContentSize) + * reuse CCtx without reset (note : requires no dictionary) */ +static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, ZSTD_parameters params, U64 pledgedSrcSize) { U32 const end = (U32)(cctx->nextSrc - cctx->base); - cctx->params = params; - cctx->frameContentSize = frameContentSize; + DEBUGLOG(5, "continue mode"); + cctx->appliedParams = params; + cctx->pledgedSrcSizePlusOne = pledgedSrcSize+1; cctx->consumedSrcSize = 0; + if (pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN) + cctx->appliedParams.fParams.contentSizeFlag = 0; + DEBUGLOG(5, "pledged content size : %u ; flag : %u", + (U32)pledgedSrcSize, cctx->appliedParams.fParams.contentSizeFlag); cctx->lowLimit = end; cctx->dictLimit = end; cctx->nextToUpdate = end+1; @@ -277,30 +626,39 @@ static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, ZSTD_parameters params, U64 fra return 0; } -typedef enum { ZSTDcrp_continue, ZSTDcrp_noMemset, ZSTDcrp_fullReset } ZSTD_compResetPolicy_e; +typedef enum { ZSTDcrp_continue, ZSTDcrp_noMemset } ZSTD_compResetPolicy_e; +typedef enum { ZSTDb_not_buffered, ZSTDb_buffered } ZSTD_buffered_policy_e; /*! ZSTD_resetCCtx_internal() : - note : `params` must be validated */ -static size_t ZSTD_resetCCtx_internal (ZSTD_CCtx* zc, - ZSTD_parameters params, U64 frameContentSize, - ZSTD_compResetPolicy_e const crp) + note : `params` are assumed fully validated at this stage */ +static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, + ZSTD_parameters params, U64 pledgedSrcSize, + ZSTD_compResetPolicy_e const crp, + ZSTD_buffered_policy_e const zbuff) { - if (crp == ZSTDcrp_continue) - if (ZSTD_equivalentParams(params, zc->params)) { + assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); + + if (crp == ZSTDcrp_continue) { + if (ZSTD_equivalentParams(params.cParams, zc->appliedParams.cParams)) { + DEBUGLOG(5, "ZSTD_equivalentParams()==1"); zc->fseCTables_ready = 0; zc->hufCTable_repeatMode = HUF_repeat_none; - return ZSTD_continueCCtx(zc, params, frameContentSize); - } + return ZSTD_continueCCtx(zc, params, pledgedSrcSize); + } } - { size_t const blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, (size_t)1 << params.cParams.windowLog); + { size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << params.cParams.windowLog); U32 const divider = (params.cParams.searchLength==3) ? 3 : 4; size_t const maxNbSeq = blockSize / divider; size_t const tokenSpace = blockSize + 11*maxNbSeq; - size_t const chainSize = (params.cParams.strategy == ZSTD_fast) ? 0 : (1 << params.cParams.chainLog); + size_t const chainSize = (params.cParams.strategy == ZSTD_fast) ? + 0 : (1 << params.cParams.chainLog); size_t const hSize = ((size_t)1) << params.cParams.hashLog; - U32 const hashLog3 = (params.cParams.searchLength>3) ? 0 : MIN(ZSTD_HASHLOG3_MAX, params.cParams.windowLog); + U32 const hashLog3 = (params.cParams.searchLength>3) ? + 0 : MIN(ZSTD_HASHLOG3_MAX, params.cParams.windowLog); size_t const h3Size = ((size_t)1) << hashLog3; size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); + size_t const buffOutSize = (zbuff==ZSTDb_buffered) ? ZSTD_compressBound(blockSize)+1 : 0; + size_t const buffInSize = (zbuff==ZSTDb_buffered) ? ((size_t)1 << params.cParams.windowLog) + blockSize : 0; void* ptr; /* Check if workSpace is large enough, alloc a new one if needed */ @@ -309,9 +667,20 @@ static size_t ZSTD_resetCCtx_internal (ZSTD_CCtx* zc, + entropyScratchSpace_size; size_t const optPotentialSpace = ((MaxML+1) + (MaxLL+1) + (MaxOff+1) + (1<<Litbits)) * sizeof(U32) + (ZSTD_OPT_NUM+1) * (sizeof(ZSTD_match_t)+sizeof(ZSTD_optimal_t)); - size_t const optSpace = ((params.cParams.strategy == ZSTD_btopt) || (params.cParams.strategy == ZSTD_btopt2)) ? optPotentialSpace : 0; - size_t const neededSpace = entropySpace + optSpace + tableSpace + tokenSpace; - if (zc->workSpaceSize < neededSpace) { + size_t const optSpace = ( (params.cParams.strategy == ZSTD_btopt) + || (params.cParams.strategy == ZSTD_btultra)) ? + optPotentialSpace : 0; + size_t const bufferSpace = buffInSize + buffOutSize; + size_t const neededSpace = entropySpace + optSpace + tableSpace + + tokenSpace + bufferSpace; + + if (zc->workSpaceSize < neededSpace) { /* too small : resize /*/ + DEBUGLOG(5, "Need to update workSpaceSize from %uK to %uK \n", + (unsigned)zc->workSpaceSize>>10, + (unsigned)neededSpace>>10); + /* static cctx : no resize, error out */ + if (zc->staticSize) return ERROR(memory_allocation); + zc->workSpaceSize = 0; ZSTD_free(zc->workSpace, zc->customMem); zc->workSpace = ZSTD_malloc(neededSpace, zc->customMem); @@ -333,10 +702,14 @@ static size_t ZSTD_resetCCtx_internal (ZSTD_CCtx* zc, } } /* init params */ - zc->params = params; - zc->blockSize = blockSize; - zc->frameContentSize = frameContentSize; + zc->appliedParams = params; + zc->pledgedSrcSizePlusOne = pledgedSrcSize+1; zc->consumedSrcSize = 0; + if (pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN) + zc->appliedParams.fParams.contentSizeFlag = 0; + DEBUGLOG(5, "pledged content size : %u ; flag : %u", + (U32)pledgedSrcSize, zc->appliedParams.fParams.contentSizeFlag); + zc->blockSize = blockSize; XXH64_reset(&zc->xxhState, 0); zc->stage = ZSTDcs_init; @@ -363,7 +736,8 @@ static size_t ZSTD_resetCCtx_internal (ZSTD_CCtx* zc, ptr = (char*)zc->entropyScratchSpace + entropyScratchSpace_size; /* opt parser space */ - if ((params.cParams.strategy == ZSTD_btopt) || (params.cParams.strategy == ZSTD_btopt2)) { + if ((params.cParams.strategy == ZSTD_btopt) || (params.cParams.strategy == ZSTD_btultra)) { + DEBUGLOG(5, "reserving optimal parser space"); assert(((size_t)ptr & 3) == 0); /* ensure ptr is properly aligned */ zc->seqStore.litFreq = (U32*)ptr; zc->seqStore.litLengthFreq = zc->seqStore.litFreq + (1<<Litbits); @@ -391,6 +765,13 @@ static size_t ZSTD_resetCCtx_internal (ZSTD_CCtx* zc, zc->seqStore.mlCode = zc->seqStore.llCode + maxNbSeq; zc->seqStore.ofCode = zc->seqStore.mlCode + maxNbSeq; zc->seqStore.litStart = zc->seqStore.ofCode + maxNbSeq; + ptr = zc->seqStore.litStart + blockSize; + + /* buffers */ + zc->inBuffSize = buffInSize; + zc->inBuff = (char*)ptr; + zc->outBuffSize = buffOutSize; + zc->outBuff = zc->inBuff + buffInSize; return 0; } @@ -411,21 +792,25 @@ void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx) { * Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()). * pledgedSrcSize=0 means "empty" if fParams.contentSizeFlag=1 * @return : 0, or an error code */ -size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, - ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize) +static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx, + const ZSTD_CCtx* srcCCtx, + ZSTD_frameParameters fParams, + unsigned long long pledgedSrcSize, + ZSTD_buffered_policy_e zbuff) { + DEBUGLOG(5, "ZSTD_copyCCtx_internal"); if (srcCCtx->stage!=ZSTDcs_init) return ERROR(stage_wrong); memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem)); - { ZSTD_parameters params = srcCCtx->params; + { ZSTD_parameters params = srcCCtx->appliedParams; params.fParams = fParams; - DEBUGLOG(5, "ZSTD_resetCCtx_internal : dictIDFlag : %u \n", !fParams.noDictIDFlag); - ZSTD_resetCCtx_internal(dstCCtx, params, pledgedSrcSize, ZSTDcrp_noMemset); + ZSTD_resetCCtx_internal(dstCCtx, params, pledgedSrcSize, + ZSTDcrp_noMemset, zbuff); } /* copy tables */ - { size_t const chainSize = (srcCCtx->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << srcCCtx->params.cParams.chainLog); - size_t const hSize = (size_t)1 << srcCCtx->params.cParams.hashLog; + { size_t const chainSize = (srcCCtx->appliedParams.cParams.strategy == ZSTD_fast) ? 0 : (1 << srcCCtx->appliedParams.cParams.chainLog); + size_t const hSize = (size_t)1 << srcCCtx->appliedParams.cParams.hashLog; size_t const h3Size = (size_t)1 << srcCCtx->hashLog3; size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); assert((U32*)dstCCtx->chainTable == (U32*)dstCCtx->hashTable + hSize); /* chainTable must follow hashTable */ @@ -467,9 +852,11 @@ size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, unsigned long long pledgedSrcSize) { ZSTD_frameParameters fParams = { 1 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ }; + ZSTD_buffered_policy_e const zbuff = (ZSTD_buffered_policy_e)(srcCCtx->inBuffSize>0); + ZSTD_STATIC_ASSERT((U32)ZSTDb_buffered==1); fParams.contentSizeFlag = pledgedSrcSize>0; - return ZSTD_copyCCtx_internal(dstCCtx, srcCCtx, fParams, pledgedSrcSize); + return ZSTD_copyCCtx_internal(dstCCtx, srcCCtx, fParams, pledgedSrcSize, zbuff); } @@ -488,10 +875,10 @@ static void ZSTD_reduceTable (U32* const table, U32 const size, U32 const reduce * rescale all indexes to avoid future overflow (indexes are U32) */ static void ZSTD_reduceIndex (ZSTD_CCtx* zc, const U32 reducerValue) { - { U32 const hSize = 1 << zc->params.cParams.hashLog; + { U32 const hSize = 1 << zc->appliedParams.cParams.hashLog; ZSTD_reduceTable(zc->hashTable, hSize, reducerValue); } - { U32 const chainSize = (zc->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << zc->params.cParams.chainLog); + { U32 const chainSize = (zc->appliedParams.cParams.strategy == ZSTD_fast) ? 0 : (1 << zc->appliedParams.cParams.chainLog); ZSTD_reduceTable(zc->chainTable, chainSize, reducerValue); } { U32 const h3Size = (zc->hashLog3) ? 1 << zc->hashLog3 : 0; @@ -529,10 +916,11 @@ static size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void case 2: /* 2 - 2 - 12 */ MEM_writeLE16(ostart, (U16)((U32)set_basic + (1<<2) + (srcSize<<4))); break; - default: /*note : should not be necessary : flSize is within {1,2,3} */ case 3: /* 2 - 2 - 20 */ MEM_writeLE32(ostart, (U32)((U32)set_basic + (3<<2) + (srcSize<<4))); break; + default: /* not necessary : flSize is {1,2,3} */ + assert(0); } memcpy(ostart + flSize, src, srcSize); @@ -554,10 +942,11 @@ static size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, cons case 2: /* 2 - 2 - 12 */ MEM_writeLE16(ostart, (U16)((U32)set_rle + (1<<2) + (srcSize<<4))); break; - default: /*note : should not be necessary : flSize is necessarily within {1,2,3} */ case 3: /* 2 - 2 - 20 */ MEM_writeLE32(ostart, (U32)((U32)set_rle + (3<<2) + (srcSize<<4))); break; + default: /* not necessary : flSize is {1,2,3} */ + assert(0); } ostart[flSize] = *(const BYTE*)src; @@ -587,7 +976,7 @@ static size_t ZSTD_compressLiterals (ZSTD_CCtx* zc, if (dstCapacity < lhSize+1) return ERROR(dstSize_tooSmall); /* not enough space for compression */ { HUF_repeat repeat = zc->hufCTable_repeatMode; - int const preferRepeat = zc->params.cParams.strategy < ZSTD_lazy ? srcSize <= 1024 : 0; + int const preferRepeat = zc->appliedParams.cParams.strategy < ZSTD_lazy ? srcSize <= 1024 : 0; if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1; cLitSize = singleStream ? HUF_compress1X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, zc->entropyScratchSpace, entropyScratchSpace_size, zc->hufCTable, &repeat, preferRepeat) @@ -619,13 +1008,14 @@ static size_t ZSTD_compressLiterals (ZSTD_CCtx* zc, MEM_writeLE32(ostart, lhc); break; } - default: /* should not be necessary, lhSize is only {3,4,5} */ case 5: /* 2 - 2 - 18 - 18 */ { U32 const lhc = hType + (3 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<22); MEM_writeLE32(ostart, lhc); ostart[4] = (BYTE)(cLitSize >> 10); break; } + default: /* not possible : lhSize is {3,4,5} */ + assert(0); } return lhSize+cLitSize; } @@ -676,7 +1066,7 @@ MEM_STATIC size_t ZSTD_compressSequences (ZSTD_CCtx* zc, void* dst, size_t dstCapacity, size_t srcSize) { - const int longOffsets = zc->params.cParams.windowLog > STREAM_ACCUMULATOR_MIN; + const int longOffsets = zc->appliedParams.cParams.windowLog > STREAM_ACCUMULATOR_MIN; const seqStore_t* seqStorePtr = &(zc->seqStore); U32 count[MaxSeq+1]; S16 norm[MaxSeq+1]; @@ -881,12 +1271,6 @@ _check_compressibility: return op - ostart; } -#if 0 /* for debug */ -# define STORESEQ_DEBUG -#include <stdio.h> /* fprintf */ -U32 g_startDebug = 0; -const BYTE* g_start = NULL; -#endif /*! ZSTD_storeSeq() : Store a sequence (literal length, literals, offset code and match length code) into seqStore_t. @@ -895,16 +1279,16 @@ const BYTE* g_start = NULL; */ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const void* literals, U32 offsetCode, size_t matchCode) { -#ifdef STORESEQ_DEBUG - if (g_startDebug) { - const U32 pos = (U32)((const BYTE*)literals - g_start); - if (g_start==NULL) g_start = (const BYTE*)literals; - if ((pos > 1895000) && (pos < 1895300)) - DEBUGLOG(5, "Cpos %6u :%5u literals & match %3u bytes at distance %6u \n", - pos, (U32)litLength, (U32)matchCode+MINMATCH, (U32)offsetCode); - } +#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG >= 6) + static const BYTE* g_start = NULL; + U32 const pos = (U32)((const BYTE*)literals - g_start); + if (g_start==NULL) g_start = (const BYTE*)literals; + if ((pos > 0) && (pos < 1000000000)) + DEBUGLOG(6, "Cpos %6u :%5u literals & match %3u bytes at distance %6u", + pos, (U32)litLength, (U32)matchCode+MINMATCH, (U32)offsetCode); #endif /* copy Literals */ + assert(seqStorePtr->lit + litLength <= seqStorePtr->litStart + 128 KB); ZSTD_wildcopy(seqStorePtr->lit, literals, litLength); seqStorePtr->lit += litLength; @@ -1078,7 +1462,7 @@ static size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls) static void ZSTD_fillHashTable (ZSTD_CCtx* zc, const void* end, const U32 mls) { U32* const hashTable = zc->hashTable; - U32 const hBits = zc->params.cParams.hashLog; + U32 const hBits = zc->appliedParams.cParams.hashLog; const BYTE* const base = zc->base; const BYTE* ip = base + zc->nextToUpdate; const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE; @@ -1097,7 +1481,7 @@ void ZSTD_compressBlock_fast_generic(ZSTD_CCtx* cctx, const U32 mls) { U32* const hashTable = cctx->hashTable; - U32 const hBits = cctx->params.cParams.hashLog; + U32 const hBits = cctx->appliedParams.cParams.hashLog; seqStore_t* seqStorePtr = &(cctx->seqStore); const BYTE* const base = cctx->base; const BYTE* const istart = (const BYTE*)src; @@ -1182,7 +1566,7 @@ void ZSTD_compressBlock_fast_generic(ZSTD_CCtx* cctx, static void ZSTD_compressBlock_fast(ZSTD_CCtx* ctx, const void* src, size_t srcSize) { - const U32 mls = ctx->params.cParams.searchLength; + const U32 mls = ctx->appliedParams.cParams.searchLength; switch(mls) { default: /* includes case 3 */ @@ -1203,7 +1587,7 @@ static void ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx, const U32 mls) { U32* hashTable = ctx->hashTable; - const U32 hBits = ctx->params.cParams.hashLog; + const U32 hBits = ctx->appliedParams.cParams.hashLog; seqStore_t* seqStorePtr = &(ctx->seqStore); const BYTE* const base = ctx->base; const BYTE* const dictBase = ctx->dictBase; @@ -1296,7 +1680,7 @@ static void ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx, static void ZSTD_compressBlock_fast_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize) { - U32 const mls = ctx->params.cParams.searchLength; + U32 const mls = ctx->appliedParams.cParams.searchLength; switch(mls) { default: /* includes case 3 */ @@ -1318,9 +1702,9 @@ static void ZSTD_compressBlock_fast_extDict(ZSTD_CCtx* ctx, static void ZSTD_fillDoubleHashTable (ZSTD_CCtx* cctx, const void* end, const U32 mls) { U32* const hashLarge = cctx->hashTable; - U32 const hBitsL = cctx->params.cParams.hashLog; + U32 const hBitsL = cctx->appliedParams.cParams.hashLog; U32* const hashSmall = cctx->chainTable; - U32 const hBitsS = cctx->params.cParams.chainLog; + U32 const hBitsS = cctx->appliedParams.cParams.chainLog; const BYTE* const base = cctx->base; const BYTE* ip = base + cctx->nextToUpdate; const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE; @@ -1340,9 +1724,9 @@ void ZSTD_compressBlock_doubleFast_generic(ZSTD_CCtx* cctx, const U32 mls) { U32* const hashLong = cctx->hashTable; - const U32 hBitsL = cctx->params.cParams.hashLog; + const U32 hBitsL = cctx->appliedParams.cParams.hashLog; U32* const hashSmall = cctx->chainTable; - const U32 hBitsS = cctx->params.cParams.chainLog; + const U32 hBitsS = cctx->appliedParams.cParams.chainLog; seqStore_t* seqStorePtr = &(cctx->seqStore); const BYTE* const base = cctx->base; const BYTE* const istart = (const BYTE*)src; @@ -1452,7 +1836,7 @@ void ZSTD_compressBlock_doubleFast_generic(ZSTD_CCtx* cctx, static void ZSTD_compressBlock_doubleFast(ZSTD_CCtx* ctx, const void* src, size_t srcSize) { - const U32 mls = ctx->params.cParams.searchLength; + const U32 mls = ctx->appliedParams.cParams.searchLength; switch(mls) { default: /* includes case 3 */ @@ -1473,9 +1857,9 @@ static void ZSTD_compressBlock_doubleFast_extDict_generic(ZSTD_CCtx* ctx, const U32 mls) { U32* const hashLong = ctx->hashTable; - U32 const hBitsL = ctx->params.cParams.hashLog; + U32 const hBitsL = ctx->appliedParams.cParams.hashLog; U32* const hashSmall = ctx->chainTable; - U32 const hBitsS = ctx->params.cParams.chainLog; + U32 const hBitsS = ctx->appliedParams.cParams.chainLog; seqStore_t* seqStorePtr = &(ctx->seqStore); const BYTE* const base = ctx->base; const BYTE* const dictBase = ctx->dictBase; @@ -1602,7 +1986,7 @@ static void ZSTD_compressBlock_doubleFast_extDict_generic(ZSTD_CCtx* ctx, static void ZSTD_compressBlock_doubleFast_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize) { - U32 const mls = ctx->params.cParams.searchLength; + U32 const mls = ctx->appliedParams.cParams.searchLength; switch(mls) { default: /* includes case 3 */ @@ -1628,10 +2012,10 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co U32 extDict) { U32* const hashTable = zc->hashTable; - U32 const hashLog = zc->params.cParams.hashLog; + U32 const hashLog = zc->appliedParams.cParams.hashLog; size_t const h = ZSTD_hashPtr(ip, hashLog, mls); U32* const bt = zc->chainTable; - U32 const btLog = zc->params.cParams.chainLog - 1; + U32 const btLog = zc->appliedParams.cParams.chainLog - 1; U32 const btMask = (1 << btLog) - 1; U32 matchIndex = hashTable[h]; size_t commonLengthSmaller=0, commonLengthLarger=0; @@ -1733,10 +2117,10 @@ static size_t ZSTD_insertBtAndFindBestMatch ( U32 extDict) { U32* const hashTable = zc->hashTable; - U32 const hashLog = zc->params.cParams.hashLog; + U32 const hashLog = zc->appliedParams.cParams.hashLog; size_t const h = ZSTD_hashPtr(ip, hashLog, mls); U32* const bt = zc->chainTable; - U32 const btLog = zc->params.cParams.chainLog - 1; + U32 const btLog = zc->appliedParams.cParams.chainLog - 1; U32 const btMask = (1 << btLog) - 1; U32 matchIndex = hashTable[h]; size_t commonLengthSmaller=0, commonLengthLarger=0; @@ -1896,9 +2280,9 @@ FORCE_INLINE U32 ZSTD_insertAndFindFirstIndex (ZSTD_CCtx* zc, const BYTE* ip, U32 mls) { U32* const hashTable = zc->hashTable; - const U32 hashLog = zc->params.cParams.hashLog; + const U32 hashLog = zc->appliedParams.cParams.hashLog; U32* const chainTable = zc->chainTable; - const U32 chainMask = (1 << zc->params.cParams.chainLog) - 1; + const U32 chainMask = (1 << zc->appliedParams.cParams.chainLog) - 1; const BYTE* const base = zc->base; const U32 target = (U32)(ip - base); U32 idx = zc->nextToUpdate; @@ -1915,8 +2299,8 @@ U32 ZSTD_insertAndFindFirstIndex (ZSTD_CCtx* zc, const BYTE* ip, U32 mls) } - -FORCE_INLINE /* inlining is important to hardwire a hot branch (template emulation) */ +/* inlining is important to hardwire a hot branch (template emulation) */ +FORCE_INLINE size_t ZSTD_HcFindBestMatch_generic ( ZSTD_CCtx* zc, /* Index table will be updated */ const BYTE* const ip, const BYTE* const iLimit, @@ -1924,7 +2308,7 @@ size_t ZSTD_HcFindBestMatch_generic ( const U32 maxNbAttempts, const U32 mls, const U32 extDict) { U32* const chainTable = zc->chainTable; - const U32 chainSize = (1 << zc->params.cParams.chainLog); + const U32 chainSize = (1 << zc->appliedParams.cParams.chainLog); const U32 chainMask = chainSize-1; const BYTE* const base = zc->base; const BYTE* const dictBase = zc->dictBase; @@ -2018,8 +2402,8 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx, const BYTE* const ilimit = iend - 8; const BYTE* const base = ctx->base + ctx->dictLimit; - U32 const maxSearches = 1 << ctx->params.cParams.searchLog; - U32 const mls = ctx->params.cParams.searchLength; + U32 const maxSearches = 1 << ctx->appliedParams.cParams.searchLog; + U32 const mls = ctx->appliedParams.cParams.searchLength; typedef size_t (*searchMax_f)(ZSTD_CCtx* zc, const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr, @@ -2101,15 +2485,19 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx, break; /* nothing found : store previous solution */ } + /* NOTE: + * start[-offset+ZSTD_REP_MOVE-1] is undefined behavior. + * (-offset+ZSTD_REP_MOVE-1) is unsigned, and is added to start, which + * overflows the pointer, which is undefined behavior. + */ /* catch up */ if (offset) { while ( (start > anchor) && (start > base+offset-ZSTD_REP_MOVE) - && (start[-1] == start[-1-offset+ZSTD_REP_MOVE]) ) /* only search for offset within prefix */ + && (start[-1] == (start-offset+ZSTD_REP_MOVE)[-1]) ) /* only search for offset within prefix */ { start--; matchLength++; } offset_2 = offset_1; offset_1 = (U32)(offset - ZSTD_REP_MOVE); } - /* store sequence */ _storeSequence: { size_t const litLength = start - anchor; @@ -2182,8 +2570,8 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx, const BYTE* const dictEnd = dictBase + dictLimit; const BYTE* const dictStart = dictBase + ctx->lowLimit; - const U32 maxSearches = 1 << ctx->params.cParams.searchLog; - const U32 mls = ctx->params.cParams.searchLength; + const U32 maxSearches = 1 << ctx->appliedParams.cParams.searchLog; + const U32 mls = ctx->appliedParams.cParams.searchLength; typedef size_t (*searchMax_f)(ZSTD_CCtx* zc, const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr, @@ -2370,7 +2758,7 @@ static void ZSTD_compressBlock_btopt(ZSTD_CCtx* ctx, const void* src, size_t src #endif } -static void ZSTD_compressBlock_btopt2(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +static void ZSTD_compressBlock_btultra(ZSTD_CCtx* ctx, const void* src, size_t srcSize) { #ifdef ZSTD_OPT_H_91842398743 ZSTD_compressBlock_opt_generic(ctx, src, srcSize, 1); @@ -2390,7 +2778,7 @@ static void ZSTD_compressBlock_btopt_extDict(ZSTD_CCtx* ctx, const void* src, si #endif } -static void ZSTD_compressBlock_btopt2_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +static void ZSTD_compressBlock_btultra_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize) { #ifdef ZSTD_OPT_H_91842398743 ZSTD_compressBlock_opt_extDict_generic(ctx, src, srcSize, 1); @@ -2401,26 +2789,32 @@ static void ZSTD_compressBlock_btopt2_extDict(ZSTD_CCtx* ctx, const void* src, s } +/* ZSTD_selectBlockCompressor() : + * assumption : strat is a valid strategy */ typedef void (*ZSTD_blockCompressor) (ZSTD_CCtx* ctx, const void* src, size_t srcSize); - static ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int extDict) { - static const ZSTD_blockCompressor blockCompressor[2][8] = { - { ZSTD_compressBlock_fast, ZSTD_compressBlock_doubleFast, ZSTD_compressBlock_greedy, + static const ZSTD_blockCompressor blockCompressor[2][(unsigned)ZSTD_btultra+1] = { + { ZSTD_compressBlock_fast /* default for 0 */, + ZSTD_compressBlock_fast, ZSTD_compressBlock_doubleFast, ZSTD_compressBlock_greedy, ZSTD_compressBlock_lazy, ZSTD_compressBlock_lazy2, ZSTD_compressBlock_btlazy2, - ZSTD_compressBlock_btopt, ZSTD_compressBlock_btopt2 }, - { ZSTD_compressBlock_fast_extDict, ZSTD_compressBlock_doubleFast_extDict, ZSTD_compressBlock_greedy_extDict, + ZSTD_compressBlock_btopt, ZSTD_compressBlock_btultra }, + { ZSTD_compressBlock_fast_extDict /* default for 0 */, + ZSTD_compressBlock_fast_extDict, ZSTD_compressBlock_doubleFast_extDict, ZSTD_compressBlock_greedy_extDict, ZSTD_compressBlock_lazy_extDict,ZSTD_compressBlock_lazy2_extDict, ZSTD_compressBlock_btlazy2_extDict, - ZSTD_compressBlock_btopt_extDict, ZSTD_compressBlock_btopt2_extDict } + ZSTD_compressBlock_btopt_extDict, ZSTD_compressBlock_btultra_extDict } }; + ZSTD_STATIC_ASSERT((unsigned)ZSTD_fast == 1); + assert((U32)strat >= (U32)ZSTD_fast); + assert((U32)strat <= (U32)ZSTD_btultra); - return blockCompressor[extDict][(U32)strat]; + return blockCompressor[extDict!=0][(U32)strat]; } static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCapacity, const void* src, size_t srcSize) { - ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->params.cParams.strategy, zc->lowLimit < zc->dictLimit); + ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->appliedParams.cParams.strategy, zc->lowLimit < zc->dictLimit); const BYTE* const base = zc->base; const BYTE* const istart = (const BYTE*)src; const U32 current = (U32)(istart-base); @@ -2433,14 +2827,14 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCa } -/*! ZSTD_compress_generic() : +/*! ZSTD_compress_frameChunk() : * Compress a chunk of data into one or multiple blocks. * All blocks will be terminated, all input will be consumed. * Function will issue an error if there is not enough `dstCapacity` to hold the compressed content. * Frame is supposed already started (header already produced) * @return : compressed size, or an error code */ -static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, +static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, U32 lastFrameChunk) @@ -2450,9 +2844,9 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, const BYTE* ip = (const BYTE*)src; BYTE* const ostart = (BYTE*)dst; BYTE* op = ostart; - U32 const maxDist = 1 << cctx->params.cParams.windowLog; + U32 const maxDist = 1 << cctx->appliedParams.cParams.windowLog; - if (cctx->params.fParams.checksumFlag && srcSize) + if (cctx->appliedParams.fParams.checksumFlag && srcSize) XXH64_update(&cctx->xxhState, src, srcSize); while (remaining) { @@ -2465,9 +2859,9 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, /* preemptive overflow correction */ if (cctx->lowLimit > (3U<<29)) { - U32 const cycleMask = (1 << ZSTD_cycleLog(cctx->params.cParams.hashLog, cctx->params.cParams.strategy)) - 1; + U32 const cycleMask = (1 << ZSTD_cycleLog(cctx->appliedParams.cParams.hashLog, cctx->appliedParams.cParams.strategy)) - 1; U32 const current = (U32)(ip - cctx->base); - U32 const newCurrent = (current & cycleMask) + (1 << cctx->params.cParams.windowLog); + U32 const newCurrent = (current & cycleMask) + (1 << cctx->appliedParams.cParams.windowLog); U32 const correction = current - newCurrent; ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_64 <= 30); ZSTD_reduceIndex(cctx, correction); @@ -2522,22 +2916,20 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity, U32 const singleSegment = params.fParams.contentSizeFlag && (windowSize >= pledgedSrcSize); BYTE const windowLogByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3); U32 const fcsCode = params.fParams.contentSizeFlag ? - (pledgedSrcSize>=256) + (pledgedSrcSize>=65536+256) + (pledgedSrcSize>=0xFFFFFFFFU) : /* 0-3 */ - 0; + (pledgedSrcSize>=256) + (pledgedSrcSize>=65536+256) + (pledgedSrcSize>=0xFFFFFFFFU) : 0; /* 0-3 */ BYTE const frameHeaderDecriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag<<2) + (singleSegment<<5) + (fcsCode<<6) ); size_t pos; if (dstCapacity < ZSTD_frameHeaderSize_max) return ERROR(dstSize_tooSmall); - DEBUGLOG(5, "ZSTD_writeFrameHeader : dictIDFlag : %u \n", !params.fParams.noDictIDFlag); - DEBUGLOG(5, "ZSTD_writeFrameHeader : dictID : %u \n", dictID); - DEBUGLOG(5, "ZSTD_writeFrameHeader : dictIDSizeCode : %u \n", dictIDSizeCode); + DEBUGLOG(5, "ZSTD_writeFrameHeader : dictIDFlag : %u ; dictID : %u ; dictIDSizeCode : %u", + !params.fParams.noDictIDFlag, dictID, dictIDSizeCode); MEM_writeLE32(dst, ZSTD_MAGICNUMBER); op[4] = frameHeaderDecriptionByte; pos=5; if (!singleSegment) op[pos++] = windowLogByte; switch(dictIDSizeCode) { - default: /* impossible */ + default: assert(0); /* impossible */ case 0 : break; case 1 : op[pos] = (BYTE)(dictID); pos++; break; case 2 : MEM_writeLE16(op+pos, (U16)dictID); pos+=2; break; @@ -2545,7 +2937,7 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity, } switch(fcsCode) { - default: /* impossible */ + default: assert(0); /* impossible */ case 0 : if (singleSegment) op[pos++] = (BYTE)(pledgedSrcSize); break; case 1 : MEM_writeLE16(op+pos, (U16)(pledgedSrcSize-256)); pos+=2; break; case 2 : MEM_writeLE32(op+pos, (U32)(pledgedSrcSize)); pos+=4; break; @@ -2563,10 +2955,13 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx, const BYTE* const ip = (const BYTE*) src; size_t fhSize = 0; + DEBUGLOG(5, "ZSTD_compressContinue_internal"); + DEBUGLOG(5, "stage: %u", cctx->stage); if (cctx->stage==ZSTDcs_created) return ERROR(stage_wrong); /* missing init (ZSTD_compressBegin) */ if (frame && (cctx->stage==ZSTDcs_init)) { - fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->params, cctx->frameContentSize, cctx->dictID); + fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->appliedParams, + cctx->pledgedSrcSizePlusOne-1, cctx->dictID); if (ZSTD_isError(fhSize)) return fhSize; dstCapacity -= fhSize; dst = (char*)dst + fhSize; @@ -2596,7 +2991,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx, if (srcSize) { size_t const cSize = frame ? - ZSTD_compress_generic (cctx, dst, dstCapacity, src, srcSize, lastFrameChunk) : + ZSTD_compress_frameChunk (cctx, dst, dstCapacity, src, srcSize, lastFrameChunk) : ZSTD_compressBlock_internal (cctx, dst, dstCapacity, src, srcSize); if (ZSTD_isError(cSize)) return cSize; cctx->consumedSrcSize += srcSize; @@ -2614,14 +3009,18 @@ size_t ZSTD_compressContinue (ZSTD_CCtx* cctx, } -size_t ZSTD_getBlockSizeMax(ZSTD_CCtx* cctx) +size_t ZSTD_getBlockSize(const ZSTD_CCtx* cctx) { - return MIN (ZSTD_BLOCKSIZE_ABSOLUTEMAX, 1 << cctx->params.cParams.windowLog); + U32 const cLevel = cctx->compressionLevel; + ZSTD_compressionParameters cParams = (cLevel == ZSTD_CLEVEL_CUSTOM) ? + cctx->appliedParams.cParams : + ZSTD_getCParams(cLevel, 0, 0); + return MIN (ZSTD_BLOCKSIZE_MAX, 1 << cParams.windowLog); } size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) { - size_t const blockSizeMax = ZSTD_getBlockSizeMax(cctx); + size_t const blockSizeMax = ZSTD_getBlockSize(cctx); if (srcSize > blockSizeMax) return ERROR(srcSize_wrong); return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 0 /* frame mode */, 0 /* last chunk */); } @@ -2645,32 +3044,32 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t zc->nextSrc = iend; if (srcSize <= HASH_READ_SIZE) return 0; - switch(zc->params.cParams.strategy) + switch(zc->appliedParams.cParams.strategy) { case ZSTD_fast: - ZSTD_fillHashTable (zc, iend, zc->params.cParams.searchLength); + ZSTD_fillHashTable (zc, iend, zc->appliedParams.cParams.searchLength); break; case ZSTD_dfast: - ZSTD_fillDoubleHashTable (zc, iend, zc->params.cParams.searchLength); + ZSTD_fillDoubleHashTable (zc, iend, zc->appliedParams.cParams.searchLength); break; case ZSTD_greedy: case ZSTD_lazy: case ZSTD_lazy2: if (srcSize >= HASH_READ_SIZE) - ZSTD_insertAndFindFirstIndex(zc, iend-HASH_READ_SIZE, zc->params.cParams.searchLength); + ZSTD_insertAndFindFirstIndex(zc, iend-HASH_READ_SIZE, zc->appliedParams.cParams.searchLength); break; case ZSTD_btlazy2: case ZSTD_btopt: - case ZSTD_btopt2: + case ZSTD_btultra: if (srcSize >= HASH_READ_SIZE) - ZSTD_updateTree(zc, iend-HASH_READ_SIZE, iend, 1 << zc->params.cParams.searchLog, zc->params.cParams.searchLength); + ZSTD_updateTree(zc, iend-HASH_READ_SIZE, iend, 1 << zc->appliedParams.cParams.searchLog, zc->appliedParams.cParams.searchLength); break; default: - return ERROR(GENERIC); /* strategy doesn't exist; impossible */ + assert(0); /* not possible : not a valid strategy id */ } zc->nextToUpdate = (U32)(iend - zc->base); @@ -2710,7 +3109,7 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_CCtx* cctx, const void* dict, size_t BYTE scratchBuffer[1<<MAX(MLFSELog,LLFSELog)]; dictPtr += 4; /* skip magic number */ - cctx->dictID = cctx->params.fParams.noDictIDFlag ? 0 : MEM_readLE32(dictPtr); + cctx->dictID = cctx->appliedParams.fParams.noDictIDFlag ? 0 : MEM_readLE32(dictPtr); dictPtr += 4; { size_t const hufHeaderSize = HUF_readCTable(cctx->hufCTable, 255, dictPtr, dictEnd-dictPtr); @@ -2781,28 +3180,56 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_CCtx* cctx, const void* dict, size_t /** ZSTD_compress_insertDictionary() : * @return : 0, or an error code */ -static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize) +static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* cctx, + const void* dict, size_t dictSize, + ZSTD_dictMode_e dictMode) { + DEBUGLOG(5, "ZSTD_compress_insertDictionary"); if ((dict==NULL) || (dictSize<=8)) return 0; - /* dict as pure content */ - if ((MEM_readLE32(dict) != ZSTD_DICT_MAGIC) || (cctx->forceRawDict)) + /* dict restricted modes */ + if (dictMode==ZSTD_dm_rawContent) return ZSTD_loadDictionaryContent(cctx, dict, dictSize); - /* dict as zstd dictionary */ + if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) { + if (dictMode == ZSTD_dm_auto) { + DEBUGLOG(5, "raw content dictionary detected"); + return ZSTD_loadDictionaryContent(cctx, dict, dictSize); + } + if (dictMode == ZSTD_dm_fullDict) + return ERROR(dictionary_wrong); + assert(0); /* impossible */ + } + + /* dict as full zstd dictionary */ return ZSTD_loadZstdDictionary(cctx, dict, dictSize); } /*! ZSTD_compressBegin_internal() : -* @return : 0, or an error code */ + * @return : 0, or an error code */ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, - ZSTD_parameters params, U64 pledgedSrcSize) + ZSTD_dictMode_e dictMode, + const ZSTD_CDict* cdict, + ZSTD_parameters params, U64 pledgedSrcSize, + ZSTD_buffered_policy_e zbuff) { - ZSTD_compResetPolicy_e const crp = dictSize ? ZSTDcrp_fullReset : ZSTDcrp_continue; + DEBUGLOG(4, "ZSTD_compressBegin_internal"); + DEBUGLOG(4, "dict ? %s", dict ? "dict" : (cdict ? "cdict" : "none")); + DEBUGLOG(4, "dictMode : %u", (U32)dictMode); + /* params are supposed to be fully validated at this point */ assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); - CHECK_F(ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, crp)); - return ZSTD_compress_insertDictionary(cctx, dict, dictSize); + assert(!((dict) && (cdict))); /* either dict or cdict, not both */ + + if (cdict && cdict->dictContentSize>0) { + return ZSTD_copyCCtx_internal(cctx, cdict->refContext, + params.fParams, pledgedSrcSize, + zbuff); + } + + CHECK_F( ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, + ZSTDcrp_continue, zbuff) ); + return ZSTD_compress_insertDictionary(cctx, dict, dictSize, dictMode); } @@ -2814,14 +3241,16 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, { /* compression parameters verification and optimization */ CHECK_F(ZSTD_checkCParams(params.cParams)); - return ZSTD_compressBegin_internal(cctx, dict, dictSize, params, pledgedSrcSize); + return ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dm_auto, NULL, + params, pledgedSrcSize, ZSTDb_not_buffered); } size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel) { ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize); - return ZSTD_compressBegin_internal(cctx, dict, dictSize, params, 0); + return ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dm_auto, NULL, + params, 0, ZSTDb_not_buffered); } @@ -2840,11 +3269,12 @@ static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity) BYTE* op = ostart; size_t fhSize = 0; + DEBUGLOG(5, "ZSTD_writeEpilogue"); if (cctx->stage == ZSTDcs_created) return ERROR(stage_wrong); /* init missing */ /* special case : empty frame */ if (cctx->stage == ZSTDcs_init) { - fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->params, 0, 0); + fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->appliedParams, 0, 0); if (ZSTD_isError(fhSize)) return fhSize; dstCapacity -= fhSize; op += fhSize; @@ -2860,7 +3290,7 @@ static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity) dstCapacity -= ZSTD_blockHeaderSize; } - if (cctx->params.fParams.checksumFlag) { + if (cctx->appliedParams.fParams.checksumFlag) { U32 const checksum = (U32) XXH64_digest(&cctx->xxhState); if (dstCapacity<4) return ERROR(dstSize_tooSmall); MEM_writeLE32(op, checksum); @@ -2877,13 +3307,19 @@ size_t ZSTD_compressEnd (ZSTD_CCtx* cctx, const void* src, size_t srcSize) { size_t endResult; - size_t const cSize = ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1 /* frame mode */, 1 /* last chunk */); + size_t const cSize = ZSTD_compressContinue_internal(cctx, + dst, dstCapacity, src, srcSize, + 1 /* frame mode */, 1 /* last chunk */); if (ZSTD_isError(cSize)) return cSize; endResult = ZSTD_writeEpilogue(cctx, (char*)dst + cSize, dstCapacity-cSize); if (ZSTD_isError(endResult)) return endResult; - if (cctx->params.fParams.contentSizeFlag) { /* control src size */ - if (cctx->frameContentSize != cctx->consumedSrcSize) return ERROR(srcSize_wrong); - } + if (cctx->appliedParams.fParams.contentSizeFlag) { /* control src size */ + DEBUGLOG(5, "end of frame : controlling src size"); + if (cctx->pledgedSrcSizePlusOne != cctx->consumedSrcSize+1) { + DEBUGLOG(5, "error : pledgedSrcSize = %u, while realSrcSize = %u", + (U32)cctx->pledgedSrcSizePlusOne-1, (U32)cctx->consumedSrcSize); + return ERROR(srcSize_wrong); + } } return cSize + endResult; } @@ -2894,7 +3330,8 @@ static size_t ZSTD_compress_internal (ZSTD_CCtx* cctx, const void* dict,size_t dictSize, ZSTD_parameters params) { - CHECK_F(ZSTD_compressBegin_internal(cctx, dict, dictSize, params, srcSize)); + CHECK_F( ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dm_auto, NULL, + params, srcSize, ZSTDb_not_buffered) ); return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize); } @@ -2926,25 +3363,36 @@ size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcS size_t result; ZSTD_CCtx ctxBody; memset(&ctxBody, 0, sizeof(ctxBody)); - memcpy(&ctxBody.customMem, &defaultCustomMem, sizeof(ZSTD_customMem)); + ctxBody.customMem = ZSTD_defaultCMem; result = ZSTD_compressCCtx(&ctxBody, dst, dstCapacity, src, srcSize, compressionLevel); - ZSTD_free(ctxBody.workSpace, defaultCustomMem); /* can't free ctxBody itself, as it's on stack; free only heap content */ + ZSTD_free(ctxBody.workSpace, ZSTD_defaultCMem); /* can't free ctxBody itself, as it's on stack; free only heap content */ return result; } /* ===== Dictionary API ===== */ -struct ZSTD_CDict_s { - void* dictBuffer; - const void* dictContent; - size_t dictContentSize; - ZSTD_CCtx* refContext; -}; /* typedef'd tp ZSTD_CDict within "zstd.h" */ +/*! ZSTD_estimateCDictSize_advanced() : + * Estimate amount of memory that will be needed to create a dictionary with following arguments */ +size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, unsigned byReference) +{ + DEBUGLOG(5, "sizeof(ZSTD_CDict) : %u", (U32)sizeof(ZSTD_CDict)); + DEBUGLOG(5, "CCtx estimate : %u", (U32)ZSTD_estimateCCtxSize_advanced(cParams)); + return sizeof(ZSTD_CDict) + ZSTD_estimateCCtxSize_advanced(cParams) + + (byReference ? 0 : dictSize); +} + +size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel) +{ + ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, 0, dictSize); + return ZSTD_estimateCDictSize_advanced(dictSize, cParams, 0); +} size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict) { if (cdict==NULL) return 0; /* support sizeof on NULL */ + DEBUGLOG(5, "sizeof(*cdict) : %u", (U32)sizeof(*cdict)); + DEBUGLOG(5, "ZSTD_sizeof_CCtx : %u", (U32)ZSTD_sizeof_CCtx(cdict->refContext)); return ZSTD_sizeof_CCtx(cdict->refContext) + (cdict->dictBuffer ? cdict->dictContentSize : 0) + sizeof(*cdict); } @@ -2956,13 +3404,46 @@ static ZSTD_parameters ZSTD_makeParams(ZSTD_compressionParameters cParams, ZSTD_ return params; } -ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, unsigned byReference, +static size_t ZSTD_initCDict_internal( + ZSTD_CDict* cdict, + const void* dictBuffer, size_t dictSize, + unsigned byReference, ZSTD_dictMode_e dictMode, + ZSTD_compressionParameters cParams) +{ + DEBUGLOG(5, "ZSTD_initCDict_internal, mode %u", (U32)dictMode); + if ((byReference) || (!dictBuffer) || (!dictSize)) { + cdict->dictBuffer = NULL; + cdict->dictContent = dictBuffer; + } else { + void* const internalBuffer = ZSTD_malloc(dictSize, cdict->refContext->customMem); + cdict->dictBuffer = internalBuffer; + cdict->dictContent = internalBuffer; + if (!internalBuffer) return ERROR(memory_allocation); + memcpy(internalBuffer, dictBuffer, dictSize); + } + cdict->dictContentSize = dictSize; + + { ZSTD_frameParameters const fParams = { 0 /* contentSizeFlag */, + 0 /* checksumFlag */, 0 /* noDictIDFlag */ }; /* dummy */ + ZSTD_parameters const params = ZSTD_makeParams(cParams, fParams); + CHECK_F( ZSTD_compressBegin_internal(cdict->refContext, + cdict->dictContent, dictSize, dictMode, + NULL, + params, ZSTD_CONTENTSIZE_UNKNOWN, + ZSTDb_not_buffered) ); + } + + return 0; +} + +ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, + unsigned byReference, ZSTD_dictMode_e dictMode, ZSTD_compressionParameters cParams, ZSTD_customMem customMem) { - if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem; - if (!customMem.customAlloc || !customMem.customFree) return NULL; + DEBUGLOG(5, "ZSTD_createCDict_advanced, mode %u", (U32)dictMode); + if (!customMem.customAlloc ^ !customMem.customFree) return NULL; - { ZSTD_CDict* const cdict = (ZSTD_CDict*) ZSTD_malloc(sizeof(ZSTD_CDict), customMem); + { ZSTD_CDict* const cdict = (ZSTD_CDict*)ZSTD_malloc(sizeof(ZSTD_CDict), customMem); ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(customMem); if (!cdict || !cctx) { @@ -2970,46 +3451,34 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, u ZSTD_freeCCtx(cctx); return NULL; } + cdict->refContext = cctx; - if ((byReference) || (!dictBuffer) || (!dictSize)) { - cdict->dictBuffer = NULL; - cdict->dictContent = dictBuffer; - } else { - void* const internalBuffer = ZSTD_malloc(dictSize, customMem); - if (!internalBuffer) { ZSTD_free(cctx, customMem); ZSTD_free(cdict, customMem); return NULL; } - memcpy(internalBuffer, dictBuffer, dictSize); - cdict->dictBuffer = internalBuffer; - cdict->dictContent = internalBuffer; + if (ZSTD_isError( ZSTD_initCDict_internal(cdict, + dictBuffer, dictSize, + byReference, dictMode, + cParams) )) { + ZSTD_freeCDict(cdict); + return NULL; } - { ZSTD_frameParameters const fParams = { 0 /* contentSizeFlag */, 0 /* checksumFlag */, 0 /* noDictIDFlag */ }; /* dummy */ - ZSTD_parameters const params = ZSTD_makeParams(cParams, fParams); - size_t const errorCode = ZSTD_compressBegin_advanced(cctx, cdict->dictContent, dictSize, params, 0); - if (ZSTD_isError(errorCode)) { - ZSTD_free(cdict->dictBuffer, customMem); - ZSTD_free(cdict, customMem); - ZSTD_freeCCtx(cctx); - return NULL; - } } - - cdict->refContext = cctx; - cdict->dictContentSize = dictSize; return cdict; } } ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel) { - ZSTD_customMem const allocator = { NULL, NULL, NULL }; ZSTD_compressionParameters cParams = ZSTD_getCParams(compressionLevel, 0, dictSize); - return ZSTD_createCDict_advanced(dict, dictSize, 0, cParams, allocator); + return ZSTD_createCDict_advanced(dict, dictSize, + 0 /* byReference */, ZSTD_dm_auto, + cParams, ZSTD_defaultCMem); } ZSTD_CDict* ZSTD_createCDict_byReference(const void* dict, size_t dictSize, int compressionLevel) { - ZSTD_customMem const allocator = { NULL, NULL, NULL }; ZSTD_compressionParameters cParams = ZSTD_getCParams(compressionLevel, 0, dictSize); - return ZSTD_createCDict_advanced(dict, dictSize, 1, cParams, allocator); + return ZSTD_createCDict_advanced(dict, dictSize, + 1 /* byReference */, ZSTD_dm_auto, + cParams, ZSTD_defaultCMem); } size_t ZSTD_freeCDict(ZSTD_CDict* cdict) @@ -3023,7 +3492,54 @@ size_t ZSTD_freeCDict(ZSTD_CDict* cdict) } } -static ZSTD_parameters ZSTD_getParamsFromCDict(const ZSTD_CDict* cdict) { +/*! ZSTD_initStaticCDict_advanced() : + * Generate a digested dictionary in provided memory area. + * workspace: The memory area to emplace the dictionary into. + * Provided pointer must 8-bytes aligned. + * It must outlive dictionary usage. + * workspaceSize: Use ZSTD_estimateCDictSize() + * to determine how large workspace must be. + * cParams : use ZSTD_getCParams() to transform a compression level + * into its relevants cParams. + * @return : pointer to ZSTD_CDict*, or NULL if error (size too small) + * Note : there is no corresponding "free" function. + * Since workspace was allocated externally, it must be freed externally. + */ +ZSTD_CDict* ZSTD_initStaticCDict(void* workspace, size_t workspaceSize, + const void* dict, size_t dictSize, + unsigned byReference, ZSTD_dictMode_e dictMode, + ZSTD_compressionParameters cParams) +{ + size_t const cctxSize = ZSTD_estimateCCtxSize_advanced(cParams); + size_t const neededSize = sizeof(ZSTD_CDict) + (byReference ? 0 : dictSize) + + cctxSize; + ZSTD_CDict* const cdict = (ZSTD_CDict*) workspace; + void* ptr; + DEBUGLOG(5, "(size_t)workspace & 7 : %u", (U32)(size_t)workspace & 7); + if ((size_t)workspace & 7) return NULL; /* 8-aligned */ + DEBUGLOG(5, "(workspaceSize < neededSize) : (%u < %u) => %u", + (U32)workspaceSize, (U32)neededSize, (U32)(workspaceSize < neededSize)); + if (workspaceSize < neededSize) return NULL; + + if (!byReference) { + memcpy(cdict+1, dict, dictSize); + dict = cdict+1; + ptr = (char*)workspace + sizeof(ZSTD_CDict) + dictSize; + } else { + ptr = cdict+1; + } + cdict->refContext = ZSTD_initStaticCCtx(ptr, cctxSize); + + if (ZSTD_isError( ZSTD_initCDict_internal(cdict, + dict, dictSize, + 1 /* byReference */, dictMode, + cParams) )) + return NULL; + + return cdict; +} + +ZSTD_parameters ZSTD_getParamsFromCDict(const ZSTD_CDict* cdict) { return ZSTD_getParamsFromCCtx(cdict->refContext); } @@ -3033,16 +3549,16 @@ size_t ZSTD_compressBegin_usingCDict_advanced( ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize) { - if (cdict==NULL) return ERROR(GENERIC); /* does not support NULL cdict */ - DEBUGLOG(5, "ZSTD_compressBegin_usingCDict_advanced : dictIDFlag == %u \n", !fParams.noDictIDFlag); - if (cdict->dictContentSize) - CHECK_F( ZSTD_copyCCtx_internal(cctx, cdict->refContext, fParams, pledgedSrcSize) ) - else { - ZSTD_parameters params = cdict->refContext->params; + if (cdict==NULL) return ERROR(dictionary_wrong); + { ZSTD_parameters params = cdict->refContext->appliedParams; params.fParams = fParams; - CHECK_F(ZSTD_compressBegin_internal(cctx, NULL, 0, params, pledgedSrcSize)); + DEBUGLOG(5, "ZSTD_compressBegin_usingCDict_advanced"); + return ZSTD_compressBegin_internal(cctx, + NULL, 0, ZSTD_dm_auto, + cdict, + params, pledgedSrcSize, + ZSTDb_not_buffered); } - return 0; } /* ZSTD_compressBegin_usingCDict() : @@ -3051,7 +3567,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced( size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) { ZSTD_frameParameters const fParams = { 0 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ }; - DEBUGLOG(5, "ZSTD_compressBegin_usingCDict : dictIDFlag == %u \n", !fParams.noDictIDFlag); + DEBUGLOG(5, "ZSTD_compressBegin_usingCDict : dictIDFlag == %u", !fParams.noDictIDFlag); return ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, 0); } @@ -3084,176 +3600,133 @@ size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, * Streaming ********************************************************************/ -typedef enum { zcss_init, zcss_load, zcss_flush, zcss_final } ZSTD_cStreamStage; - -struct ZSTD_CStream_s { - ZSTD_CCtx* cctx; - ZSTD_CDict* cdictLocal; - const ZSTD_CDict* cdict; - char* inBuff; - size_t inBuffSize; - size_t inToCompress; - size_t inBuffPos; - size_t inBuffTarget; - size_t blockSize; - char* outBuff; - size_t outBuffSize; - size_t outBuffContentSize; - size_t outBuffFlushedSize; - ZSTD_cStreamStage stage; - U32 checksum; - U32 frameEnded; - U64 pledgedSrcSize; - ZSTD_parameters params; - ZSTD_customMem customMem; -}; /* typedef'd to ZSTD_CStream within "zstd.h" */ - ZSTD_CStream* ZSTD_createCStream(void) { - return ZSTD_createCStream_advanced(defaultCustomMem); + return ZSTD_createCStream_advanced(ZSTD_defaultCMem); } -ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem) +ZSTD_CStream* ZSTD_initStaticCStream(void *workspace, size_t workspaceSize) { - ZSTD_CStream* zcs; - - if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem; - if (!customMem.customAlloc || !customMem.customFree) return NULL; + return ZSTD_initStaticCCtx(workspace, workspaceSize); +} - zcs = (ZSTD_CStream*)ZSTD_malloc(sizeof(ZSTD_CStream), customMem); - if (zcs==NULL) return NULL; - memset(zcs, 0, sizeof(ZSTD_CStream)); - memcpy(&zcs->customMem, &customMem, sizeof(ZSTD_customMem)); - zcs->cctx = ZSTD_createCCtx_advanced(customMem); - if (zcs->cctx == NULL) { ZSTD_freeCStream(zcs); return NULL; } - return zcs; +ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem) +{ /* CStream and CCtx are now same object */ + return ZSTD_createCCtx_advanced(customMem); } size_t ZSTD_freeCStream(ZSTD_CStream* zcs) { - if (zcs==NULL) return 0; /* support free on NULL */ - { ZSTD_customMem const cMem = zcs->customMem; - ZSTD_freeCCtx(zcs->cctx); - zcs->cctx = NULL; - ZSTD_freeCDict(zcs->cdictLocal); - zcs->cdictLocal = NULL; - ZSTD_free(zcs->inBuff, cMem); - zcs->inBuff = NULL; - ZSTD_free(zcs->outBuff, cMem); - zcs->outBuff = NULL; - ZSTD_free(zcs, cMem); - return 0; - } + return ZSTD_freeCCtx(zcs); /* same object */ } + /*====== Initialization ======*/ -size_t ZSTD_CStreamInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; } +size_t ZSTD_CStreamInSize(void) { return ZSTD_BLOCKSIZE_MAX; } size_t ZSTD_CStreamOutSize(void) { - return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSOLUTEMAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */ ; + return ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */ ; } -static size_t ZSTD_resetCStream_internal(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize) +static size_t ZSTD_resetCStream_internal(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, ZSTD_dictMode_e dictMode, + const ZSTD_CDict* cdict, + ZSTD_parameters params, unsigned long long pledgedSrcSize) { - if (zcs->inBuffSize==0) return ERROR(stage_wrong); /* zcs has not been init at least once => can't reset */ - - DEBUGLOG(5, "ZSTD_resetCStream_internal : dictIDFlag == %u \n", !zcs->params.fParams.noDictIDFlag); + DEBUGLOG(4, "ZSTD_resetCStream_internal"); + /* params are supposed to be fully validated at this point */ + assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); + assert(!((dict) && (cdict))); /* either dict or cdict, not both */ - if (zcs->cdict) CHECK_F(ZSTD_compressBegin_usingCDict_advanced(zcs->cctx, zcs->cdict, zcs->params.fParams, pledgedSrcSize)) - else CHECK_F(ZSTD_compressBegin_internal(zcs->cctx, NULL, 0, zcs->params, pledgedSrcSize)); + CHECK_F( ZSTD_compressBegin_internal(zcs, + dict, dictSize, dictMode, + cdict, + params, pledgedSrcSize, + ZSTDb_buffered) ); zcs->inToCompress = 0; zcs->inBuffPos = 0; zcs->inBuffTarget = zcs->blockSize; zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0; - zcs->stage = zcss_load; + zcs->streamStage = zcss_load; zcs->frameEnded = 0; - zcs->pledgedSrcSize = pledgedSrcSize; return 0; /* ready to go */ } size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize) { - - zcs->params.fParams.contentSizeFlag = (pledgedSrcSize > 0); - DEBUGLOG(5, "ZSTD_resetCStream : dictIDFlag == %u \n", !zcs->params.fParams.noDictIDFlag); - return ZSTD_resetCStream_internal(zcs, pledgedSrcSize); + ZSTD_parameters params = zcs->requestedParams; + params.fParams.contentSizeFlag = (pledgedSrcSize > 0); + DEBUGLOG(5, "ZSTD_resetCStream"); + if (zcs->compressionLevel != ZSTD_CLEVEL_CUSTOM) { + params.cParams = ZSTD_getCParams(zcs->compressionLevel, pledgedSrcSize, 0 /* dictSize */); + } + return ZSTD_resetCStream_internal(zcs, NULL, 0, zcs->dictMode, zcs->cdict, params, pledgedSrcSize); } -/* ZSTD_initCStream_internal() : - * params are supposed validated at this stage - * and zcs->cdict is supposed to be correct */ -static size_t ZSTD_initCStream_stage2(ZSTD_CStream* zcs, - const ZSTD_parameters params, - unsigned long long pledgedSrcSize) +/*! ZSTD_initCStream_internal() : + * Note : not static, but hidden (not exposed). Used by zstdmt_compress.c + * Assumption 1 : params are valid + * Assumption 2 : either dict, or cdict, is defined, not both */ +size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, const ZSTD_CDict* cdict, + ZSTD_parameters params, unsigned long long pledgedSrcSize) { + DEBUGLOG(5, "ZSTD_initCStream_internal"); assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); + assert(!((dict) && (cdict))); /* either dict or cdict, not both */ - /* allocate buffers */ - { size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog; - if (zcs->inBuffSize < neededInBuffSize) { - zcs->inBuffSize = 0; - ZSTD_free(zcs->inBuff, zcs->customMem); - zcs->inBuff = (char*) ZSTD_malloc(neededInBuffSize, zcs->customMem); - if (zcs->inBuff == NULL) return ERROR(memory_allocation); - zcs->inBuffSize = neededInBuffSize; + if (dict && dictSize >= 8) { + DEBUGLOG(5, "loading dictionary of size %u", (U32)dictSize); + if (zcs->staticSize) { /* static CCtx : never uses malloc */ + /* incompatible with internal cdict creation */ + return ERROR(memory_allocation); } - zcs->blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, neededInBuffSize); - } - if (zcs->outBuffSize < ZSTD_compressBound(zcs->blockSize)+1) { - size_t const outBuffSize = ZSTD_compressBound(zcs->blockSize)+1; - zcs->outBuffSize = 0; - ZSTD_free(zcs->outBuff, zcs->customMem); - zcs->outBuff = (char*) ZSTD_malloc(outBuffSize, zcs->customMem); - if (zcs->outBuff == NULL) return ERROR(memory_allocation); - zcs->outBuffSize = outBuffSize; + ZSTD_freeCDict(zcs->cdictLocal); + zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, + zcs->dictContentByRef, zcs->dictMode, + params.cParams, zcs->customMem); + zcs->cdict = zcs->cdictLocal; + if (zcs->cdictLocal == NULL) return ERROR(memory_allocation); + } else { + if (cdict) { + ZSTD_parameters const cdictParams = ZSTD_getParamsFromCDict(cdict); + params.cParams = cdictParams.cParams; /* cParams are enforced from cdict */ + } + ZSTD_freeCDict(zcs->cdictLocal); + zcs->cdictLocal = NULL; + zcs->cdict = cdict; } - zcs->checksum = params.fParams.checksumFlag > 0; - zcs->params = params; - - DEBUGLOG(5, "ZSTD_initCStream_stage2 : dictIDFlag == %u \n", !params.fParams.noDictIDFlag); - return ZSTD_resetCStream_internal(zcs, pledgedSrcSize); + zcs->requestedParams = params; + zcs->compressionLevel = ZSTD_CLEVEL_CUSTOM; + return ZSTD_resetCStream_internal(zcs, NULL, 0, zcs->dictMode, zcs->cdict, params, pledgedSrcSize); } /* ZSTD_initCStream_usingCDict_advanced() : * same as ZSTD_initCStream_usingCDict(), with control over frame parameters */ -size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize, ZSTD_frameParameters fParams) -{ - if (!cdict) return ERROR(GENERIC); /* cannot handle NULL cdict (does not know what to do) */ +size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, + const ZSTD_CDict* cdict, + ZSTD_frameParameters fParams, + unsigned long long pledgedSrcSize) +{ /* cannot handle NULL cdict (does not know what to do) */ + if (!cdict) return ERROR(dictionary_wrong); { ZSTD_parameters params = ZSTD_getParamsFromCDict(cdict); params.fParams = fParams; - zcs->cdict = cdict; - return ZSTD_initCStream_stage2(zcs, params, pledgedSrcSize); + return ZSTD_initCStream_internal(zcs, + NULL, 0, cdict, + params, pledgedSrcSize); } } /* note : cdict must outlive compression session */ size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict) { - ZSTD_frameParameters const fParams = { 0 /* content */, 0 /* checksum */, 0 /* noDictID */ }; - return ZSTD_initCStream_usingCDict_advanced(zcs, cdict, 0, fParams); -} - -static size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs, - const void* dict, size_t dictSize, - ZSTD_parameters params, unsigned long long pledgedSrcSize) -{ - assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); - zcs->cdict = NULL; - - if (dict && dictSize >= 8) { - ZSTD_freeCDict(zcs->cdictLocal); - zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, 0 /* copy */, params.cParams, zcs->customMem); - if (zcs->cdictLocal == NULL) return ERROR(memory_allocation); - zcs->cdict = zcs->cdictLocal; - } - - DEBUGLOG(5, "ZSTD_initCStream_internal : dictIDFlag == %u \n", !params.fParams.noDictIDFlag); - return ZSTD_initCStream_stage2(zcs, params, pledgedSrcSize); + ZSTD_frameParameters const fParams = { 0 /* contentSize */, 0 /* checksum */, 0 /* hideDictID */ }; + return ZSTD_initCStream_usingCDict_advanced(zcs, cdict, fParams, 0); /* note : will check that cdict != NULL */ } size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, @@ -3261,120 +3734,180 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, ZSTD_parameters params, unsigned long long pledgedSrcSize) { CHECK_F( ZSTD_checkCParams(params.cParams) ); - DEBUGLOG(5, "ZSTD_initCStream_advanced : dictIDFlag == %u \n", !params.fParams.noDictIDFlag); - return ZSTD_initCStream_internal(zcs, dict, dictSize, params, pledgedSrcSize); + zcs->requestedParams = params; + zcs->compressionLevel = ZSTD_CLEVEL_CUSTOM; + return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL, params, pledgedSrcSize); } size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel) { ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize); - return ZSTD_initCStream_internal(zcs, dict, dictSize, params, 0); + zcs->compressionLevel = compressionLevel; + return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL, params, 0); } size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize) { ZSTD_parameters params = ZSTD_getParams(compressionLevel, pledgedSrcSize, 0); params.fParams.contentSizeFlag = (pledgedSrcSize>0); - return ZSTD_initCStream_internal(zcs, NULL, 0, params, pledgedSrcSize); + return ZSTD_initCStream_internal(zcs, NULL, 0, NULL, params, pledgedSrcSize); } size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel) { ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, 0); - return ZSTD_initCStream_internal(zcs, NULL, 0, params, 0); -} - -size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs) -{ - if (zcs==NULL) return 0; /* support sizeof on NULL */ - return sizeof(*zcs) + ZSTD_sizeof_CCtx(zcs->cctx) + ZSTD_sizeof_CDict(zcs->cdictLocal) + zcs->outBuffSize + zcs->inBuffSize; + return ZSTD_initCStream_internal(zcs, NULL, 0, NULL, params, 0); } /*====== Compression ======*/ -typedef enum { zsf_gather, zsf_flush, zsf_end } ZSTD_flush_e; - -MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, + const void* src, size_t srcSize) { size_t const length = MIN(dstCapacity, srcSize); - memcpy(dst, src, length); + if (length) memcpy(dst, src, length); return length; } -static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, - void* dst, size_t* dstCapacityPtr, - const void* src, size_t* srcSizePtr, - ZSTD_flush_e const flush) +/** ZSTD_compressStream_generic(): + * internal function for all *compressStream*() variants and *compress_generic() + * @return : hint size for next input */ +size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, + ZSTD_outBuffer* output, + ZSTD_inBuffer* input, + ZSTD_EndDirective const flushMode) { + const char* const istart = (const char*)input->src; + const char* const iend = istart + input->size; + const char* ip = istart + input->pos; + char* const ostart = (char*)output->dst; + char* const oend = ostart + output->size; + char* op = ostart + output->pos; U32 someMoreWork = 1; - const char* const istart = (const char*)src; - const char* const iend = istart + *srcSizePtr; - const char* ip = istart; - char* const ostart = (char*)dst; - char* const oend = ostart + *dstCapacityPtr; - char* op = ostart; + + /* check expectations */ + DEBUGLOG(5, "ZSTD_compressStream_generic, flush=%u", (U32)flushMode); + assert(zcs->inBuff != NULL); + assert(zcs->inBuffSize>0); + assert(zcs->outBuff!= NULL); + assert(zcs->outBuffSize>0); + assert(output->pos <= output->size); + assert(input->pos <= input->size); while (someMoreWork) { - switch(zcs->stage) + switch(zcs->streamStage) { - case zcss_init: return ERROR(init_missing); /* call ZBUFF_compressInit() first ! */ + case zcss_init: + /* call ZSTD_initCStream() first ! */ + return ERROR(init_missing); case zcss_load: - /* complete inBuffer */ + if ( (flushMode == ZSTD_e_end) + && ((size_t)(oend-op) >= ZSTD_compressBound(iend-ip)) /* enough dstCapacity */ + && (zcs->inBuffPos == 0) ) { + /* shortcut to compression pass directly into output buffer */ + size_t const cSize = ZSTD_compressEnd(zcs, + op, oend-op, ip, iend-ip); + DEBUGLOG(4, "ZSTD_compressEnd : %u", (U32)cSize); + if (ZSTD_isError(cSize)) return cSize; + ip = iend; + op += cSize; + zcs->frameEnded = 1; + ZSTD_startNewCompression(zcs); + someMoreWork = 0; break; + } + /* complete loading into inBuffer */ { size_t const toLoad = zcs->inBuffTarget - zcs->inBuffPos; - size_t const loaded = ZSTD_limitCopy(zcs->inBuff + zcs->inBuffPos, toLoad, ip, iend-ip); + size_t const loaded = ZSTD_limitCopy( + zcs->inBuff + zcs->inBuffPos, toLoad, + ip, iend-ip); zcs->inBuffPos += loaded; ip += loaded; - if ( (zcs->inBuffPos==zcs->inToCompress) || (!flush && (toLoad != loaded)) ) { - someMoreWork = 0; break; /* not enough input to get a full block : stop there, wait for more */ - } } + if ( (flushMode == ZSTD_e_continue) + && (zcs->inBuffPos < zcs->inBuffTarget) ) { + /* not enough input to fill full block : stop here */ + someMoreWork = 0; break; + } + if ( (flushMode == ZSTD_e_flush) + && (zcs->inBuffPos == zcs->inToCompress) ) { + /* empty */ + someMoreWork = 0; break; + } + } /* compress current block (note : this stage cannot be stopped in the middle) */ + DEBUGLOG(5, "stream compression stage (flushMode==%u)", flushMode); { void* cDst; size_t cSize; size_t const iSize = zcs->inBuffPos - zcs->inToCompress; size_t oSize = oend-op; + unsigned const lastBlock = (flushMode == ZSTD_e_end) && (ip==iend); if (oSize >= ZSTD_compressBound(iSize)) - cDst = op; /* compress directly into output buffer (avoid flush stage) */ + cDst = op; /* compress into output buffer, to skip flush stage */ else cDst = zcs->outBuff, oSize = zcs->outBuffSize; - cSize = (flush == zsf_end) ? - ZSTD_compressEnd(zcs->cctx, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize) : - ZSTD_compressContinue(zcs->cctx, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize); + cSize = lastBlock ? + ZSTD_compressEnd(zcs, cDst, oSize, + zcs->inBuff + zcs->inToCompress, iSize) : + ZSTD_compressContinue(zcs, cDst, oSize, + zcs->inBuff + zcs->inToCompress, iSize); if (ZSTD_isError(cSize)) return cSize; - if (flush == zsf_end) zcs->frameEnded = 1; + zcs->frameEnded = lastBlock; /* prepare next block */ zcs->inBuffTarget = zcs->inBuffPos + zcs->blockSize; if (zcs->inBuffTarget > zcs->inBuffSize) - zcs->inBuffPos = 0, zcs->inBuffTarget = zcs->blockSize; /* note : inBuffSize >= blockSize */ + zcs->inBuffPos = 0, zcs->inBuffTarget = zcs->blockSize; + DEBUGLOG(5, "inBuffTarget:%u / inBuffSize:%u", + (U32)zcs->inBuffTarget, (U32)zcs->inBuffSize); + if (!lastBlock) + assert(zcs->inBuffTarget <= zcs->inBuffSize); zcs->inToCompress = zcs->inBuffPos; - if (cDst == op) { op += cSize; break; } /* no need to flush */ + if (cDst == op) { /* no need to flush */ + op += cSize; + if (zcs->frameEnded) { + DEBUGLOG(5, "Frame completed directly in outBuffer"); + someMoreWork = 0; + ZSTD_startNewCompression(zcs); + } + break; + } zcs->outBuffContentSize = cSize; zcs->outBuffFlushedSize = 0; - zcs->stage = zcss_flush; /* pass-through to flush stage */ + zcs->streamStage = zcss_flush; /* pass-through to flush stage */ } - + /* fall-through */ case zcss_flush: + DEBUGLOG(5, "flush stage"); { size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize; - size_t const flushed = ZSTD_limitCopy(op, oend-op, zcs->outBuff + zcs->outBuffFlushedSize, toFlush); + size_t const flushed = ZSTD_limitCopy(op, oend-op, + zcs->outBuff + zcs->outBuffFlushedSize, toFlush); + DEBUGLOG(5, "toFlush: %u into %u ==> flushed: %u", + (U32)toFlush, (U32)(oend-op), (U32)flushed); op += flushed; zcs->outBuffFlushedSize += flushed; - if (toFlush!=flushed) { someMoreWork = 0; break; } /* dst too small to store flushed data : stop there */ + if (toFlush!=flushed) { + /* flush not fully completed, presumably because dst is too small */ + assert(op==oend); + someMoreWork = 0; + break; + } zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0; - zcs->stage = zcss_load; + if (zcs->frameEnded) { + DEBUGLOG(5, "Frame completed on flush"); + someMoreWork = 0; + ZSTD_startNewCompression(zcs); + break; + } + zcs->streamStage = zcss_load; break; } - case zcss_final: - someMoreWork = 0; /* do nothing */ - break; - - default: - return ERROR(GENERIC); /* impossible */ + default: /* impossible */ + assert(0); } } - *srcSizePtr = ip - istart; - *dstCapacityPtr = op - ostart; + input->pos = ip - istart; + output->pos = op - ostart; if (zcs->frameEnded) return 0; { size_t hintInSize = zcs->inBuffTarget - zcs->inBuffPos; if (hintInSize==0) hintInSize = zcs->blockSize; @@ -3384,14 +3917,86 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input) { - size_t sizeRead = input->size - input->pos; - size_t sizeWritten = output->size - output->pos; - size_t const result = ZSTD_compressStream_generic(zcs, - (char*)(output->dst) + output->pos, &sizeWritten, - (const char*)(input->src) + input->pos, &sizeRead, zsf_gather); - input->pos += sizeRead; - output->pos += sizeWritten; - return result; + /* check conditions */ + if (output->pos > output->size) return ERROR(GENERIC); + if (input->pos > input->size) return ERROR(GENERIC); + + return ZSTD_compressStream_generic(zcs, output, input, ZSTD_e_continue); +} + +/*! ZSTDMT_initCStream_internal() : + * Private use only. Init streaming operation. + * expects params to be valid. + * must receive dict, or cdict, or none, but not both. + * @return : 0, or an error code */ +size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs, + const void* dict, size_t dictSize, const ZSTD_CDict* cdict, + ZSTD_parameters params, unsigned long long pledgedSrcSize); + + +size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, + ZSTD_outBuffer* output, + ZSTD_inBuffer* input, + ZSTD_EndDirective endOp) +{ + /* check conditions */ + if (output->pos > output->size) return ERROR(GENERIC); + if (input->pos > input->size) return ERROR(GENERIC); + assert(cctx!=NULL); + + /* transparent initialization stage */ + if (cctx->streamStage == zcss_init) { + const void* const prefix = cctx->prefix; + size_t const prefixSize = cctx->prefixSize; + ZSTD_parameters params = cctx->requestedParams; + if (cctx->compressionLevel != ZSTD_CLEVEL_CUSTOM) + params.cParams = ZSTD_getCParams(cctx->compressionLevel, + cctx->pledgedSrcSizePlusOne-1, 0 /*dictSize*/); + cctx->prefix = NULL; cctx->prefixSize = 0; /* single usage */ + assert(prefix==NULL || cctx->cdict==NULL); /* only one can be set */ + +#ifdef ZSTD_MULTITHREAD + if (cctx->nbThreads > 1) { + DEBUGLOG(4, "call ZSTDMT_initCStream_internal as nbThreads=%u", cctx->nbThreads); + CHECK_F( ZSTDMT_initCStream_internal(cctx->mtctx, prefix, prefixSize, cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) ); + cctx->streamStage = zcss_load; + } else +#endif + { + CHECK_F( ZSTD_resetCStream_internal(cctx, prefix, prefixSize, cctx->dictMode, cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) ); + } } + + /* compression stage */ +#ifdef ZSTD_MULTITHREAD + if (cctx->nbThreads > 1) { + size_t const flushMin = ZSTDMT_compressStream_generic(cctx->mtctx, output, input, endOp); + DEBUGLOG(5, "ZSTDMT_compressStream_generic : %u", (U32)flushMin); + if ( ZSTD_isError(flushMin) + || (endOp == ZSTD_e_end && flushMin == 0) ) { /* compression completed */ + ZSTD_startNewCompression(cctx); + } + return flushMin; + } +#endif + + CHECK_F( ZSTD_compressStream_generic(cctx, output, input, endOp) ); + DEBUGLOG(5, "completed ZSTD_compress_generic"); + return cctx->outBuffContentSize - cctx->outBuffFlushedSize; /* remaining to flush */ +} + +size_t ZSTD_compress_generic_simpleArgs ( + ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, size_t* dstPos, + const void* src, size_t srcSize, size_t* srcPos, + ZSTD_EndDirective endOp) +{ + ZSTD_outBuffer output = { dst, dstCapacity, *dstPos }; + ZSTD_inBuffer input = { src, srcSize, *srcPos }; + /* ZSTD_compress_generic() will check validity of dstPos and srcPos */ + size_t const cErr = ZSTD_compress_generic(cctx, &output, &input, endOp); + *dstPos = output.pos; + *srcPos = input.pos; + return cErr; } @@ -3401,89 +4006,59 @@ size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuf * @return : amount of data remaining to flush */ size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) { - size_t srcSize = 0; - size_t sizeWritten = output->size - output->pos; - size_t const result = ZSTD_compressStream_generic(zcs, - (char*)(output->dst) + output->pos, &sizeWritten, - &srcSize, &srcSize, /* use a valid src address instead of NULL */ - zsf_flush); - output->pos += sizeWritten; - if (ZSTD_isError(result)) return result; - return zcs->outBuffContentSize - zcs->outBuffFlushedSize; /* remaining to flush */ + ZSTD_inBuffer input = { NULL, 0, 0 }; + if (output->pos > output->size) return ERROR(GENERIC); + CHECK_F( ZSTD_compressStream_generic(zcs, output, &input, ZSTD_e_flush) ); + return zcs->outBuffContentSize - zcs->outBuffFlushedSize; /* remaining to flush */ } size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) { - BYTE* const ostart = (BYTE*)(output->dst) + output->pos; - BYTE* const oend = (BYTE*)(output->dst) + output->size; - BYTE* op = ostart; - - if (zcs->stage != zcss_final) { - /* flush whatever remains */ - size_t srcSize = 0; - size_t sizeWritten = output->size - output->pos; - size_t const notEnded = ZSTD_compressStream_generic(zcs, ostart, &sizeWritten, - &srcSize /* use a valid src address instead of NULL */, &srcSize, zsf_end); - size_t const remainingToFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize; - op += sizeWritten; - if (remainingToFlush) { - output->pos += sizeWritten; - return remainingToFlush + ZSTD_BLOCKHEADERSIZE /* final empty block */ + (zcs->checksum * 4); - } - /* create epilogue */ - zcs->stage = zcss_final; - zcs->outBuffContentSize = !notEnded ? 0 : - /* write epilogue, including final empty block, into outBuff */ - ZSTD_compressEnd(zcs->cctx, zcs->outBuff, zcs->outBuffSize, NULL, 0); - if (ZSTD_isError(zcs->outBuffContentSize)) return zcs->outBuffContentSize; - } - - /* flush epilogue */ - { size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize; - size_t const flushed = ZSTD_limitCopy(op, oend-op, zcs->outBuff + zcs->outBuffFlushedSize, toFlush); - op += flushed; - zcs->outBuffFlushedSize += flushed; - output->pos += op-ostart; - if (toFlush==flushed) zcs->stage = zcss_init; /* end reached */ - return toFlush - flushed; + ZSTD_inBuffer input = { NULL, 0, 0 }; + if (output->pos > output->size) return ERROR(GENERIC); + CHECK_F( ZSTD_compressStream_generic(zcs, output, &input, ZSTD_e_end) ); + { size_t const lastBlockSize = zcs->frameEnded ? 0 : ZSTD_BLOCKHEADERSIZE; + size_t const checksumSize = zcs->frameEnded ? 0 : zcs->appliedParams.fParams.checksumFlag * 4; + size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize + lastBlockSize + checksumSize; + DEBUGLOG(5, "ZSTD_endStream : remaining to flush : %u", + (unsigned)toFlush); + return toFlush; } } - /*-===== Pre-defined compression levels =====-*/ -#define ZSTD_DEFAULT_CLEVEL 1 #define ZSTD_MAX_CLEVEL 22 int ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; } static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEVEL+1] = { -{ /* "default" */ +{ /* "default" - guarantees a monotonically increasing memory budget */ /* W, C, H, S, L, TL, strat */ { 18, 12, 12, 1, 7, 16, ZSTD_fast }, /* level 0 - never used */ { 19, 13, 14, 1, 7, 16, ZSTD_fast }, /* level 1 */ { 19, 15, 16, 1, 6, 16, ZSTD_fast }, /* level 2 */ - { 20, 16, 17, 1, 5, 16, ZSTD_dfast }, /* level 3.*/ - { 20, 18, 18, 1, 5, 16, ZSTD_dfast }, /* level 4.*/ - { 20, 15, 18, 3, 5, 16, ZSTD_greedy }, /* level 5 */ - { 21, 16, 19, 2, 5, 16, ZSTD_lazy }, /* level 6 */ - { 21, 17, 20, 3, 5, 16, ZSTD_lazy }, /* level 7 */ + { 20, 16, 17, 1, 5, 16, ZSTD_dfast }, /* level 3 */ + { 20, 17, 18, 1, 5, 16, ZSTD_dfast }, /* level 4 */ + { 20, 17, 18, 2, 5, 16, ZSTD_greedy }, /* level 5 */ + { 21, 17, 19, 2, 5, 16, ZSTD_lazy }, /* level 6 */ + { 21, 18, 19, 3, 5, 16, ZSTD_lazy }, /* level 7 */ { 21, 18, 20, 3, 5, 16, ZSTD_lazy2 }, /* level 8 */ - { 21, 20, 20, 3, 5, 16, ZSTD_lazy2 }, /* level 9 */ + { 21, 19, 20, 3, 5, 16, ZSTD_lazy2 }, /* level 9 */ { 21, 19, 21, 4, 5, 16, ZSTD_lazy2 }, /* level 10 */ { 22, 20, 22, 4, 5, 16, ZSTD_lazy2 }, /* level 11 */ { 22, 20, 22, 5, 5, 16, ZSTD_lazy2 }, /* level 12 */ { 22, 21, 22, 5, 5, 16, ZSTD_lazy2 }, /* level 13 */ { 22, 21, 22, 6, 5, 16, ZSTD_lazy2 }, /* level 14 */ - { 22, 21, 21, 5, 5, 16, ZSTD_btlazy2 }, /* level 15 */ + { 22, 21, 22, 5, 5, 16, ZSTD_btlazy2 }, /* level 15 */ { 23, 22, 22, 5, 5, 16, ZSTD_btlazy2 }, /* level 16 */ - { 23, 21, 22, 4, 5, 24, ZSTD_btopt }, /* level 17 */ + { 23, 22, 22, 4, 5, 24, ZSTD_btopt }, /* level 17 */ { 23, 22, 22, 5, 4, 32, ZSTD_btopt }, /* level 18 */ { 23, 23, 22, 6, 3, 48, ZSTD_btopt }, /* level 19 */ - { 25, 25, 23, 7, 3, 64, ZSTD_btopt2 }, /* level 20 */ - { 26, 26, 23, 7, 3,256, ZSTD_btopt2 }, /* level 21 */ - { 27, 27, 25, 9, 3,512, ZSTD_btopt2 }, /* level 22 */ + { 25, 25, 23, 7, 3, 64, ZSTD_btultra }, /* level 20 */ + { 26, 26, 24, 7, 3,256, ZSTD_btultra }, /* level 21 */ + { 27, 27, 25, 9, 3,512, ZSTD_btultra }, /* level 22 */ }, { /* for srcSize <= 256 KB */ /* W, C, H, S, L, T, strat */ @@ -3507,9 +4082,9 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV { 18, 19, 18, 8, 3, 64, ZSTD_btopt }, /* level 17.*/ { 18, 19, 18, 9, 3,128, ZSTD_btopt }, /* level 18.*/ { 18, 19, 18, 10, 3,256, ZSTD_btopt }, /* level 19.*/ - { 18, 19, 18, 11, 3,512, ZSTD_btopt2 }, /* level 20.*/ - { 18, 19, 18, 12, 3,512, ZSTD_btopt2 }, /* level 21.*/ - { 18, 19, 18, 13, 3,512, ZSTD_btopt2 }, /* level 22.*/ + { 18, 19, 18, 11, 3,512, ZSTD_btultra }, /* level 20.*/ + { 18, 19, 18, 12, 3,512, ZSTD_btultra }, /* level 21.*/ + { 18, 19, 18, 13, 3,512, ZSTD_btultra }, /* level 22.*/ }, { /* for srcSize <= 128 KB */ /* W, C, H, S, L, T, strat */ @@ -3533,9 +4108,9 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV { 17, 18, 17, 7, 3, 64, ZSTD_btopt }, /* level 17.*/ { 17, 18, 17, 7, 3,256, ZSTD_btopt }, /* level 18.*/ { 17, 18, 17, 8, 3,256, ZSTD_btopt }, /* level 19.*/ - { 17, 18, 17, 9, 3,256, ZSTD_btopt2 }, /* level 20.*/ - { 17, 18, 17, 10, 3,256, ZSTD_btopt2 }, /* level 21.*/ - { 17, 18, 17, 11, 3,512, ZSTD_btopt2 }, /* level 22.*/ + { 17, 18, 17, 9, 3,256, ZSTD_btultra }, /* level 20.*/ + { 17, 18, 17, 10, 3,256, ZSTD_btultra }, /* level 21.*/ + { 17, 18, 17, 11, 3,512, ZSTD_btultra }, /* level 22.*/ }, { /* for srcSize <= 16 KB */ /* W, C, H, S, L, T, strat */ @@ -3559,39 +4134,59 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV { 14, 15, 15, 6, 3,128, ZSTD_btopt }, /* level 17.*/ { 14, 15, 15, 6, 3,256, ZSTD_btopt }, /* level 18.*/ { 14, 15, 15, 7, 3,256, ZSTD_btopt }, /* level 19.*/ - { 14, 15, 15, 8, 3,256, ZSTD_btopt2 }, /* level 20.*/ - { 14, 15, 15, 9, 3,256, ZSTD_btopt2 }, /* level 21.*/ - { 14, 15, 15, 10, 3,256, ZSTD_btopt2 }, /* level 22.*/ + { 14, 15, 15, 8, 3,256, ZSTD_btultra }, /* level 20.*/ + { 14, 15, 15, 9, 3,256, ZSTD_btultra }, /* level 21.*/ + { 14, 15, 15, 10, 3,256, ZSTD_btultra }, /* level 22.*/ }, }; +#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=1) +/* This function just controls + * the monotonic memory budget increase of ZSTD_defaultCParameters[0]. + * Run once, on first ZSTD_getCParams() usage, if ZSTD_DEBUG is enabled + */ +MEM_STATIC void ZSTD_check_compressionLevel_monotonicIncrease_memoryBudget(void) +{ + int level; + for (level=1; level<ZSTD_maxCLevel(); level++) { + ZSTD_compressionParameters const c1 = ZSTD_defaultCParameters[0][level]; + ZSTD_compressionParameters const c2 = ZSTD_defaultCParameters[0][level+1]; + assert(c1.windowLog <= c2.windowLog); +# define ZSTD_TABLECOST(h,c) ((1<<(h)) + (1<<(c))) + assert(ZSTD_TABLECOST(c1.hashLog, c1.chainLog) <= ZSTD_TABLECOST(c2.hashLog, c2.chainLog)); + } +} +#endif + /*! ZSTD_getCParams() : * @return ZSTD_compressionParameters structure for a selected compression level, `srcSize` and `dictSize`. * Size values are optional, provide 0 if not known or unused */ -ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long srcSize, size_t dictSize) +ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize) { - ZSTD_compressionParameters cp; - size_t const addedSize = srcSize ? 0 : 500; - U64 const rSize = srcSize+dictSize ? srcSize+dictSize+addedSize : (U64)-1; + size_t const addedSize = srcSizeHint ? 0 : 500; + U64 const rSize = srcSizeHint+dictSize ? srcSizeHint+dictSize+addedSize : (U64)-1; U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB); /* intentional underflow for srcSizeHint == 0 */ - if (compressionLevel <= 0) compressionLevel = ZSTD_DEFAULT_CLEVEL; /* 0 == default; no negative compressionLevel yet */ - if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL; - cp = ZSTD_defaultCParameters[tableID][compressionLevel]; - if (MEM_32bits()) { /* auto-correction, for 32-bits mode */ - if (cp.windowLog > ZSTD_WINDOWLOG_MAX) cp.windowLog = ZSTD_WINDOWLOG_MAX; - if (cp.chainLog > ZSTD_CHAINLOG_MAX) cp.chainLog = ZSTD_CHAINLOG_MAX; - if (cp.hashLog > ZSTD_HASHLOG_MAX) cp.hashLog = ZSTD_HASHLOG_MAX; + +#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=1) + static int g_monotonicTest = 1; + if (g_monotonicTest) { + ZSTD_check_compressionLevel_monotonicIncrease_memoryBudget(); + g_monotonicTest=0; } - cp = ZSTD_adjustCParams(cp, srcSize, dictSize); - return cp; +#endif + + if (compressionLevel <= 0) compressionLevel = ZSTD_CLEVEL_DEFAULT; /* 0 == default; no negative compressionLevel yet */ + if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL; + { ZSTD_compressionParameters const cp = ZSTD_defaultCParameters[tableID][compressionLevel]; + return ZSTD_adjustCParams_internal(cp, srcSizeHint, dictSize); } } /*! ZSTD_getParams() : * same as ZSTD_getCParams(), but @return a `ZSTD_parameters` object (instead of `ZSTD_compressionParameters`). * All fields of `ZSTD_frameParameters` are set to default (0) */ -ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSize, size_t dictSize) { +ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize) { ZSTD_parameters params; - ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, srcSize, dictSize); + ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, srcSizeHint, dictSize); memset(¶ms, 0, sizeof(params)); params.cParams = cParams; return params; diff --git a/thirdparty/zstd/compress/zstd_opt.h b/thirdparty/zstd/compress/zstd_opt.h index 5437611912..e8e98915ea 100644 --- a/thirdparty/zstd/compress/zstd_opt.h +++ b/thirdparty/zstd/compress/zstd_opt.h @@ -43,6 +43,7 @@ MEM_STATIC void ZSTD_rescaleFreqs(seqStore_t* ssPtr, const BYTE* src, size_t src if (ssPtr->litLengthSum == 0) { if (srcSize <= 1024) ssPtr->staticPrices = 1; + assert(ssPtr->litFreq!=NULL); for (u=0; u<=MaxLit; u++) ssPtr->litFreq[u] = 0; for (u=0; u<srcSize; u++) @@ -201,6 +202,20 @@ MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const B } +/* function safe only for comparisons */ +MEM_STATIC U32 ZSTD_readMINMATCH(const void* memPtr, U32 length) +{ + switch (length) + { + default : + case 4 : return MEM_read32(memPtr); + case 3 : if (MEM_isLittleEndian()) + return MEM_read32(memPtr)<<8; + else + return MEM_read32(memPtr)>>8; + } +} + /* Update hashTable3 up to ip (excluded) Assumption : always within prefix (i.e. not within extDict) */ @@ -234,12 +249,12 @@ static U32 ZSTD_insertBtAndGetAllMatches ( { const BYTE* const base = zc->base; const U32 current = (U32)(ip-base); - const U32 hashLog = zc->params.cParams.hashLog; + const U32 hashLog = zc->appliedParams.cParams.hashLog; const size_t h = ZSTD_hashPtr(ip, hashLog, mls); U32* const hashTable = zc->hashTable; U32 matchIndex = hashTable[h]; U32* const bt = zc->chainTable; - const U32 btLog = zc->params.cParams.chainLog - 1; + const U32 btLog = zc->appliedParams.cParams.chainLog - 1; const U32 btMask= (1U << btLog) - 1; size_t commonLengthSmaller=0, commonLengthLarger=0; const BYTE* const dictBase = zc->dictBase; @@ -267,7 +282,7 @@ static U32 ZSTD_insertBtAndGetAllMatches ( if (match[bestLength] == ip[bestLength]) currentMl = ZSTD_count(ip, match, iLimit); } else { match = dictBase + matchIndex3; - if (MEM_readMINMATCH(match, MINMATCH) == MEM_readMINMATCH(ip, MINMATCH)) /* assumption : matchIndex3 <= dictLimit-4 (by table construction) */ + if (ZSTD_readMINMATCH(match, MINMATCH) == ZSTD_readMINMATCH(ip, MINMATCH)) /* assumption : matchIndex3 <= dictLimit-4 (by table construction) */ currentMl = ZSTD_count_2segments(ip+MINMATCH, match+MINMATCH, iLimit, dictEnd, prefixStart) + MINMATCH; } @@ -410,10 +425,10 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, const BYTE* const base = ctx->base; const BYTE* const prefixStart = base + ctx->dictLimit; - const U32 maxSearches = 1U << ctx->params.cParams.searchLog; - const U32 sufficient_len = ctx->params.cParams.targetLength; - const U32 mls = ctx->params.cParams.searchLength; - const U32 minMatch = (ctx->params.cParams.searchLength == 3) ? 3 : 4; + const U32 maxSearches = 1U << ctx->appliedParams.cParams.searchLog; + const U32 sufficient_len = ctx->appliedParams.cParams.targetLength; + const U32 mls = ctx->appliedParams.cParams.searchLength; + const U32 minMatch = (ctx->appliedParams.cParams.searchLength == 3) ? 3 : 4; ZSTD_optimal_t* opt = seqStorePtr->priceTable; ZSTD_match_t* matches = seqStorePtr->matchTable; @@ -439,7 +454,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, for (i=(ip == anchor); i<last_i; i++) { const S32 repCur = (i==ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : rep[i]; if ( (repCur > 0) && (repCur < (S32)(ip-prefixStart)) - && (MEM_readMINMATCH(ip, minMatch) == MEM_readMINMATCH(ip - repCur, minMatch))) { + && (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(ip - repCur, minMatch))) { mlen = (U32)ZSTD_count(ip+minMatch, ip+minMatch-repCur, iend) + minMatch; if (mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) { best_mlen = mlen; best_off = i; cur = 0; last_pos = 1; @@ -524,7 +539,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, for (i=(opt[cur].mlen != 1); i<last_i; i++) { /* check rep */ const S32 repCur = (i==ZSTD_REP_MOVE_OPT) ? (opt[cur].rep[0] - 1) : opt[cur].rep[i]; if ( (repCur > 0) && (repCur < (S32)(inr-prefixStart)) - && (MEM_readMINMATCH(inr, minMatch) == MEM_readMINMATCH(inr - repCur, minMatch))) { + && (ZSTD_readMINMATCH(inr, minMatch) == ZSTD_readMINMATCH(inr - repCur, minMatch))) { mlen = (U32)ZSTD_count(inr+minMatch, inr+minMatch - repCur, iend) + minMatch; if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) { @@ -663,10 +678,10 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, const BYTE* const dictBase = ctx->dictBase; const BYTE* const dictEnd = dictBase + dictLimit; - const U32 maxSearches = 1U << ctx->params.cParams.searchLog; - const U32 sufficient_len = ctx->params.cParams.targetLength; - const U32 mls = ctx->params.cParams.searchLength; - const U32 minMatch = (ctx->params.cParams.searchLength == 3) ? 3 : 4; + const U32 maxSearches = 1U << ctx->appliedParams.cParams.searchLog; + const U32 sufficient_len = ctx->appliedParams.cParams.targetLength; + const U32 mls = ctx->appliedParams.cParams.searchLength; + const U32 minMatch = (ctx->appliedParams.cParams.searchLength == 3) ? 3 : 4; ZSTD_optimal_t* opt = seqStorePtr->priceTable; ZSTD_match_t* matches = seqStorePtr->matchTable; @@ -698,7 +713,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, const BYTE* const repMatch = repBase + repIndex; if ( (repCur > 0 && repCur <= (S32)current) && (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex>lowestIndex)) /* intentional overflow */ - && (MEM_readMINMATCH(ip, minMatch) == MEM_readMINMATCH(repMatch, minMatch)) ) { + && (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) ) { /* repcode detected we should take it */ const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; mlen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iend, repEnd, prefixStart) + minMatch; @@ -794,7 +809,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, const BYTE* const repMatch = repBase + repIndex; if ( (repCur > 0 && repCur <= (S32)(current+cur)) && (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex>lowestIndex)) /* intentional overflow */ - && (MEM_readMINMATCH(inr, minMatch) == MEM_readMINMATCH(repMatch, minMatch)) ) { + && (ZSTD_readMINMATCH(inr, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) ) { /* repcode detected */ const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; mlen = (U32)ZSTD_count_2segments(inr+minMatch, repMatch+minMatch, iend, repEnd, prefixStart) + minMatch; diff --git a/thirdparty/zstd/compress/zstdmt_compress.c b/thirdparty/zstd/compress/zstdmt_compress.c index fc7f52a290..0cee01eacb 100644 --- a/thirdparty/zstd/compress/zstdmt_compress.c +++ b/thirdparty/zstd/compress/zstdmt_compress.c @@ -14,34 +14,31 @@ /* ====== Compiler specifics ====== */ #if defined(_MSC_VER) -# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */ +# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */ #endif /* ====== Dependencies ====== */ -#include <stdlib.h> /* malloc */ -#include <string.h> /* memcpy */ -#include "pool.h" /* threadpool */ -#include "threading.h" /* mutex */ -#include "zstd_internal.h" /* MIN, ERROR, ZSTD_*, ZSTD_highbit32 */ +#include <string.h> /* memcpy, memset */ +#include "pool.h" /* threadpool */ +#include "threading.h" /* mutex */ +#include "zstd_internal.h" /* MIN, ERROR, ZSTD_*, ZSTD_highbit32 */ #include "zstdmt_compress.h" /* ====== Debug ====== */ -#if 0 +#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=2) # include <stdio.h> # include <unistd.h> # include <sys/times.h> - static unsigned g_debugLevel = 5; -# define DEBUGLOGRAW(l, ...) if (l<=g_debugLevel) { fprintf(stderr, __VA_ARGS__); } -# define DEBUGLOG(l, ...) if (l<=g_debugLevel) { fprintf(stderr, __FILE__ ": "); fprintf(stderr, __VA_ARGS__); fprintf(stderr, " \n"); } +# define DEBUGLOGRAW(l, ...) if (l<=ZSTD_DEBUG) { fprintf(stderr, __VA_ARGS__); } -# define DEBUG_PRINTHEX(l,p,n) { \ - unsigned debug_u; \ - for (debug_u=0; debug_u<(n); debug_u++) \ +# define DEBUG_PRINTHEX(l,p,n) { \ + unsigned debug_u; \ + for (debug_u=0; debug_u<(n); debug_u++) \ DEBUGLOGRAW(l, "%02X ", ((const unsigned char*)(p))[debug_u]); \ - DEBUGLOGRAW(l, " \n"); \ + DEBUGLOGRAW(l, " \n"); \ } static unsigned long long GetCurrentClockTimeMicroseconds(void) @@ -53,22 +50,22 @@ static unsigned long long GetCurrentClockTimeMicroseconds(void) return ((((unsigned long long)newTicks)*(1000000))/_ticksPerSecond); } } -#define MUTEX_WAIT_TIME_DLEVEL 5 -#define PTHREAD_MUTEX_LOCK(mutex) \ -if (g_debugLevel>=MUTEX_WAIT_TIME_DLEVEL) { \ - unsigned long long const beforeTime = GetCurrentClockTimeMicroseconds(); \ - pthread_mutex_lock(mutex); \ - { unsigned long long const afterTime = GetCurrentClockTimeMicroseconds(); \ - unsigned long long const elapsedTime = (afterTime-beforeTime); \ - if (elapsedTime > 1000) { /* or whatever threshold you like; I'm using 1 millisecond here */ \ - DEBUGLOG(MUTEX_WAIT_TIME_DLEVEL, "Thread took %llu microseconds to acquire mutex %s \n", \ - elapsedTime, #mutex); \ - } } \ -} else pthread_mutex_lock(mutex); +#define MUTEX_WAIT_TIME_DLEVEL 6 +#define PTHREAD_MUTEX_LOCK(mutex) { \ + if (ZSTD_DEBUG>=MUTEX_WAIT_TIME_DLEVEL) { \ + unsigned long long const beforeTime = GetCurrentClockTimeMicroseconds(); \ + pthread_mutex_lock(mutex); \ + { unsigned long long const afterTime = GetCurrentClockTimeMicroseconds(); \ + unsigned long long const elapsedTime = (afterTime-beforeTime); \ + if (elapsedTime > 1000) { /* or whatever threshold you like; I'm using 1 millisecond here */ \ + DEBUGLOG(MUTEX_WAIT_TIME_DLEVEL, "Thread took %llu microseconds to acquire mutex %s \n", \ + elapsedTime, #mutex); \ + } } \ + } else pthread_mutex_lock(mutex); \ +} #else -# define DEBUGLOG(l, ...) {} /* disabled */ # define PTHREAD_MUTEX_LOCK(m) pthread_mutex_lock(m) # define DEBUG_PRINTHEX(l,p,n) {} @@ -87,16 +84,19 @@ static const buffer_t g_nullBuffer = { NULL, 0 }; typedef struct ZSTDMT_bufferPool_s { unsigned totalBuffers; unsigned nbBuffers; + ZSTD_customMem cMem; buffer_t bTable[1]; /* variable size */ } ZSTDMT_bufferPool; -static ZSTDMT_bufferPool* ZSTDMT_createBufferPool(unsigned nbThreads) +static ZSTDMT_bufferPool* ZSTDMT_createBufferPool(unsigned nbThreads, ZSTD_customMem cMem) { unsigned const maxNbBuffers = 2*nbThreads + 2; - ZSTDMT_bufferPool* const bufPool = (ZSTDMT_bufferPool*)calloc(1, sizeof(ZSTDMT_bufferPool) + (maxNbBuffers-1) * sizeof(buffer_t)); + ZSTDMT_bufferPool* const bufPool = (ZSTDMT_bufferPool*)ZSTD_calloc( + sizeof(ZSTDMT_bufferPool) + (maxNbBuffers-1) * sizeof(buffer_t), cMem); if (bufPool==NULL) return NULL; bufPool->totalBuffers = maxNbBuffers; bufPool->nbBuffers = 0; + bufPool->cMem = cMem; return bufPool; } @@ -105,23 +105,39 @@ static void ZSTDMT_freeBufferPool(ZSTDMT_bufferPool* bufPool) unsigned u; if (!bufPool) return; /* compatibility with free on NULL */ for (u=0; u<bufPool->totalBuffers; u++) - free(bufPool->bTable[u].start); - free(bufPool); + ZSTD_free(bufPool->bTable[u].start, bufPool->cMem); + ZSTD_free(bufPool, bufPool->cMem); +} + +/* only works at initialization, not during compression */ +static size_t ZSTDMT_sizeof_bufferPool(ZSTDMT_bufferPool* bufPool) +{ + size_t const poolSize = sizeof(*bufPool) + + (bufPool->totalBuffers - 1) * sizeof(buffer_t); + unsigned u; + size_t totalBufferSize = 0; + for (u=0; u<bufPool->totalBuffers; u++) + totalBufferSize += bufPool->bTable[u].size; + + return poolSize + totalBufferSize; } -/* assumption : invocation from main thread only ! */ +/** ZSTDMT_getBuffer() : + * assumption : invocation from main thread only ! */ static buffer_t ZSTDMT_getBuffer(ZSTDMT_bufferPool* pool, size_t bSize) { if (pool->nbBuffers) { /* try to use an existing buffer */ buffer_t const buf = pool->bTable[--(pool->nbBuffers)]; size_t const availBufferSize = buf.size; - if ((availBufferSize >= bSize) & (availBufferSize <= 10*bSize)) /* large enough, but not too much */ + if ((availBufferSize >= bSize) & (availBufferSize <= 10*bSize)) + /* large enough, but not too much */ return buf; - free(buf.start); /* size conditions not respected : scratch this buffer and create a new one */ + /* size conditions not respected : scratch this buffer, create new one */ + ZSTD_free(buf.start, pool->cMem); } /* create new buffer */ { buffer_t buffer; - void* const start = malloc(bSize); + void* const start = ZSTD_malloc(bSize, pool->cMem); if (start==NULL) bSize = 0; buffer.start = start; /* note : start can be NULL if malloc fails ! */ buffer.size = bSize; @@ -138,7 +154,7 @@ static void ZSTDMT_releaseBuffer(ZSTDMT_bufferPool* pool, buffer_t buf) return; } /* Reached bufferPool capacity (should not happen) */ - free(buf.start); + ZSTD_free(buf.start, pool->cMem); } @@ -147,6 +163,7 @@ static void ZSTDMT_releaseBuffer(ZSTDMT_bufferPool* pool, buffer_t buf) typedef struct { unsigned totalCCtx; unsigned availCCtx; + ZSTD_customMem cMem; ZSTD_CCtx* cctx[1]; /* variable size */ } ZSTDMT_CCtxPool; @@ -158,23 +175,40 @@ static void ZSTDMT_freeCCtxPool(ZSTDMT_CCtxPool* pool) unsigned u; for (u=0; u<pool->totalCCtx; u++) ZSTD_freeCCtx(pool->cctx[u]); /* note : compatible with free on NULL */ - free(pool); + ZSTD_free(pool, pool->cMem); } /* ZSTDMT_createCCtxPool() : * implies nbThreads >= 1 , checked by caller ZSTDMT_createCCtx() */ -static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(unsigned nbThreads) +static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(unsigned nbThreads, + ZSTD_customMem cMem) { - ZSTDMT_CCtxPool* const cctxPool = (ZSTDMT_CCtxPool*) calloc(1, sizeof(ZSTDMT_CCtxPool) + (nbThreads-1)*sizeof(ZSTD_CCtx*)); + ZSTDMT_CCtxPool* const cctxPool = (ZSTDMT_CCtxPool*) ZSTD_calloc( + sizeof(ZSTDMT_CCtxPool) + (nbThreads-1)*sizeof(ZSTD_CCtx*), cMem); if (!cctxPool) return NULL; + cctxPool->cMem = cMem; cctxPool->totalCCtx = nbThreads; cctxPool->availCCtx = 1; /* at least one cctx for single-thread mode */ - cctxPool->cctx[0] = ZSTD_createCCtx(); + cctxPool->cctx[0] = ZSTD_createCCtx_advanced(cMem); if (!cctxPool->cctx[0]) { ZSTDMT_freeCCtxPool(cctxPool); return NULL; } - DEBUGLOG(1, "cctxPool created, with %u threads", nbThreads); + DEBUGLOG(3, "cctxPool created, with %u threads", nbThreads); return cctxPool; } +/* only works during initialization phase, not during compression */ +static size_t ZSTDMT_sizeof_CCtxPool(ZSTDMT_CCtxPool* cctxPool) +{ + unsigned const nbThreads = cctxPool->totalCCtx; + size_t const poolSize = sizeof(*cctxPool) + + (nbThreads-1)*sizeof(ZSTD_CCtx*); + unsigned u; + size_t totalCCtxSize = 0; + for (u=0; u<nbThreads; u++) + totalCCtxSize += ZSTD_sizeof_CCtx(cctxPool->cctx[u]); + + return poolSize + totalCCtxSize; +} + static ZSTD_CCtx* ZSTDMT_getCCtx(ZSTDMT_CCtxPool* pool) { if (pool->availCCtx) { @@ -218,7 +252,7 @@ typedef struct { pthread_mutex_t* jobCompleted_mutex; pthread_cond_t* jobCompleted_cond; ZSTD_parameters params; - ZSTD_CDict* cdict; + const ZSTD_CDict* cdict; unsigned long long fullFrameSize; } ZSTDMT_jobDescription; @@ -228,11 +262,11 @@ void ZSTDMT_compressChunk(void* jobDescription) ZSTDMT_jobDescription* const job = (ZSTDMT_jobDescription*)jobDescription; const void* const src = (const char*)job->srcStart + job->dictSize; buffer_t const dstBuff = job->dstBuff; - DEBUGLOG(3, "job (first:%u) (last:%u) : dictSize %u, srcSize %u", + DEBUGLOG(5, "job (first:%u) (last:%u) : dictSize %u, srcSize %u", job->firstChunk, job->lastChunk, (U32)job->dictSize, (U32)job->srcSize); if (job->cdict) { /* should only happen for first segment */ size_t const initError = ZSTD_compressBegin_usingCDict_advanced(job->cctx, job->cdict, job->params.fParams, job->fullFrameSize); - if (job->cdict) DEBUGLOG(3, "using CDict "); + DEBUGLOG(5, "using CDict"); if (ZSTD_isError(initError)) { job->cSize = initError; goto _endJob; } } else { /* srcStart points at reloaded section */ if (!job->firstChunk) job->params.fParams.contentSizeFlag = 0; /* ensure no srcSize control */ @@ -247,12 +281,12 @@ void ZSTDMT_compressChunk(void* jobDescription) ZSTD_invalidateRepCodes(job->cctx); } - DEBUGLOG(4, "Compressing : "); + DEBUGLOG(5, "Compressing : "); DEBUG_PRINTHEX(4, job->srcStart, 12); job->cSize = (job->lastChunk) ? ZSTD_compressEnd (job->cctx, dstBuff.start, dstBuff.size, src, job->srcSize) : ZSTD_compressContinue(job->cctx, dstBuff.start, dstBuff.size, src, job->srcSize); - DEBUGLOG(3, "compressed %u bytes into %u bytes (first:%u) (last:%u)", + DEBUGLOG(5, "compressed %u bytes into %u bytes (first:%u) (last:%u)", (unsigned)job->srcSize, (unsigned)job->cSize, job->firstChunk, job->lastChunk); DEBUGLOG(5, "dstBuff.size : %u ; => %s", (U32)dstBuff.size, ZSTD_getErrorName(job->cSize)); @@ -271,6 +305,7 @@ _endJob: struct ZSTDMT_CCtx_s { POOL_ctx* factory; + ZSTDMT_jobDescription* jobs; ZSTDMT_bufferPool* buffPool; ZSTDMT_CCtxPool* cctxPool; pthread_mutex_t jobCompleted_mutex; @@ -292,50 +327,64 @@ struct ZSTDMT_CCtx_s { unsigned overlapRLog; unsigned long long frameContentSize; size_t sectionSize; - ZSTD_CDict* cdict; - ZSTD_CStream* cstream; - ZSTDMT_jobDescription jobs[1]; /* variable size (must lies at the end) */ + ZSTD_customMem cMem; + ZSTD_CDict* cdictLocal; + const ZSTD_CDict* cdict; }; -ZSTDMT_CCtx *ZSTDMT_createCCtx(unsigned nbThreads) +static ZSTDMT_jobDescription* ZSTDMT_allocJobsTable(U32* nbJobsPtr, ZSTD_customMem cMem) { - ZSTDMT_CCtx* cctx; - U32 const minNbJobs = nbThreads + 2; - U32 const nbJobsLog2 = ZSTD_highbit32(minNbJobs) + 1; + U32 const nbJobsLog2 = ZSTD_highbit32(*nbJobsPtr) + 1; U32 const nbJobs = 1 << nbJobsLog2; - DEBUGLOG(5, "nbThreads : %u ; minNbJobs : %u ; nbJobsLog2 : %u ; nbJobs : %u \n", - nbThreads, minNbJobs, nbJobsLog2, nbJobs); + *nbJobsPtr = nbJobs; + return (ZSTDMT_jobDescription*) ZSTD_calloc( + nbJobs * sizeof(ZSTDMT_jobDescription), cMem); +} + +ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbThreads, ZSTD_customMem cMem) +{ + ZSTDMT_CCtx* mtctx; + U32 nbJobs = nbThreads + 2; + DEBUGLOG(3, "ZSTDMT_createCCtx_advanced"); + if ((nbThreads < 1) | (nbThreads > ZSTDMT_NBTHREADS_MAX)) return NULL; - cctx = (ZSTDMT_CCtx*) calloc(1, sizeof(ZSTDMT_CCtx) + nbJobs*sizeof(ZSTDMT_jobDescription)); - if (!cctx) return NULL; - cctx->nbThreads = nbThreads; - cctx->jobIDMask = nbJobs - 1; - cctx->allJobsCompleted = 1; - cctx->sectionSize = 0; - cctx->overlapRLog = 3; - cctx->factory = POOL_create(nbThreads, 1); - cctx->buffPool = ZSTDMT_createBufferPool(nbThreads); - cctx->cctxPool = ZSTDMT_createCCtxPool(nbThreads); - if (!cctx->factory | !cctx->buffPool | !cctx->cctxPool) { /* one object was not created */ - ZSTDMT_freeCCtx(cctx); + if ((cMem.customAlloc!=NULL) ^ (cMem.customFree!=NULL)) + /* invalid custom allocator */ + return NULL; + + mtctx = (ZSTDMT_CCtx*) ZSTD_calloc(sizeof(ZSTDMT_CCtx), cMem); + if (!mtctx) return NULL; + mtctx->cMem = cMem; + mtctx->nbThreads = nbThreads; + mtctx->allJobsCompleted = 1; + mtctx->sectionSize = 0; + mtctx->overlapRLog = 3; + mtctx->factory = POOL_create(nbThreads, 1); + mtctx->jobs = ZSTDMT_allocJobsTable(&nbJobs, cMem); + mtctx->jobIDMask = nbJobs - 1; + mtctx->buffPool = ZSTDMT_createBufferPool(nbThreads, cMem); + mtctx->cctxPool = ZSTDMT_createCCtxPool(nbThreads, cMem); + if (!mtctx->factory | !mtctx->jobs | !mtctx->buffPool | !mtctx->cctxPool) { + ZSTDMT_freeCCtx(mtctx); return NULL; } - if (nbThreads==1) { - cctx->cstream = ZSTD_createCStream(); - if (!cctx->cstream) { - ZSTDMT_freeCCtx(cctx); return NULL; - } } - pthread_mutex_init(&cctx->jobCompleted_mutex, NULL); /* Todo : check init function return */ - pthread_cond_init(&cctx->jobCompleted_cond, NULL); - DEBUGLOG(4, "mt_cctx created, for %u threads \n", nbThreads); - return cctx; + pthread_mutex_init(&mtctx->jobCompleted_mutex, NULL); /* Todo : check init function return */ + pthread_cond_init(&mtctx->jobCompleted_cond, NULL); + DEBUGLOG(3, "mt_cctx created, for %u threads", nbThreads); + return mtctx; +} + +ZSTDMT_CCtx* ZSTDMT_createCCtx(unsigned nbThreads) +{ + return ZSTDMT_createCCtx_advanced(nbThreads, ZSTD_defaultCMem); } /* ZSTDMT_releaseAllJobResources() : - * Ensure all workers are killed first. */ + * note : ensure all workers are killed first ! */ static void ZSTDMT_releaseAllJobResources(ZSTDMT_CCtx* mtctx) { unsigned jobID; + DEBUGLOG(3, "ZSTDMT_releaseAllJobResources"); for (jobID=0; jobID <= mtctx->jobIDMask; jobID++) { ZSTDMT_releaseBuffer(mtctx->buffPool, mtctx->jobs[jobID].dstBuff); mtctx->jobs[jobID].dstBuff = g_nullBuffer; @@ -356,15 +405,26 @@ size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx) POOL_free(mtctx->factory); if (!mtctx->allJobsCompleted) ZSTDMT_releaseAllJobResources(mtctx); /* stop workers first */ ZSTDMT_freeBufferPool(mtctx->buffPool); /* release job resources into pools first */ + ZSTD_free(mtctx->jobs, mtctx->cMem); ZSTDMT_freeCCtxPool(mtctx->cctxPool); - ZSTD_freeCDict(mtctx->cdict); - ZSTD_freeCStream(mtctx->cstream); + ZSTD_freeCDict(mtctx->cdictLocal); pthread_mutex_destroy(&mtctx->jobCompleted_mutex); pthread_cond_destroy(&mtctx->jobCompleted_cond); - free(mtctx); + ZSTD_free(mtctx, mtctx->cMem); return 0; } +size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx) +{ + if (mtctx == NULL) return 0; /* supports sizeof NULL */ + return sizeof(*mtctx) + + POOL_sizeof(mtctx->factory) + + ZSTDMT_sizeof_bufferPool(mtctx->buffPool) + + (mtctx->jobIDMask+1) * sizeof(ZSTDMT_jobDescription) + + ZSTDMT_sizeof_CCtxPool(mtctx->cctxPool) + + ZSTD_sizeof_CDict(mtctx->cdictLocal); +} + size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSDTMT_parameter parameter, unsigned value) { switch(parameter) @@ -373,7 +433,7 @@ size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSDTMT_parameter parameter, mtctx->sectionSize = value; return 0; case ZSTDMT_p_overlapSectionLog : - DEBUGLOG(4, "ZSTDMT_p_overlapSectionLog : %u", value); + DEBUGLOG(5, "ZSTDMT_p_overlapSectionLog : %u", value); mtctx->overlapRLog = (value >= 9) ? 0 : 9 - value; return 0; default : @@ -386,31 +446,49 @@ size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSDTMT_parameter parameter, /* ===== Multi-threaded compression ===== */ /* ------------------------------------------ */ -size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx, +static unsigned computeNbChunks(size_t srcSize, unsigned windowLog, unsigned nbThreads) { + size_t const chunkSizeTarget = (size_t)1 << (windowLog + 2); + size_t const chunkMaxSize = chunkSizeTarget << 2; + size_t const passSizeMax = chunkMaxSize * nbThreads; + unsigned const multiplier = (unsigned)(srcSize / passSizeMax) + 1; + unsigned const nbChunksLarge = multiplier * nbThreads; + unsigned const nbChunksMax = (unsigned)(srcSize / chunkSizeTarget) + 1; + unsigned const nbChunksSmall = MIN(nbChunksMax, nbThreads); + return (multiplier>1) ? nbChunksLarge : nbChunksSmall; +} + + +size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, - int compressionLevel) + const ZSTD_CDict* cdict, + ZSTD_parameters const params, + unsigned overlapRLog) { - ZSTD_parameters params = ZSTD_getParams(compressionLevel, srcSize, 0); - U32 const overlapLog = (compressionLevel >= ZSTD_maxCLevel()) ? 0 : 3; - size_t const overlapSize = (size_t)1 << (params.cParams.windowLog - overlapLog); - size_t const chunkTargetSize = (size_t)1 << (params.cParams.windowLog + 2); - unsigned const nbChunksMax = (unsigned)(srcSize / chunkTargetSize) + 1; - unsigned nbChunks = MIN(nbChunksMax, mtctx->nbThreads); + size_t const overlapSize = (overlapRLog>=9) ? 0 : (size_t)1 << (params.cParams.windowLog - overlapRLog); + unsigned nbChunks = computeNbChunks(srcSize, params.cParams.windowLog, mtctx->nbThreads); size_t const proposedChunkSize = (srcSize + (nbChunks-1)) / nbChunks; - size_t const avgChunkSize = ((proposedChunkSize & 0x1FFFF) < 0xFFFF) ? proposedChunkSize + 0xFFFF : proposedChunkSize; /* avoid too small last block */ - size_t remainingSrcSize = srcSize; + size_t const avgChunkSize = ((proposedChunkSize & 0x1FFFF) < 0x7FFF) ? proposedChunkSize + 0xFFFF : proposedChunkSize; /* avoid too small last block */ const char* const srcStart = (const char*)src; + size_t remainingSrcSize = srcSize; unsigned const compressWithinDst = (dstCapacity >= ZSTD_compressBound(srcSize)) ? nbChunks : (unsigned)(dstCapacity / ZSTD_compressBound(avgChunkSize)); /* presumes avgChunkSize >= 256 KB, which should be the case */ size_t frameStartPos = 0, dstBufferPos = 0; - DEBUGLOG(3, "windowLog : %2u => chunkTargetSize : %u bytes ", params.cParams.windowLog, (U32)chunkTargetSize); - DEBUGLOG(2, "nbChunks : %2u (chunkSize : %u bytes) ", nbChunks, (U32)avgChunkSize); - params.fParams.contentSizeFlag = 1; - + DEBUGLOG(4, "nbChunks : %2u (chunkSize : %u bytes) ", nbChunks, (U32)avgChunkSize); if (nbChunks==1) { /* fallback to single-thread mode */ ZSTD_CCtx* const cctx = mtctx->cctxPool->cctx[0]; - return ZSTD_compressCCtx(cctx, dst, dstCapacity, src, srcSize, compressionLevel); + if (cdict) return ZSTD_compress_usingCDict_advanced(cctx, dst, dstCapacity, src, srcSize, cdict, params.fParams); + return ZSTD_compress_advanced(cctx, dst, dstCapacity, src, srcSize, NULL, 0, params); + } + assert(avgChunkSize >= 256 KB); /* condition for ZSTD_compressBound(A) + ZSTD_compressBound(B) <= ZSTD_compressBound(A+B), which is useful to avoid allocating extra buffers */ + + if (nbChunks > mtctx->jobIDMask+1) { /* enlarge job table */ + U32 nbJobs = nbChunks; + ZSTD_free(mtctx->jobs, mtctx->cMem); + mtctx->jobIDMask = 0; + mtctx->jobs = ZSTDMT_allocJobsTable(&nbJobs, mtctx->cMem); + if (mtctx->jobs==NULL) return ERROR(memory_allocation); + mtctx->jobIDMask = nbJobs - 1; } { unsigned u; @@ -425,15 +503,18 @@ size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx, if ((cctx==NULL) || (dstBuffer.start==NULL)) { mtctx->jobs[u].cSize = ERROR(memory_allocation); /* job result */ mtctx->jobs[u].jobCompleted = 1; - nbChunks = u+1; + nbChunks = u+1; /* only wait and free u jobs, instead of initially expected nbChunks ones */ break; /* let's wait for previous jobs to complete, but don't start new ones */ } mtctx->jobs[u].srcStart = srcStart + frameStartPos - dictSize; mtctx->jobs[u].dictSize = dictSize; mtctx->jobs[u].srcSize = chunkSize; + mtctx->jobs[u].cdict = mtctx->nextJobID==0 ? cdict : NULL; mtctx->jobs[u].fullFrameSize = srcSize; mtctx->jobs[u].params = params; + /* do not calculate checksum within sections, but write it in header for first section */ + if (u!=0) mtctx->jobs[u].params.fParams.checksumFlag = 0; mtctx->jobs[u].dstBuff = dstBuffer; mtctx->jobs[u].cctx = cctx; mtctx->jobs[u].firstChunk = (u==0); @@ -442,27 +523,27 @@ size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx, mtctx->jobs[u].jobCompleted_mutex = &mtctx->jobCompleted_mutex; mtctx->jobs[u].jobCompleted_cond = &mtctx->jobCompleted_cond; - DEBUGLOG(3, "posting job %u (%u bytes)", u, (U32)chunkSize); - DEBUG_PRINTHEX(3, mtctx->jobs[u].srcStart, 12); + DEBUGLOG(5, "posting job %u (%u bytes)", u, (U32)chunkSize); + DEBUG_PRINTHEX(6, mtctx->jobs[u].srcStart, 12); POOL_add(mtctx->factory, ZSTDMT_compressChunk, &mtctx->jobs[u]); frameStartPos += chunkSize; dstBufferPos += dstBufferCapacity; remainingSrcSize -= chunkSize; } } - /* note : since nbChunks <= nbThreads, all jobs should be running immediately in parallel */ + /* collect result */ { unsigned chunkID; size_t error = 0, dstPos = 0; for (chunkID=0; chunkID<nbChunks; chunkID++) { - DEBUGLOG(3, "waiting for chunk %u ", chunkID); + DEBUGLOG(5, "waiting for chunk %u ", chunkID); PTHREAD_MUTEX_LOCK(&mtctx->jobCompleted_mutex); while (mtctx->jobs[chunkID].jobCompleted==0) { - DEBUGLOG(4, "waiting for jobCompleted signal from chunk %u", chunkID); + DEBUGLOG(5, "waiting for jobCompleted signal from chunk %u", chunkID); pthread_cond_wait(&mtctx->jobCompleted_cond, &mtctx->jobCompleted_mutex); } pthread_mutex_unlock(&mtctx->jobCompleted_mutex); - DEBUGLOG(3, "ready to write chunk %u ", chunkID); + DEBUGLOG(5, "ready to write chunk %u ", chunkID); ZSTDMT_releaseCCtx(mtctx->cctxPool, mtctx->jobs[chunkID].cctx); mtctx->jobs[chunkID].cctx = NULL; @@ -470,20 +551,33 @@ size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx, { size_t const cSize = mtctx->jobs[chunkID].cSize; if (ZSTD_isError(cSize)) error = cSize; if ((!error) && (dstPos + cSize > dstCapacity)) error = ERROR(dstSize_tooSmall); - if (chunkID) { /* note : chunk 0 is already written directly into dst */ + if (chunkID) { /* note : chunk 0 is written directly at dst, which is correct position */ if (!error) - memmove((char*)dst + dstPos, mtctx->jobs[chunkID].dstBuff.start, cSize); /* may overlap if chunk decompressed within dst */ - if (chunkID >= compressWithinDst) /* otherwise, it decompresses within dst */ + memmove((char*)dst + dstPos, mtctx->jobs[chunkID].dstBuff.start, cSize); /* may overlap when chunk compressed within dst */ + if (chunkID >= compressWithinDst) { /* chunk compressed into its own buffer, which must be released */ + DEBUGLOG(5, "releasing buffer %u>=%u", chunkID, compressWithinDst); ZSTDMT_releaseBuffer(mtctx->buffPool, mtctx->jobs[chunkID].dstBuff); + } mtctx->jobs[chunkID].dstBuff = g_nullBuffer; } dstPos += cSize ; } } - if (!error) DEBUGLOG(3, "compressed size : %u ", (U32)dstPos); + if (!error) DEBUGLOG(4, "compressed size : %u ", (U32)dstPos); return error ? error : dstPos; } +} + +size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + int compressionLevel) +{ + U32 const overlapRLog = (compressionLevel >= ZSTD_maxCLevel()) ? 0 : 3; + ZSTD_parameters params = ZSTD_getParams(compressionLevel, srcSize, 0); + params.fParams.contentSizeFlag = 1; + return ZSTDMT_compress_advanced(mtctx, dst, dstCapacity, src, srcSize, NULL, params, overlapRLog); } @@ -491,12 +585,14 @@ size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx, /* ======= Streaming API ======= */ /* ====================================== */ -static void ZSTDMT_waitForAllJobsCompleted(ZSTDMT_CCtx* zcs) { +static void ZSTDMT_waitForAllJobsCompleted(ZSTDMT_CCtx* zcs) +{ + DEBUGLOG(4, "ZSTDMT_waitForAllJobsCompleted"); while (zcs->doneJobID < zcs->nextJobID) { unsigned const jobID = zcs->doneJobID & zcs->jobIDMask; PTHREAD_MUTEX_LOCK(&zcs->jobCompleted_mutex); while (zcs->jobs[jobID].jobCompleted==0) { - DEBUGLOG(4, "waiting for jobCompleted signal from chunk %u", zcs->doneJobID); /* we want to block when waiting for data to flush */ + DEBUGLOG(5, "waiting for jobCompleted signal from chunk %u", zcs->doneJobID); /* we want to block when waiting for data to flush */ pthread_cond_wait(&zcs->jobCompleted_cond, &zcs->jobCompleted_mutex); } pthread_mutex_unlock(&zcs->jobCompleted_mutex); @@ -505,33 +601,54 @@ static void ZSTDMT_waitForAllJobsCompleted(ZSTDMT_CCtx* zcs) { } -static size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs, - const void* dict, size_t dictSize, unsigned updateDict, - ZSTD_parameters params, unsigned long long pledgedSrcSize) +/** ZSTDMT_initCStream_internal() : + * internal usage only */ +size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs, + const void* dict, size_t dictSize, const ZSTD_CDict* cdict, + ZSTD_parameters params, unsigned long long pledgedSrcSize) { - ZSTD_customMem const cmem = { NULL, NULL, NULL }; - DEBUGLOG(3, "Started new compression, with windowLog : %u", params.cParams.windowLog); - if (zcs->nbThreads==1) return ZSTD_initCStream_advanced(zcs->cstream, dict, dictSize, params, pledgedSrcSize); - if (zcs->allJobsCompleted == 0) { /* previous job not correctly finished */ + DEBUGLOG(4, "ZSTDMT_initCStream_internal"); + /* params are supposed to be fully validated at this point */ + assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); + assert(!((dict) && (cdict))); /* either dict or cdict, not both */ + + if (zcs->nbThreads==1) { + DEBUGLOG(4, "single thread mode"); + return ZSTD_initCStream_internal(zcs->cctxPool->cctx[0], + dict, dictSize, cdict, + params, pledgedSrcSize); + } + + if (zcs->allJobsCompleted == 0) { /* previous compression not correctly finished */ ZSTDMT_waitForAllJobsCompleted(zcs); ZSTDMT_releaseAllJobResources(zcs); zcs->allJobsCompleted = 1; } + zcs->params = params; - if (updateDict) { - ZSTD_freeCDict(zcs->cdict); zcs->cdict = NULL; - if (dict && dictSize) { - zcs->cdict = ZSTD_createCDict_advanced(dict, dictSize, 0, params.cParams, cmem); - if (zcs->cdict == NULL) return ERROR(memory_allocation); - } } zcs->frameContentSize = pledgedSrcSize; + if (dict) { + DEBUGLOG(4,"cdictLocal: %08X", (U32)(size_t)zcs->cdictLocal); + ZSTD_freeCDict(zcs->cdictLocal); + zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, + 0 /* byRef */, ZSTD_dm_auto, /* note : a loadPrefix becomes an internal CDict */ + params.cParams, zcs->cMem); + zcs->cdict = zcs->cdictLocal; + if (zcs->cdictLocal == NULL) return ERROR(memory_allocation); + } else { + DEBUGLOG(4,"cdictLocal: %08X", (U32)(size_t)zcs->cdictLocal); + ZSTD_freeCDict(zcs->cdictLocal); + zcs->cdictLocal = NULL; + zcs->cdict = cdict; + } + zcs->targetDictSize = (zcs->overlapRLog>=9) ? 0 : (size_t)1 << (zcs->params.cParams.windowLog - zcs->overlapRLog); DEBUGLOG(4, "overlapRLog : %u ", zcs->overlapRLog); - DEBUGLOG(3, "overlap Size : %u KB", (U32)(zcs->targetDictSize>>10)); + DEBUGLOG(4, "overlap Size : %u KB", (U32)(zcs->targetDictSize>>10)); zcs->targetSectionSize = zcs->sectionSize ? zcs->sectionSize : (size_t)1 << (zcs->params.cParams.windowLog + 2); zcs->targetSectionSize = MAX(ZSTDMT_SECTION_SIZE_MIN, zcs->targetSectionSize); zcs->targetSectionSize = MAX(zcs->targetDictSize, zcs->targetSectionSize); - DEBUGLOG(3, "Section Size : %u KB", (U32)(zcs->targetSectionSize>>10)); + DEBUGLOG(4, "Section Size : %u KB", (U32)(zcs->targetSectionSize>>10)); zcs->marginSize = zcs->targetSectionSize >> 2; zcs->inBuffSize = zcs->targetDictSize + zcs->targetSectionSize + zcs->marginSize; zcs->inBuff.buffer = ZSTDMT_getBuffer(zcs->buffPool, zcs->inBuffSize); @@ -546,24 +663,39 @@ static size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs, return 0; } -size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* zcs, +size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize) { - return ZSTDMT_initCStream_internal(zcs, dict, dictSize, 1, params, pledgedSrcSize); + DEBUGLOG(5, "ZSTDMT_initCStream_advanced"); + return ZSTDMT_initCStream_internal(mtctx, dict, dictSize, NULL, params, pledgedSrcSize); } +size_t ZSTDMT_initCStream_usingCDict(ZSTDMT_CCtx* mtctx, + const ZSTD_CDict* cdict, + ZSTD_frameParameters fParams, + unsigned long long pledgedSrcSize) +{ + ZSTD_parameters params = ZSTD_getParamsFromCDict(cdict); + if (cdict==NULL) return ERROR(dictionary_wrong); /* method incompatible with NULL cdict */ + params.fParams = fParams; + return ZSTDMT_initCStream_internal(mtctx, NULL, 0 /*dictSize*/, cdict, + params, pledgedSrcSize); +} + + /* ZSTDMT_resetCStream() : * pledgedSrcSize is optional and can be zero == unknown */ size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* zcs, unsigned long long pledgedSrcSize) { - if (zcs->nbThreads==1) return ZSTD_resetCStream(zcs->cstream, pledgedSrcSize); + if (zcs->nbThreads==1) + return ZSTD_resetCStream(zcs->cctxPool->cctx[0], pledgedSrcSize); return ZSTDMT_initCStream_internal(zcs, NULL, 0, 0, zcs->params, pledgedSrcSize); } size_t ZSTDMT_initCStream(ZSTDMT_CCtx* zcs, int compressionLevel) { ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, 0); - return ZSTDMT_initCStream_internal(zcs, NULL, 0, 1, params, 0); + return ZSTDMT_initCStream_internal(zcs, NULL, 0, NULL, params, 0); } @@ -582,13 +714,16 @@ static size_t ZSTDMT_createCompressionJob(ZSTDMT_CCtx* zcs, size_t srcSize, unsi return ERROR(memory_allocation); } - DEBUGLOG(4, "preparing job %u to compress %u bytes with %u preload ", zcs->nextJobID, (U32)srcSize, (U32)zcs->dictSize); + DEBUGLOG(4, "preparing job %u to compress %u bytes with %u preload ", + zcs->nextJobID, (U32)srcSize, (U32)zcs->dictSize); zcs->jobs[jobID].src = zcs->inBuff.buffer; zcs->jobs[jobID].srcStart = zcs->inBuff.buffer.start; zcs->jobs[jobID].srcSize = srcSize; - zcs->jobs[jobID].dictSize = zcs->dictSize; /* note : zcs->inBuff.filled is presumed >= srcSize + dictSize */ + zcs->jobs[jobID].dictSize = zcs->dictSize; + assert(zcs->inBuff.filled >= srcSize + zcs->dictSize); zcs->jobs[jobID].params = zcs->params; - if (zcs->nextJobID) zcs->jobs[jobID].params.fParams.checksumFlag = 0; /* do not calculate checksum within sections, just keep it in header for first section */ + /* do not calculate checksum within sections, but write it in header for first section */ + if (zcs->nextJobID) zcs->jobs[jobID].params.fParams.checksumFlag = 0; zcs->jobs[jobID].cdict = zcs->nextJobID==0 ? zcs->cdict : NULL; zcs->jobs[jobID].fullFrameSize = zcs->frameContentSize; zcs->jobs[jobID].dstBuff = dstBuffer; @@ -603,6 +738,7 @@ static size_t ZSTDMT_createCompressionJob(ZSTDMT_CCtx* zcs, size_t srcSize, unsi /* get a new buffer for next input */ if (!endFrame) { size_t const newDictSize = MIN(srcSize + zcs->dictSize, zcs->targetDictSize); + DEBUGLOG(5, "ZSTDMT_createCompressionJob::endFrame = %u", endFrame); zcs->inBuff.buffer = ZSTDMT_getBuffer(zcs->buffPool, zcs->inBuffSize); if (zcs->inBuff.buffer.start == NULL) { /* not enough memory to allocate next input buffer */ zcs->jobs[jobID].jobCompleted = 1; @@ -611,22 +747,33 @@ static size_t ZSTDMT_createCompressionJob(ZSTDMT_CCtx* zcs, size_t srcSize, unsi ZSTDMT_releaseAllJobResources(zcs); return ERROR(memory_allocation); } - DEBUGLOG(5, "inBuff filled to %u", (U32)zcs->inBuff.filled); + DEBUGLOG(5, "inBuff currently filled to %u", (U32)zcs->inBuff.filled); zcs->inBuff.filled -= srcSize + zcs->dictSize - newDictSize; - DEBUGLOG(5, "new job : filled to %u, with %u dict and %u src", (U32)zcs->inBuff.filled, (U32)newDictSize, (U32)(zcs->inBuff.filled - newDictSize)); - memmove(zcs->inBuff.buffer.start, (const char*)zcs->jobs[jobID].srcStart + zcs->dictSize + srcSize - newDictSize, zcs->inBuff.filled); + DEBUGLOG(5, "new job : inBuff filled to %u, with %u dict and %u src", + (U32)zcs->inBuff.filled, (U32)newDictSize, + (U32)(zcs->inBuff.filled - newDictSize)); + memmove(zcs->inBuff.buffer.start, + (const char*)zcs->jobs[jobID].srcStart + zcs->dictSize + srcSize - newDictSize, + zcs->inBuff.filled); DEBUGLOG(5, "new inBuff pre-filled"); zcs->dictSize = newDictSize; - } else { + } else { /* if (endFrame==1) */ + DEBUGLOG(5, "ZSTDMT_createCompressionJob::endFrame = %u", endFrame); zcs->inBuff.buffer = g_nullBuffer; zcs->inBuff.filled = 0; zcs->dictSize = 0; zcs->frameEnded = 1; if (zcs->nextJobID == 0) - zcs->params.fParams.checksumFlag = 0; /* single chunk : checksum is calculated directly within worker thread */ + /* single chunk exception : checksum is calculated directly within worker thread */ + zcs->params.fParams.checksumFlag = 0; } - DEBUGLOG(3, "posting job %u : %u bytes (end:%u) (note : doneJob = %u=>%u)", zcs->nextJobID, (U32)zcs->jobs[jobID].srcSize, zcs->jobs[jobID].lastChunk, zcs->doneJobID, zcs->doneJobID & zcs->jobIDMask); + DEBUGLOG(4, "posting job %u : %u bytes (end:%u) (note : doneJob = %u=>%u)", + zcs->nextJobID, + (U32)zcs->jobs[jobID].srcSize, + zcs->jobs[jobID].lastChunk, + zcs->doneJobID, + zcs->doneJobID & zcs->jobIDMask); POOL_add(zcs->factory, ZSTDMT_compressChunk, &zcs->jobs[jobID]); /* this call is blocking when thread worker pool is exhausted */ zcs->nextJobID++; return 0; @@ -664,7 +811,7 @@ static size_t ZSTDMT_flushNextJob(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output, unsi XXH64_update(&zcs->xxhState, (const char*)job.srcStart + job.dictSize, job.srcSize); if (zcs->frameEnded && (zcs->doneJobID+1 == zcs->nextJobID)) { /* write checksum at end of last section */ U32 const checksum = (U32)XXH64_digest(&zcs->xxhState); - DEBUGLOG(4, "writing checksum : %08X \n", checksum); + DEBUGLOG(5, "writing checksum : %08X \n", checksum); MEM_writeLE32((char*)job.dstBuff.start + job.cSize, checksum); job.cSize += 4; zcs->jobs[wJobID].cSize += 4; @@ -675,7 +822,7 @@ static size_t ZSTDMT_flushNextJob(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output, unsi zcs->jobs[wJobID].jobScanned = 1; } { size_t const toWrite = MIN(job.cSize - job.dstFlushed, output->size - output->pos); - DEBUGLOG(4, "Flushing %u bytes from job %u ", (U32)toWrite, zcs->doneJobID); + DEBUGLOG(5, "Flushing %u bytes from job %u ", (U32)toWrite, zcs->doneJobID); memcpy((char*)output->dst + output->pos, (const char*)job.dstBuff.start + job.dstFlushed, toWrite); output->pos += toWrite; job.dstFlushed += toWrite; @@ -696,26 +843,81 @@ static size_t ZSTDMT_flushNextJob(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output, unsi } } -size_t ZSTDMT_compressStream(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input) +/** ZSTDMT_compressStream_generic() : + * internal use only + * assumption : output and input are valid (pos <= size) + * @return : minimum amount of data remaining to flush, 0 if none */ +size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx, + ZSTD_outBuffer* output, + ZSTD_inBuffer* input, + ZSTD_EndDirective endOp) { - size_t const newJobThreshold = zcs->dictSize + zcs->targetSectionSize + zcs->marginSize; - if (zcs->frameEnded) return ERROR(stage_wrong); /* current frame being ended. Only flush is allowed. Restart with init */ - if (zcs->nbThreads==1) return ZSTD_compressStream(zcs->cstream, output, input); + size_t const newJobThreshold = mtctx->dictSize + mtctx->targetSectionSize + mtctx->marginSize; + assert(output->pos <= output->size); + assert(input->pos <= input->size); + if ((mtctx->frameEnded) && (endOp==ZSTD_e_continue)) { + /* current frame being ended. Only flush/end are allowed. Or start new frame with init */ + return ERROR(stage_wrong); + } + if (mtctx->nbThreads==1) { + return ZSTD_compressStream_generic(mtctx->cctxPool->cctx[0], output, input, endOp); + } + + /* single-pass shortcut (note : this is blocking-mode) */ + if ( (mtctx->nextJobID==0) /* just started */ + && (mtctx->inBuff.filled==0) /* nothing buffered */ + && (endOp==ZSTD_e_end) /* end order */ + && (output->size - output->pos >= ZSTD_compressBound(input->size - input->pos)) ) { /* enough room */ + size_t const cSize = ZSTDMT_compress_advanced(mtctx, + (char*)output->dst + output->pos, output->size - output->pos, + (const char*)input->src + input->pos, input->size - input->pos, + mtctx->cdict, mtctx->params, mtctx->overlapRLog); + if (ZSTD_isError(cSize)) return cSize; + input->pos = input->size; + output->pos += cSize; + ZSTDMT_releaseBuffer(mtctx->buffPool, mtctx->inBuff.buffer); /* was allocated in initStream */ + mtctx->allJobsCompleted = 1; + mtctx->frameEnded = 1; + return 0; + } /* fill input buffer */ - { size_t const toLoad = MIN(input->size - input->pos, zcs->inBuffSize - zcs->inBuff.filled); - memcpy((char*)zcs->inBuff.buffer.start + zcs->inBuff.filled, input->src, toLoad); + if ((input->src) && (mtctx->inBuff.buffer.start)) { /* support NULL input */ + size_t const toLoad = MIN(input->size - input->pos, mtctx->inBuffSize - mtctx->inBuff.filled); + DEBUGLOG(2, "inBuff:%08X; inBuffSize=%u; ToCopy=%u", (U32)(size_t)mtctx->inBuff.buffer.start, (U32)mtctx->inBuffSize, (U32)toLoad); + memcpy((char*)mtctx->inBuff.buffer.start + mtctx->inBuff.filled, (const char*)input->src + input->pos, toLoad); input->pos += toLoad; - zcs->inBuff.filled += toLoad; + mtctx->inBuff.filled += toLoad; } - if ( (zcs->inBuff.filled >= newJobThreshold) /* filled enough : let's compress */ - && (zcs->nextJobID <= zcs->doneJobID + zcs->jobIDMask) ) { /* avoid overwriting job round buffer */ - CHECK_F( ZSTDMT_createCompressionJob(zcs, zcs->targetSectionSize, 0) ); + if ( (mtctx->inBuff.filled >= newJobThreshold) /* filled enough : let's compress */ + && (mtctx->nextJobID <= mtctx->doneJobID + mtctx->jobIDMask) ) { /* avoid overwriting job round buffer */ + CHECK_F( ZSTDMT_createCompressionJob(mtctx, mtctx->targetSectionSize, 0 /* endFrame */) ); } - /* check for data to flush */ - CHECK_F( ZSTDMT_flushNextJob(zcs, output, (zcs->inBuff.filled == zcs->inBuffSize)) ); /* block if it wasn't possible to create new job due to saturation */ + /* check for potential compressed data ready to be flushed */ + CHECK_F( ZSTDMT_flushNextJob(mtctx, output, (mtctx->inBuff.filled == mtctx->inBuffSize) /* blockToFlush */) ); /* block if it wasn't possible to create new job due to saturation */ + + if (input->pos < input->size) /* input not consumed : do not flush yet */ + endOp = ZSTD_e_continue; + + switch(endOp) + { + case ZSTD_e_flush: + return ZSTDMT_flushStream(mtctx, output); + case ZSTD_e_end: + return ZSTDMT_endStream(mtctx, output); + case ZSTD_e_continue: + return 1; + default: + return ERROR(GENERIC); /* invalid endDirective */ + } +} + + +size_t ZSTDMT_compressStream(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input) +{ + CHECK_F( ZSTDMT_compressStream_generic(zcs, output, input, ZSTD_e_continue) ); /* recommended next input size : fill current input buffer */ return zcs->inBuffSize - zcs->inBuff.filled; /* note : could be zero when input buffer is fully filled and no more availability to create new job */ @@ -726,26 +928,28 @@ static size_t ZSTDMT_flushStream_internal(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* outp { size_t const srcSize = zcs->inBuff.filled - zcs->dictSize; - if (srcSize) DEBUGLOG(4, "flushing : %u bytes left to compress", (U32)srcSize); if ( ((srcSize > 0) || (endFrame && !zcs->frameEnded)) && (zcs->nextJobID <= zcs->doneJobID + zcs->jobIDMask) ) { CHECK_F( ZSTDMT_createCompressionJob(zcs, srcSize, endFrame) ); } /* check if there is any data available to flush */ - DEBUGLOG(5, "zcs->doneJobID : %u ; zcs->nextJobID : %u ", zcs->doneJobID, zcs->nextJobID); - return ZSTDMT_flushNextJob(zcs, output, 1); + return ZSTDMT_flushNextJob(zcs, output, 1 /* blockToFlush */); } size_t ZSTDMT_flushStream(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output) { - if (zcs->nbThreads==1) return ZSTD_flushStream(zcs->cstream, output); - return ZSTDMT_flushStream_internal(zcs, output, 0); + DEBUGLOG(5, "ZSTDMT_flushStream"); + if (zcs->nbThreads==1) + return ZSTD_flushStream(zcs->cctxPool->cctx[0], output); + return ZSTDMT_flushStream_internal(zcs, output, 0 /* endFrame */); } size_t ZSTDMT_endStream(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output) { - if (zcs->nbThreads==1) return ZSTD_endStream(zcs->cstream, output); - return ZSTDMT_flushStream_internal(zcs, output, 1); + DEBUGLOG(4, "ZSTDMT_endStream"); + if (zcs->nbThreads==1) + return ZSTD_endStream(zcs->cctxPool->cctx[0], output); + return ZSTDMT_flushStream_internal(zcs, output, 1 /* endFrame */); } diff --git a/thirdparty/zstd/compress/zstdmt_compress.h b/thirdparty/zstd/compress/zstdmt_compress.h index 27f78ee031..fad63b6d86 100644 --- a/thirdparty/zstd/compress/zstdmt_compress.h +++ b/thirdparty/zstd/compress/zstdmt_compress.h @@ -15,25 +15,34 @@ #endif -/* Note : All prototypes defined in this file shall be considered experimental. - * There is no guarantee of API continuity (yet) on any of these prototypes */ +/* Note : All prototypes defined in this file are labelled experimental. + * No guarantee of API continuity is provided on any of them. + * In fact, the expectation is that these prototypes will be replaced + * by ZSTD_compress_generic() API in the near future */ /* === Dependencies === */ -#include <stddef.h> /* size_t */ +#include <stddef.h> /* size_t */ #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters */ -#include "zstd.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTDLIB_API */ +#include "zstd.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTDLIB_API */ -/* === Simple one-pass functions === */ - +/* === Memory management === */ typedef struct ZSTDMT_CCtx_s ZSTDMT_CCtx; ZSTDLIB_API ZSTDMT_CCtx* ZSTDMT_createCCtx(unsigned nbThreads); -ZSTDLIB_API size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* cctx); +ZSTDLIB_API ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbThreads, + ZSTD_customMem cMem); +ZSTDLIB_API size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx); + +ZSTDLIB_API size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx); + + +/* === Simple buffer-to-butter one-pass function === */ + +ZSTDLIB_API size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + int compressionLevel); -ZSTDLIB_API size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* cctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - int compressionLevel); /* === Streaming functions === */ @@ -53,8 +62,22 @@ ZSTDLIB_API size_t ZSTDMT_endStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output); # define ZSTDMT_SECTION_SIZE_MIN (1U << 20) /* 1 MB - Minimum size of each compression job */ #endif -ZSTDLIB_API size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx, const void* dict, size_t dictSize, /**< dict can be released after init, a local copy is preserved within zcs */ - ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be zero == unknown */ +ZSTDLIB_API size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict, + ZSTD_parameters const params, + unsigned overlapRLog); + +ZSTDLIB_API size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx, + const void* dict, size_t dictSize, /* dict can be released after init, a local copy is preserved within zcs */ + ZSTD_parameters params, + unsigned long long pledgedSrcSize); /* pledgedSrcSize is optional and can be zero == unknown */ + +ZSTDLIB_API size_t ZSTDMT_initCStream_usingCDict(ZSTDMT_CCtx* mtctx, + const ZSTD_CDict* cdict, + ZSTD_frameParameters fparams, + unsigned long long pledgedSrcSize); /* note : zero means empty */ /* ZSDTMT_parameter : * List of parameters that can be set using ZSTDMT_setMTCtxParameter() */ @@ -71,6 +94,19 @@ typedef enum { ZSTDLIB_API size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSDTMT_parameter parameter, unsigned value); +/*! ZSTDMT_compressStream_generic() : + * Combines ZSTDMT_compressStream() with ZSTDMT_flushStream() or ZSTDMT_endStream() + * depending on flush directive. + * @return : minimum amount of data still to be flushed + * 0 if fully flushed + * or an error code */ +ZSTDLIB_API size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx, + ZSTD_outBuffer* output, + ZSTD_inBuffer* input, + ZSTD_EndDirective endOp); + + + #if defined (__cplusplus) } #endif |