diff options
Diffstat (limited to 'thirdparty/zstd/compress/zstd_compress.c')
| -rw-r--r-- | thirdparty/zstd/compress/zstd_compress.c | 771 |
1 files changed, 434 insertions, 337 deletions
diff --git a/thirdparty/zstd/compress/zstd_compress.c b/thirdparty/zstd/compress/zstd_compress.c index 2c46c79f1c..8d1629246d 100644 --- a/thirdparty/zstd/compress/zstd_compress.c +++ b/thirdparty/zstd/compress/zstd_compress.c @@ -26,7 +26,7 @@ #include "fse.h" #define HUF_STATIC_LINKING_ONLY #include "huf.h" -#include "zstd_compress.h" +#include "zstd_compress_internal.h" #include "zstd_fast.h" #include "zstd_double_fast.h" #include "zstd_lazy.h" @@ -43,17 +43,6 @@ size_t ZSTD_compressBound(size_t srcSize) { /*-************************************* -* Sequence storage -***************************************/ -static void ZSTD_resetSeqStore(seqStore_t* ssPtr) -{ - ssPtr->lit = ssPtr->litStart; - ssPtr->sequences = ssPtr->sequencesStart; - ssPtr->longLengthID = 0; -} - - -/*-************************************* * Context memory management ***************************************/ struct ZSTD_CDict_s { @@ -78,6 +67,7 @@ ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) if (!cctx) return NULL; cctx->customMem = customMem; cctx->requestedParams.compressionLevel = ZSTD_CLEVEL_DEFAULT; + cctx->requestedParams.fParams.contentSizeFlag = 1; ZSTD_STATIC_ASSERT(zcss_init==0); ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN==(0ULL - 1)); return cctx; @@ -152,28 +142,34 @@ const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) { return &(ctx->seqStor #define ZSTD_CLEVEL_CUSTOM 999 static ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams( - ZSTD_CCtx_params params, U64 srcSizeHint, size_t dictSize) + ZSTD_CCtx_params CCtxParams, U64 srcSizeHint, size_t dictSize) { - return (params.compressionLevel == ZSTD_CLEVEL_CUSTOM ? - params.cParams : - ZSTD_getCParams(params.compressionLevel, srcSizeHint, dictSize)); + DEBUGLOG(4, "ZSTD_getCParamsFromCCtxParams: srcSize = %u, dictSize = %u", + (U32)srcSizeHint, (U32)dictSize); + return (CCtxParams.compressionLevel == ZSTD_CLEVEL_CUSTOM) ? + CCtxParams.cParams : + ZSTD_getCParams(CCtxParams.compressionLevel, srcSizeHint, dictSize); } -static void ZSTD_cLevelToCCtxParams_srcSize(ZSTD_CCtx_params* params, U64 srcSize) +static void ZSTD_cLevelToCCtxParams_srcSize(ZSTD_CCtx_params* CCtxParams, U64 srcSize) { - params->cParams = ZSTD_getCParamsFromCCtxParams(*params, srcSize, 0); - params->compressionLevel = ZSTD_CLEVEL_CUSTOM; + DEBUGLOG(4, "ZSTD_cLevelToCCtxParams_srcSize: srcSize = %u", + (U32)srcSize); + CCtxParams->cParams = ZSTD_getCParamsFromCCtxParams(*CCtxParams, srcSize, 0); + CCtxParams->compressionLevel = ZSTD_CLEVEL_CUSTOM; } static void ZSTD_cLevelToCParams(ZSTD_CCtx* cctx) { + DEBUGLOG(4, "ZSTD_cLevelToCParams: level=%i", cctx->requestedParams.compressionLevel); ZSTD_cLevelToCCtxParams_srcSize( &cctx->requestedParams, cctx->pledgedSrcSizePlusOne-1); } -static void ZSTD_cLevelToCCtxParams(ZSTD_CCtx_params* params) +static void ZSTD_cLevelToCCtxParams(ZSTD_CCtx_params* CCtxParams) { - ZSTD_cLevelToCCtxParams_srcSize(params, 0); + DEBUGLOG(4, "ZSTD_cLevelToCCtxParams"); + ZSTD_cLevelToCCtxParams_srcSize(CCtxParams, ZSTD_CONTENTSIZE_UNKNOWN); } static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromCParams( @@ -251,6 +247,7 @@ static ZSTD_CCtx_params ZSTD_assignParamsToCCtxParams( size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value) { + DEBUGLOG(4, "ZSTD_CCtx_setParameter (%u, %u)", (U32)param, value); if (cctx->streamStage != zcss_init) return ERROR(stage_wrong); switch(param) @@ -259,7 +256,6 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); case ZSTD_p_compressionLevel: - if (value == 0) return 0; /* special value : 0 means "don't change anything" */ if (cctx->cdict) return ERROR(stage_wrong); return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); @@ -270,9 +266,8 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v case ZSTD_p_minMatch: case ZSTD_p_targetLength: case ZSTD_p_compressionStrategy: - if (value == 0) return 0; /* special value : 0 means "don't change anything" */ if (cctx->cdict) return ERROR(stage_wrong); - ZSTD_cLevelToCParams(cctx); /* Can optimize if srcSize is known */ + if (value>0) ZSTD_cLevelToCParams(cctx); /* Can optimize if srcSize is known */ return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); case ZSTD_p_contentSizeFlag: @@ -281,15 +276,12 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); case ZSTD_p_forceMaxWindow : /* Force back-references to remain < windowSize, - * even when referencing into Dictionary content + * even when referencing into Dictionary content. * default : 0 when using a CDict, 1 when using a Prefix */ - cctx->loadedDictEnd = 0; return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); case ZSTD_p_nbThreads: - if (value==0) return 0; - DEBUGLOG(5, " setting nbThreads : %u", value); - if (value > 1 && cctx->staticSize) { + if ((value > 1) && cctx->staticSize) { return ERROR(parameter_unsupported); /* MT not compatible with static alloc */ } return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); @@ -298,22 +290,15 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); case ZSTD_p_overlapSizeLog: - DEBUGLOG(5, " setting overlap with nbThreads == %u", cctx->requestedParams.nbThreads); return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); case ZSTD_p_enableLongDistanceMatching: if (cctx->cdict) return ERROR(stage_wrong); - if (value != 0) { - ZSTD_cLevelToCParams(cctx); - } + if (value>0) ZSTD_cLevelToCParams(cctx); return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); case ZSTD_p_ldmHashLog: case ZSTD_p_ldmMinMatch: - if (value == 0) return 0; /* special value : 0 means "don't change anything" */ - if (cctx->cdict) return ERROR(stage_wrong); - return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); - case ZSTD_p_ldmBucketSizeLog: case ZSTD_p_ldmHashEveryLog: if (cctx->cdict) return ERROR(stage_wrong); @@ -324,160 +309,167 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v } size_t ZSTD_CCtxParam_setParameter( - ZSTD_CCtx_params* params, ZSTD_cParameter param, unsigned value) + ZSTD_CCtx_params* CCtxParams, ZSTD_cParameter param, unsigned value) { + DEBUGLOG(4, "ZSTD_CCtxParam_setParameter (%u, %u)", (U32)param, value); switch(param) { case ZSTD_p_format : if (value > (unsigned)ZSTD_f_zstd1_magicless) return ERROR(parameter_unsupported); - params->format = (ZSTD_format_e)value; - return 0; + CCtxParams->format = (ZSTD_format_e)value; + return (size_t)CCtxParams->format; case ZSTD_p_compressionLevel : if ((int)value > ZSTD_maxCLevel()) value = ZSTD_maxCLevel(); - if (value == 0) return 0; - params->compressionLevel = value; - return 0; + if (value) /* 0 : does not change current level */ + CCtxParams->compressionLevel = value; + return CCtxParams->compressionLevel; case ZSTD_p_windowLog : - if (value == 0) return 0; - CLAMPCHECK(value, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX); - ZSTD_cLevelToCCtxParams(params); - params->cParams.windowLog = value; - return 0; + DEBUGLOG(4, "ZSTD_CCtxParam_setParameter: set windowLog=%u", value); + if (value) { /* 0 : does not change current windowLog */ + CLAMPCHECK(value, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX); + ZSTD_cLevelToCCtxParams(CCtxParams); + CCtxParams->cParams.windowLog = value; + } + return CCtxParams->cParams.windowLog; case ZSTD_p_hashLog : - if (value == 0) return 0; - CLAMPCHECK(value, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX); - ZSTD_cLevelToCCtxParams(params); - params->cParams.hashLog = value; - return 0; + if (value) { /* 0 : does not change current hashLog */ + CLAMPCHECK(value, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX); + ZSTD_cLevelToCCtxParams(CCtxParams); + CCtxParams->cParams.hashLog = value; + } + return CCtxParams->cParams.hashLog; case ZSTD_p_chainLog : - if (value == 0) return 0; - CLAMPCHECK(value, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX); - ZSTD_cLevelToCCtxParams(params); - params->cParams.chainLog = value; - return 0; + if (value) { /* 0 : does not change current chainLog */ + CLAMPCHECK(value, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX); + ZSTD_cLevelToCCtxParams(CCtxParams); + CCtxParams->cParams.chainLog = value; + } + return CCtxParams->cParams.chainLog; case ZSTD_p_searchLog : - if (value == 0) return 0; - CLAMPCHECK(value, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX); - ZSTD_cLevelToCCtxParams(params); - params->cParams.searchLog = value; - return 0; + if (value) { /* 0 : does not change current searchLog */ + CLAMPCHECK(value, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX); + ZSTD_cLevelToCCtxParams(CCtxParams); + CCtxParams->cParams.searchLog = value; + } + return value; case ZSTD_p_minMatch : - if (value == 0) return 0; - CLAMPCHECK(value, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX); - ZSTD_cLevelToCCtxParams(params); - params->cParams.searchLength = value; - return 0; + if (value) { /* 0 : does not change current minMatch length */ + CLAMPCHECK(value, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX); + ZSTD_cLevelToCCtxParams(CCtxParams); + CCtxParams->cParams.searchLength = value; + } + return CCtxParams->cParams.searchLength; case ZSTD_p_targetLength : - if (value == 0) return 0; - CLAMPCHECK(value, ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX); - ZSTD_cLevelToCCtxParams(params); - params->cParams.targetLength = value; - return 0; + if (value) { /* 0 : does not change current sufficient_len */ + CLAMPCHECK(value, ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX); + ZSTD_cLevelToCCtxParams(CCtxParams); + CCtxParams->cParams.targetLength = value; + } + return CCtxParams->cParams.targetLength; case ZSTD_p_compressionStrategy : - if (value == 0) return 0; - CLAMPCHECK(value, (unsigned)ZSTD_fast, (unsigned)ZSTD_btultra); - ZSTD_cLevelToCCtxParams(params); - params->cParams.strategy = (ZSTD_strategy)value; - return 0; + if (value) { /* 0 : does not change currentstrategy */ + CLAMPCHECK(value, (unsigned)ZSTD_fast, (unsigned)ZSTD_btultra); + ZSTD_cLevelToCCtxParams(CCtxParams); + CCtxParams->cParams.strategy = (ZSTD_strategy)value; + } + return (size_t)CCtxParams->cParams.strategy; case ZSTD_p_contentSizeFlag : /* Content size written in frame header _when known_ (default:1) */ - DEBUGLOG(5, "set content size flag = %u", (value>0)); - params->fParams.contentSizeFlag = value > 0; - return 0; + DEBUGLOG(4, "set content size flag = %u", (value>0)); + CCtxParams->fParams.contentSizeFlag = value > 0; + return CCtxParams->fParams.contentSizeFlag; case ZSTD_p_checksumFlag : /* A 32-bits content checksum will be calculated and written at end of frame (default:0) */ - params->fParams.checksumFlag = value > 0; - return 0; + CCtxParams->fParams.checksumFlag = value > 0; + return CCtxParams->fParams.checksumFlag; case ZSTD_p_dictIDFlag : /* When applicable, dictionary's dictID is provided in frame header (default:1) */ - DEBUGLOG(5, "set dictIDFlag = %u", (value>0)); - params->fParams.noDictIDFlag = (value == 0); - return 0; + DEBUGLOG(4, "set dictIDFlag = %u", (value>0)); + CCtxParams->fParams.noDictIDFlag = (value == 0); + return !CCtxParams->fParams.noDictIDFlag; case ZSTD_p_forceMaxWindow : - params->forceWindow = value > 0; - return 0; + CCtxParams->forceWindow = (value > 0); + return CCtxParams->forceWindow; case ZSTD_p_nbThreads : - if (value == 0) return 0; + if (value == 0) return CCtxParams->nbThreads; #ifndef ZSTD_MULTITHREAD if (value > 1) return ERROR(parameter_unsupported); - return 0; + return 1; #else - return ZSTDMT_initializeCCtxParameters(params, value); + return ZSTDMT_CCtxParam_setNbThreads(CCtxParams, value); #endif case ZSTD_p_jobSize : #ifndef ZSTD_MULTITHREAD return ERROR(parameter_unsupported); #else - if (params->nbThreads <= 1) return ERROR(parameter_unsupported); - return ZSTDMT_CCtxParam_setMTCtxParameter(params, ZSTDMT_p_sectionSize, value); + if (CCtxParams->nbThreads <= 1) return ERROR(parameter_unsupported); + return ZSTDMT_CCtxParam_setMTCtxParameter(CCtxParams, ZSTDMT_p_jobSize, value); #endif case ZSTD_p_overlapSizeLog : #ifndef ZSTD_MULTITHREAD return ERROR(parameter_unsupported); #else - if (params->nbThreads <= 1) return ERROR(parameter_unsupported); - return ZSTDMT_CCtxParam_setMTCtxParameter(params, ZSTDMT_p_overlapSectionLog, value); + if (CCtxParams->nbThreads <= 1) return ERROR(parameter_unsupported); + return ZSTDMT_CCtxParam_setMTCtxParameter(CCtxParams, ZSTDMT_p_overlapSectionLog, value); #endif case ZSTD_p_enableLongDistanceMatching : - if (value != 0) { - ZSTD_cLevelToCCtxParams(params); - params->cParams.windowLog = ZSTD_LDM_DEFAULT_WINDOW_LOG; + if (value) { + ZSTD_cLevelToCCtxParams(CCtxParams); + CCtxParams->cParams.windowLog = ZSTD_LDM_DEFAULT_WINDOW_LOG; } - return ZSTD_ldm_initializeParameters(¶ms->ldmParams, value); + return ZSTD_ldm_initializeParameters(&CCtxParams->ldmParams, value); case ZSTD_p_ldmHashLog : - if (value == 0) return 0; - CLAMPCHECK(value, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX); - params->ldmParams.hashLog = value; - return 0; + if (value) { /* 0 : does not change current ldmHashLog */ + CLAMPCHECK(value, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX); + CCtxParams->ldmParams.hashLog = value; + } + return CCtxParams->ldmParams.hashLog; case ZSTD_p_ldmMinMatch : - if (value == 0) return 0; - CLAMPCHECK(value, ZSTD_LDM_MINMATCH_MIN, ZSTD_LDM_MINMATCH_MAX); - params->ldmParams.minMatchLength = value; - return 0; + if (value) { /* 0 : does not change current ldmMinMatch */ + CLAMPCHECK(value, ZSTD_LDM_MINMATCH_MIN, ZSTD_LDM_MINMATCH_MAX); + CCtxParams->ldmParams.minMatchLength = value; + } + return CCtxParams->ldmParams.minMatchLength; case ZSTD_p_ldmBucketSizeLog : if (value > ZSTD_LDM_BUCKETSIZELOG_MAX) { return ERROR(parameter_outOfBound); } - params->ldmParams.bucketSizeLog = value; - return 0; + CCtxParams->ldmParams.bucketSizeLog = value; + return value; case ZSTD_p_ldmHashEveryLog : if (value > ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN) { return ERROR(parameter_outOfBound); } - params->ldmParams.hashEveryLog = value; - return 0; + CCtxParams->ldmParams.hashEveryLog = value; + return value; default: return ERROR(parameter_unsupported); } } -/** - * This function should be updated whenever ZSTD_CCtx_params is updated. - * Parameters are copied manually before the dictionary is loaded. - * The multithreading parameters jobSize and overlapSizeLog are set only if - * nbThreads > 1. - * - * Pledged srcSize is treated as unknown. +/** ZSTD_CCtx_setParametersUsingCCtxParams() : + * just applies `params` into `cctx` + * no action is performed, parameters are merely stored. */ size_t ZSTD_CCtx_setParametersUsingCCtxParams( ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params) @@ -485,33 +477,14 @@ size_t ZSTD_CCtx_setParametersUsingCCtxParams( if (cctx->streamStage != zcss_init) return ERROR(stage_wrong); if (cctx->cdict) return ERROR(stage_wrong); - /* Assume the compression and frame parameters are validated */ - cctx->requestedParams.cParams = params->cParams; - cctx->requestedParams.fParams = params->fParams; - cctx->requestedParams.compressionLevel = params->compressionLevel; - - /* Set force window explicitly since it sets cctx->loadedDictEnd */ - CHECK_F( ZSTD_CCtx_setParameter( - cctx, ZSTD_p_forceMaxWindow, params->forceWindow) ); - - /* Set multithreading parameters explicitly */ - CHECK_F( ZSTD_CCtx_setParameter(cctx, ZSTD_p_nbThreads, params->nbThreads) ); - if (params->nbThreads > 1) { - CHECK_F( ZSTD_CCtx_setParameter(cctx, ZSTD_p_jobSize, params->jobSize) ); - CHECK_F( ZSTD_CCtx_setParameter( - cctx, ZSTD_p_overlapSizeLog, params->overlapSizeLog) ); - } + cctx->requestedParams = *params; - /* Copy long distance matching parameters */ - cctx->requestedParams.ldmParams = params->ldmParams; - - /* customMem is used only for create/free params and can be ignored */ return 0; } ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize) { - DEBUGLOG(4, " setting pledgedSrcSize to %u", (U32)pledgedSrcSize); + DEBUGLOG(4, "ZSTD_CCtx_setPledgedSrcSize to %u bytes", (U32)pledgedSrcSize); if (cctx->streamStage != zcss_init) return ERROR(stage_wrong); cctx->pledgedSrcSizePlusOne = pledgedSrcSize+1; return 0; @@ -523,14 +496,14 @@ size_t ZSTD_CCtx_loadDictionary_advanced( { if (cctx->streamStage != zcss_init) return ERROR(stage_wrong); if (cctx->staticSize) return ERROR(memory_allocation); /* no malloc for static CCtx */ - DEBUGLOG(4, "load dictionary of size %u", (U32)dictSize); + DEBUGLOG(4, "ZSTD_CCtx_loadDictionary_advanced (size: %u)", (U32)dictSize); ZSTD_freeCDict(cctx->cdictLocal); /* in case one already exists */ if (dict==NULL || dictSize==0) { /* no dictionary mode */ cctx->cdictLocal = NULL; cctx->cdict = NULL; } else { ZSTD_compressionParameters const cParams = - ZSTD_getCParamsFromCCtxParams(cctx->requestedParams, 0, dictSize); + ZSTD_getCParamsFromCCtxParams(cctx->requestedParams, cctx->pledgedSrcSizePlusOne-1, dictSize); cctx->cdictLocal = ZSTD_createCDict_advanced( dict, dictSize, dictLoadMethod, dictMode, @@ -756,10 +729,7 @@ size_t ZSTD_estimateCStreamSize(int compressionLevel) { static U32 ZSTD_equivalentCParams(ZSTD_compressionParameters cParams1, ZSTD_compressionParameters cParams2) { - U32 bslog1 = MIN(cParams1.windowLog, ZSTD_BLOCKSIZELOG_MAX); - U32 bslog2 = MIN(cParams2.windowLog, ZSTD_BLOCKSIZELOG_MAX); - return (bslog1 == bslog2) /* same block size */ - & (cParams1.hashLog == cParams2.hashLog) + return (cParams1.hashLog == cParams2.hashLog) & (cParams1.chainLog == cParams2.chainLog) & (cParams1.strategy == cParams2.strategy) /* opt parser space */ & ((cParams1.searchLength==3) == (cParams2.searchLength==3)); /* hashlog3 space */ @@ -778,12 +748,38 @@ static U32 ZSTD_equivalentLdmParams(ldmParams_t ldmParams1, ldmParams1.hashEveryLog == ldmParams2.hashEveryLog); } +typedef enum { ZSTDb_not_buffered, ZSTDb_buffered } ZSTD_buffered_policy_e; + +/* ZSTD_sufficientBuff() : + * check internal buffers exist for streaming if buffPol == ZSTDb_buffered . + * Note : they are assumed to be correctly sized if ZSTD_equivalentCParams()==1 */ +static U32 ZSTD_sufficientBuff(size_t bufferSize1, size_t blockSize1, + ZSTD_buffered_policy_e buffPol2, + ZSTD_compressionParameters cParams2, + U64 pledgedSrcSize) +{ + size_t const windowSize2 = MAX(1, (size_t)MIN(((U64)1 << cParams2.windowLog), pledgedSrcSize)); + size_t const blockSize2 = MIN(ZSTD_BLOCKSIZE_MAX, windowSize2); + size_t const neededBufferSize2 = (buffPol2==ZSTDb_buffered) ? windowSize2 + blockSize2 : 0; + DEBUGLOG(4, "ZSTD_sufficientBuff: windowSize2=%u from wlog=%u", + (U32)windowSize2, cParams2.windowLog); + DEBUGLOG(4, "ZSTD_sufficientBuff: blockSize2 %u <=? blockSize1 %u", + (U32)blockSize2, (U32)blockSize1); + return (blockSize2 <= blockSize1) /* seqStore space depends on blockSize */ + & (neededBufferSize2 <= bufferSize1); +} + /** Equivalence for resetCCtx purposes */ static U32 ZSTD_equivalentParams(ZSTD_CCtx_params params1, - ZSTD_CCtx_params params2) + ZSTD_CCtx_params params2, + size_t buffSize1, size_t blockSize1, + ZSTD_buffered_policy_e buffPol2, + U64 pledgedSrcSize) { + DEBUGLOG(4, "ZSTD_equivalentParams: pledgedSrcSize=%u", (U32)pledgedSrcSize); return ZSTD_equivalentCParams(params1.cParams, params2.cParams) && - ZSTD_equivalentLdmParams(params1.ldmParams, params2.ldmParams); + ZSTD_equivalentLdmParams(params1.ldmParams, params2.ldmParams) && + ZSTD_sufficientBuff(buffSize1, blockSize1, buffPol2, params2.cParams, pledgedSrcSize); } /*! ZSTD_continueCCtx() : @@ -791,7 +787,11 @@ static U32 ZSTD_equivalentParams(ZSTD_CCtx_params params1, static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, ZSTD_CCtx_params params, U64 pledgedSrcSize) { U32 const end = (U32)(cctx->nextSrc - cctx->base); - DEBUGLOG(4, "continue mode"); + size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << params.cParams.windowLog), pledgedSrcSize)); + size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize); + DEBUGLOG(4, "ZSTD_continueCCtx"); + + cctx->blockSize = blockSize; /* previous block size could be different even for same windowLog, due to pledgedSrcSize */ cctx->appliedParams = params; cctx->pledgedSrcSizePlusOne = pledgedSrcSize+1; cctx->consumedSrcSize = 0; @@ -812,7 +812,6 @@ static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, ZSTD_CCtx_params params, U64 pl } typedef enum { ZSTDcrp_continue, ZSTDcrp_noMemset } ZSTD_compResetPolicy_e; -typedef enum { ZSTDb_not_buffered, ZSTDb_buffered } ZSTD_buffered_policy_e; /*! ZSTD_resetCCtx_internal() : note : `params` are assumed fully validated at this stage */ @@ -821,13 +820,16 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, ZSTD_compResetPolicy_e const crp, ZSTD_buffered_policy_e const zbuff) { - DEBUGLOG(4, "ZSTD_resetCCtx_internal"); + DEBUGLOG(4, "ZSTD_resetCCtx_internal: pledgedSrcSize=%u, wlog=%u", + (U32)pledgedSrcSize, params.cParams.windowLog); assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); - DEBUGLOG(4, "pledgedSrcSize: %u", (U32)pledgedSrcSize); if (crp == ZSTDcrp_continue) { - if (ZSTD_equivalentParams(params, zc->appliedParams)) { - DEBUGLOG(4, "ZSTD_equivalentParams()==1"); + if (ZSTD_equivalentParams(zc->appliedParams, params, + zc->inBuffSize, zc->blockSize, + zbuff, pledgedSrcSize)) { + DEBUGLOG(4, "ZSTD_equivalentParams()==1 -> continue mode (wLog1=%u, blockSize1=%u)", + zc->appliedParams.cParams.windowLog, (U32)zc->blockSize); assert(!(params.ldmParams.enableLdm && params.ldmParams.hashEveryLog == ZSTD_LDM_HASHEVERYLOG_NOTSET)); zc->entropy->hufCTable_repeatMode = HUF_repeat_none; @@ -836,6 +838,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, zc->entropy->litlength_repeatMode = FSE_repeat_none; return ZSTD_continueCCtx(zc, params, pledgedSrcSize); } } + DEBUGLOG(4, "ZSTD_equivalentParams()==0 -> reset CCtx"); if (params.ldmParams.enableLdm) { /* Adjust long distance matching parameters */ @@ -846,7 +849,8 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, ZSTD_ldm_getHashPower(params.ldmParams.minMatchLength); } - { size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << params.cParams.windowLog); + { size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << params.cParams.windowLog), pledgedSrcSize)); + size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize); U32 const divider = (params.cParams.searchLength==3) ? 3 : 4; size_t const maxNbSeq = blockSize / divider; size_t const tokenSpace = blockSize + 11*maxNbSeq; @@ -858,7 +862,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, size_t const h3Size = ((size_t)1) << hashLog3; size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); size_t const buffOutSize = (zbuff==ZSTDb_buffered) ? ZSTD_compressBound(blockSize)+1 : 0; - size_t const buffInSize = (zbuff==ZSTDb_buffered) ? ((size_t)1 << params.cParams.windowLog) + blockSize : 0; + size_t const buffInSize = (zbuff==ZSTDb_buffered) ? windowSize + blockSize : 0; void* ptr; /* Check if workSpace is large enough, alloc a new one if needed */ @@ -874,11 +878,15 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, : 0; size_t const neededSpace = entropySpace + optSpace + ldmSpace + tableSpace + tokenSpace + bufferSpace; + DEBUGLOG(4, "Need %uKB workspace, including %uKB for tables, and %uKB for buffers", + (U32)(neededSpace>>10), (U32)(tableSpace>>10), (U32)(bufferSpace>>10)); + DEBUGLOG(4, "chainSize: %u - hSize: %u - h3Size: %u - windowSize: %u - blockSize: %u", + (U32)chainSize, (U32)hSize, (U32)h3Size, (U32)windowSize, (U32)blockSize); if (zc->workSpaceSize < neededSpace) { /* too small : resize */ - DEBUGLOG(5, "Need to update workSpaceSize from %uK to %uK \n", - (unsigned)zc->workSpaceSize>>10, - (unsigned)neededSpace>>10); + DEBUGLOG(4, "Need to update workSpaceSize from %uK to %uK", + (unsigned)(zc->workSpaceSize>>10), + (unsigned)(neededSpace>>10)); /* static cctx : no resize, error out */ if (zc->staticSize) return ERROR(memory_allocation); @@ -901,7 +909,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, zc->consumedSrcSize = 0; if (pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN) zc->appliedParams.fParams.contentSizeFlag = 0; - DEBUGLOG(5, "pledged content size : %u ; flag : %u", + DEBUGLOG(4, "pledged content size : %u ; flag : %u", (U32)pledgedSrcSize, zc->appliedParams.fParams.contentSizeFlag); zc->blockSize = blockSize; @@ -927,7 +935,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, /* opt parser space */ if ((params.cParams.strategy == ZSTD_btopt) || (params.cParams.strategy == ZSTD_btultra)) { - DEBUGLOG(5, "reserving optimal parser space"); + DEBUGLOG(4, "reserving optimal parser space"); assert(((size_t)ptr & 3) == 0); /* ensure ptr is properly aligned */ zc->optState.litFreq = (U32*)ptr; zc->optState.litLengthFreq = zc->optState.litFreq + (1<<Litbits); @@ -951,6 +959,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, } /* table Space */ + DEBUGLOG(4, "reset table : %u", crp!=ZSTDcrp_noMemset); if (crp!=ZSTDcrp_noMemset) memset(ptr, 0, tableSpace); /* reset tables only */ assert(((size_t)ptr & 3) == 0); /* ensure ptr is properly aligned */ zc->hashTable = (U32*)(ptr); @@ -999,15 +1008,16 @@ void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx) { /*! ZSTD_copyCCtx_internal() : * Duplicate an existing context `srcCCtx` into another one `dstCCtx`. - * The "context", in this case, refers to the hash and chain tables, entropy - * tables, and dictionary offsets. * Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()). - * pledgedSrcSize=0 means "empty" if fParams.contentSizeFlag=1 - * @return : 0, or an error code */ + * The "context", in this case, refers to the hash and chain tables, + * entropy tables, and dictionary references. + * `windowLog` value is enforced if != 0, otherwise value is copied from srcCCtx. + * @return : 0, or an error code */ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, + unsigned windowLog, ZSTD_frameParameters fParams, - unsigned long long pledgedSrcSize, + U64 pledgedSrcSize, ZSTD_buffered_policy_e zbuff) { DEBUGLOG(5, "ZSTD_copyCCtx_internal"); @@ -1017,6 +1027,7 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx, { ZSTD_CCtx_params params = dstCCtx->requestedParams; /* Copy only compression parameters related to tables. */ params.cParams = srcCCtx->appliedParams.cParams; + if (windowLog) params.cParams.windowLog = windowLog; params.fParams = fParams; ZSTD_resetCCtx_internal(dstCCtx, params, pledgedSrcSize, ZSTDcrp_noMemset, zbuff); @@ -1045,6 +1056,12 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx, /* copy entropy tables */ memcpy(dstCCtx->entropy, srcCCtx->entropy, sizeof(ZSTD_entropyCTables_t)); + /* copy repcodes */ + { + int i; + for (i = 0; i < ZSTD_REP_NUM; ++i) + dstCCtx->seqStore.rep[i] = srcCCtx->seqStore.rep[i]; + } return 0; } @@ -1059,9 +1076,12 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, unsigned long ZSTD_frameParameters fParams = { 1 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ }; ZSTD_buffered_policy_e const zbuff = (ZSTD_buffered_policy_e)(srcCCtx->inBuffSize>0); ZSTD_STATIC_ASSERT((U32)ZSTDb_buffered==1); - fParams.contentSizeFlag = pledgedSrcSize>0; + if (pledgedSrcSize==0) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN; + fParams.contentSizeFlag = (pledgedSrcSize != ZSTD_CONTENTSIZE_UNKNOWN); - return ZSTD_copyCCtx_internal(dstCCtx, srcCCtx, fParams, pledgedSrcSize, zbuff); + return ZSTD_copyCCtx_internal(dstCCtx, srcCCtx, + 0 /*windowLog from srcCCtx*/, fParams, pledgedSrcSize, + zbuff); } @@ -1238,7 +1258,7 @@ static size_t ZSTD_compressLiterals (ZSTD_entropyCTables_t * entropy, ostart[4] = (BYTE)(cLitSize >> 10); break; } - default: /* not possible : lhSize is {3,4,5} */ + default: /* not possible : lhSize is {3,4,5} */ assert(0); } return lhSize+cLitSize; @@ -1247,8 +1267,6 @@ static size_t ZSTD_compressLiterals (ZSTD_entropyCTables_t * entropy, void ZSTD_seqToCodes(const seqStore_t* seqStorePtr) { - BYTE const LL_deltaCode = 19; - BYTE const ML_deltaCode = 36; const seqDef* const sequences = seqStorePtr->sequencesStart; BYTE* const llCodeTable = seqStorePtr->llCode; BYTE* const ofCodeTable = seqStorePtr->ofCode; @@ -1258,9 +1276,9 @@ void ZSTD_seqToCodes(const seqStore_t* seqStorePtr) for (u=0; u<nbSeq; u++) { U32 const llv = sequences[u].litLength; U32 const mlv = sequences[u].matchLength; - llCodeTable[u] = (llv> 63) ? (BYTE)ZSTD_highbit32(llv) + LL_deltaCode : LL_Code[llv]; + llCodeTable[u] = (BYTE)ZSTD_LLcode(llv); ofCodeTable[u] = (BYTE)ZSTD_highbit32(sequences[u].offset); - mlCodeTable[u] = (mlv>127) ? (BYTE)ZSTD_highbit32(mlv) + ML_deltaCode : ML_Code[mlv]; + mlCodeTable[u] = (BYTE)ZSTD_MLcode(mlv); } if (seqStorePtr->longLengthID==1) llCodeTable[seqStorePtr->longLengthPos] = MaxLL; @@ -1273,7 +1291,8 @@ typedef enum { ZSTD_defaultAllowed = 1 } ZSTD_defaultPolicy_e; -MEM_STATIC symbolEncodingType_e ZSTD_selectEncodingType( +MEM_STATIC +symbolEncodingType_e ZSTD_selectEncodingType( FSE_repeat* repeatMode, size_t const mostFrequent, size_t nbSeq, U32 defaultNormLog, ZSTD_defaultPolicy_e const isDefaultAllowed) { @@ -1281,6 +1300,7 @@ MEM_STATIC symbolEncodingType_e ZSTD_selectEncodingType( #define MAX_SEQ_FOR_STATIC_FSE 1000 ZSTD_STATIC_ASSERT(ZSTD_defaultDisallowed == 0 && ZSTD_defaultAllowed != 0); if ((mostFrequent == nbSeq) && (!isDefaultAllowed || nbSeq > 2)) { + DEBUGLOG(5, "Selected set_rle"); /* Prefer set_basic over set_rle when there are 2 or less symbols, * since RLE uses 1 byte, but set_basic uses 5-6 bits per symbol. * If basic encoding isn't possible, always choose RLE. @@ -1288,18 +1308,30 @@ MEM_STATIC symbolEncodingType_e ZSTD_selectEncodingType( *repeatMode = FSE_repeat_check; return set_rle; } - if (isDefaultAllowed && (*repeatMode == FSE_repeat_valid) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) { + if ( isDefaultAllowed + && (*repeatMode == FSE_repeat_valid) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) { + DEBUGLOG(5, "Selected set_repeat"); return set_repeat; } - if (isDefaultAllowed && ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (defaultNormLog-1))))) { - *repeatMode = FSE_repeat_valid; + if ( isDefaultAllowed + && ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (defaultNormLog-1)))) ) { + DEBUGLOG(5, "Selected set_basic"); + /* The format allows default tables to be repeated, but it isn't useful. + * When using simple heuristics to select encoding type, we don't want + * to confuse these tables with dictionaries. When running more careful + * analysis, we don't need to waste time checking both repeating tables + * and default tables. + */ + *repeatMode = FSE_repeat_none; return set_basic; } + DEBUGLOG(5, "Selected set_compressed"); *repeatMode = FSE_repeat_check; return set_compressed; } -MEM_STATIC size_t ZSTD_buildCTable(void* dst, size_t dstCapacity, +MEM_STATIC +size_t ZSTD_buildCTable(void* dst, size_t dstCapacity, FSE_CTable* CTable, U32 FSELog, symbolEncodingType_e type, U32* count, U32 max, BYTE const* codeTable, size_t nbSeq, @@ -1317,7 +1349,7 @@ MEM_STATIC size_t ZSTD_buildCTable(void* dst, size_t dstCapacity, case set_repeat: return 0; case set_basic: - CHECK_F(FSE_buildCTable_wksp(CTable, defaultNorm, defaultMax, defaultNormLog, workspace, workspaceSize)); + CHECK_F(FSE_buildCTable_wksp(CTable, defaultNorm, defaultMax, defaultNormLog, workspace, workspaceSize)); /* note : could be pre-calculated */ return 0; case set_compressed: { S16 norm[MaxSeq + 1]; @@ -1339,11 +1371,13 @@ MEM_STATIC size_t ZSTD_buildCTable(void* dst, size_t dstCapacity, } } -MEM_STATIC size_t ZSTD_encodeSequences(void* dst, size_t dstCapacity, - FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, - FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, - FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, - seqDef const* sequences, size_t nbSeq, int longOffsets) +MEM_STATIC +size_t ZSTD_encodeSequences( + void* dst, size_t dstCapacity, + FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, + FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, + FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, + seqDef const* sequences, size_t nbSeq, int longOffsets) { BIT_CStream_t blockStream; FSE_CState_t stateMatchLength; @@ -1380,8 +1414,12 @@ MEM_STATIC size_t ZSTD_encodeSequences(void* dst, size_t dstCapacity, BYTE const ofCode = ofCodeTable[n]; BYTE const mlCode = mlCodeTable[n]; U32 const llBits = LL_bits[llCode]; - U32 const ofBits = ofCode; /* 32b*/ /* 64b*/ + U32 const ofBits = ofCode; U32 const mlBits = ML_bits[mlCode]; + DEBUGLOG(6, "encoding: litlen:%2u - matchlen:%2u - offCode:%7u", + sequences[n].litLength, + sequences[n].matchLength + MINMATCH, + sequences[n].offset); /* 32b*/ /* 64b*/ /* (7)*/ /* (7)*/ FSE_encodeSymbol(&blockStream, &stateOffsetBits, ofCode); /* 15 */ /* 15 */ FSE_encodeSymbol(&blockStream, &stateMatchLength, mlCode); /* 24 */ /* 24 */ @@ -1447,14 +1485,18 @@ MEM_STATIC size_t ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, entropy, cParams->strategy, op, dstCapacity, literals, litSize); if (ZSTD_isError(cSize)) return cSize; + assert(cSize <= dstCapacity); op += cSize; } /* Sequences Header */ - if ((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead */) return ERROR(dstSize_tooSmall); - if (nbSeq < 0x7F) *op++ = (BYTE)nbSeq; - else if (nbSeq < LONGNBSEQ) op[0] = (BYTE)((nbSeq>>8) + 0x80), op[1] = (BYTE)nbSeq, op+=2; - else op[0]=0xFF, MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)), op+=3; + if ((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead*/) return ERROR(dstSize_tooSmall); + if (nbSeq < 0x7F) + *op++ = (BYTE)nbSeq; + else if (nbSeq < LONGNBSEQ) + op[0] = (BYTE)((nbSeq>>8) + 0x80), op[1] = (BYTE)nbSeq, op+=2; + else + op[0]=0xFF, MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)), op+=3; if (nbSeq==0) return op - ostart; /* seqHead : flags for FSE encoding type */ @@ -1462,9 +1504,10 @@ MEM_STATIC size_t ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, /* convert length/distances into codes */ ZSTD_seqToCodes(seqStorePtr); - /* CTable for Literal Lengths */ + /* build CTable for Literal Lengths */ { U32 max = MaxLL; size_t const mostFrequent = FSE_countFast_wksp(count, &max, llCodeTable, nbSeq, entropy->workspace); + DEBUGLOG(5, "Building LL table"); LLtype = ZSTD_selectEncodingType(&entropy->litlength_repeatMode, mostFrequent, nbSeq, LL_defaultNormLog, ZSTD_defaultAllowed); { size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_LitLength, LLFSELog, (symbolEncodingType_e)LLtype, count, max, llCodeTable, nbSeq, LL_defaultNorm, LL_defaultNormLog, MaxLL, @@ -1472,11 +1515,12 @@ MEM_STATIC size_t ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, if (ZSTD_isError(countSize)) return countSize; op += countSize; } } - /* CTable for Offsets */ + /* build CTable for Offsets */ { U32 max = MaxOff; size_t const mostFrequent = FSE_countFast_wksp(count, &max, ofCodeTable, nbSeq, entropy->workspace); /* We can only use the basic table if max <= DefaultMaxOff, otherwise the offsets are too large */ - ZSTD_defaultPolicy_e const defaultPolicy = max <= DefaultMaxOff ? ZSTD_defaultAllowed : ZSTD_defaultDisallowed; + ZSTD_defaultPolicy_e const defaultPolicy = (max <= DefaultMaxOff) ? ZSTD_defaultAllowed : ZSTD_defaultDisallowed; + DEBUGLOG(5, "Building OF table"); Offtype = ZSTD_selectEncodingType(&entropy->offcode_repeatMode, mostFrequent, nbSeq, OF_defaultNormLog, defaultPolicy); { size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_OffsetBits, OffFSELog, (symbolEncodingType_e)Offtype, count, max, ofCodeTable, nbSeq, OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, @@ -1484,9 +1528,10 @@ MEM_STATIC size_t ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, if (ZSTD_isError(countSize)) return countSize; op += countSize; } } - /* CTable for MatchLengths */ + /* build CTable for MatchLengths */ { U32 max = MaxML; size_t const mostFrequent = FSE_countFast_wksp(count, &max, mlCodeTable, nbSeq, entropy->workspace); + DEBUGLOG(5, "Building ML table"); MLtype = ZSTD_selectEncodingType(&entropy->matchlength_repeatMode, mostFrequent, nbSeq, ML_defaultNormLog, ZSTD_defaultAllowed); { size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_MatchLength, MLFSELog, (symbolEncodingType_e)MLtype, count, max, mlCodeTable, nbSeq, ML_defaultNorm, ML_defaultNormLog, MaxML, @@ -1497,13 +1542,15 @@ MEM_STATIC size_t ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, *seqHead = (BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2)); - { size_t const streamSize = ZSTD_encodeSequences(op, oend - op, - CTable_MatchLength, mlCodeTable, - CTable_OffsetBits, ofCodeTable, - CTable_LitLength, llCodeTable, - sequences, nbSeq, longOffsets); - if (ZSTD_isError(streamSize)) return streamSize; - op += streamSize; + { size_t const bitstreamSize = ZSTD_encodeSequences( + op, oend - op, + CTable_MatchLength, mlCodeTable, + CTable_OffsetBits, ofCodeTable, + CTable_LitLength, llCodeTable, + sequences, nbSeq, + longOffsets); + if (ZSTD_isError(bitstreamSize)) return bitstreamSize; + op += bitstreamSize; } return op - ostart; @@ -1517,27 +1564,33 @@ MEM_STATIC size_t ZSTD_compressSequences(seqStore_t* seqStorePtr, { size_t const cSize = ZSTD_compressSequences_internal(seqStorePtr, entropy, cParams, dst, dstCapacity); - size_t const minGain = ZSTD_minGain(srcSize); - size_t const maxCSize = srcSize - minGain; /* If the srcSize <= dstCapacity, then there is enough space to write a * raw uncompressed block. Since we ran out of space, the block must not * be compressible, so fall back to a raw uncompressed block. */ - int const uncompressibleError = cSize == ERROR(dstSize_tooSmall) && srcSize <= dstCapacity; - + int const uncompressibleError = (cSize == ERROR(dstSize_tooSmall)) && (srcSize <= dstCapacity); if (ZSTD_isError(cSize) && !uncompressibleError) return cSize; + /* We check that dictionaries have offset codes available for the first + * block. After the first block, the offcode table might not have large + * enough codes to represent the offsets in the data. + */ + if (entropy->offcode_repeatMode == FSE_repeat_valid) + entropy->offcode_repeatMode = FSE_repeat_check; + /* Check compressibility */ - if (cSize >= maxCSize || uncompressibleError) { - entropy->hufCTable_repeatMode = HUF_repeat_none; - entropy->offcode_repeatMode = FSE_repeat_none; - entropy->matchlength_repeatMode = FSE_repeat_none; - entropy->litlength_repeatMode = FSE_repeat_none; - return 0; - } + { size_t const minGain = ZSTD_minGain(srcSize); /* note : fixed formula, maybe should depend on compression level, or strategy */ + size_t const maxCSize = srcSize - minGain; + if (cSize >= maxCSize || uncompressibleError) { + entropy->hufCTable_repeatMode = HUF_repeat_none; + entropy->offcode_repeatMode = FSE_repeat_none; + entropy->matchlength_repeatMode = FSE_repeat_none; + entropy->litlength_repeatMode = FSE_repeat_none; + return 0; /* block not compressed */ + } } assert(!ZSTD_isError(cSize)); - /* confirm repcodes */ + /* block is compressed => confirm repcodes in history */ { int i; for (i=0; i<ZSTD_REP_NUM; i++) seqStorePtr->rep[i] = seqStorePtr->repToConfirm[i]; } return cSize; } @@ -1559,9 +1612,9 @@ ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int extDict ZSTD_compressBlock_btopt_extDict, ZSTD_compressBlock_btultra_extDict } }; ZSTD_STATIC_ASSERT((unsigned)ZSTD_fast == 1); + assert((U32)strat >= (U32)ZSTD_fast); assert((U32)strat <= (U32)ZSTD_btultra); - return blockCompressor[extDict!=0][(U32)strat]; } @@ -1572,30 +1625,38 @@ static void ZSTD_storeLastLiterals(seqStore_t* seqStorePtr, seqStorePtr->lit += lastLLSize; } +static void ZSTD_resetSeqStore(seqStore_t* ssPtr) +{ + ssPtr->lit = ssPtr->litStart; + ssPtr->sequences = ssPtr->sequencesStart; + ssPtr->longLengthID = 0; +} + static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCapacity, const void* src, size_t srcSize) { - const BYTE* const base = zc->base; - const BYTE* const istart = (const BYTE*)src; - const U32 current = (U32)(istart-base); - size_t lastLLSize; - const BYTE* anchor; - U32 const extDict = zc->lowLimit < zc->dictLimit; - const ZSTD_blockCompressor blockCompressor = - zc->appliedParams.ldmParams.enableLdm - ? (extDict ? ZSTD_compressBlock_ldm_extDict : ZSTD_compressBlock_ldm) - : ZSTD_selectBlockCompressor(zc->appliedParams.cParams.strategy, extDict); - - if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) return 0; /* don't even attempt compression below a certain srcSize */ + DEBUGLOG(5, "ZSTD_compressBlock_internal : dstCapacity = %u", (U32)dstCapacity); + if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) + return 0; /* don't even attempt compression below a certain srcSize */ ZSTD_resetSeqStore(&(zc->seqStore)); - if (current > zc->nextToUpdate + 384) - zc->nextToUpdate = current - MIN(192, (U32)(current - zc->nextToUpdate - 384)); /* limited update after finding a very long match */ - - lastLLSize = blockCompressor(zc, src, srcSize); - - /* Last literals */ - anchor = (const BYTE*)src + srcSize - lastLLSize; - ZSTD_storeLastLiterals(&zc->seqStore, anchor, lastLLSize); + /* limited update after a very long match */ + { const BYTE* const base = zc->base; + const BYTE* const istart = (const BYTE*)src; + const U32 current = (U32)(istart-base); + if (current > zc->nextToUpdate + 384) + zc->nextToUpdate = current - MIN(192, (U32)(current - zc->nextToUpdate - 384)); + } + /* find and store sequences */ + { U32 const extDict = zc->lowLimit < zc->dictLimit; + const ZSTD_blockCompressor blockCompressor = + zc->appliedParams.ldmParams.enableLdm + ? (extDict ? ZSTD_compressBlock_ldm_extDict : ZSTD_compressBlock_ldm) + : ZSTD_selectBlockCompressor(zc->appliedParams.cParams.strategy, extDict); + size_t const lastLLSize = blockCompressor(zc, src, srcSize); + const BYTE* const anchor = (const BYTE*)src + srcSize - lastLLSize; + ZSTD_storeLastLiterals(&zc->seqStore, anchor, lastLLSize); + } + /* encode */ return ZSTD_compressSequences(&zc->seqStore, zc->entropy, &zc->appliedParams.cParams, dst, dstCapacity, srcSize); } @@ -1618,13 +1679,14 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, BYTE* const ostart = (BYTE*)dst; BYTE* op = ostart; U32 const maxDist = (U32)1 << cctx->appliedParams.cParams.windowLog; + assert(cctx->appliedParams.cParams.windowLog <= 31); + DEBUGLOG(5, "ZSTD_compress_frameChunk (blockSize=%u)", (U32)blockSize); if (cctx->appliedParams.fParams.checksumFlag && srcSize) XXH64_update(&cctx->xxhState, src, srcSize); while (remaining) { U32 const lastBlock = lastFrameChunk & (blockSize >= remaining); - size_t cSize; if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */ @@ -1666,34 +1728,39 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, else cctx->nextToUpdate -= correction; DEBUGLOG(4, "Correction of 0x%x bytes to lowLimit=0x%x\n", correction, cctx->lowLimit); } - + /* enforce maxDist */ if ((U32)(ip+blockSize - cctx->base) > cctx->loadedDictEnd + maxDist) { - /* enforce maxDist */ U32 const newLowLimit = (U32)(ip+blockSize - cctx->base) - maxDist; if (cctx->lowLimit < newLowLimit) cctx->lowLimit = newLowLimit; if (cctx->dictLimit < cctx->lowLimit) cctx->dictLimit = cctx->lowLimit; } - cSize = ZSTD_compressBlock_internal(cctx, op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, ip, blockSize); - if (ZSTD_isError(cSize)) return cSize; - - if (cSize == 0) { /* block is not compressible */ - U32 const cBlockHeader24 = lastBlock + (((U32)bt_raw)<<1) + (U32)(blockSize << 3); - if (blockSize + ZSTD_blockHeaderSize > dstCapacity) return ERROR(dstSize_tooSmall); - MEM_writeLE32(op, cBlockHeader24); /* no pb, 4th byte will be overwritten */ - memcpy(op + ZSTD_blockHeaderSize, ip, blockSize); - cSize = ZSTD_blockHeaderSize+blockSize; - } else { - U32 const cBlockHeader24 = lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3); - MEM_writeLE24(op, cBlockHeader24); - cSize += ZSTD_blockHeaderSize; - } + { size_t cSize = ZSTD_compressBlock_internal(cctx, + op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, + ip, blockSize); + if (ZSTD_isError(cSize)) return cSize; + + if (cSize == 0) { /* block is not compressible */ + U32 const cBlockHeader24 = lastBlock + (((U32)bt_raw)<<1) + (U32)(blockSize << 3); + if (blockSize + ZSTD_blockHeaderSize > dstCapacity) return ERROR(dstSize_tooSmall); + MEM_writeLE32(op, cBlockHeader24); /* 4th byte will be overwritten */ + memcpy(op + ZSTD_blockHeaderSize, ip, blockSize); + cSize = ZSTD_blockHeaderSize + blockSize; + } else { + U32 const cBlockHeader24 = lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3); + MEM_writeLE24(op, cBlockHeader24); + cSize += ZSTD_blockHeaderSize; + } - remaining -= blockSize; - dstCapacity -= cSize; - ip += blockSize; - op += cSize; - } + ip += blockSize; + assert(remaining >= blockSize); + remaining -= blockSize; + op += cSize; + assert(dstCapacity >= cSize); + dstCapacity -= cSize; + DEBUGLOG(5, "ZSTD_compress_frameChunk: adding a block of size %u", + (U32)cSize); + } } if (lastFrameChunk && (op>ostart)) cctx->stage = ZSTDcs_ending; return op-ostart; @@ -1719,7 +1786,6 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity, !params.fParams.noDictIDFlag, dictID, dictIDSizeCode); if (params.format == ZSTD_f_zstd1) { - DEBUGLOG(4, "writing zstd magic number"); MEM_writeLE32(dst, ZSTD_MAGICNUMBER); pos = 4; } @@ -1753,8 +1819,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx, const BYTE* const ip = (const BYTE*) src; size_t fhSize = 0; - DEBUGLOG(5, "ZSTD_compressContinue_internal"); - DEBUGLOG(5, "stage: %u", cctx->stage); + DEBUGLOG(5, "ZSTD_compressContinue_internal, stage: %u", cctx->stage); if (cctx->stage==ZSTDcs_created) return ERROR(stage_wrong); /* missing init (ZSTD_compressBegin) */ if (frame && (cctx->stage==ZSTDcs_init)) { @@ -1766,17 +1831,21 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx, cctx->stage = ZSTDcs_ongoing; } + if (!srcSize) return fhSize; /* do not generate an empty block if no input */ + /* Check if blocks follow each other */ if (src != cctx->nextSrc) { /* not contiguous */ - ptrdiff_t const delta = cctx->nextSrc - ip; + size_t const distanceFromBase = (size_t)(cctx->nextSrc - cctx->base); cctx->lowLimit = cctx->dictLimit; - cctx->dictLimit = (U32)(cctx->nextSrc - cctx->base); + assert(distanceFromBase == (size_t)(U32)distanceFromBase); /* should never overflow */ + cctx->dictLimit = (U32)distanceFromBase; cctx->dictBase = cctx->base; - cctx->base -= delta; + cctx->base = ip - distanceFromBase; cctx->nextToUpdate = cctx->dictLimit; if (cctx->dictLimit - cctx->lowLimit < HASH_READ_SIZE) cctx->lowLimit = cctx->dictLimit; /* too small extDict */ } + cctx->nextSrc = ip + srcSize; /* if input and dictionary overlap : reduce dictionary (area presumed modified by input) */ if ((ip+srcSize > cctx->dictBase + cctx->lowLimit) & (ip < cctx->dictBase + cctx->dictLimit)) { @@ -1785,17 +1854,14 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx, cctx->lowLimit = lowLimitMax; } - cctx->nextSrc = ip + srcSize; - - if (srcSize) { - size_t const cSize = frame ? + DEBUGLOG(5, "ZSTD_compressContinue_internal (blockSize=%u)", (U32)cctx->blockSize); + { size_t const cSize = frame ? ZSTD_compress_frameChunk (cctx, dst, dstCapacity, src, srcSize, lastFrameChunk) : ZSTD_compressBlock_internal (cctx, dst, dstCapacity, src, srcSize); if (ZSTD_isError(cSize)) return cSize; cctx->consumedSrcSize += srcSize; return cSize + fhSize; - } else - return fhSize; + } } size_t ZSTD_compressContinue (ZSTD_CCtx* cctx, @@ -1832,7 +1898,7 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t zc->lowLimit = zc->dictLimit; zc->dictLimit = (U32)(zc->nextSrc - zc->base); zc->dictBase = zc->base; - zc->base += ip - zc->nextSrc; + zc->base = ip - zc->dictLimit; zc->nextToUpdate = zc->dictLimit; zc->loadedDictEnd = zc->appliedParams.forceWindow ? 0 : (U32)(iend - zc->base); @@ -1983,7 +2049,7 @@ static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictMode_e dictMode) { - DEBUGLOG(5, "ZSTD_compress_insertDictionary"); + DEBUGLOG(4, "ZSTD_compress_insertDictionary (dictSize=%u)", (U32)dictSize); if ((dict==NULL) || (dictSize<=8)) return 0; /* dict restricted modes */ @@ -1992,7 +2058,7 @@ static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* cctx, if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) { if (dictMode == ZSTD_dm_auto) { - DEBUGLOG(5, "raw content dictionary detected"); + DEBUGLOG(4, "raw content dictionary detected"); return ZSTD_loadDictionaryContent(cctx, dict, dictSize); } if (dictMode == ZSTD_dm_fullDict) @@ -2006,21 +2072,22 @@ static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* cctx, /*! ZSTD_compressBegin_internal() : * @return : 0, or an error code */ -static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, +size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictMode_e dictMode, const ZSTD_CDict* cdict, - ZSTD_CCtx_params params, U64 pledgedSrcSize, - ZSTD_buffered_policy_e zbuff) + ZSTD_CCtx_params params, U64 pledgedSrcSize, + ZSTD_buffered_policy_e zbuff) { - DEBUGLOG(4, "ZSTD_compressBegin_internal"); + DEBUGLOG(4, "ZSTD_compressBegin_internal: wlog=%u", params.cParams.windowLog); /* params are supposed to be fully validated at this point */ assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); assert(!((dict) && (cdict))); /* either dict or cdict, not both */ if (cdict && cdict->dictContentSize>0) { + cctx->requestedParams = params; return ZSTD_copyCCtx_internal(cctx, cdict->refContext, - params.fParams, pledgedSrcSize, + params.cParams.windowLog, params.fParams, pledgedSrcSize, zbuff); } @@ -2029,16 +2096,19 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, return ZSTD_compress_insertDictionary(cctx, dict, dictSize, dictMode); } -size_t ZSTD_compressBegin_advanced_internal( - ZSTD_CCtx* cctx, +size_t ZSTD_compressBegin_advanced_internal(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictMode_e dictMode, + const ZSTD_CDict* cdict, ZSTD_CCtx_params params, unsigned long long pledgedSrcSize) { + DEBUGLOG(4, "ZSTD_compressBegin_advanced_internal: wlog=%u", params.cParams.windowLog); /* compression parameters verification and optimization */ CHECK_F( ZSTD_checkCParams(params.cParams) ); - return ZSTD_compressBegin_internal(cctx, dict, dictSize, dictMode, NULL, + return ZSTD_compressBegin_internal(cctx, + dict, dictSize, dictMode, + cdict, params, pledgedSrcSize, ZSTDb_not_buffered); } @@ -2051,9 +2121,10 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, { ZSTD_CCtx_params const cctxParams = ZSTD_assignParamsToCCtxParams(cctx->requestedParams, params); - return ZSTD_compressBegin_advanced_internal(cctx, dict, dictSize, ZSTD_dm_auto, - cctxParams, - pledgedSrcSize); + return ZSTD_compressBegin_advanced_internal(cctx, + dict, dictSize, ZSTD_dm_auto, + NULL /*cdict*/, + cctxParams, pledgedSrcSize); } size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel) @@ -2061,8 +2132,9 @@ size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t di ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize); ZSTD_CCtx_params const cctxParams = ZSTD_assignParamsToCCtxParams(cctx->requestedParams, params); + DEBUGLOG(4, "ZSTD_compressBegin_usingDict"); return ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dm_auto, NULL, - cctxParams, 0, ZSTDb_not_buffered); + cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, ZSTDb_not_buffered); } size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel) @@ -2143,6 +2215,7 @@ static size_t ZSTD_compress_internal (ZSTD_CCtx* cctx, { ZSTD_CCtx_params const cctxParams = ZSTD_assignParamsToCCtxParams(cctx->requestedParams, params); + DEBUGLOG(4, "ZSTD_compress_internal"); return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, @@ -2156,6 +2229,7 @@ size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx, const void* dict,size_t dictSize, ZSTD_parameters params) { + DEBUGLOG(4, "ZSTD_compress_advanced"); CHECK_F(ZSTD_checkCParams(params.cParams)); return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, params); } @@ -2168,6 +2242,7 @@ size_t ZSTD_compress_advanced_internal( const void* dict,size_t dictSize, ZSTD_CCtx_params params) { + DEBUGLOG(4, "ZSTD_compress_advanced_internal"); CHECK_F( ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dm_auto, NULL, params, srcSize, ZSTDb_not_buffered) ); return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize); @@ -2176,8 +2251,10 @@ size_t ZSTD_compress_advanced_internal( size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const void* dict, size_t dictSize, int compressionLevel) { - ZSTD_parameters params = ZSTD_getParams(compressionLevel, srcSize, dict ? dictSize : 0); + ZSTD_parameters params = ZSTD_getParams(compressionLevel, srcSize ? srcSize : 1, dict ? dictSize : 0); params.fParams.contentSizeFlag = 1; + DEBUGLOG(4, "ZSTD_compress_usingDict (level=%i, srcSize=%u, dictSize=%u)", + compressionLevel, (U32)srcSize, (U32)dictSize); return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, params); } @@ -2234,7 +2311,7 @@ static size_t ZSTD_initCDict_internal( ZSTD_dictMode_e dictMode, ZSTD_compressionParameters cParams) { - DEBUGLOG(5, "ZSTD_initCDict_internal, mode %u", (U32)dictMode); + DEBUGLOG(3, "ZSTD_initCDict_internal, mode %u", (U32)dictMode); if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dictBuffer) || (!dictSize)) { cdict->dictBuffer = NULL; cdict->dictContent = dictBuffer; @@ -2264,7 +2341,7 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, ZSTD_dictMode_e dictMode, ZSTD_compressionParameters cParams, ZSTD_customMem customMem) { - DEBUGLOG(5, "ZSTD_createCDict_advanced, mode %u", (U32)dictMode); + DEBUGLOG(3, "ZSTD_createCDict_advanced, mode %u", (U32)dictMode); if (!customMem.customAlloc ^ !customMem.customFree) return NULL; { ZSTD_CDict* const cdict = (ZSTD_CDict*)ZSTD_malloc(sizeof(ZSTD_CDict), customMem); @@ -2339,9 +2416,9 @@ ZSTD_CDict* ZSTD_initStaticCDict(void* workspace, size_t workspaceSize, + cctxSize; ZSTD_CDict* const cdict = (ZSTD_CDict*) workspace; void* ptr; - DEBUGLOG(5, "(size_t)workspace & 7 : %u", (U32)(size_t)workspace & 7); + DEBUGLOG(4, "(size_t)workspace & 7 : %u", (U32)(size_t)workspace & 7); if ((size_t)workspace & 7) return NULL; /* 8-aligned */ - DEBUGLOG(5, "(workspaceSize < neededSize) : (%u < %u) => %u", + DEBUGLOG(4, "(workspaceSize < neededSize) : (%u < %u) => %u", (U32)workspaceSize, (U32)neededSize, (U32)(workspaceSize < neededSize)); if (workspaceSize < neededSize) return NULL; @@ -2373,11 +2450,11 @@ size_t ZSTD_compressBegin_usingCDict_advanced( ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize) { + DEBUGLOG(4, "ZSTD_compressBegin_usingCDict_advanced"); if (cdict==NULL) return ERROR(dictionary_wrong); { ZSTD_CCtx_params params = cctx->requestedParams; params.cParams = ZSTD_getCParamsFromCDict(cdict); params.fParams = fParams; - DEBUGLOG(5, "ZSTD_compressBegin_usingCDict_advanced"); return ZSTD_compressBegin_internal(cctx, NULL, 0, ZSTD_dm_auto, cdict, @@ -2392,7 +2469,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced( size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) { ZSTD_frameParameters const fParams = { 0 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ }; - DEBUGLOG(5, "ZSTD_compressBegin_usingCDict : dictIDFlag == %u", !fParams.noDictIDFlag); + DEBUGLOG(4, "ZSTD_compressBegin_usingCDict : dictIDFlag == %u", !fParams.noDictIDFlag); return ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, 0); } @@ -2427,6 +2504,7 @@ size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, ZSTD_CStream* ZSTD_createCStream(void) { + DEBUGLOG(3, "ZSTD_createCStream"); return ZSTD_createCStream_advanced(ZSTD_defaultCMem); } @@ -2457,9 +2535,9 @@ size_t ZSTD_CStreamOutSize(void) } static size_t ZSTD_resetCStream_internal(ZSTD_CStream* zcs, - const void* dict, size_t dictSize, ZSTD_dictMode_e dictMode, - const ZSTD_CDict* cdict, - const ZSTD_CCtx_params params, unsigned long long pledgedSrcSize) + const void* const dict, size_t const dictSize, ZSTD_dictMode_e const dictMode, + const ZSTD_CDict* const cdict, + ZSTD_CCtx_params const params, unsigned long long const pledgedSrcSize) { DEBUGLOG(4, "ZSTD_resetCStream_internal"); /* params are supposed to be fully validated at this point */ @@ -2467,31 +2545,35 @@ static size_t ZSTD_resetCStream_internal(ZSTD_CStream* zcs, assert(!((dict) && (cdict))); /* either dict or cdict, not both */ CHECK_F( ZSTD_compressBegin_internal(zcs, - dict, dictSize, dictMode, - cdict, - params, pledgedSrcSize, - ZSTDb_buffered) ); + dict, dictSize, dictMode, + cdict, + params, pledgedSrcSize, + ZSTDb_buffered) ); zcs->inToCompress = 0; zcs->inBuffPos = 0; - zcs->inBuffTarget = zcs->blockSize; + zcs->inBuffTarget = zcs->blockSize + + (zcs->blockSize == pledgedSrcSize); /* for small input: avoid automatic flush on reaching end of block, since it would require to add a 3-bytes null block to end frame */ zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0; zcs->streamStage = zcss_load; zcs->frameEnded = 0; return 0; /* ready to go */ } +/* ZSTD_resetCStream(): + * pledgedSrcSize == 0 means "unknown" */ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize) { ZSTD_CCtx_params params = zcs->requestedParams; - params.fParams.contentSizeFlag = (pledgedSrcSize > 0); + DEBUGLOG(4, "ZSTD_resetCStream: pledgedSrcSize = %u", (U32)pledgedSrcSize); + if (pledgedSrcSize==0) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN; + params.fParams.contentSizeFlag = 1; params.cParams = ZSTD_getCParamsFromCCtxParams(params, pledgedSrcSize, 0); - DEBUGLOG(4, "ZSTD_resetCStream"); return ZSTD_resetCStream_internal(zcs, NULL, 0, ZSTD_dm_auto, zcs->cdict, params, pledgedSrcSize); } /*! ZSTD_initCStream_internal() : - * Note : not static, but hidden (not exposed). Used by zstdmt_compress.c + * Note : for lib/compress only. Used by zstdmt_compress.c. * Assumption 1 : params are valid * Assumption 2 : either dict, or cdict, is defined, not both */ size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs, @@ -2503,7 +2585,7 @@ size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs, assert(!((dict) && (cdict))); /* either dict or cdict, not both */ if (dict && dictSize >= 8) { - DEBUGLOG(5, "loading dictionary of size %u", (U32)dictSize); + DEBUGLOG(4, "loading dictionary of size %u", (U32)dictSize); if (zcs->staticSize) { /* static CCtx : never uses malloc */ /* incompatible with internal cdict creation */ return ERROR(memory_allocation); @@ -2516,14 +2598,14 @@ size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs, if (zcs->cdictLocal == NULL) return ERROR(memory_allocation); } else { if (cdict) { - params.cParams = ZSTD_getCParamsFromCDict(cdict); /* cParams are enforced from cdict */ + params.cParams = ZSTD_getCParamsFromCDict(cdict); /* cParams are enforced from cdict; it includes windowLog */ } ZSTD_freeCDict(zcs->cdictLocal); zcs->cdictLocal = NULL; zcs->cdict = cdict; } - params.compressionLevel = ZSTD_CLEVEL_CUSTOM; + params.compressionLevel = ZSTD_CLEVEL_CUSTOM; /* enforce usage of cParams, instead of a dynamic derivation from cLevel (but does that happen ?) */ zcs->requestedParams = params; return ZSTD_resetCStream_internal(zcs, NULL, 0, ZSTD_dm_auto, zcs->cdict, params, pledgedSrcSize); @@ -2535,8 +2617,9 @@ size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize) -{ /* cannot handle NULL cdict (does not know what to do) */ - if (!cdict) return ERROR(dictionary_wrong); +{ + DEBUGLOG(4, "ZSTD_initCStream_usingCDict_advanced"); + if (!cdict) return ERROR(dictionary_wrong); /* cannot handle NULL cdict (does not know what to do) */ { ZSTD_CCtx_params params = zcs->requestedParams; params.cParams = ZSTD_getCParamsFromCDict(cdict); params.fParams = fParams; @@ -2549,18 +2632,25 @@ size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, /* note : cdict must outlive compression session */ size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict) { - ZSTD_frameParameters const fParams = { 0 /* contentSize */, 0 /* checksum */, 0 /* hideDictID */ }; - return ZSTD_initCStream_usingCDict_advanced(zcs, cdict, fParams, 0); /* note : will check that cdict != NULL */ + ZSTD_frameParameters const fParams = { 0 /* contentSizeFlag */, 0 /* checksum */, 0 /* hideDictID */ }; + DEBUGLOG(4, "ZSTD_initCStream_usingCDict"); + return ZSTD_initCStream_usingCDict_advanced(zcs, cdict, fParams, ZSTD_CONTENTSIZE_UNKNOWN); /* note : will check that cdict != NULL */ } +/* ZSTD_initCStream_advanced() : + * pledgedSrcSize must be correct. + * if srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. + * dict is loaded with default parameters ZSTD_dm_auto and ZSTD_dlm_byCopy. */ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize) { - ZSTD_CCtx_params const cctxParams = - ZSTD_assignParamsToCCtxParams(zcs->requestedParams, params); + ZSTD_CCtx_params const cctxParams = ZSTD_assignParamsToCCtxParams(zcs->requestedParams, params); + DEBUGLOG(4, "ZSTD_initCStream_advanced: pledgedSrcSize=%u, flag=%u", + (U32)pledgedSrcSize, params.fParams.contentSizeFlag); CHECK_F( ZSTD_checkCParams(params.cParams) ); - return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL, cctxParams, pledgedSrcSize); + if ((pledgedSrcSize==0) && (params.fParams.contentSizeFlag==0)) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN; /* for compatibility with older programs relying on this behavior. Users should now specify ZSTD_CONTENTSIZE_UNKNOWN. This line will be removed in the future. */ + return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL /*cdict*/, cctxParams, pledgedSrcSize); } size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel) @@ -2568,21 +2658,21 @@ size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t di ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize); ZSTD_CCtx_params const cctxParams = ZSTD_assignParamsToCCtxParams(zcs->requestedParams, params); - return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL, cctxParams, 0); + return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL, cctxParams, ZSTD_CONTENTSIZE_UNKNOWN); } -size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize) +size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pss) { - ZSTD_CCtx_params cctxParams; + U64 const pledgedSrcSize = (pss==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss; /* temporary : 0 interpreted as "unknown" during transition period. Users willing to specify "unknown" **must** use ZSTD_CONTENTSIZE_UNKNOWN. `0` will be interpreted as "empty" in the future */ ZSTD_parameters const params = ZSTD_getParams(compressionLevel, pledgedSrcSize, 0); - cctxParams = ZSTD_assignParamsToCCtxParams(zcs->requestedParams, params); - cctxParams.fParams.contentSizeFlag = (pledgedSrcSize>0); + ZSTD_CCtx_params const cctxParams = ZSTD_assignParamsToCCtxParams(zcs->requestedParams, params); return ZSTD_initCStream_internal(zcs, NULL, 0, NULL, cctxParams, pledgedSrcSize); } size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel) { - return ZSTD_initCStream_srcSize(zcs, compressionLevel, 0); + DEBUGLOG(4, "ZSTD_initCStream"); + return ZSTD_initCStream_srcSize(zcs, compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN); } /*====== Compression ======*/ @@ -2615,9 +2705,9 @@ size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, /* check expectations */ DEBUGLOG(5, "ZSTD_compressStream_generic, flush=%u", (U32)flushMode); assert(zcs->inBuff != NULL); - assert(zcs->inBuffSize>0); - assert(zcs->outBuff!= NULL); - assert(zcs->outBuffSize>0); + assert(zcs->inBuffSize > 0); + assert(zcs->outBuff != NULL); + assert(zcs->outBuffSize > 0); assert(output->pos <= output->size); assert(input->pos <= input->size); @@ -2757,7 +2847,7 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, ZSTD_inBuffer* input, ZSTD_EndDirective endOp) { - DEBUGLOG(5, "ZSTD_compress_generic"); + DEBUGLOG(5, "ZSTD_compress_generic, endOp=%u ", (U32)endOp); /* check conditions */ if (output->pos > output->size) return ERROR(GENERIC); if (input->pos > input->size) return ERROR(GENERIC); @@ -2765,42 +2855,47 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, /* transparent initialization stage */ if (cctx->streamStage == zcss_init) { - ZSTD_prefixDict const prefixDict = cctx->prefixDict; ZSTD_CCtx_params params = cctx->requestedParams; - params.cParams = ZSTD_getCParamsFromCCtxParams( - cctx->requestedParams, cctx->pledgedSrcSizePlusOne-1, 0 /*dictSize*/); + ZSTD_prefixDict const prefixDict = cctx->prefixDict; memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict)); /* single usage */ assert(prefixDict.dict==NULL || cctx->cdict==NULL); /* only one can be set */ DEBUGLOG(4, "ZSTD_compress_generic : transparent init stage"); + if (endOp == ZSTD_e_end) cctx->pledgedSrcSizePlusOne = input->size + 1; /* auto-fix pledgedSrcSize */ + params.cParams = ZSTD_getCParamsFromCCtxParams( + cctx->requestedParams, cctx->pledgedSrcSizePlusOne-1, 0 /*dictSize*/); #ifdef ZSTD_MULTITHREAD + if ((cctx->pledgedSrcSizePlusOne-1) <= ZSTDMT_JOBSIZE_MIN) + params.nbThreads = 1; /* do not invoke multi-threading when src size is too small */ if (params.nbThreads > 1) { - if (cctx->mtctx == NULL || cctx->appliedParams.nbThreads != params.nbThreads) { + if (cctx->mtctx == NULL || (params.nbThreads != ZSTDMT_getNbThreads(cctx->mtctx))) { + DEBUGLOG(4, "ZSTD_compress_generic: creating new mtctx for nbThreads=%u (previous: %u)", + params.nbThreads, ZSTDMT_getNbThreads(cctx->mtctx)); ZSTDMT_freeCCtx(cctx->mtctx); cctx->mtctx = ZSTDMT_createCCtx_advanced(params.nbThreads, cctx->customMem); if (cctx->mtctx == NULL) return ERROR(memory_allocation); } DEBUGLOG(4, "call ZSTDMT_initCStream_internal as nbThreads=%u", params.nbThreads); CHECK_F( ZSTDMT_initCStream_internal( - cctx->mtctx, - prefixDict.dict, prefixDict.dictSize, ZSTD_dm_rawContent, - cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) ); + cctx->mtctx, + prefixDict.dict, prefixDict.dictSize, ZSTD_dm_rawContent, + cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) ); cctx->streamStage = zcss_load; cctx->appliedParams.nbThreads = params.nbThreads; } else #endif - { - CHECK_F( ZSTD_resetCStream_internal( + { CHECK_F( ZSTD_resetCStream_internal( cctx, prefixDict.dict, prefixDict.dictSize, prefixDict.dictMode, cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) ); + assert(cctx->streamStage == zcss_load); + assert(cctx->appliedParams.nbThreads <= 1); } } /* compression stage */ #ifdef ZSTD_MULTITHREAD if (cctx->appliedParams.nbThreads > 1) { size_t const flushMin = ZSTDMT_compressStream_generic(cctx->mtctx, output, input, endOp); - DEBUGLOG(5, "ZSTDMT_compressStream_generic result : %u", (U32)flushMin); if ( ZSTD_isError(flushMin) || (endOp == ZSTD_e_end && flushMin == 0) ) { /* compression completed */ ZSTD_startNewCompression(cctx); @@ -2850,8 +2945,7 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) { size_t const lastBlockSize = zcs->frameEnded ? 0 : ZSTD_BLOCKHEADERSIZE; size_t const checksumSize = zcs->frameEnded ? 0 : zcs->appliedParams.fParams.checksumFlag * 4; size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize + lastBlockSize + checksumSize; - DEBUGLOG(5, "ZSTD_endStream : remaining to flush : %u", - (unsigned)toFlush); + DEBUGLOG(4, "ZSTD_endStream : remaining to flush : %u", (U32)toFlush); return toFlush; } } @@ -2880,12 +2974,12 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV { 22, 20, 22, 5, 5, 16, ZSTD_lazy2 }, /* level 12 */ { 22, 21, 22, 5, 5, 16, ZSTD_lazy2 }, /* level 13 */ { 22, 21, 22, 6, 5, 16, ZSTD_lazy2 }, /* level 14 */ - { 22, 21, 22, 5, 5, 16, ZSTD_btlazy2 }, /* level 15 */ - { 23, 22, 22, 5, 5, 16, ZSTD_btlazy2 }, /* level 16 */ - { 23, 22, 22, 4, 5, 24, ZSTD_btopt }, /* level 17 */ - { 23, 22, 22, 5, 4, 32, ZSTD_btopt }, /* level 18 */ - { 23, 23, 22, 6, 3, 48, ZSTD_btopt }, /* level 19 */ - { 25, 25, 23, 7, 3, 64, ZSTD_btultra }, /* level 20 */ + { 22, 21, 22, 4, 5, 16, ZSTD_btlazy2 }, /* level 15 */ + { 22, 21, 22, 4, 5, 48, ZSTD_btopt }, /* level 16 */ + { 23, 22, 22, 4, 4, 48, ZSTD_btopt }, /* level 17 */ + { 23, 22, 22, 5, 3, 64, ZSTD_btopt }, /* level 18 */ + { 23, 23, 22, 7, 3,128, ZSTD_btopt }, /* level 19 */ + { 25, 25, 23, 7, 3,128, ZSTD_btultra }, /* level 20 */ { 26, 26, 24, 7, 3,256, ZSTD_btultra }, /* level 21 */ { 27, 27, 25, 9, 3,512, ZSTD_btultra }, /* level 22 */ }, @@ -3004,6 +3098,8 @@ ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long l } #endif + DEBUGLOG(4, "ZSTD_getCParams: cLevel=%i, srcSize=%u, dictSize=%u => table %u", + compressionLevel, (U32)srcSizeHint, (U32)dictSize, tableID); if (compressionLevel <= 0) compressionLevel = ZSTD_CLEVEL_DEFAULT; /* 0 == default; no negative compressionLevel yet */ if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL; { ZSTD_compressionParameters const cp = ZSTD_defaultCParameters[tableID][compressionLevel]; @@ -3019,5 +3115,6 @@ ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSizeH ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, srcSizeHint, dictSize); memset(¶ms, 0, sizeof(params)); params.cParams = cParams; + params.fParams.contentSizeFlag = 1; return params; } |