From 3328348c63b8b1bd5a6eaf53683b8ed68bf01f6e Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Tue, 15 Oct 2019 10:22:06 -0400 Subject: [PATCH 01/13] Add compressionlevel to cdict --- lib/compress/zstd_compress.c | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 7facbeff0..f16da6a85 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -50,6 +50,7 @@ struct ZSTD_CDict_s { ZSTD_compressedBlockState_t cBlockState; ZSTD_customMem customMem; U32 dictID; + BYTE compressionLevel; /* 0 indicates that advanced API was used to select params */ }; /* typedef'd to ZSTD_CDict within "zstd.h" */ ZSTD_CCtx* ZSTD_createCCtx(void) From 3fa4daaa55a95efdb985b3919fa67dddc4211c49 Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Tue, 15 Oct 2019 12:28:23 -0400 Subject: [PATCH 02/13] Fix error --- lib/compress/zstd_compress.c | 44 ++++++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index f16da6a85..5d4648790 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -50,7 +50,7 @@ struct ZSTD_CDict_s { ZSTD_compressedBlockState_t cBlockState; ZSTD_customMem customMem; U32 dictID; - BYTE compressionLevel; /* 0 indicates that advanced API was used to select params */ + BYTE compressionLevel; /* 0 indicates that advanced API was used to select CDict params */ }; /* typedef'd to ZSTD_CDict within "zstd.h" */ ZSTD_CCtx* ZSTD_createCCtx(void) @@ -2890,6 +2890,8 @@ ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs, bs, ms, ws, params, dict, dictSize, dtlm, workspace); } +#define ZSTD_USE_CDICT_PARAMS_CUTOFF (1 MB) + /*! ZSTD_compressBegin_internal() : * @return : 0, or an error code */ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, @@ -2904,17 +2906,22 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, /* params are supposed to be fully validated at this point */ assert(!ZSTD_isError(ZSTD_checkCParams(params->cParams))); assert(!((dict) && (cdict))); /* either dict or cdict, not both */ - - if (cdict && cdict->dictContentSize>0) { + if ( (cdict) + && (cdict->dictContentSize > 0) + && (pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_CUTOFF || cdict->compressionLevel == 0) ) { return ZSTD_resetCCtx_usingCDict(cctx, cdict, params, pledgedSrcSize, zbuff); } FORWARD_IF_ERROR( ZSTD_resetCCtx_internal(cctx, *params, pledgedSrcSize, - ZSTDcrp_makeClean, zbuff) ); - { size_t const dictID = ZSTD_compress_insertDictionary( - cctx->blockState.prevCBlock, &cctx->blockState.matchState, - &cctx->workspace, params, dict, dictSize, dictContentType, dtlm, - cctx->entropyWorkspace); + ZSTDcrp_continue, zbuff) ); + { size_t const dictID = cdict ? + ZSTD_compress_insertDictionary( + cctx->blockState.prevCBlock, &cctx->blockState.matchState, + params, cdict->dictContent, cdict->dictContentSize, + dictContentType, dtlm, cctx->entropyWorkspace) + : ZSTD_compress_insertDictionary( + cctx->blockState.prevCBlock, &cctx->blockState.matchState, + params, dict, dictSize, dictContentType, dtlm, cctx->entropyWorkspace); FORWARD_IF_ERROR(dictID); assert(dictID <= UINT_MAX); cctx->dictID = (U32)dictID; @@ -3229,6 +3236,13 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, assert(cdict != NULL); ZSTD_cwksp_move(&cdict->workspace, &ws); cdict->customMem = customMem; +<<<<<<< HEAD +======= + cdict->workspace = workspace; + cdict->workspaceSize = workspaceSize; + cdict->compressionLevel = 0; /* signals advanced API usage */ + +>>>>>>> Fix error if (ZSTD_isError( ZSTD_initCDict_internal(cdict, dictBuffer, dictSize, dictLoadMethod, dictContentType, @@ -3244,9 +3258,11 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel) { ZSTD_compressionParameters cParams = ZSTD_getCParams(compressionLevel, 0, dictSize); - return ZSTD_createCDict_advanced(dict, dictSize, - ZSTD_dlm_byCopy, ZSTD_dct_auto, - cParams, ZSTD_defaultCMem); + ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dict, dictSize, + ZSTD_dlm_byCopy, ZSTD_dct_auto, + cParams, ZSTD_defaultCMem); + cdict->compressionLevel = compressionLevel == 0 ? ZSTD_CLEVEL_DEFAULT : compressionLevel; + return cdict; } ZSTD_CDict* ZSTD_createCDict_byReference(const void* dict, size_t dictSize, int compressionLevel) @@ -3334,7 +3350,11 @@ size_t ZSTD_compressBegin_usingCDict_advanced( DEBUGLOG(4, "ZSTD_compressBegin_usingCDict_advanced"); RETURN_ERROR_IF(cdict==NULL, dictionary_wrong); { ZSTD_CCtx_params params = cctx->requestedParams; - params.cParams = ZSTD_getCParamsFromCDict(cdict); + params.cParams = (pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_CUTOFF) || (cdict->compressionLevel == 0) ? + ZSTD_getCParamsFromCDict(cdict) + : ZSTD_getCParams(cdict->compressionLevel, + pledgedSrcSize, + cdict->dictContentSize); /* Increase window log to fit the entire dictionary and source if the * source size is known. Limit the increase to 19, which is the * window log for compression level 1 with the largest source size. From 053a35fd6428cef16fcbf4c39d8746e82f932e3c Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Tue, 15 Oct 2019 12:44:48 -0400 Subject: [PATCH 03/13] formatting --- lib/compress/zstd_compress.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 5d4648790..9002616b6 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -3259,8 +3259,8 @@ ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionL { ZSTD_compressionParameters cParams = ZSTD_getCParams(compressionLevel, 0, dictSize); ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dict, dictSize, - ZSTD_dlm_byCopy, ZSTD_dct_auto, - cParams, ZSTD_defaultCMem); + ZSTD_dlm_byCopy, ZSTD_dct_auto, + cParams, ZSTD_defaultCMem); cdict->compressionLevel = compressionLevel == 0 ? ZSTD_CLEVEL_DEFAULT : compressionLevel; return cdict; } @@ -3353,8 +3353,8 @@ size_t ZSTD_compressBegin_usingCDict_advanced( params.cParams = (pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_CUTOFF) || (cdict->compressionLevel == 0) ? ZSTD_getCParamsFromCDict(cdict) : ZSTD_getCParams(cdict->compressionLevel, - pledgedSrcSize, - cdict->dictContentSize); + pledgedSrcSize, + cdict->dictContentSize); /* Increase window log to fit the entire dictionary and source if the * source size is known. Limit the increase to 19, which is the * window log for compression level 1 with the largest source size. From a727a85a7eb60f72d375b90c6003b64ae54f2930 Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Tue, 15 Oct 2019 13:10:05 -0400 Subject: [PATCH 04/13] merge conflicts round 2 --- lib/compress/zstd_compress.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 9002616b6..9dbc7f8fb 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2917,7 +2917,7 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, { size_t const dictID = cdict ? ZSTD_compress_insertDictionary( cctx->blockState.prevCBlock, &cctx->blockState.matchState, - params, cdict->dictContent, cdict->dictContentSize, + &cctx->workspace, params, cdict->dictContent, cdict->dictContentSize, dictContentType, dtlm, cctx->entropyWorkspace) : ZSTD_compress_insertDictionary( cctx->blockState.prevCBlock, &cctx->blockState.matchState, @@ -3237,12 +3237,17 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, ZSTD_cwksp_move(&cdict->workspace, &ws); cdict->customMem = customMem; <<<<<<< HEAD +<<<<<<< HEAD ======= cdict->workspace = workspace; cdict->workspaceSize = workspaceSize; cdict->compressionLevel = 0; /* signals advanced API usage */ >>>>>>> Fix error +======= + cdict->compressionLevel = 0; /* signals advanced API usage */ + +>>>>>>> merge conflicts round 2 if (ZSTD_isError( ZSTD_initCDict_internal(cdict, dictBuffer, dictSize, dictLoadMethod, dictContentType, From ea3cb6988f49d96e158f8180bf9aa2f4a8351058 Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Tue, 15 Oct 2019 13:58:44 -0400 Subject: [PATCH 05/13] Cast to BYTE to appease appveyor --- lib/compress/zstd_compress.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 9dbc7f8fb..7c4e299a7 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -3266,7 +3266,7 @@ ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionL ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto, cParams, ZSTD_defaultCMem); - cdict->compressionLevel = compressionLevel == 0 ? ZSTD_CLEVEL_DEFAULT : compressionLevel; + cdict->compressionLevel = compressionLevel == 0 ? (BYTE)ZSTD_CLEVEL_DEFAULT : (BYTE)compressionLevel; return cdict; } From cf51501d2f6b83b5dbf958775827592febbf9071 Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Wed, 16 Oct 2019 10:29:31 -0400 Subject: [PATCH 06/13] Fix test --- lib/compress/zstd_compress.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 7c4e299a7..3e1e370ef 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -3266,7 +3266,9 @@ ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionL ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto, cParams, ZSTD_defaultCMem); - cdict->compressionLevel = compressionLevel == 0 ? (BYTE)ZSTD_CLEVEL_DEFAULT : (BYTE)compressionLevel; + if (cdict) + cdict->compressionLevel = compressionLevel == 0 ? (BYTE)ZSTD_CLEVEL_DEFAULT : (BYTE)compressionLevel; + return cdict; } From bb2df8c4999f8e9679678d3c0c5188d3c239f692 Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Wed, 16 Oct 2019 10:31:27 -0400 Subject: [PATCH 07/13] Trailing whitespace --- lib/compress/zstd_compress.c | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 3e1e370ef..d0b01c073 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -3268,7 +3268,6 @@ ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionL cParams, ZSTD_defaultCMem); if (cdict) cdict->compressionLevel = compressionLevel == 0 ? (BYTE)ZSTD_CLEVEL_DEFAULT : (BYTE)compressionLevel; - return cdict; } From f0fccc8847dcd2e279a215a4d8024e46e32f8de6 Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Wed, 16 Oct 2019 15:05:29 -0400 Subject: [PATCH 08/13] Changed to int from BYTE --- lib/compress/zstd_compress.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index d0b01c073..2daa588f1 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -50,7 +50,7 @@ struct ZSTD_CDict_s { ZSTD_compressedBlockState_t cBlockState; ZSTD_customMem customMem; U32 dictID; - BYTE compressionLevel; /* 0 indicates that advanced API was used to select CDict params */ + int compressionLevel; /* 0 indicates that advanced API was used to select CDict params */ }; /* typedef'd to ZSTD_CDict within "zstd.h" */ ZSTD_CCtx* ZSTD_createCCtx(void) From 9294f4826b3bbcd1581574af859cb0678d60c7a1 Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Wed, 16 Oct 2019 15:06:02 -0400 Subject: [PATCH 09/13] Changed to int from BYTE --- lib/compress/zstd_compress.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 2daa588f1..634a0f857 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -3267,7 +3267,7 @@ ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionL ZSTD_dlm_byCopy, ZSTD_dct_auto, cParams, ZSTD_defaultCMem); if (cdict) - cdict->compressionLevel = compressionLevel == 0 ? (BYTE)ZSTD_CLEVEL_DEFAULT : (BYTE)compressionLevel; + cdict->compressionLevel = compressionLevel == 0 ? ZSTD_CLEVEL_DEFAULT : compressionLevel; return cdict; } From e4de8b098af8e4e375d6af5ce17bdffb986209a1 Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Sun, 20 Oct 2019 14:03:09 -0400 Subject: [PATCH 10/13] Added support for forcing new CDict behavior and updated enum --- lib/compress/zstd_compress.c | 3 ++- lib/zstd.h | 7 ++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 634a0f857..ab5b13d01 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2908,7 +2908,8 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, assert(!((dict) && (cdict))); /* either dict or cdict, not both */ if ( (cdict) && (cdict->dictContentSize > 0) - && (pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_CUTOFF || cdict->compressionLevel == 0) ) { + && (pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_CUTOFF || cdict->compressionLevel == 0) + && (params->attachDictPref != ZSTD_dictForceInputParams) ) { return ZSTD_resetCCtx_usingCDict(cctx, cdict, params, pledgedSrcSize, zbuff); } diff --git a/lib/zstd.h b/lib/zstd.h index 667845627..4859733bb 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -1149,7 +1149,7 @@ typedef enum { * to evolve and should be considered only in the context of extremely * advanced performance tuning. * - * Zstd currently supports the use of a CDict in two ways: + * Zstd currently supports the use of a CDict in three ways: * * - The contents of the CDict can be copied into the working context. This * means that the compression can search both the dictionary and input @@ -1164,6 +1164,10 @@ typedef enum { * tables. However, this model incurs no start-up cost (as long as the * working context's tables can be reused). For small inputs, this can be * faster than copying the CDict's tables. + * + * - The CDict's tables are not used at all, and instead we use the working + * context alone to determine how our tables are initialized. This method + * should be used when using a small dictionary to compress a large input. * * Zstd has a simple internal heuristic that selects which strategy to use * at the beginning of a compression. However, if experimentation shows that @@ -1173,6 +1177,7 @@ typedef enum { ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */ ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */ ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */ + ZSTD_dictForceInputParams = 3, /* Always use input to determine tables */ } ZSTD_dictAttachPref_e; typedef enum { From 8f69c476439b6492161ea798832a708a3b877164 Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Sun, 20 Oct 2019 19:02:47 -0400 Subject: [PATCH 11/13] Add enum to decision process --- lib/compress/zstd_compress.c | 5 +++-- lib/zstd.h | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index ab5b13d01..84af6feea 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2909,7 +2909,7 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, if ( (cdict) && (cdict->dictContentSize > 0) && (pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_CUTOFF || cdict->compressionLevel == 0) - && (params->attachDictPref != ZSTD_dictForceInputParams) ) { + && (params->attachDictPref != ZSTD_dictForceSource) ) { return ZSTD_resetCCtx_usingCDict(cctx, cdict, params, pledgedSrcSize, zbuff); } @@ -3357,7 +3357,8 @@ size_t ZSTD_compressBegin_usingCDict_advanced( DEBUGLOG(4, "ZSTD_compressBegin_usingCDict_advanced"); RETURN_ERROR_IF(cdict==NULL, dictionary_wrong); { ZSTD_CCtx_params params = cctx->requestedParams; - params.cParams = (pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_CUTOFF) || (cdict->compressionLevel == 0) ? + params.cParams = ( (pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_CUTOFF) || (cdict->compressionLevel == 0) ) + && (params->attachDictPref != ZSTD_dictForceSource) ? ZSTD_getCParamsFromCDict(cdict) : ZSTD_getCParams(cdict->compressionLevel, pledgedSrcSize, diff --git a/lib/zstd.h b/lib/zstd.h index 4859733bb..c0521c1b8 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -1177,7 +1177,7 @@ typedef enum { ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */ ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */ ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */ - ZSTD_dictForceInputParams = 3, /* Always use input to determine tables */ + ZSTD_dictForceSource = 3, /* Always use src input to determine tables */ } ZSTD_dictAttachPref_e; typedef enum { From 1f3a51fb52034af5bd2a662b27977dfe0d9bf2e4 Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Sun, 20 Oct 2019 22:01:08 -0400 Subject: [PATCH 12/13] Updated forceAttachDict param bounds --- lib/compress/zstd_compress.c | 4 ++-- lib/zstd.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 84af6feea..3aa111ddb 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -387,7 +387,7 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param) case ZSTD_c_forceAttachDict: ZSTD_STATIC_ASSERT(ZSTD_dictDefaultAttach < ZSTD_dictForceCopy); bounds.lowerBound = ZSTD_dictDefaultAttach; - bounds.upperBound = ZSTD_dictForceCopy; /* note : how to ensure at compile time that this is the highest value enum ? */ + bounds.upperBound = ZSTD_dictForceSource; /* note : how to ensure at compile time that this is the highest value enum ? */ return bounds; case ZSTD_c_literalCompressionMode: @@ -3358,7 +3358,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced( RETURN_ERROR_IF(cdict==NULL, dictionary_wrong); { ZSTD_CCtx_params params = cctx->requestedParams; params.cParams = ( (pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_CUTOFF) || (cdict->compressionLevel == 0) ) - && (params->attachDictPref != ZSTD_dictForceSource) ? + && (params.attachDictPref != ZSTD_dictForceSource) ? ZSTD_getCParamsFromCDict(cdict) : ZSTD_getCParams(cdict->compressionLevel, pledgedSrcSize, diff --git a/lib/zstd.h b/lib/zstd.h index c0521c1b8..5468c34cc 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -1177,7 +1177,7 @@ typedef enum { ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */ ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */ ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */ - ZSTD_dictForceSource = 3, /* Always use src input to determine tables */ + ZSTD_dictForceSource = 3, /* Always use src input to determine tables */ } ZSTD_dictAttachPref_e; typedef enum { From 676f89902acbb2fb8294e601e31ae396e125c297 Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Mon, 21 Oct 2019 15:29:55 -0400 Subject: [PATCH 13/13] Added multiplier, renamed new enum to something more useful --- lib/compress/zstd_compress.c | 17 +++++++++++------ lib/zstd.h | 8 +++++--- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 3aa111ddb..73e6618c1 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -387,7 +387,7 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param) case ZSTD_c_forceAttachDict: ZSTD_STATIC_ASSERT(ZSTD_dictDefaultAttach < ZSTD_dictForceCopy); bounds.lowerBound = ZSTD_dictDefaultAttach; - bounds.upperBound = ZSTD_dictForceSource; /* note : how to ensure at compile time that this is the highest value enum ? */ + bounds.upperBound = ZSTD_dictForceLoad; /* note : how to ensure at compile time that this is the highest value enum ? */ return bounds; case ZSTD_c_literalCompressionMode: @@ -2890,7 +2890,8 @@ ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs, bs, ms, ws, params, dict, dictSize, dtlm, workspace); } -#define ZSTD_USE_CDICT_PARAMS_CUTOFF (1 MB) +#define ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF (128 KB) +#define ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER (6) /*! ZSTD_compressBegin_internal() : * @return : 0, or an error code */ @@ -2908,8 +2909,10 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, assert(!((dict) && (cdict))); /* either dict or cdict, not both */ if ( (cdict) && (cdict->dictContentSize > 0) - && (pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_CUTOFF || cdict->compressionLevel == 0) - && (params->attachDictPref != ZSTD_dictForceSource) ) { + && ( (pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF + || pledgedSrcSize < cdict->dictContentSize * ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER) + || cdict->compressionLevel == 0) + && (params->attachDictPref != ZSTD_dictForceLoad) ) { return ZSTD_resetCCtx_usingCDict(cctx, cdict, params, pledgedSrcSize, zbuff); } @@ -3357,8 +3360,10 @@ size_t ZSTD_compressBegin_usingCDict_advanced( DEBUGLOG(4, "ZSTD_compressBegin_usingCDict_advanced"); RETURN_ERROR_IF(cdict==NULL, dictionary_wrong); { ZSTD_CCtx_params params = cctx->requestedParams; - params.cParams = ( (pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_CUTOFF) || (cdict->compressionLevel == 0) ) - && (params.attachDictPref != ZSTD_dictForceSource) ? + params.cParams = ( (pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF + || pledgedSrcSize < cdict->dictContentSize * ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER) + || (cdict->compressionLevel == 0) ) + && (params.attachDictPref != ZSTD_dictForceLoad) ? ZSTD_getCParamsFromCDict(cdict) : ZSTD_getCParams(cdict->compressionLevel, pledgedSrcSize, diff --git a/lib/zstd.h b/lib/zstd.h index 5468c34cc..3ba476e88 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -1166,8 +1166,10 @@ typedef enum { * faster than copying the CDict's tables. * * - The CDict's tables are not used at all, and instead we use the working - * context alone to determine how our tables are initialized. This method - * should be used when using a small dictionary to compress a large input. + * context alone to reload the dictionary and use params based on the source + * size. See ZSTD_compress_insertDictionary() and ZSTD_compress_usingDict(). + * This method is effective when the dictionary sizes are very small relative + * to the input size, and the input size is fairly large to begin with. * * Zstd has a simple internal heuristic that selects which strategy to use * at the beginning of a compression. However, if experimentation shows that @@ -1177,7 +1179,7 @@ typedef enum { ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */ ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */ ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */ - ZSTD_dictForceSource = 3, /* Always use src input to determine tables */ + ZSTD_dictForceLoad = 3, /* Always reload the dictionary */ } ZSTD_dictAttachPref_e; typedef enum {