From a65eb39f9d79a895ae246ac4b053828711b0eeed Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Tue, 15 Oct 2019 10:22:06 -0400 Subject: [PATCH 01/29] Add compressionlevel to cdict --- lib/compress/zstd_compress.c | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index df4e828cd..acbd3a288 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -51,6 +51,7 @@ struct ZSTD_CDict_s { ZSTD_compressedBlockState_t cBlockState; ZSTD_customMem customMem; U32 dictID; + BYTE compressionLevel; /* 0 indicates that advanced API was used to select params */ }; /* typedef'd to ZSTD_CDict within "zstd.h" */ ZSTD_CCtx* ZSTD_createCCtx(void) From 0c8df5c928e3d26e601a39423cc47e9ce4a9ddf5 Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Tue, 15 Oct 2019 12:28:23 -0400 Subject: [PATCH 02/29] Fix error --- lib/compress/zstd_compress.c | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index acbd3a288..a4e73fcda 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -51,7 +51,7 @@ struct ZSTD_CDict_s { ZSTD_compressedBlockState_t cBlockState; ZSTD_customMem customMem; U32 dictID; - BYTE compressionLevel; /* 0 indicates that advanced API was used to select params */ + BYTE compressionLevel; /* 0 indicates that advanced API was used to select CDict params */ }; /* typedef'd to ZSTD_CDict within "zstd.h" */ ZSTD_CCtx* ZSTD_createCCtx(void) @@ -2826,6 +2826,8 @@ ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs, return ZSTD_loadZstdDictionary(bs, ms, params, dict, dictSize, dtlm, workspace); } +#define ZSTD_USE_CDICT_PARAMS_CUTOFF (1 MB) + /*! ZSTD_compressBegin_internal() : * @return : 0, or an error code */ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, @@ -2840,16 +2842,22 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, /* params are supposed to be fully validated at this point */ assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); assert(!((dict) && (cdict))); /* either dict or cdict, not both */ - - if (cdict && cdict->dictContentSize>0) { + if ( (cdict) + && (cdict->dictContentSize > 0) + && (pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_CUTOFF || cdict->compressionLevel == 0) ) { return ZSTD_resetCCtx_usingCDict(cctx, cdict, params, pledgedSrcSize, zbuff); } FORWARD_IF_ERROR( ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, ZSTDcrp_continue, zbuff) ); - { size_t const dictID = ZSTD_compress_insertDictionary( - cctx->blockState.prevCBlock, &cctx->blockState.matchState, - ¶ms, dict, dictSize, dictContentType, dtlm, cctx->entropyWorkspace); + { size_t const dictID = cdict ? + ZSTD_compress_insertDictionary( + cctx->blockState.prevCBlock, &cctx->blockState.matchState, + ¶ms, cdict->dictContent, cdict->dictContentSize, + dictContentType, dtlm, cctx->entropyWorkspace) + : ZSTD_compress_insertDictionary( + cctx->blockState.prevCBlock, &cctx->blockState.matchState, + ¶ms, dict, dictSize, dictContentType, dtlm, cctx->entropyWorkspace); FORWARD_IF_ERROR(dictID); assert(dictID <= UINT_MAX); cctx->dictID = (U32)dictID; @@ -3152,6 +3160,8 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, cdict->customMem = customMem; cdict->workspace = workspace; cdict->workspaceSize = workspaceSize; + cdict->compressionLevel = 0; /* signals advanced API usage */ + if (ZSTD_isError( ZSTD_initCDict_internal(cdict, dictBuffer, dictSize, dictLoadMethod, dictContentType, @@ -3167,9 +3177,11 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel) { ZSTD_compressionParameters cParams = ZSTD_getCParams(compressionLevel, 0, dictSize); - return ZSTD_createCDict_advanced(dict, dictSize, - ZSTD_dlm_byCopy, ZSTD_dct_auto, - cParams, ZSTD_defaultCMem); + ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dict, dictSize, + ZSTD_dlm_byCopy, ZSTD_dct_auto, + cParams, ZSTD_defaultCMem); + cdict->compressionLevel = compressionLevel == 0 ? ZSTD_CLEVEL_DEFAULT : compressionLevel; + return cdict; } ZSTD_CDict* ZSTD_createCDict_byReference(const void* dict, size_t dictSize, int compressionLevel) @@ -3255,7 +3267,11 @@ size_t ZSTD_compressBegin_usingCDict_advanced( DEBUGLOG(4, "ZSTD_compressBegin_usingCDict_advanced"); RETURN_ERROR_IF(cdict==NULL, dictionary_wrong); { ZSTD_CCtx_params params = cctx->requestedParams; - params.cParams = ZSTD_getCParamsFromCDict(cdict); + params.cParams = (pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_CUTOFF) || (cdict->compressionLevel == 0) ? + ZSTD_getCParamsFromCDict(cdict) + : ZSTD_getCParams(cdict->compressionLevel, + pledgedSrcSize, + cdict->dictContentSize); /* Increase window log to fit the entire dictionary and source if the * source size is known. Limit the increase to 19, which is the * window log for compression level 1 with the largest source size. From 23dac23a49a859e275b37d32e00cf23a63917303 Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Tue, 15 Oct 2019 12:44:48 -0400 Subject: [PATCH 03/29] formatting --- lib/compress/zstd_compress.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index a4e73fcda..55e9c8a65 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -3178,8 +3178,8 @@ ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionL { ZSTD_compressionParameters cParams = ZSTD_getCParams(compressionLevel, 0, dictSize); ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dict, dictSize, - ZSTD_dlm_byCopy, ZSTD_dct_auto, - cParams, ZSTD_defaultCMem); + ZSTD_dlm_byCopy, ZSTD_dct_auto, + cParams, ZSTD_defaultCMem); cdict->compressionLevel = compressionLevel == 0 ? ZSTD_CLEVEL_DEFAULT : compressionLevel; return cdict; } @@ -3270,8 +3270,8 @@ size_t ZSTD_compressBegin_usingCDict_advanced( params.cParams = (pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_CUTOFF) || (cdict->compressionLevel == 0) ? ZSTD_getCParamsFromCDict(cdict) : ZSTD_getCParams(cdict->compressionLevel, - pledgedSrcSize, - cdict->dictContentSize); + pledgedSrcSize, + cdict->dictContentSize); /* Increase window log to fit the entire dictionary and source if the * source size is known. Limit the increase to 19, which is the * window log for compression level 1 with the largest source size. From 5c010c9d2d9c78622fff02c539159c94d185347b Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Tue, 15 Oct 2019 13:10:05 -0400 Subject: [PATCH 04/29] merge conflicts round 2 --- lib/compress/zstd_compress.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index c6ab40ecd..adf0ccab7 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2917,7 +2917,7 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, { size_t const dictID = cdict ? ZSTD_compress_insertDictionary( cctx->blockState.prevCBlock, &cctx->blockState.matchState, - ¶ms, cdict->dictContent, cdict->dictContentSize, + &cctx->workspace, params, cdict->dictContent, cdict->dictContentSize, dictContentType, dtlm, cctx->entropyWorkspace) : ZSTD_compress_insertDictionary( cctx->blockState.prevCBlock, &cctx->blockState.matchState, @@ -3237,13 +3237,8 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, assert(cdict != NULL); ZSTD_cwksp_move(&cdict->workspace, &ws); cdict->customMem = customMem; -<<<<<<< HEAD - cdict->workspace = workspace; - cdict->workspaceSize = workspaceSize; cdict->compressionLevel = 0; /* signals advanced API usage */ -======= ->>>>>>> 8b6d96827c24dd09109830272f413254833317d9 if (ZSTD_isError( ZSTD_initCDict_internal(cdict, dictBuffer, dictSize, dictLoadMethod, dictContentType, From 5e901b6f32932aa00fda4dc118b0cef1c2029461 Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Tue, 15 Oct 2019 13:58:44 -0400 Subject: [PATCH 05/29] Cast to BYTE to appease appveyor --- lib/compress/zstd_compress.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index adf0ccab7..2737b05f7 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -3257,7 +3257,7 @@ ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionL ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto, cParams, ZSTD_defaultCMem); - cdict->compressionLevel = compressionLevel == 0 ? ZSTD_CLEVEL_DEFAULT : compressionLevel; + cdict->compressionLevel = compressionLevel == 0 ? (BYTE)ZSTD_CLEVEL_DEFAULT : (BYTE)compressionLevel; return cdict; } From 8cb217444617f0432a33a3cd307487cb026e62d3 Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Wed, 16 Oct 2019 10:29:31 -0400 Subject: [PATCH 06/29] Fix test --- lib/compress/zstd_compress.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 2737b05f7..0207effe9 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -3257,7 +3257,9 @@ ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionL ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto, cParams, ZSTD_defaultCMem); - cdict->compressionLevel = compressionLevel == 0 ? (BYTE)ZSTD_CLEVEL_DEFAULT : (BYTE)compressionLevel; + if (cdict) + cdict->compressionLevel = compressionLevel == 0 ? (BYTE)ZSTD_CLEVEL_DEFAULT : (BYTE)compressionLevel; + return cdict; } From cf00ea367a1c9056bad1c86038a4cb35a44e08f1 Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Wed, 16 Oct 2019 10:31:27 -0400 Subject: [PATCH 07/29] Trailing whitespace --- lib/compress/zstd_compress.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 0207effe9..4de67777f 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2911,7 +2911,7 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, && (pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_CUTOFF || cdict->compressionLevel == 0) ) { return ZSTD_resetCCtx_usingCDict(cctx, cdict, params, pledgedSrcSize, zbuff); } - + FORWARD_IF_ERROR( ZSTD_resetCCtx_internal(cctx, *params, pledgedSrcSize, ZSTDcrp_makeClean, zbuff) ); { size_t const dictID = cdict ? @@ -3259,7 +3259,6 @@ ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionL cParams, ZSTD_defaultCMem); if (cdict) cdict->compressionLevel = compressionLevel == 0 ? (BYTE)ZSTD_CLEVEL_DEFAULT : (BYTE)compressionLevel; - return cdict; } From 4f7d26b0eec5eff6fc30a3927f80bd317f8e763a Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Wed, 16 Oct 2019 15:05:29 -0400 Subject: [PATCH 08/29] Changed to int from BYTE --- lib/compress/zstd_compress.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 4de67777f..a8d2b3a47 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -50,7 +50,7 @@ struct ZSTD_CDict_s { ZSTD_compressedBlockState_t cBlockState; ZSTD_customMem customMem; U32 dictID; - BYTE compressionLevel; /* 0 indicates that advanced API was used to select CDict params */ + int compressionLevel; /* 0 indicates that advanced API was used to select CDict params */ }; /* typedef'd to ZSTD_CDict within "zstd.h" */ ZSTD_CCtx* ZSTD_createCCtx(void) From 4455f00cb8b476c152e8681835a94ce8674439fc Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Wed, 16 Oct 2019 15:06:02 -0400 Subject: [PATCH 09/29] Changed to int from BYTE --- lib/compress/zstd_compress.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index a8d2b3a47..672f76080 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -3258,7 +3258,7 @@ ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionL ZSTD_dlm_byCopy, ZSTD_dct_auto, cParams, ZSTD_defaultCMem); if (cdict) - cdict->compressionLevel = compressionLevel == 0 ? (BYTE)ZSTD_CLEVEL_DEFAULT : (BYTE)compressionLevel; + cdict->compressionLevel = compressionLevel == 0 ? ZSTD_CLEVEL_DEFAULT : compressionLevel; return cdict; } From 1daa898c93545d659ae4987a428bac4883961dc6 Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Sun, 20 Oct 2019 14:03:09 -0400 Subject: [PATCH 10/29] Added support for forcing new CDict behavior and updated enum --- lib/compress/zstd_compress.c | 3 ++- lib/zstd.h | 7 ++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 672f76080..f0e45f518 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2908,7 +2908,8 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, assert(!((dict) && (cdict))); /* either dict or cdict, not both */ if ( (cdict) && (cdict->dictContentSize > 0) - && (pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_CUTOFF || cdict->compressionLevel == 0) ) { + && (pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_CUTOFF || cdict->compressionLevel == 0) + && (params->attachDictPref != ZSTD_dictForceInputParams) ) { return ZSTD_resetCCtx_usingCDict(cctx, cdict, params, pledgedSrcSize, zbuff); } diff --git a/lib/zstd.h b/lib/zstd.h index 667845627..4859733bb 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -1149,7 +1149,7 @@ typedef enum { * to evolve and should be considered only in the context of extremely * advanced performance tuning. * - * Zstd currently supports the use of a CDict in two ways: + * Zstd currently supports the use of a CDict in three ways: * * - The contents of the CDict can be copied into the working context. This * means that the compression can search both the dictionary and input @@ -1164,6 +1164,10 @@ typedef enum { * tables. However, this model incurs no start-up cost (as long as the * working context's tables can be reused). For small inputs, this can be * faster than copying the CDict's tables. + * + * - The CDict's tables are not used at all, and instead we use the working + * context alone to determine how our tables are initialized. This method + * should be used when using a small dictionary to compress a large input. * * Zstd has a simple internal heuristic that selects which strategy to use * at the beginning of a compression. However, if experimentation shows that @@ -1173,6 +1177,7 @@ typedef enum { ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */ ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */ ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */ + ZSTD_dictForceInputParams = 3, /* Always use input to determine tables */ } ZSTD_dictAttachPref_e; typedef enum { From 6d297265f910dc08e99b4bb81a8da22524240305 Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Sun, 20 Oct 2019 19:02:47 -0400 Subject: [PATCH 11/29] Add enum to decision process --- lib/compress/zstd_compress.c | 5 +++-- lib/zstd.h | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index f0e45f518..51e8f59c0 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2909,7 +2909,7 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, if ( (cdict) && (cdict->dictContentSize > 0) && (pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_CUTOFF || cdict->compressionLevel == 0) - && (params->attachDictPref != ZSTD_dictForceInputParams) ) { + && (params->attachDictPref != ZSTD_dictForceSource) ) { return ZSTD_resetCCtx_usingCDict(cctx, cdict, params, pledgedSrcSize, zbuff); } @@ -3348,7 +3348,8 @@ size_t ZSTD_compressBegin_usingCDict_advanced( DEBUGLOG(4, "ZSTD_compressBegin_usingCDict_advanced"); RETURN_ERROR_IF(cdict==NULL, dictionary_wrong); { ZSTD_CCtx_params params = cctx->requestedParams; - params.cParams = (pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_CUTOFF) || (cdict->compressionLevel == 0) ? + params.cParams = ( (pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_CUTOFF) || (cdict->compressionLevel == 0) ) + && (params->attachDictPref != ZSTD_dictForceSource) ? ZSTD_getCParamsFromCDict(cdict) : ZSTD_getCParams(cdict->compressionLevel, pledgedSrcSize, diff --git a/lib/zstd.h b/lib/zstd.h index 4859733bb..c0521c1b8 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -1177,7 +1177,7 @@ typedef enum { ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */ ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */ ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */ - ZSTD_dictForceInputParams = 3, /* Always use input to determine tables */ + ZSTD_dictForceSource = 3, /* Always use src input to determine tables */ } ZSTD_dictAttachPref_e; typedef enum { From e8aa3e486d7ad23bafb19cc39be9db5bce4ee3c7 Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Sun, 20 Oct 2019 22:01:08 -0400 Subject: [PATCH 12/29] Updated forceAttachDict param bounds --- lib/compress/zstd_compress.c | 4 ++-- lib/zstd.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 51e8f59c0..98d064397 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -387,7 +387,7 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param) case ZSTD_c_forceAttachDict: ZSTD_STATIC_ASSERT(ZSTD_dictDefaultAttach < ZSTD_dictForceCopy); bounds.lowerBound = ZSTD_dictDefaultAttach; - bounds.upperBound = ZSTD_dictForceCopy; /* note : how to ensure at compile time that this is the highest value enum ? */ + bounds.upperBound = ZSTD_dictForceSource; /* note : how to ensure at compile time that this is the highest value enum ? */ return bounds; case ZSTD_c_literalCompressionMode: @@ -3349,7 +3349,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced( RETURN_ERROR_IF(cdict==NULL, dictionary_wrong); { ZSTD_CCtx_params params = cctx->requestedParams; params.cParams = ( (pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_CUTOFF) || (cdict->compressionLevel == 0) ) - && (params->attachDictPref != ZSTD_dictForceSource) ? + && (params.attachDictPref != ZSTD_dictForceSource) ? ZSTD_getCParamsFromCDict(cdict) : ZSTD_getCParams(cdict->compressionLevel, pledgedSrcSize, diff --git a/lib/zstd.h b/lib/zstd.h index c0521c1b8..5468c34cc 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -1177,7 +1177,7 @@ typedef enum { ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */ ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */ ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */ - ZSTD_dictForceSource = 3, /* Always use src input to determine tables */ + ZSTD_dictForceSource = 3, /* Always use src input to determine tables */ } ZSTD_dictAttachPref_e; typedef enum { From 3328348c63b8b1bd5a6eaf53683b8ed68bf01f6e Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Tue, 15 Oct 2019 10:22:06 -0400 Subject: [PATCH 13/29] Add compressionlevel to cdict --- lib/compress/zstd_compress.c | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 7facbeff0..f16da6a85 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -50,6 +50,7 @@ struct ZSTD_CDict_s { ZSTD_compressedBlockState_t cBlockState; ZSTD_customMem customMem; U32 dictID; + BYTE compressionLevel; /* 0 indicates that advanced API was used to select params */ }; /* typedef'd to ZSTD_CDict within "zstd.h" */ ZSTD_CCtx* ZSTD_createCCtx(void) From 3fa4daaa55a95efdb985b3919fa67dddc4211c49 Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Tue, 15 Oct 2019 12:28:23 -0400 Subject: [PATCH 14/29] Fix error --- lib/compress/zstd_compress.c | 44 ++++++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index f16da6a85..5d4648790 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -50,7 +50,7 @@ struct ZSTD_CDict_s { ZSTD_compressedBlockState_t cBlockState; ZSTD_customMem customMem; U32 dictID; - BYTE compressionLevel; /* 0 indicates that advanced API was used to select params */ + BYTE compressionLevel; /* 0 indicates that advanced API was used to select CDict params */ }; /* typedef'd to ZSTD_CDict within "zstd.h" */ ZSTD_CCtx* ZSTD_createCCtx(void) @@ -2890,6 +2890,8 @@ ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs, bs, ms, ws, params, dict, dictSize, dtlm, workspace); } +#define ZSTD_USE_CDICT_PARAMS_CUTOFF (1 MB) + /*! ZSTD_compressBegin_internal() : * @return : 0, or an error code */ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, @@ -2904,17 +2906,22 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, /* params are supposed to be fully validated at this point */ assert(!ZSTD_isError(ZSTD_checkCParams(params->cParams))); assert(!((dict) && (cdict))); /* either dict or cdict, not both */ - - if (cdict && cdict->dictContentSize>0) { + if ( (cdict) + && (cdict->dictContentSize > 0) + && (pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_CUTOFF || cdict->compressionLevel == 0) ) { return ZSTD_resetCCtx_usingCDict(cctx, cdict, params, pledgedSrcSize, zbuff); } FORWARD_IF_ERROR( ZSTD_resetCCtx_internal(cctx, *params, pledgedSrcSize, - ZSTDcrp_makeClean, zbuff) ); - { size_t const dictID = ZSTD_compress_insertDictionary( - cctx->blockState.prevCBlock, &cctx->blockState.matchState, - &cctx->workspace, params, dict, dictSize, dictContentType, dtlm, - cctx->entropyWorkspace); + ZSTDcrp_continue, zbuff) ); + { size_t const dictID = cdict ? + ZSTD_compress_insertDictionary( + cctx->blockState.prevCBlock, &cctx->blockState.matchState, + params, cdict->dictContent, cdict->dictContentSize, + dictContentType, dtlm, cctx->entropyWorkspace) + : ZSTD_compress_insertDictionary( + cctx->blockState.prevCBlock, &cctx->blockState.matchState, + params, dict, dictSize, dictContentType, dtlm, cctx->entropyWorkspace); FORWARD_IF_ERROR(dictID); assert(dictID <= UINT_MAX); cctx->dictID = (U32)dictID; @@ -3229,6 +3236,13 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, assert(cdict != NULL); ZSTD_cwksp_move(&cdict->workspace, &ws); cdict->customMem = customMem; +<<<<<<< HEAD +======= + cdict->workspace = workspace; + cdict->workspaceSize = workspaceSize; + cdict->compressionLevel = 0; /* signals advanced API usage */ + +>>>>>>> Fix error if (ZSTD_isError( ZSTD_initCDict_internal(cdict, dictBuffer, dictSize, dictLoadMethod, dictContentType, @@ -3244,9 +3258,11 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel) { ZSTD_compressionParameters cParams = ZSTD_getCParams(compressionLevel, 0, dictSize); - return ZSTD_createCDict_advanced(dict, dictSize, - ZSTD_dlm_byCopy, ZSTD_dct_auto, - cParams, ZSTD_defaultCMem); + ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dict, dictSize, + ZSTD_dlm_byCopy, ZSTD_dct_auto, + cParams, ZSTD_defaultCMem); + cdict->compressionLevel = compressionLevel == 0 ? ZSTD_CLEVEL_DEFAULT : compressionLevel; + return cdict; } ZSTD_CDict* ZSTD_createCDict_byReference(const void* dict, size_t dictSize, int compressionLevel) @@ -3334,7 +3350,11 @@ size_t ZSTD_compressBegin_usingCDict_advanced( DEBUGLOG(4, "ZSTD_compressBegin_usingCDict_advanced"); RETURN_ERROR_IF(cdict==NULL, dictionary_wrong); { ZSTD_CCtx_params params = cctx->requestedParams; - params.cParams = ZSTD_getCParamsFromCDict(cdict); + params.cParams = (pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_CUTOFF) || (cdict->compressionLevel == 0) ? + ZSTD_getCParamsFromCDict(cdict) + : ZSTD_getCParams(cdict->compressionLevel, + pledgedSrcSize, + cdict->dictContentSize); /* Increase window log to fit the entire dictionary and source if the * source size is known. Limit the increase to 19, which is the * window log for compression level 1 with the largest source size. From 053a35fd6428cef16fcbf4c39d8746e82f932e3c Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Tue, 15 Oct 2019 12:44:48 -0400 Subject: [PATCH 15/29] formatting --- lib/compress/zstd_compress.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 5d4648790..9002616b6 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -3259,8 +3259,8 @@ ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionL { ZSTD_compressionParameters cParams = ZSTD_getCParams(compressionLevel, 0, dictSize); ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dict, dictSize, - ZSTD_dlm_byCopy, ZSTD_dct_auto, - cParams, ZSTD_defaultCMem); + ZSTD_dlm_byCopy, ZSTD_dct_auto, + cParams, ZSTD_defaultCMem); cdict->compressionLevel = compressionLevel == 0 ? ZSTD_CLEVEL_DEFAULT : compressionLevel; return cdict; } @@ -3353,8 +3353,8 @@ size_t ZSTD_compressBegin_usingCDict_advanced( params.cParams = (pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_CUTOFF) || (cdict->compressionLevel == 0) ? ZSTD_getCParamsFromCDict(cdict) : ZSTD_getCParams(cdict->compressionLevel, - pledgedSrcSize, - cdict->dictContentSize); + pledgedSrcSize, + cdict->dictContentSize); /* Increase window log to fit the entire dictionary and source if the * source size is known. Limit the increase to 19, which is the * window log for compression level 1 with the largest source size. From a727a85a7eb60f72d375b90c6003b64ae54f2930 Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Tue, 15 Oct 2019 13:10:05 -0400 Subject: [PATCH 16/29] merge conflicts round 2 --- lib/compress/zstd_compress.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 9002616b6..9dbc7f8fb 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2917,7 +2917,7 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, { size_t const dictID = cdict ? ZSTD_compress_insertDictionary( cctx->blockState.prevCBlock, &cctx->blockState.matchState, - params, cdict->dictContent, cdict->dictContentSize, + &cctx->workspace, params, cdict->dictContent, cdict->dictContentSize, dictContentType, dtlm, cctx->entropyWorkspace) : ZSTD_compress_insertDictionary( cctx->blockState.prevCBlock, &cctx->blockState.matchState, @@ -3237,12 +3237,17 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, ZSTD_cwksp_move(&cdict->workspace, &ws); cdict->customMem = customMem; <<<<<<< HEAD +<<<<<<< HEAD ======= cdict->workspace = workspace; cdict->workspaceSize = workspaceSize; cdict->compressionLevel = 0; /* signals advanced API usage */ >>>>>>> Fix error +======= + cdict->compressionLevel = 0; /* signals advanced API usage */ + +>>>>>>> merge conflicts round 2 if (ZSTD_isError( ZSTD_initCDict_internal(cdict, dictBuffer, dictSize, dictLoadMethod, dictContentType, From ea3cb6988f49d96e158f8180bf9aa2f4a8351058 Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Tue, 15 Oct 2019 13:58:44 -0400 Subject: [PATCH 17/29] Cast to BYTE to appease appveyor --- lib/compress/zstd_compress.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 9dbc7f8fb..7c4e299a7 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -3266,7 +3266,7 @@ ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionL ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto, cParams, ZSTD_defaultCMem); - cdict->compressionLevel = compressionLevel == 0 ? ZSTD_CLEVEL_DEFAULT : compressionLevel; + cdict->compressionLevel = compressionLevel == 0 ? (BYTE)ZSTD_CLEVEL_DEFAULT : (BYTE)compressionLevel; return cdict; } From cf51501d2f6b83b5dbf958775827592febbf9071 Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Wed, 16 Oct 2019 10:29:31 -0400 Subject: [PATCH 18/29] Fix test --- lib/compress/zstd_compress.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 7c4e299a7..3e1e370ef 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -3266,7 +3266,9 @@ ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionL ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto, cParams, ZSTD_defaultCMem); - cdict->compressionLevel = compressionLevel == 0 ? (BYTE)ZSTD_CLEVEL_DEFAULT : (BYTE)compressionLevel; + if (cdict) + cdict->compressionLevel = compressionLevel == 0 ? (BYTE)ZSTD_CLEVEL_DEFAULT : (BYTE)compressionLevel; + return cdict; } From bb2df8c4999f8e9679678d3c0c5188d3c239f692 Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Wed, 16 Oct 2019 10:31:27 -0400 Subject: [PATCH 19/29] Trailing whitespace --- lib/compress/zstd_compress.c | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 3e1e370ef..d0b01c073 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -3268,7 +3268,6 @@ ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionL cParams, ZSTD_defaultCMem); if (cdict) cdict->compressionLevel = compressionLevel == 0 ? (BYTE)ZSTD_CLEVEL_DEFAULT : (BYTE)compressionLevel; - return cdict; } From f0fccc8847dcd2e279a215a4d8024e46e32f8de6 Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Wed, 16 Oct 2019 15:05:29 -0400 Subject: [PATCH 20/29] Changed to int from BYTE --- lib/compress/zstd_compress.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index d0b01c073..2daa588f1 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -50,7 +50,7 @@ struct ZSTD_CDict_s { ZSTD_compressedBlockState_t cBlockState; ZSTD_customMem customMem; U32 dictID; - BYTE compressionLevel; /* 0 indicates that advanced API was used to select CDict params */ + int compressionLevel; /* 0 indicates that advanced API was used to select CDict params */ }; /* typedef'd to ZSTD_CDict within "zstd.h" */ ZSTD_CCtx* ZSTD_createCCtx(void) From 9294f4826b3bbcd1581574af859cb0678d60c7a1 Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Wed, 16 Oct 2019 15:06:02 -0400 Subject: [PATCH 21/29] Changed to int from BYTE --- lib/compress/zstd_compress.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 2daa588f1..634a0f857 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -3267,7 +3267,7 @@ ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionL ZSTD_dlm_byCopy, ZSTD_dct_auto, cParams, ZSTD_defaultCMem); if (cdict) - cdict->compressionLevel = compressionLevel == 0 ? (BYTE)ZSTD_CLEVEL_DEFAULT : (BYTE)compressionLevel; + cdict->compressionLevel = compressionLevel == 0 ? ZSTD_CLEVEL_DEFAULT : compressionLevel; return cdict; } From e4de8b098af8e4e375d6af5ce17bdffb986209a1 Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Sun, 20 Oct 2019 14:03:09 -0400 Subject: [PATCH 22/29] Added support for forcing new CDict behavior and updated enum --- lib/compress/zstd_compress.c | 3 ++- lib/zstd.h | 7 ++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 634a0f857..ab5b13d01 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2908,7 +2908,8 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, assert(!((dict) && (cdict))); /* either dict or cdict, not both */ if ( (cdict) && (cdict->dictContentSize > 0) - && (pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_CUTOFF || cdict->compressionLevel == 0) ) { + && (pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_CUTOFF || cdict->compressionLevel == 0) + && (params->attachDictPref != ZSTD_dictForceInputParams) ) { return ZSTD_resetCCtx_usingCDict(cctx, cdict, params, pledgedSrcSize, zbuff); } diff --git a/lib/zstd.h b/lib/zstd.h index 667845627..4859733bb 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -1149,7 +1149,7 @@ typedef enum { * to evolve and should be considered only in the context of extremely * advanced performance tuning. * - * Zstd currently supports the use of a CDict in two ways: + * Zstd currently supports the use of a CDict in three ways: * * - The contents of the CDict can be copied into the working context. This * means that the compression can search both the dictionary and input @@ -1164,6 +1164,10 @@ typedef enum { * tables. However, this model incurs no start-up cost (as long as the * working context's tables can be reused). For small inputs, this can be * faster than copying the CDict's tables. + * + * - The CDict's tables are not used at all, and instead we use the working + * context alone to determine how our tables are initialized. This method + * should be used when using a small dictionary to compress a large input. * * Zstd has a simple internal heuristic that selects which strategy to use * at the beginning of a compression. However, if experimentation shows that @@ -1173,6 +1177,7 @@ typedef enum { ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */ ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */ ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */ + ZSTD_dictForceInputParams = 3, /* Always use input to determine tables */ } ZSTD_dictAttachPref_e; typedef enum { From 8f69c476439b6492161ea798832a708a3b877164 Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Sun, 20 Oct 2019 19:02:47 -0400 Subject: [PATCH 23/29] Add enum to decision process --- lib/compress/zstd_compress.c | 5 +++-- lib/zstd.h | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index ab5b13d01..84af6feea 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2909,7 +2909,7 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, if ( (cdict) && (cdict->dictContentSize > 0) && (pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_CUTOFF || cdict->compressionLevel == 0) - && (params->attachDictPref != ZSTD_dictForceInputParams) ) { + && (params->attachDictPref != ZSTD_dictForceSource) ) { return ZSTD_resetCCtx_usingCDict(cctx, cdict, params, pledgedSrcSize, zbuff); } @@ -3357,7 +3357,8 @@ size_t ZSTD_compressBegin_usingCDict_advanced( DEBUGLOG(4, "ZSTD_compressBegin_usingCDict_advanced"); RETURN_ERROR_IF(cdict==NULL, dictionary_wrong); { ZSTD_CCtx_params params = cctx->requestedParams; - params.cParams = (pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_CUTOFF) || (cdict->compressionLevel == 0) ? + params.cParams = ( (pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_CUTOFF) || (cdict->compressionLevel == 0) ) + && (params->attachDictPref != ZSTD_dictForceSource) ? ZSTD_getCParamsFromCDict(cdict) : ZSTD_getCParams(cdict->compressionLevel, pledgedSrcSize, diff --git a/lib/zstd.h b/lib/zstd.h index 4859733bb..c0521c1b8 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -1177,7 +1177,7 @@ typedef enum { ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */ ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */ ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */ - ZSTD_dictForceInputParams = 3, /* Always use input to determine tables */ + ZSTD_dictForceSource = 3, /* Always use src input to determine tables */ } ZSTD_dictAttachPref_e; typedef enum { From 1f3a51fb52034af5bd2a662b27977dfe0d9bf2e4 Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Sun, 20 Oct 2019 22:01:08 -0400 Subject: [PATCH 24/29] Updated forceAttachDict param bounds --- lib/compress/zstd_compress.c | 4 ++-- lib/zstd.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 84af6feea..3aa111ddb 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -387,7 +387,7 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param) case ZSTD_c_forceAttachDict: ZSTD_STATIC_ASSERT(ZSTD_dictDefaultAttach < ZSTD_dictForceCopy); bounds.lowerBound = ZSTD_dictDefaultAttach; - bounds.upperBound = ZSTD_dictForceCopy; /* note : how to ensure at compile time that this is the highest value enum ? */ + bounds.upperBound = ZSTD_dictForceSource; /* note : how to ensure at compile time that this is the highest value enum ? */ return bounds; case ZSTD_c_literalCompressionMode: @@ -3358,7 +3358,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced( RETURN_ERROR_IF(cdict==NULL, dictionary_wrong); { ZSTD_CCtx_params params = cctx->requestedParams; params.cParams = ( (pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_CUTOFF) || (cdict->compressionLevel == 0) ) - && (params->attachDictPref != ZSTD_dictForceSource) ? + && (params.attachDictPref != ZSTD_dictForceSource) ? ZSTD_getCParamsFromCDict(cdict) : ZSTD_getCParams(cdict->compressionLevel, pledgedSrcSize, diff --git a/lib/zstd.h b/lib/zstd.h index c0521c1b8..5468c34cc 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -1177,7 +1177,7 @@ typedef enum { ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */ ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */ ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */ - ZSTD_dictForceSource = 3, /* Always use src input to determine tables */ + ZSTD_dictForceSource = 3, /* Always use src input to determine tables */ } ZSTD_dictAttachPref_e; typedef enum { From 676f89902acbb2fb8294e601e31ae396e125c297 Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Mon, 21 Oct 2019 15:29:55 -0400 Subject: [PATCH 25/29] Added multiplier, renamed new enum to something more useful --- lib/compress/zstd_compress.c | 17 +++++++++++------ lib/zstd.h | 8 +++++--- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 3aa111ddb..73e6618c1 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -387,7 +387,7 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param) case ZSTD_c_forceAttachDict: ZSTD_STATIC_ASSERT(ZSTD_dictDefaultAttach < ZSTD_dictForceCopy); bounds.lowerBound = ZSTD_dictDefaultAttach; - bounds.upperBound = ZSTD_dictForceSource; /* note : how to ensure at compile time that this is the highest value enum ? */ + bounds.upperBound = ZSTD_dictForceLoad; /* note : how to ensure at compile time that this is the highest value enum ? */ return bounds; case ZSTD_c_literalCompressionMode: @@ -2890,7 +2890,8 @@ ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs, bs, ms, ws, params, dict, dictSize, dtlm, workspace); } -#define ZSTD_USE_CDICT_PARAMS_CUTOFF (1 MB) +#define ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF (128 KB) +#define ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER (6) /*! ZSTD_compressBegin_internal() : * @return : 0, or an error code */ @@ -2908,8 +2909,10 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, assert(!((dict) && (cdict))); /* either dict or cdict, not both */ if ( (cdict) && (cdict->dictContentSize > 0) - && (pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_CUTOFF || cdict->compressionLevel == 0) - && (params->attachDictPref != ZSTD_dictForceSource) ) { + && ( (pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF + || pledgedSrcSize < cdict->dictContentSize * ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER) + || cdict->compressionLevel == 0) + && (params->attachDictPref != ZSTD_dictForceLoad) ) { return ZSTD_resetCCtx_usingCDict(cctx, cdict, params, pledgedSrcSize, zbuff); } @@ -3357,8 +3360,10 @@ size_t ZSTD_compressBegin_usingCDict_advanced( DEBUGLOG(4, "ZSTD_compressBegin_usingCDict_advanced"); RETURN_ERROR_IF(cdict==NULL, dictionary_wrong); { ZSTD_CCtx_params params = cctx->requestedParams; - params.cParams = ( (pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_CUTOFF) || (cdict->compressionLevel == 0) ) - && (params.attachDictPref != ZSTD_dictForceSource) ? + params.cParams = ( (pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF + || pledgedSrcSize < cdict->dictContentSize * ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER) + || (cdict->compressionLevel == 0) ) + && (params.attachDictPref != ZSTD_dictForceLoad) ? ZSTD_getCParamsFromCDict(cdict) : ZSTD_getCParams(cdict->compressionLevel, pledgedSrcSize, diff --git a/lib/zstd.h b/lib/zstd.h index 5468c34cc..3ba476e88 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -1166,8 +1166,10 @@ typedef enum { * faster than copying the CDict's tables. * * - The CDict's tables are not used at all, and instead we use the working - * context alone to determine how our tables are initialized. This method - * should be used when using a small dictionary to compress a large input. + * context alone to reload the dictionary and use params based on the source + * size. See ZSTD_compress_insertDictionary() and ZSTD_compress_usingDict(). + * This method is effective when the dictionary sizes are very small relative + * to the input size, and the input size is fairly large to begin with. * * Zstd has a simple internal heuristic that selects which strategy to use * at the beginning of a compression. However, if experimentation shows that @@ -1177,7 +1179,7 @@ typedef enum { ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */ ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */ ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */ - ZSTD_dictForceSource = 3, /* Always use src input to determine tables */ + ZSTD_dictForceLoad = 3, /* Always reload the dictionary */ } ZSTD_dictAttachPref_e; typedef enum { From 2ab484a5f90f95a6a1b063b6da27657243d66635 Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Mon, 21 Oct 2019 18:55:17 -0400 Subject: [PATCH 26/29] Fix bad merge --- lib/compress/zstd_compress.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index c0ec593f3..d03a2cefc 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2917,7 +2917,7 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, } FORWARD_IF_ERROR( ZSTD_resetCCtx_internal(cctx, *params, pledgedSrcSize, - ZSTDcrp_continue, zbuff) ); + ZSTDcrp_makeClean, zbuff) ); { size_t const dictID = cdict ? ZSTD_compress_insertDictionary( cctx->blockState.prevCBlock, &cctx->blockState.matchState, @@ -2925,7 +2925,8 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, dictContentType, dtlm, cctx->entropyWorkspace) : ZSTD_compress_insertDictionary( cctx->blockState.prevCBlock, &cctx->blockState.matchState, - params, dict, dictSize, dictContentType, dtlm, cctx->entropyWorkspace); + &cctx->workspace, params, dict, dictSize, + dictContentType, dtlm, cctx->entropyWorkspace); FORWARD_IF_ERROR(dictID); assert(dictID <= UINT_MAX); cctx->dictID = (U32)dictID; From dbda8c318a9c0c9dc226ab4eef9b2c5c0b5966e4 Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Mon, 21 Oct 2019 19:10:13 -0400 Subject: [PATCH 27/29] Trailing comma --- lib/zstd.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/zstd.h b/lib/zstd.h index 788f5d9ec..da29ad222 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -1179,7 +1179,7 @@ typedef enum { ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */ ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */ ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */ - ZSTD_dictForceLoad = 3, /* Always reload the dictionary */ + ZSTD_dictForceLoad = 3 /* Always reload the dictionary */ } ZSTD_dictAttachPref_e; typedef enum { From 59c81aa31b689b3d40dc1eb559b566a521a07fef Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Mon, 21 Oct 2019 19:12:15 -0400 Subject: [PATCH 28/29] Line up comments :) --- lib/zstd.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/zstd.h b/lib/zstd.h index da29ad222..71012afe2 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -1179,7 +1179,7 @@ typedef enum { ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */ ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */ ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */ - ZSTD_dictForceLoad = 3 /* Always reload the dictionary */ + ZSTD_dictForceLoad = 3 /* Always reload the dictionary */ } ZSTD_dictAttachPref_e; typedef enum { From c2e1e54f2445c7ec329e812340762f8be858d38b Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Mon, 21 Oct 2019 19:16:50 -0400 Subject: [PATCH 29/29] ((x or y) or z) == (x or y or z), remove brackets --- lib/compress/zstd_compress.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index c03944941..dd411aba9 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2932,8 +2932,8 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, assert(!((dict) && (cdict))); /* either dict or cdict, not both */ if ( (cdict) && (cdict->dictContentSize > 0) - && ( (pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF - || pledgedSrcSize < cdict->dictContentSize * ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER) + && ( pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF + || pledgedSrcSize < cdict->dictContentSize * ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER || cdict->compressionLevel == 0) && (params->attachDictPref != ZSTD_dictForceLoad) ) { return ZSTD_resetCCtx_usingCDict(cctx, cdict, params, pledgedSrcSize, zbuff); @@ -3379,9 +3379,9 @@ size_t ZSTD_compressBegin_usingCDict_advanced( DEBUGLOG(4, "ZSTD_compressBegin_usingCDict_advanced"); RETURN_ERROR_IF(cdict==NULL, dictionary_wrong); { ZSTD_CCtx_params params = cctx->requestedParams; - params.cParams = ( (pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF - || pledgedSrcSize < cdict->dictContentSize * ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER) - || (cdict->compressionLevel == 0) ) + params.cParams = ( pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF + || pledgedSrcSize < cdict->dictContentSize * ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER + || cdict->compressionLevel == 0 ) && (params.attachDictPref != ZSTD_dictForceLoad) ? ZSTD_getCParamsFromCDict(cdict) : ZSTD_getCParams(cdict->compressionLevel,