From a96e67af6cab104728ff28b50a8121306bdfdf53 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Fri, 15 Feb 2019 10:28:44 -0800 Subject: [PATCH 1/5] [regression] Test level 19 with uncompressed literals --- tests/regression/config.c | 11 +++++++++++ tests/regression/results.csv | 15 +++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/tests/regression/config.c b/tests/regression/config.c index f04d32d83..4d5d19105 100644 --- a/tests/regression/config.c +++ b/tests/regression/config.c @@ -132,6 +132,16 @@ static config_t uncompressed_literals = { .param_values = PARAM_VALUES(uncompressed_literals_param_values), }; +static param_value_t const uncompressed_literals_opt_param_values[] = { + {.param = ZSTD_c_compressionLevel, .value = 19}, + {.param = ZSTD_c_literalCompressionMode, .value = ZSTD_lcm_uncompressed}, +}; + +static config_t uncompressed_literals_opt = { + .name = "uncompressed literals optimal", + .param_values = PARAM_VALUES(uncompressed_literals_opt_param_values), +}; + static param_value_t const huffman_literals_param_values[] = { {.param = ZSTD_c_compressionLevel, .value = -1}, {.param = ZSTD_c_literalCompressionMode, .value = ZSTD_lcm_huffman}, @@ -176,6 +186,7 @@ static config_t const* g_configs[] = { &small_clog, &explicit_params, &uncompressed_literals, + &uncompressed_literals_opt, &huffman_literals, NULL, }; diff --git a/tests/regression/results.csv b/tests/regression/results.csv index d9c5a4a0a..076e3b454 100644 --- a/tests/regression/results.csv +++ b/tests/regression/results.csv @@ -14,6 +14,7 @@ silesia.tar, level 13, compress sim silesia.tar, level 16, compress simple, 4381277 silesia.tar, level 19, compress simple, 4281514 silesia.tar, uncompressed literals, compress simple, 4875008 +silesia.tar, uncompressed literals optimal, compress simple, 4281514 silesia.tar, huffman literals, compress simple, 6195462 silesia, level -5, compress cctx, 7152294 silesia, level -3, compress cctx, 6789969 @@ -37,6 +38,7 @@ silesia, small hash log, compress cct silesia, small chain log, compress cctx, 4931093 silesia, explicit params, compress cctx, 4813352 silesia, uncompressed literals, compress cctx, 4862377 +silesia, uncompressed literals optimal, compress cctx, 4293262 silesia, huffman literals, compress cctx, 6191548 github, level -5, compress cctx, 232744 github, level -5 with dict, compress cctx, 47294 @@ -74,6 +76,7 @@ github, small hash log, compress cct github, small chain log, compress cctx, decompression error github, explicit params, compress cctx, decompression error github, uncompressed literals, compress cctx, 136397 +github, uncompressed literals optimal, compress cctx, 133717 github, huffman literals, compress cctx, 176575 silesia, level -5, zstdcli, 7152342 silesia, level -3, zstdcli, 6790021 @@ -176,6 +179,7 @@ silesia, small hash log, advanced one silesia, small chain log, advanced one pass, 4931093 silesia, explicit params, advanced one pass, 4815369 silesia, uncompressed literals, advanced one pass, 5155424 +silesia, uncompressed literals optimal, advanced one pass, 4426654 silesia, huffman literals, advanced one pass, 5341356 silesia.tar, level -5, advanced one pass, 7160438 silesia.tar, level -3, advanced one pass, 6789024 @@ -200,6 +204,7 @@ silesia.tar, small hash log, advanced one silesia.tar, small chain log, advanced one pass, 4943255 silesia.tar, explicit params, advanced one pass, 4829974 silesia.tar, uncompressed literals, advanced one pass, 5157992 +silesia.tar, uncompressed literals optimal, advanced one pass, 4372744 silesia.tar, huffman literals, advanced one pass, 5358079 github, level -5, advanced one pass, 232744 github, level -5 with dict, advanced one pass, 46718 @@ -238,6 +243,7 @@ github, small hash log, advanced one github, small chain log, advanced one pass, 136314 github, explicit params, advanced one pass, 137670 github, uncompressed literals, advanced one pass, 167004 +github, uncompressed literals optimal, advanced one pass, 164600 github, huffman literals, advanced one pass, 143457 silesia, level -5, advanced one pass small out, 7152294 silesia, level -3, advanced one pass small out, 6789969 @@ -262,6 +268,7 @@ silesia, small hash log, advanced one silesia, small chain log, advanced one pass small out, 4931093 silesia, explicit params, advanced one pass small out, 4815369 silesia, uncompressed literals, advanced one pass small out, 5155424 +silesia, uncompressed literals optimal, advanced one pass small out, 4426654 silesia, huffman literals, advanced one pass small out, 5341356 silesia.tar, level -5, advanced one pass small out, 7160438 silesia.tar, level -3, advanced one pass small out, 6789024 @@ -286,6 +293,7 @@ silesia.tar, small hash log, advanced one silesia.tar, small chain log, advanced one pass small out, 4943255 silesia.tar, explicit params, advanced one pass small out, 4829974 silesia.tar, uncompressed literals, advanced one pass small out, 5157992 +silesia.tar, uncompressed literals optimal, advanced one pass small out, 4372744 silesia.tar, huffman literals, advanced one pass small out, 5358079 github, level -5, advanced one pass small out, 232744 github, level -5 with dict, advanced one pass small out, 46718 @@ -324,6 +332,7 @@ github, small hash log, advanced one github, small chain log, advanced one pass small out, 136314 github, explicit params, advanced one pass small out, 137670 github, uncompressed literals, advanced one pass small out, 167004 +github, uncompressed literals optimal, advanced one pass small out, 164600 github, huffman literals, advanced one pass small out, 143457 silesia, level -5, advanced streaming, 7152294 silesia, level -3, advanced streaming, 6789973 @@ -348,6 +357,7 @@ silesia, small hash log, advanced str silesia, small chain log, advanced streaming, 4931093 silesia, explicit params, advanced streaming, 4815380 silesia, uncompressed literals, advanced streaming, 5155424 +silesia, uncompressed literals optimal, advanced streaming, 4426654 silesia, huffman literals, advanced streaming, 5341357 silesia.tar, level -5, advanced streaming, 7160440 silesia.tar, level -3, advanced streaming, 6789026 @@ -372,6 +382,7 @@ silesia.tar, small hash log, advanced str silesia.tar, small chain log, advanced streaming, 4943260 silesia.tar, explicit params, advanced streaming, 4830002 silesia.tar, uncompressed literals, advanced streaming, 5157995 +silesia.tar, uncompressed literals optimal, advanced streaming, 4372744 silesia.tar, huffman literals, advanced streaming, 5358083 github, level -5, advanced streaming, 232744 github, level -5 with dict, advanced streaming, 46718 @@ -410,6 +421,7 @@ github, small hash log, advanced str github, small chain log, advanced streaming, 136314 github, explicit params, advanced streaming, 137670 github, uncompressed literals, advanced streaming, 167004 +github, uncompressed literals optimal, advanced streaming, 164600 github, huffman literals, advanced streaming, 143457 silesia, level -5, old streaming, 7152294 silesia, level -3, old streaming, 6789973 @@ -427,6 +439,7 @@ silesia, level 16, old streamin silesia, level 19, old streaming, 4293262 silesia, no source size, old streaming, 4862341 silesia, uncompressed literals, old streaming, 4862377 +silesia, uncompressed literals optimal, old streaming, 4293262 silesia, huffman literals, old streaming, 6191549 silesia.tar, level -5, old streaming, 7160440 silesia.tar, level -3, old streaming, 6789026 @@ -444,6 +457,7 @@ silesia.tar, level 16, old streamin silesia.tar, level 19, old streaming, 4281514 silesia.tar, no source size, old streaming, 4875006 silesia.tar, uncompressed literals, old streaming, 4875010 +silesia.tar, uncompressed literals optimal, old streaming, 4281514 silesia.tar, huffman literals, old streaming, 6195465 github, level -5, old streaming, 232744 github, level -5 with dict, old streaming, 46718 @@ -475,4 +489,5 @@ github, level 19, old streamin github, level 19 with dict, old streaming, 37576 github, no source size, old streaming, 141003 github, uncompressed literals, old streaming, 136397 +github, uncompressed literals optimal, old streaming, 133717 github, huffman literals, old streaming, 176575 From 3d7377b87424877c59e068e97f3d9b930aca630a Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Fri, 15 Feb 2019 10:29:03 -0800 Subject: [PATCH 2/5] [libzstd] Handle uncompressed literals --- lib/compress/zstd_compress.c | 7 ++++-- lib/compress/zstd_compress_internal.h | 1 + lib/compress/zstd_opt.c | 34 ++++++++++++++++++++------- tests/regression/results.csv | 18 +++++++------- 4 files changed, 40 insertions(+), 20 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index e18051f00..9ea7f04ef 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -402,7 +402,6 @@ static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param) case ZSTD_c_minMatch: case ZSTD_c_targetLength: case ZSTD_c_strategy: - case ZSTD_c_literalCompressionMode: return 1; case ZSTD_c_format: @@ -421,6 +420,7 @@ static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param) case ZSTD_c_ldmBucketSizeLog: case ZSTD_c_ldmHashRateLog: case ZSTD_c_forceAttachDict: + case ZSTD_c_literalCompressionMode: default: return 0; } @@ -2677,7 +2677,10 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, goto out; /* don't even attempt compression below a certain srcSize */ } ZSTD_resetSeqStore(&(zc->seqStore)); - ms->opt.symbolCosts = &zc->blockState.prevCBlock->entropy; /* required for optimal parser to read stats from dictionary */ + /* required for optimal parser to read stats from dictionary */ + ms->opt.symbolCosts = &zc->blockState.prevCBlock->entropy; + /* tell the optimal parser how we expect to compress literals */ + ms->opt.literalCompressionMode = zc->appliedParams.literalCompressionMode; /* a gap between an attached dict and the current window is not safe, * they must remain adjacent, diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index e0b54299d..a828de3b4 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -107,6 +107,7 @@ typedef struct { U32 offCodeSumBasePrice; /* to compare to log2(offreq) */ ZSTD_OptPrice_e priceType; /* prices can be determined dynamically, or follow a pre-defined cost structure */ const ZSTD_entropyCTables_t* symbolCosts; /* pre-calculated dictionary statistics */ + ZSTD_literalCompressionMode_e literalCompressionMode; } optState_t; typedef struct { diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 44de6e97f..cf2f70b11 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -64,9 +64,15 @@ MEM_STATIC double ZSTD_fCost(U32 price) } #endif +static int ZSTD_compressedLiterals(optState_t const* const optPtr) +{ + return optPtr->literalCompressionMode != ZSTD_lcm_uncompressed; +} + static void ZSTD_setBasePrices(optState_t* optPtr, int optLevel) { - optPtr->litSumBasePrice = WEIGHT(optPtr->litSum, optLevel); + if (ZSTD_compressedLiterals(optPtr)) + optPtr->litSumBasePrice = WEIGHT(optPtr->litSum, optLevel); optPtr->litLengthSumBasePrice = WEIGHT(optPtr->litLengthSum, optLevel); optPtr->matchLengthSumBasePrice = WEIGHT(optPtr->matchLengthSum, optLevel); optPtr->offCodeSumBasePrice = WEIGHT(optPtr->offCodeSum, optLevel); @@ -99,6 +105,7 @@ ZSTD_rescaleFreqs(optState_t* const optPtr, const BYTE* const src, size_t const srcSize, int const optLevel) { + int const compressedLiterals = ZSTD_compressedLiterals(optPtr); DEBUGLOG(5, "ZSTD_rescaleFreqs (srcSize=%u)", (unsigned)srcSize); optPtr->priceType = zop_dynamic; @@ -113,9 +120,10 @@ ZSTD_rescaleFreqs(optState_t* const optPtr, /* huffman table presumed generated by dictionary */ optPtr->priceType = zop_dynamic; - assert(optPtr->litFreq != NULL); - optPtr->litSum = 0; - { unsigned lit; + if (compressedLiterals) { + unsigned lit; + assert(optPtr->litFreq != NULL); + optPtr->litSum = 0; for (lit=0; lit<=MaxLit; lit++) { U32 const scaleLog = 11; /* scale to 2K */ U32 const bitCost = HUF_getNbBits(optPtr->symbolCosts->huf.CTable, lit); @@ -163,10 +171,11 @@ ZSTD_rescaleFreqs(optState_t* const optPtr, } else { /* not a dictionary */ assert(optPtr->litFreq != NULL); - { unsigned lit = MaxLit; + if (compressedLiterals) { + unsigned lit = MaxLit; HIST_count_simple(optPtr->litFreq, &lit, src, srcSize); /* use raw first block to init statistics */ + optPtr->litSum = ZSTD_downscaleStat(optPtr->litFreq, MaxLit, 1); } - optPtr->litSum = ZSTD_downscaleStat(optPtr->litFreq, MaxLit, 1); { unsigned ll; for (ll=0; ll<=MaxLL; ll++) @@ -190,7 +199,8 @@ ZSTD_rescaleFreqs(optState_t* const optPtr, } else { /* new block : re-use previous statistics, scaled down */ - optPtr->litSum = ZSTD_downscaleStat(optPtr->litFreq, MaxLit, 1); + if (compressedLiterals) + optPtr->litSum = ZSTD_downscaleStat(optPtr->litFreq, MaxLit, 1); optPtr->litLengthSum = ZSTD_downscaleStat(optPtr->litLengthFreq, MaxLL, 0); optPtr->matchLengthSum = ZSTD_downscaleStat(optPtr->matchLengthFreq, MaxML, 0); optPtr->offCodeSum = ZSTD_downscaleStat(optPtr->offCodeFreq, MaxOff, 0); @@ -207,6 +217,10 @@ static U32 ZSTD_rawLiteralsCost(const BYTE* const literals, U32 const litLength, int optLevel) { if (litLength == 0) return 0; + + if (!ZSTD_compressedLiterals(optPtr)) + return (litLength << 3) * BITCOST_MULTIPLIER; /* Uncompressed - 8 bytes per literal. */ + if (optPtr->priceType == zop_predef) return (litLength*6) * BITCOST_MULTIPLIER; /* 6 bit per literal - no statistic used */ @@ -310,7 +324,8 @@ static void ZSTD_updateStats(optState_t* const optPtr, U32 offsetCode, U32 matchLength) { /* literals */ - { U32 u; + if (ZSTD_compressedLiterals(optPtr)) { + U32 u; for (u=0; u < litLength; u++) optPtr->litFreq[literals[u]] += ZSTD_LITFREQ_ADD; optPtr->litSum += litLength*ZSTD_LITFREQ_ADD; @@ -1108,7 +1123,8 @@ static U32 ZSTD_upscaleStat(unsigned* table, U32 lastEltIndex, int bonus) /* used in 2-pass strategy */ MEM_STATIC void ZSTD_upscaleStats(optState_t* optPtr) { - optPtr->litSum = ZSTD_upscaleStat(optPtr->litFreq, MaxLit, 0); + if (ZSTD_compressedLiterals(optPtr)) + optPtr->litSum = ZSTD_upscaleStat(optPtr->litFreq, MaxLit, 0); optPtr->litLengthSum = ZSTD_upscaleStat(optPtr->litLengthFreq, MaxLL, 0); optPtr->matchLengthSum = ZSTD_upscaleStat(optPtr->matchLengthFreq, MaxML, 0); optPtr->offCodeSum = ZSTD_upscaleStat(optPtr->offCodeFreq, MaxOff, 0); diff --git a/tests/regression/results.csv b/tests/regression/results.csv index 076e3b454..6d0d1b515 100644 --- a/tests/regression/results.csv +++ b/tests/regression/results.csv @@ -179,7 +179,7 @@ silesia, small hash log, advanced one silesia, small chain log, advanced one pass, 4931093 silesia, explicit params, advanced one pass, 4815369 silesia, uncompressed literals, advanced one pass, 5155424 -silesia, uncompressed literals optimal, advanced one pass, 4426654 +silesia, uncompressed literals optimal, advanced one pass, 4325427 silesia, huffman literals, advanced one pass, 5341356 silesia.tar, level -5, advanced one pass, 7160438 silesia.tar, level -3, advanced one pass, 6789024 @@ -204,7 +204,7 @@ silesia.tar, small hash log, advanced one silesia.tar, small chain log, advanced one pass, 4943255 silesia.tar, explicit params, advanced one pass, 4829974 silesia.tar, uncompressed literals, advanced one pass, 5157992 -silesia.tar, uncompressed literals optimal, advanced one pass, 4372744 +silesia.tar, uncompressed literals optimal, advanced one pass, 4321094 silesia.tar, huffman literals, advanced one pass, 5358079 github, level -5, advanced one pass, 232744 github, level -5 with dict, advanced one pass, 46718 @@ -243,7 +243,7 @@ github, small hash log, advanced one github, small chain log, advanced one pass, 136314 github, explicit params, advanced one pass, 137670 github, uncompressed literals, advanced one pass, 167004 -github, uncompressed literals optimal, advanced one pass, 164600 +github, uncompressed literals optimal, advanced one pass, 156824 github, huffman literals, advanced one pass, 143457 silesia, level -5, advanced one pass small out, 7152294 silesia, level -3, advanced one pass small out, 6789969 @@ -268,7 +268,7 @@ silesia, small hash log, advanced one silesia, small chain log, advanced one pass small out, 4931093 silesia, explicit params, advanced one pass small out, 4815369 silesia, uncompressed literals, advanced one pass small out, 5155424 -silesia, uncompressed literals optimal, advanced one pass small out, 4426654 +silesia, uncompressed literals optimal, advanced one pass small out, 4325427 silesia, huffman literals, advanced one pass small out, 5341356 silesia.tar, level -5, advanced one pass small out, 7160438 silesia.tar, level -3, advanced one pass small out, 6789024 @@ -293,7 +293,7 @@ silesia.tar, small hash log, advanced one silesia.tar, small chain log, advanced one pass small out, 4943255 silesia.tar, explicit params, advanced one pass small out, 4829974 silesia.tar, uncompressed literals, advanced one pass small out, 5157992 -silesia.tar, uncompressed literals optimal, advanced one pass small out, 4372744 +silesia.tar, uncompressed literals optimal, advanced one pass small out, 4321094 silesia.tar, huffman literals, advanced one pass small out, 5358079 github, level -5, advanced one pass small out, 232744 github, level -5 with dict, advanced one pass small out, 46718 @@ -332,7 +332,7 @@ github, small hash log, advanced one github, small chain log, advanced one pass small out, 136314 github, explicit params, advanced one pass small out, 137670 github, uncompressed literals, advanced one pass small out, 167004 -github, uncompressed literals optimal, advanced one pass small out, 164600 +github, uncompressed literals optimal, advanced one pass small out, 156824 github, huffman literals, advanced one pass small out, 143457 silesia, level -5, advanced streaming, 7152294 silesia, level -3, advanced streaming, 6789973 @@ -357,7 +357,7 @@ silesia, small hash log, advanced str silesia, small chain log, advanced streaming, 4931093 silesia, explicit params, advanced streaming, 4815380 silesia, uncompressed literals, advanced streaming, 5155424 -silesia, uncompressed literals optimal, advanced streaming, 4426654 +silesia, uncompressed literals optimal, advanced streaming, 4325427 silesia, huffman literals, advanced streaming, 5341357 silesia.tar, level -5, advanced streaming, 7160440 silesia.tar, level -3, advanced streaming, 6789026 @@ -382,7 +382,7 @@ silesia.tar, small hash log, advanced str silesia.tar, small chain log, advanced streaming, 4943260 silesia.tar, explicit params, advanced streaming, 4830002 silesia.tar, uncompressed literals, advanced streaming, 5157995 -silesia.tar, uncompressed literals optimal, advanced streaming, 4372744 +silesia.tar, uncompressed literals optimal, advanced streaming, 4321094 silesia.tar, huffman literals, advanced streaming, 5358083 github, level -5, advanced streaming, 232744 github, level -5 with dict, advanced streaming, 46718 @@ -421,7 +421,7 @@ github, small hash log, advanced str github, small chain log, advanced streaming, 136314 github, explicit params, advanced streaming, 137670 github, uncompressed literals, advanced streaming, 167004 -github, uncompressed literals optimal, advanced streaming, 164600 +github, uncompressed literals optimal, advanced streaming, 156824 github, huffman literals, advanced streaming, 143457 silesia, level -5, old streaming, 7152294 silesia, level -3, old streaming, 6789973 From 5261a288d1fcecdbae988468d8d3191572bb7f49 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Fri, 15 Feb 2019 14:14:32 -0800 Subject: [PATCH 3/5] Fix a bug in the compress cctx method --- tests/regression/method.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/regression/method.c b/tests/regression/method.c index 51a0fb9d1..16701c0b4 100644 --- a/tests/regression/method.c +++ b/tests/regression/method.c @@ -175,8 +175,8 @@ static result_t compress_cctx_compress( state->compressed.capacity, input.data, input.size, - state->dictionary.data, - state->dictionary.size, + config->use_dictionary ? state->dictionary.data : NULL, + config->use_dictionary ? state->dictionary.size : 0, params); else if (config->use_dictionary) state->compressed.size = ZSTD_compress_usingDict( From 0c53c5ad4aa419d1f3aab0022ad181d1efec6f30 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Fri, 15 Feb 2019 14:15:36 -0800 Subject: [PATCH 4/5] [zstdcli] Add a flag to control literals compression --- programs/fileio.c | 9 +++++++++ programs/fileio.h | 3 +++ programs/zstdcli.c | 5 ++++- tests/playTests.sh | 7 +++++++ tests/regression/config.c | 3 +++ tests/regression/results.csv | 23 ++++++++++++++++------- 6 files changed, 42 insertions(+), 8 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index dd47a1e32..8a5715113 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -296,6 +296,7 @@ struct FIO_prefs_s { int ldmMinMatch; int ldmBucketSizeLog; int ldmHashRateLog; + ZSTD_literalCompressionMode_e literalCompressionMode; /* IO preferences */ U32 removeSrcFile; @@ -339,6 +340,7 @@ FIO_prefs_t* FIO_createPreferences(void) ret->ldmMinMatch = 0; ret->ldmBucketSizeLog = FIO_LDM_PARAM_NOTSET; ret->ldmHashRateLog = FIO_LDM_PARAM_NOTSET; + ret->literalCompressionMode = ZSTD_lcm_auto; return ret; } @@ -406,6 +408,12 @@ void FIO_setRsyncable(FIO_prefs_t* const prefs, int rsyncable) { prefs->rsyncable = rsyncable; } +void FIO_setLiteralCompressionMode( + FIO_prefs_t* const prefs, + ZSTD_literalCompressionMode_e mode) { + prefs->literalCompressionMode = mode; +} + void FIO_setAdaptMin(FIO_prefs_t* const prefs, int minCLevel) { #ifndef ZSTD_NOCOMPRESS @@ -674,6 +682,7 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs, CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_minMatch, (int)comprParams.minMatch) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_targetLength, (int)comprParams.targetLength) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_strategy, comprParams.strategy) ); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_literalCompressionMode, (int)prefs->literalCompressionMode) ); /* multi-threading */ #ifdef ZSTD_MULTITHREAD DISPLAYLEVEL(5,"set nb workers = %u \n", prefs->nbWorkers); diff --git a/programs/fileio.h b/programs/fileio.h index b20570bcb..e46633752 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -71,6 +71,9 @@ void FIO_setOverlapLog(FIO_prefs_t* const prefs, int overlapLog); void FIO_setRemoveSrcFile(FIO_prefs_t* const prefs, unsigned flag); void FIO_setSparseWrite(FIO_prefs_t* const prefs, unsigned sparse); /**< 0: no sparse; 1: disable on stdout; 2: always enabled */ void FIO_setRsyncable(FIO_prefs_t* const prefs, int rsyncable); +void FIO_setLiteralCompressionMode( + FIO_prefs_t* const prefs, + ZSTD_literalCompressionMode_e mode); void FIO_setNoProgress(unsigned noProgress); void FIO_setNotificationLevel(int level); diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 794e2a811..60035e566 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -148,6 +148,7 @@ static int usage_advanced(const char* programName) #endif DISPLAY( "--no-dictID : don't write dictID into header (dictionary compression)\n"); DISPLAY( "--[no-]check : integrity check (default: enabled) \n"); + DISPLAY( "--[no-]compress-literals : force (un)compressed literals \n"); #endif #ifdef UTIL_HAS_CREATEFILELIST DISPLAY( " -r : operate recursively on directories \n"); @@ -483,7 +484,7 @@ static int init_cLevel(void) { if ((*ptr>='0') && (*ptr<='9')) { unsigned absLevel; - if (readU32FromCharChecked(&ptr, &absLevel)) { + if (readU32FromCharChecked(&ptr, &absLevel)) { DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: numeric value too large\n", ENV_CLEVEL, env); return ZSTDCLI_CLEVEL_DEFAULT; } else if (*ptr == 0) { @@ -659,6 +660,8 @@ int main(int argCount, const char* argv[]) if (!strcmp(argument, "--format=lz4")) { suffix = LZ4_EXTENSION; FIO_setCompressionType(prefs, FIO_lz4Compression); continue; } #endif if (!strcmp(argument, "--rsyncable")) { rsyncable = 1; continue; } + if (!strcmp(argument, "--compress-literals")) { FIO_setLiteralCompressionMode(prefs, ZSTD_lcm_huffman); continue; } + if (!strcmp(argument, "--no-compress-literals")) { FIO_setLiteralCompressionMode(prefs, ZSTD_lcm_uncompressed); continue; } if (!strcmp(argument, "--no-progress")) { FIO_setNoProgress(1); continue; } /* long commands with arguments */ diff --git a/tests/playTests.sh b/tests/playTests.sh index 5be065fa8..62b3057e9 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -200,6 +200,13 @@ $ZSTD tmp -fo tmp && die "zstd compression overwrote the input file" $ZSTD tmp.zst -dfo tmp.zst && die "zstd decompression overwrote the input file" $ECHO "test: detect that input file does not exist" $ZSTD nothere && die "zstd hasn't detected that input file does not exist" +$ECHO "test: --[no-]compress-literals" +$ZSTD tmp -c --no-compress-literals -1 | $ZSTD -t +$ZSTD tmp -c --no-compress-literals --fast=1 | $ZSTD -t +$ZSTD tmp -c --no-compress-literals -19 | $ZSTD -t +$ZSTD tmp -c --compress-literals -1 | $ZSTD -t +$ZSTD tmp -c --compress-literals --fast=1 | $ZSTD -t +$ZSTD tmp -c --compress-literals -19 | $ZSTD -t $ECHO "test : file removal" $ZSTD -f --rm tmp diff --git a/tests/regression/config.c b/tests/regression/config.c index 4d5d19105..bd3640099 100644 --- a/tests/regression/config.c +++ b/tests/regression/config.c @@ -129,6 +129,7 @@ static param_value_t const uncompressed_literals_param_values[] = { static config_t uncompressed_literals = { .name = "uncompressed literals", + .cli_args = "-3 --no-compress-literals", .param_values = PARAM_VALUES(uncompressed_literals_param_values), }; @@ -139,6 +140,7 @@ static param_value_t const uncompressed_literals_opt_param_values[] = { static config_t uncompressed_literals_opt = { .name = "uncompressed literals optimal", + .cli_args = "-19 --no-compress-literals", .param_values = PARAM_VALUES(uncompressed_literals_opt_param_values), }; @@ -149,6 +151,7 @@ static param_value_t const huffman_literals_param_values[] = { static config_t huffman_literals = { .name = "huffman literals", + .cli_args = "--fast=1 --compress-literals", .param_values = PARAM_VALUES(huffman_literals_param_values), }; diff --git a/tests/regression/results.csv b/tests/regression/results.csv index 6d0d1b515..23afae5ee 100644 --- a/tests/regression/results.csv +++ b/tests/regression/results.csv @@ -68,13 +68,13 @@ github, level 16, compress cct github, level 16 with dict, compress cctx, 37568 github, level 19, compress cctx, 133717 github, level 19 with dict, compress cctx, 37567 -github, long distance mode, compress cctx, decompression error -github, multithreaded, compress cctx, decompression error -github, multithreaded long distance mode, compress cctx, decompression error -github, small window log, compress cctx, decompression error -github, small hash log, compress cctx, decompression error -github, small chain log, compress cctx, decompression error -github, explicit params, compress cctx, decompression error +github, long distance mode, compress cctx, 141473 +github, multithreaded, compress cctx, 141473 +github, multithreaded long distance mode, compress cctx, 141473 +github, small window log, compress cctx, 141473 +github, small hash log, compress cctx, 138943 +github, small chain log, compress cctx, 139239 +github, explicit params, compress cctx, 140924 github, uncompressed literals, compress cctx, 136397 github, uncompressed literals optimal, compress cctx, 133717 github, huffman literals, compress cctx, 176575 @@ -99,6 +99,9 @@ silesia, small window log, zstdcli, silesia, small hash log, zstdcli, 6554946 silesia, small chain log, zstdcli, 4931141 silesia, explicit params, zstdcli, 4815380 +silesia, uncompressed literals, zstdcli, 5155472 +silesia, uncompressed literals optimal, zstdcli, 4325475 +silesia, huffman literals, zstdcli, 5341405 silesia.tar, level -5, zstdcli, 7161160 silesia.tar, level -3, zstdcli, 6789865 silesia.tar, level -1, zstdcli, 6196433 @@ -121,6 +124,9 @@ silesia.tar, small window log, zstdcli, silesia.tar, small hash log, zstdcli, 6587841 silesia.tar, small chain log, zstdcli, 4943259 silesia.tar, explicit params, zstdcli, 4839202 +silesia.tar, uncompressed literals, zstdcli, 5158134 +silesia.tar, uncompressed literals optimal, zstdcli, 4321098 +silesia.tar, huffman literals, zstdcli, 5358479 github, level -5, zstdcli, 234744 github, level -5 with dict, zstdcli, 48718 github, level -3, zstdcli, 222611 @@ -156,6 +162,9 @@ github, small window log, zstdcli, github, small hash log, zstdcli, 137467 github, small chain log, zstdcli, 138314 github, explicit params, zstdcli, 136140 +github, uncompressed literals, zstdcli, 169004 +github, uncompressed literals optimal, zstdcli, 158824 +github, huffman literals, zstdcli, 145457 silesia, level -5, advanced one pass, 7152294 silesia, level -3, advanced one pass, 6789969 silesia, level -1, advanced one pass, 6191548 From 19ca3fbc037e2708b8f604c13a6ed176480f3642 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Fri, 15 Feb 2019 15:24:55 -0800 Subject: [PATCH 5/5] [zstdcli] Respect --[no-]compress-literals in benchmark mode --- programs/benchzstd.c | 4 +++- programs/benchzstd.h | 1 + programs/zstdcli.c | 9 ++++++--- tests/playTests.sh | 2 ++ 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/programs/benchzstd.c b/programs/benchzstd.c index 0be74a965..4bd42cfe7 100644 --- a/programs/benchzstd.c +++ b/programs/benchzstd.c @@ -135,7 +135,8 @@ BMK_advancedParams_t BMK_initAdvancedParams(void) { 0, /* ldmMinMatch */ 0, /* ldmHashLog */ 0, /* ldmBuckSizeLog */ - 0 /* ldmHashRateLog */ + 0, /* ldmHashRateLog */ + ZSTD_lcm_auto /* literalCompressionMode */ }; return res; } @@ -174,6 +175,7 @@ static void BMK_initCCtx(ZSTD_CCtx* ctx, CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmHashLog, adv->ldmHashLog)); CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmBucketSizeLog, adv->ldmBucketSizeLog)); CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmHashRateLog, adv->ldmHashRateLog)); + CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_literalCompressionMode, (int)adv->literalCompressionMode)); CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_windowLog, comprParams->windowLog)); CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_hashLog, comprParams->hashLog)); CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_chainLog, comprParams->chainLog)); diff --git a/programs/benchzstd.h b/programs/benchzstd.h index 3a8b893e7..d6f5486ab 100644 --- a/programs/benchzstd.h +++ b/programs/benchzstd.h @@ -116,6 +116,7 @@ typedef struct { unsigned ldmHashLog; unsigned ldmBucketSizeLog; unsigned ldmHashRateLog; + ZSTD_literalCompressionMode_e literalCompressionMode; } BMK_advancedParams_t; /* returns default parameters used by nonAdvanced functions */ diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 60035e566..f57821742 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -568,6 +568,7 @@ int main(int argCount, const char* argv[]) #ifndef ZSTD_NOBENCH BMK_advancedParams_t benchParams = BMK_initAdvancedParams(); #endif + ZSTD_literalCompressionMode_e literalCompressionMode = ZSTD_lcm_auto; /* init */ @@ -660,8 +661,8 @@ int main(int argCount, const char* argv[]) if (!strcmp(argument, "--format=lz4")) { suffix = LZ4_EXTENSION; FIO_setCompressionType(prefs, FIO_lz4Compression); continue; } #endif if (!strcmp(argument, "--rsyncable")) { rsyncable = 1; continue; } - if (!strcmp(argument, "--compress-literals")) { FIO_setLiteralCompressionMode(prefs, ZSTD_lcm_huffman); continue; } - if (!strcmp(argument, "--no-compress-literals")) { FIO_setLiteralCompressionMode(prefs, ZSTD_lcm_uncompressed); continue; } + if (!strcmp(argument, "--compress-literals")) { literalCompressionMode = ZSTD_lcm_huffman; continue; } + if (!strcmp(argument, "--no-compress-literals")) { literalCompressionMode = ZSTD_lcm_uncompressed; continue; } if (!strcmp(argument, "--no-progress")) { FIO_setNoProgress(1); continue; } /* long commands with arguments */ @@ -998,6 +999,7 @@ int main(int argCount, const char* argv[]) if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) { benchParams.ldmHashRateLog = g_ldmHashRateLog; } + benchParams.literalCompressionMode = literalCompressionMode; if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel(); if (cLevelLast > ZSTD_maxCLevel()) cLevelLast = ZSTD_maxCLevel(); @@ -1111,6 +1113,7 @@ int main(int argCount, const char* argv[]) FIO_setAdaptMin(prefs, adaptMin); FIO_setAdaptMax(prefs, adaptMax); FIO_setRsyncable(prefs, rsyncable); + FIO_setLiteralCompressionMode(prefs, literalCompressionMode); if (adaptMin > cLevel) cLevel = adaptMin; if (adaptMax < cLevel) cLevel = adaptMax; @@ -1119,7 +1122,7 @@ int main(int argCount, const char* argv[]) else operationResult = FIO_compressMultipleFilenames(prefs, filenameTable, filenameIdx, outFileName, suffix, dictFileName, cLevel, compressionParams); #else - (void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; /* not used when ZSTD_NOCOMPRESS set */ + (void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; /* not used when ZSTD_NOCOMPRESS set */ DISPLAY("Compression not supported \n"); #endif } else { /* decompression or test */ diff --git a/tests/playTests.sh b/tests/playTests.sh index 62b3057e9..493542960 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -207,6 +207,8 @@ $ZSTD tmp -c --no-compress-literals -19 | $ZSTD -t $ZSTD tmp -c --compress-literals -1 | $ZSTD -t $ZSTD tmp -c --compress-literals --fast=1 | $ZSTD -t $ZSTD tmp -c --compress-literals -19 | $ZSTD -t +$ZSTD -b --fast=1 -i1e1 tmp --compress-literals +$ZSTD -b --fast=1 -i1e1 tmp --no-compress-literals $ECHO "test : file removal" $ZSTD -f --rm tmp