From 4609a40b89b94cc61cc6a6a833725d6f89bf9de6 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 7 Feb 2025 15:58:42 -0800 Subject: [PATCH 01/10] dynamically adjust hratelog and ldmml based on strategy --- lib/compress/zstd_ldm.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c index 42b38c0d0..6f866ea16 100644 --- a/lib/compress/zstd_ldm.c +++ b/lib/compress/zstd_ldm.c @@ -139,15 +139,19 @@ void ZSTD_ldm_adjustParameters(ldmParams_t* params, ZSTD_STATIC_ASSERT(LDM_BUCKET_SIZE_LOG <= ZSTD_LDM_BUCKETSIZELOG_MAX); DEBUGLOG(4, "ZSTD_ldm_adjustParameters"); if (!params->bucketSizeLog) params->bucketSizeLog = LDM_BUCKET_SIZE_LOG; - if (!params->minMatchLength) params->minMatchLength = LDM_MIN_MATCH_LENGTH; + if (params->hashRateLog == 0) { + assert(1 <= (int)cParams->strategy && (int)cParams->strategy <= 9); + /* mapping: strat1 -> rate8 ... strat9 -> rate4*/ + params->hashRateLog = 9 - ((cParams->strategy+1)/2); + } if (params->hashLog == 0) { - params->hashLog = MAX(ZSTD_HASHLOG_MIN, params->windowLog - LDM_HASH_RLOG); + params->hashLog = MAX(ZSTD_HASHLOG_MIN, params->windowLog - params->hashRateLog); assert(params->hashLog <= ZSTD_HASHLOG_MAX); } - if (params->hashRateLog == 0) { - params->hashRateLog = params->windowLog < params->hashLog - ? 0 - : params->windowLog - params->hashLog; + if (params->minMatchLength == 0) { + params->minMatchLength = (params->hashRateLog < 6) ? + LDM_MIN_MATCH_LENGTH : + LDM_MIN_MATCH_LENGTH * 2; } params->bucketSizeLog = MIN(params->bucketSizeLog, params->hashLog); } From f26cc54f37c614d3351b6873f0ee6e3fff00f6f6 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 7 Feb 2025 16:59:34 -0800 Subject: [PATCH 02/10] dynamic bucket sizes --- lib/compress/zstd_ldm.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c index 6f866ea16..39f23c565 100644 --- a/lib/compress/zstd_ldm.c +++ b/lib/compress/zstd_ldm.c @@ -16,7 +16,7 @@ #include "zstd_double_fast.h" /* ZSTD_fillDoubleHashTable() */ #include "zstd_ldm_geartab.h" -#define LDM_BUCKET_SIZE_LOG 3 +#define LDM_BUCKET_SIZE_LOG 4 #define LDM_MIN_MATCH_LENGTH 64 #define LDM_HASH_RLOG 7 @@ -138,7 +138,6 @@ void ZSTD_ldm_adjustParameters(ldmParams_t* params, params->windowLog = cParams->windowLog; ZSTD_STATIC_ASSERT(LDM_BUCKET_SIZE_LOG <= ZSTD_LDM_BUCKETSIZELOG_MAX); DEBUGLOG(4, "ZSTD_ldm_adjustParameters"); - if (!params->bucketSizeLog) params->bucketSizeLog = LDM_BUCKET_SIZE_LOG; if (params->hashRateLog == 0) { assert(1 <= (int)cParams->strategy && (int)cParams->strategy <= 9); /* mapping: strat1 -> rate8 ... strat9 -> rate4*/ @@ -152,6 +151,15 @@ void ZSTD_ldm_adjustParameters(ldmParams_t* params, params->minMatchLength = (params->hashRateLog < 6) ? LDM_MIN_MATCH_LENGTH : LDM_MIN_MATCH_LENGTH * 2; + if (cParams->strategy >= ZSTD_btultra2) + params->minMatchLength /= 2; + } + if (params->bucketSizeLog==0) { + params->bucketSizeLog = LDM_BUCKET_SIZE_LOG; + if (cParams->strategy > ZSTD_lazy) { + params->bucketSizeLog += (U32)cParams->strategy - (U32)ZSTD_lazy; + } + params->bucketSizeLog = MIN(params->bucketSizeLog, ZSTD_LDM_BUCKETSIZELOG_MAX); } params->bucketSizeLog = MIN(params->bucketSizeLog, params->hashLog); } From bf218c142aade4aa842205e93bc8260dcbfb372d Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 7 Feb 2025 17:15:24 -0800 Subject: [PATCH 03/10] updated LDM documentation --- programs/fileio.c | 11 ++++++----- programs/zstd.1.md | 27 ++++++++++++++------------- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 1b8aa8a99..0ecca40d2 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1100,11 +1100,12 @@ static void FIO_adjustParamsForPatchFromMode(FIO_prefs_t* const prefs, FIO_setLdmFlag(prefs, 1); } if (cParams.strategy >= ZSTD_btopt) { - DISPLAYLEVEL(3, "[Optimal parser notes] Consider the following to improve patch size at the cost of speed:\n"); - DISPLAYLEVEL(3, "- Use --single-thread mode in the zstd cli\n"); - DISPLAYLEVEL(3, "- Set a larger targetLength (e.g. --zstd=targetLength=4096)\n"); - DISPLAYLEVEL(3, "- Set a larger chainLog (e.g. --zstd=chainLog=%u)\n", ZSTD_CHAINLOG_MAX); - DISPLAYLEVEL(3, "Also consider playing around with searchLog and hashLog\n"); + DISPLAYLEVEL(4, "[Optimal parser notes] Consider the following to improve patch size at the cost of speed:\n"); + DISPLAYLEVEL(4, "- Set a larger targetLength (e.g. --zstd=targetLength=4096)\n"); + DISPLAYLEVEL(4, "- Set a larger chainLog (e.g. --zstd=chainLog=%u)\n", ZSTD_CHAINLOG_MAX); + DISPLAYLEVEL(4, "- Set a larger LDM hashLog (e.g. --zstd=ldmHashLog=%u)\n", ZSTD_LDM_HASHLOG_MAX); + DISPLAYLEVEL(4, "- Set a smaller LDM rateLog (e.g. --zstd=ldmHashRateLog=%u)\n", ZSTD_LDM_HASHRATELOG_MIN); + DISPLAYLEVEL(4, "Also consider playing around with searchLog and hashLog\n"); } } diff --git a/programs/zstd.1.md b/programs/zstd.1.md index 29b7a5bb7..be2179e5d 100644 --- a/programs/zstd.1.md +++ b/programs/zstd.1.md @@ -455,6 +455,17 @@ The list of available _options_: Value 0 is special and means "default": _ovlog_ is automatically determined by `zstd`. In which case, _ovlog_ will range from 6 to 9, depending on selected _strat_. +- `ldmHashRateLog`=_lhrlog_, `lhrlog`=_lhrlog_: + Specify the frequency of inserting entries into the long distance matching + hash table. + + This option is ignored unless long distance matching is enabled. + + Larger values will improve compression speed. Deviating far from the + default value will likely result in a decrease in compression ratio. + + The default value varies between 4 and 8, depending on `strategy`. + - `ldmHashLog`=_lhlog_, `lhlog`=_lhlog_: Specify the maximum size for a hash table used for long distance matching. @@ -463,7 +474,7 @@ The list of available _options_: Bigger hash tables usually improve compression ratio at the expense of more memory during compression and a decrease in compression speed. - The minimum _lhlog_ is 6 and the maximum is 30 (default: 20). + The minimum _lhlog_ is 6 and the maximum is 30 (default: `windowLog - ldmHashRateLog`). - `ldmMinMatch`=_lmml_, `lmml`=_lmml_: Specify the minimum searched length of a match for long distance matching. @@ -472,7 +483,7 @@ The list of available _options_: Larger/very small values usually decrease compression ratio. - The minimum _lmml_ is 4 and the maximum is 4096 (default: 64). + The minimum _lmml_ is 4 and the maximum is 4096 (default: 32 to 128, depending on `strategy`). - `ldmBucketSizeLog`=_lblog_, `lblog`=_lblog_: Specify the size of each bucket for the hash table used for long distance @@ -483,18 +494,8 @@ The list of available _options_: Larger bucket sizes improve collision resolution but decrease compression speed. - The minimum _lblog_ is 1 and the maximum is 8 (default: 3). + The minimum _lblog_ is 1 and the maximum is 8 (default: 4 to 8, depending on `strategy`). -- `ldmHashRateLog`=_lhrlog_, `lhrlog`=_lhrlog_: - Specify the frequency of inserting entries into the long distance matching - hash table. - - This option is ignored unless long distance matching is enabled. - - Larger values will improve compression speed. Deviating far from the - default value will likely result in a decrease in compression ratio. - - The default value is `wlog - lhlog`. ### Example The following parameters sets advanced compression options to something From 7c5b6002c9c40936fbfd13bd6e7737b059f16d4a Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 7 Feb 2025 17:40:54 -0800 Subject: [PATCH 04/10] update results from the --long mode --- tests/regression/results.csv | 48 ++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/tests/regression/results.csv b/tests/regression/results.csv index 505f4755b..467073ebf 100644 --- a/tests/regression/results.csv +++ b/tests/regression/results.csv @@ -111,9 +111,9 @@ silesia, level 9, zstdcli, silesia, level 13, zstdcli, 4488438 silesia, level 16, zstdcli, 4358150 silesia, level 19, zstdcli, 4265929 -silesia, long distance mode, zstdcli, 4824875 +silesia, long distance mode, zstdcli, 4828930 silesia, multithreaded, zstdcli, 4833113 -silesia, multithreaded long distance mode, zstdcli, 4824875 +silesia, multithreaded long distance mode, zstdcli, 4828930 silesia, small window log, zstdcli, 7094528 silesia, small hash log, zstdcli, 6527214 silesia, small chain log, zstdcli, 4911647 @@ -137,9 +137,9 @@ silesia.tar, level 13, zstdcli, silesia.tar, level 16, zstdcli, 4357018 silesia.tar, level 19, zstdcli, 4259593 silesia.tar, no source size, zstdcli, 4836000 -silesia.tar, long distance mode, zstdcli, 4828171 +silesia.tar, long distance mode, zstdcli, 4833141 silesia.tar, multithreaded, zstdcli, 4836004 -silesia.tar, multithreaded long distance mode, zstdcli, 4828171 +silesia.tar, multithreaded long distance mode, zstdcli, 4833141 silesia.tar, small window log, zstdcli, 7100110 silesia.tar, small hash log, zstdcli, 6530127 silesia.tar, small chain log, zstdcli, 4915865 @@ -217,9 +217,9 @@ github.tar, level 19, zstdcli, github.tar, level 19 with dict, zstdcli, 32705 github.tar, no source size, zstdcli, 38885 github.tar, no source size with dict, zstdcli, 38115 -github.tar, long distance mode, zstdcli, 40227 +github.tar, long distance mode, zstdcli, 38801 github.tar, multithreaded, zstdcli, 38888 -github.tar, multithreaded long distance mode, zstdcli, 40227 +github.tar, multithreaded long distance mode, zstdcli, 38801 github.tar, small window log, zstdcli, 198539 github.tar, small hash log, zstdcli, 129874 github.tar, small chain log, zstdcli, 41673 @@ -251,9 +251,9 @@ silesia, level 13, advanced silesia, level 16, advanced one pass, 4356799 silesia, level 19, advanced one pass, 4265851 silesia, no source size, advanced one pass, 4832054 -silesia, long distance mode, advanced one pass, 4823833 +silesia, long distance mode, advanced one pass, 4827748 silesia, multithreaded, advanced one pass, 4833065 -silesia, multithreaded long distance mode, advanced one pass, 4824827 +silesia, multithreaded long distance mode, advanced one pass, 4828882 silesia, small window log, advanced one pass, 7094480 silesia, small hash log, advanced one pass, 6525510 silesia, small chain log, advanced one pass, 4912248 @@ -285,9 +285,9 @@ silesia.tar, level 13, advanced silesia.tar, level 16, advanced one pass, 4355572 silesia.tar, level 19, advanced one pass, 4257629 silesia.tar, no source size, advanced one pass, 4829268 -silesia.tar, long distance mode, advanced one pass, 4816169 +silesia.tar, long distance mode, advanced one pass, 4821065 silesia.tar, multithreaded, advanced one pass, 4836000 -silesia.tar, multithreaded long distance mode, advanced one pass, 4828167 +silesia.tar, multithreaded long distance mode, advanced one pass, 4833137 silesia.tar, small window log, advanced one pass, 7100064 silesia.tar, small hash log, advanced one pass, 6530222 silesia.tar, small chain log, advanced one pass, 4915689 @@ -535,9 +535,9 @@ github.tar, level 19 with dict copy, advanced github.tar, level 19 with dict load, advanced one pass, 32428 github.tar, no source size, advanced one pass, 38884 github.tar, no source size with dict, advanced one pass, 37995 -github.tar, long distance mode, advanced one pass, 40242 +github.tar, long distance mode, advanced one pass, 38797 github.tar, multithreaded, advanced one pass, 38884 -github.tar, multithreaded long distance mode, advanced one pass, 40223 +github.tar, multithreaded long distance mode, advanced one pass, 38797 github.tar, small window log, advanced one pass, 198535 github.tar, small hash log, advanced one pass, 129870 github.tar, small chain log, advanced one pass, 41669 @@ -569,9 +569,9 @@ silesia, level 13, advanced silesia, level 16, advanced one pass small out, 4356799 silesia, level 19, advanced one pass small out, 4265851 silesia, no source size, advanced one pass small out, 4832054 -silesia, long distance mode, advanced one pass small out, 4823833 +silesia, long distance mode, advanced one pass small out, 4827748 silesia, multithreaded, advanced one pass small out, 4833065 -silesia, multithreaded long distance mode, advanced one pass small out, 4824827 +silesia, multithreaded long distance mode, advanced one pass small out, 4828882 silesia, small window log, advanced one pass small out, 7094480 silesia, small hash log, advanced one pass small out, 6525510 silesia, small chain log, advanced one pass small out, 4912248 @@ -603,9 +603,9 @@ silesia.tar, level 13, advanced silesia.tar, level 16, advanced one pass small out, 4355572 silesia.tar, level 19, advanced one pass small out, 4257629 silesia.tar, no source size, advanced one pass small out, 4829268 -silesia.tar, long distance mode, advanced one pass small out, 4816169 +silesia.tar, long distance mode, advanced one pass small out, 4821065 silesia.tar, multithreaded, advanced one pass small out, 4836000 -silesia.tar, multithreaded long distance mode, advanced one pass small out, 4828167 +silesia.tar, multithreaded long distance mode, advanced one pass small out, 4833137 silesia.tar, small window log, advanced one pass small out, 7100064 silesia.tar, small hash log, advanced one pass small out, 6530222 silesia.tar, small chain log, advanced one pass small out, 4915689 @@ -853,9 +853,9 @@ github.tar, level 19 with dict copy, advanced github.tar, level 19 with dict load, advanced one pass small out, 32428 github.tar, no source size, advanced one pass small out, 38884 github.tar, no source size with dict, advanced one pass small out, 37995 -github.tar, long distance mode, advanced one pass small out, 40242 +github.tar, long distance mode, advanced one pass small out, 38797 github.tar, multithreaded, advanced one pass small out, 38884 -github.tar, multithreaded long distance mode, advanced one pass small out, 40223 +github.tar, multithreaded long distance mode, advanced one pass small out, 38797 github.tar, small window log, advanced one pass small out, 198535 github.tar, small hash log, advanced one pass small out, 129870 github.tar, small chain log, advanced one pass small out, 41669 @@ -887,9 +887,9 @@ silesia, level 13, advanced silesia, level 16, advanced streaming, 4358094 silesia, level 19, advanced streaming, 4265908 silesia, no source size, advanced streaming, 4835768 -silesia, long distance mode, advanced streaming, 4827592 +silesia, long distance mode, advanced streaming, 4831544 silesia, multithreaded, advanced streaming, 4833065 -silesia, multithreaded long distance mode, advanced streaming, 4824827 +silesia, multithreaded long distance mode, advanced streaming, 4828882 silesia, small window log, advanced streaming, 7110591 silesia, small hash log, advanced streaming, 6525259 silesia, small chain log, advanced streaming, 4911577 @@ -921,9 +921,9 @@ silesia.tar, level 13, advanced silesia.tar, level 16, advanced streaming, 4358029 silesia.tar, level 19, advanced streaming, 4258228 silesia.tar, no source size, advanced streaming, 4846779 -silesia.tar, long distance mode, advanced streaming, 4826177 +silesia.tar, long distance mode, advanced streaming, 4831119 silesia.tar, multithreaded, advanced streaming, 4836000 -silesia.tar, multithreaded long distance mode, advanced streaming, 4828167 +silesia.tar, multithreaded long distance mode, advanced streaming, 4833137 silesia.tar, small window log, advanced streaming, 7117024 silesia.tar, small hash log, advanced streaming, 6529503 silesia.tar, small chain log, advanced streaming, 4915956 @@ -1171,9 +1171,9 @@ github.tar, level 19 with dict copy, advanced github.tar, level 19 with dict load, advanced streaming, 32428 github.tar, no source size, advanced streaming, 38881 github.tar, no source size with dict, advanced streaming, 38111 -github.tar, long distance mode, advanced streaming, 40242 +github.tar, long distance mode, advanced streaming, 38797 github.tar, multithreaded, advanced streaming, 38884 -github.tar, multithreaded long distance mode, advanced streaming, 40223 +github.tar, multithreaded long distance mode, advanced streaming, 38797 github.tar, small window log, advanced streaming, 199553 github.tar, small hash log, advanced streaming, 129870 github.tar, small chain log, advanced streaming, 41669 From 72406b71c30efbfe865611a79f50117254820c40 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 7 Feb 2025 18:57:44 -0800 Subject: [PATCH 05/10] update hrlog rule to favor compression ratio a bit more at low levels --- lib/compress/zstd_ldm.c | 2 +- programs/zstd.1.md | 2 +- tests/regression/results.csv | 48 ++++++++++++++++++------------------ 3 files changed, 26 insertions(+), 26 deletions(-) diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c index 39f23c565..e85e904da 100644 --- a/lib/compress/zstd_ldm.c +++ b/lib/compress/zstd_ldm.c @@ -141,7 +141,7 @@ void ZSTD_ldm_adjustParameters(ldmParams_t* params, if (params->hashRateLog == 0) { assert(1 <= (int)cParams->strategy && (int)cParams->strategy <= 9); /* mapping: strat1 -> rate8 ... strat9 -> rate4*/ - params->hashRateLog = 9 - ((cParams->strategy+1)/2); + params->hashRateLog = 7 - (cParams->strategy/3); } if (params->hashLog == 0) { params->hashLog = MAX(ZSTD_HASHLOG_MIN, params->windowLog - params->hashRateLog); diff --git a/programs/zstd.1.md b/programs/zstd.1.md index be2179e5d..1459abb9f 100644 --- a/programs/zstd.1.md +++ b/programs/zstd.1.md @@ -464,7 +464,7 @@ The list of available _options_: Larger values will improve compression speed. Deviating far from the default value will likely result in a decrease in compression ratio. - The default value varies between 4 and 8, depending on `strategy`. + The default value varies between 4 and 7, depending on `strategy`. - `ldmHashLog`=_lhlog_, `lhlog`=_lhlog_: Specify the maximum size for a hash table used for long distance matching. diff --git a/tests/regression/results.csv b/tests/regression/results.csv index 467073ebf..cc7500d96 100644 --- a/tests/regression/results.csv +++ b/tests/regression/results.csv @@ -111,9 +111,9 @@ silesia, level 9, zstdcli, silesia, level 13, zstdcli, 4488438 silesia, level 16, zstdcli, 4358150 silesia, level 19, zstdcli, 4265929 -silesia, long distance mode, zstdcli, 4828930 +silesia, long distance mode, zstdcli, 4827199 silesia, multithreaded, zstdcli, 4833113 -silesia, multithreaded long distance mode, zstdcli, 4828930 +silesia, multithreaded long distance mode, zstdcli, 4827199 silesia, small window log, zstdcli, 7094528 silesia, small hash log, zstdcli, 6527214 silesia, small chain log, zstdcli, 4911647 @@ -137,9 +137,9 @@ silesia.tar, level 13, zstdcli, silesia.tar, level 16, zstdcli, 4357018 silesia.tar, level 19, zstdcli, 4259593 silesia.tar, no source size, zstdcli, 4836000 -silesia.tar, long distance mode, zstdcli, 4833141 +silesia.tar, long distance mode, zstdcli, 4831415 silesia.tar, multithreaded, zstdcli, 4836004 -silesia.tar, multithreaded long distance mode, zstdcli, 4833141 +silesia.tar, multithreaded long distance mode, zstdcli, 4831415 silesia.tar, small window log, zstdcli, 7100110 silesia.tar, small hash log, zstdcli, 6530127 silesia.tar, small chain log, zstdcli, 4915865 @@ -217,9 +217,9 @@ github.tar, level 19, zstdcli, github.tar, level 19 with dict, zstdcli, 32705 github.tar, no source size, zstdcli, 38885 github.tar, no source size with dict, zstdcli, 38115 -github.tar, long distance mode, zstdcli, 38801 +github.tar, long distance mode, zstdcli, 38775 github.tar, multithreaded, zstdcli, 38888 -github.tar, multithreaded long distance mode, zstdcli, 38801 +github.tar, multithreaded long distance mode, zstdcli, 38775 github.tar, small window log, zstdcli, 198539 github.tar, small hash log, zstdcli, 129874 github.tar, small chain log, zstdcli, 41673 @@ -251,9 +251,9 @@ silesia, level 13, advanced silesia, level 16, advanced one pass, 4356799 silesia, level 19, advanced one pass, 4265851 silesia, no source size, advanced one pass, 4832054 -silesia, long distance mode, advanced one pass, 4827748 +silesia, long distance mode, advanced one pass, 4826152 silesia, multithreaded, advanced one pass, 4833065 -silesia, multithreaded long distance mode, advanced one pass, 4828882 +silesia, multithreaded long distance mode, advanced one pass, 4827151 silesia, small window log, advanced one pass, 7094480 silesia, small hash log, advanced one pass, 6525510 silesia, small chain log, advanced one pass, 4912248 @@ -285,9 +285,9 @@ silesia.tar, level 13, advanced silesia.tar, level 16, advanced one pass, 4355572 silesia.tar, level 19, advanced one pass, 4257629 silesia.tar, no source size, advanced one pass, 4829268 -silesia.tar, long distance mode, advanced one pass, 4821065 +silesia.tar, long distance mode, advanced one pass, 4819329 silesia.tar, multithreaded, advanced one pass, 4836000 -silesia.tar, multithreaded long distance mode, advanced one pass, 4833137 +silesia.tar, multithreaded long distance mode, advanced one pass, 4831411 silesia.tar, small window log, advanced one pass, 7100064 silesia.tar, small hash log, advanced one pass, 6530222 silesia.tar, small chain log, advanced one pass, 4915689 @@ -535,9 +535,9 @@ github.tar, level 19 with dict copy, advanced github.tar, level 19 with dict load, advanced one pass, 32428 github.tar, no source size, advanced one pass, 38884 github.tar, no source size with dict, advanced one pass, 37995 -github.tar, long distance mode, advanced one pass, 38797 +github.tar, long distance mode, advanced one pass, 38771 github.tar, multithreaded, advanced one pass, 38884 -github.tar, multithreaded long distance mode, advanced one pass, 38797 +github.tar, multithreaded long distance mode, advanced one pass, 38771 github.tar, small window log, advanced one pass, 198535 github.tar, small hash log, advanced one pass, 129870 github.tar, small chain log, advanced one pass, 41669 @@ -569,9 +569,9 @@ silesia, level 13, advanced silesia, level 16, advanced one pass small out, 4356799 silesia, level 19, advanced one pass small out, 4265851 silesia, no source size, advanced one pass small out, 4832054 -silesia, long distance mode, advanced one pass small out, 4827748 +silesia, long distance mode, advanced one pass small out, 4826152 silesia, multithreaded, advanced one pass small out, 4833065 -silesia, multithreaded long distance mode, advanced one pass small out, 4828882 +silesia, multithreaded long distance mode, advanced one pass small out, 4827151 silesia, small window log, advanced one pass small out, 7094480 silesia, small hash log, advanced one pass small out, 6525510 silesia, small chain log, advanced one pass small out, 4912248 @@ -603,9 +603,9 @@ silesia.tar, level 13, advanced silesia.tar, level 16, advanced one pass small out, 4355572 silesia.tar, level 19, advanced one pass small out, 4257629 silesia.tar, no source size, advanced one pass small out, 4829268 -silesia.tar, long distance mode, advanced one pass small out, 4821065 +silesia.tar, long distance mode, advanced one pass small out, 4819329 silesia.tar, multithreaded, advanced one pass small out, 4836000 -silesia.tar, multithreaded long distance mode, advanced one pass small out, 4833137 +silesia.tar, multithreaded long distance mode, advanced one pass small out, 4831411 silesia.tar, small window log, advanced one pass small out, 7100064 silesia.tar, small hash log, advanced one pass small out, 6530222 silesia.tar, small chain log, advanced one pass small out, 4915689 @@ -853,9 +853,9 @@ github.tar, level 19 with dict copy, advanced github.tar, level 19 with dict load, advanced one pass small out, 32428 github.tar, no source size, advanced one pass small out, 38884 github.tar, no source size with dict, advanced one pass small out, 37995 -github.tar, long distance mode, advanced one pass small out, 38797 +github.tar, long distance mode, advanced one pass small out, 38771 github.tar, multithreaded, advanced one pass small out, 38884 -github.tar, multithreaded long distance mode, advanced one pass small out, 38797 +github.tar, multithreaded long distance mode, advanced one pass small out, 38771 github.tar, small window log, advanced one pass small out, 198535 github.tar, small hash log, advanced one pass small out, 129870 github.tar, small chain log, advanced one pass small out, 41669 @@ -887,9 +887,9 @@ silesia, level 13, advanced silesia, level 16, advanced streaming, 4358094 silesia, level 19, advanced streaming, 4265908 silesia, no source size, advanced streaming, 4835768 -silesia, long distance mode, advanced streaming, 4831544 +silesia, long distance mode, advanced streaming, 4829919 silesia, multithreaded, advanced streaming, 4833065 -silesia, multithreaded long distance mode, advanced streaming, 4828882 +silesia, multithreaded long distance mode, advanced streaming, 4827151 silesia, small window log, advanced streaming, 7110591 silesia, small hash log, advanced streaming, 6525259 silesia, small chain log, advanced streaming, 4911577 @@ -921,9 +921,9 @@ silesia.tar, level 13, advanced silesia.tar, level 16, advanced streaming, 4358029 silesia.tar, level 19, advanced streaming, 4258228 silesia.tar, no source size, advanced streaming, 4846779 -silesia.tar, long distance mode, advanced streaming, 4831119 +silesia.tar, long distance mode, advanced streaming, 4829404 silesia.tar, multithreaded, advanced streaming, 4836000 -silesia.tar, multithreaded long distance mode, advanced streaming, 4833137 +silesia.tar, multithreaded long distance mode, advanced streaming, 4831411 silesia.tar, small window log, advanced streaming, 7117024 silesia.tar, small hash log, advanced streaming, 6529503 silesia.tar, small chain log, advanced streaming, 4915956 @@ -1171,9 +1171,9 @@ github.tar, level 19 with dict copy, advanced github.tar, level 19 with dict load, advanced streaming, 32428 github.tar, no source size, advanced streaming, 38881 github.tar, no source size with dict, advanced streaming, 38111 -github.tar, long distance mode, advanced streaming, 38797 +github.tar, long distance mode, advanced streaming, 38771 github.tar, multithreaded, advanced streaming, 38884 -github.tar, multithreaded long distance mode, advanced streaming, 38797 +github.tar, multithreaded long distance mode, advanced streaming, 38771 github.tar, small window log, advanced streaming, 199553 github.tar, small hash log, advanced streaming, 129870 github.tar, small chain log, advanced streaming, 41669 From d5e4698267b970545c349b3ed30a9168bcd165c3 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 7 Feb 2025 22:47:57 -0800 Subject: [PATCH 06/10] fix boundary condition --- lib/compress/zstd_ldm.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c index e85e904da..aa5743c5c 100644 --- a/lib/compress/zstd_ldm.c +++ b/lib/compress/zstd_ldm.c @@ -144,8 +144,7 @@ void ZSTD_ldm_adjustParameters(ldmParams_t* params, params->hashRateLog = 7 - (cParams->strategy/3); } if (params->hashLog == 0) { - params->hashLog = MAX(ZSTD_HASHLOG_MIN, params->windowLog - params->hashRateLog); - assert(params->hashLog <= ZSTD_HASHLOG_MAX); + params->hashLog = MIN(MAX(ZSTD_HASHLOG_MIN, params->windowLog - params->hashRateLog), ZSTD_HASHLOG_MAX); } if (params->minMatchLength == 0) { params->minMatchLength = (params->hashRateLog < 6) ? From 09d7e34ed8c138e913d8c11724f2d5fb5a1436bd Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 8 Feb 2025 14:27:29 -0800 Subject: [PATCH 07/10] adjust mml --- lib/compress/zstd_ldm.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c index aa5743c5c..981287d85 100644 --- a/lib/compress/zstd_ldm.c +++ b/lib/compress/zstd_ldm.c @@ -147,10 +147,8 @@ void ZSTD_ldm_adjustParameters(ldmParams_t* params, params->hashLog = MIN(MAX(ZSTD_HASHLOG_MIN, params->windowLog - params->hashRateLog), ZSTD_HASHLOG_MAX); } if (params->minMatchLength == 0) { - params->minMatchLength = (params->hashRateLog < 6) ? - LDM_MIN_MATCH_LENGTH : - LDM_MIN_MATCH_LENGTH * 2; - if (cParams->strategy >= ZSTD_btultra2) + params->minMatchLength = LDM_MIN_MATCH_LENGTH; + if (cParams->strategy >= ZSTD_btultra) params->minMatchLength /= 2; } if (params->bucketSizeLog==0) { From 339bca66066e4a099bc91d04265c9bfbf15001bb Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 8 Feb 2025 16:16:35 -0800 Subject: [PATCH 08/10] update ldm compression results --- programs/zstd.1.md | 2 +- tests/regression/results.csv | 48 ++++++++++++++++++------------------ 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/programs/zstd.1.md b/programs/zstd.1.md index 1459abb9f..e5c1b7fd2 100644 --- a/programs/zstd.1.md +++ b/programs/zstd.1.md @@ -483,7 +483,7 @@ The list of available _options_: Larger/very small values usually decrease compression ratio. - The minimum _lmml_ is 4 and the maximum is 4096 (default: 32 to 128, depending on `strategy`). + The minimum _lmml_ is 4 and the maximum is 4096 (default: 32 to 64, depending on `strategy`). - `ldmBucketSizeLog`=_lblog_, `lblog`=_lblog_: Specify the size of each bucket for the hash table used for long distance diff --git a/tests/regression/results.csv b/tests/regression/results.csv index cc7500d96..c0d4f4ae7 100644 --- a/tests/regression/results.csv +++ b/tests/regression/results.csv @@ -111,9 +111,9 @@ silesia, level 9, zstdcli, silesia, level 13, zstdcli, 4488438 silesia, level 16, zstdcli, 4358150 silesia, level 19, zstdcli, 4265929 -silesia, long distance mode, zstdcli, 4827199 +silesia, long distance mode, zstdcli, 4824341 silesia, multithreaded, zstdcli, 4833113 -silesia, multithreaded long distance mode, zstdcli, 4827199 +silesia, multithreaded long distance mode, zstdcli, 4824341 silesia, small window log, zstdcli, 7094528 silesia, small hash log, zstdcli, 6527214 silesia, small chain log, zstdcli, 4911647 @@ -137,9 +137,9 @@ silesia.tar, level 13, zstdcli, silesia.tar, level 16, zstdcli, 4357018 silesia.tar, level 19, zstdcli, 4259593 silesia.tar, no source size, zstdcli, 4836000 -silesia.tar, long distance mode, zstdcli, 4831415 +silesia.tar, long distance mode, zstdcli, 4827830 silesia.tar, multithreaded, zstdcli, 4836004 -silesia.tar, multithreaded long distance mode, zstdcli, 4831415 +silesia.tar, multithreaded long distance mode, zstdcli, 4827830 silesia.tar, small window log, zstdcli, 7100110 silesia.tar, small hash log, zstdcli, 6530127 silesia.tar, small chain log, zstdcli, 4915865 @@ -217,9 +217,9 @@ github.tar, level 19, zstdcli, github.tar, level 19 with dict, zstdcli, 32705 github.tar, no source size, zstdcli, 38885 github.tar, no source size with dict, zstdcli, 38115 -github.tar, long distance mode, zstdcli, 38775 +github.tar, long distance mode, zstdcli, 40143 github.tar, multithreaded, zstdcli, 38888 -github.tar, multithreaded long distance mode, zstdcli, 38775 +github.tar, multithreaded long distance mode, zstdcli, 40143 github.tar, small window log, zstdcli, 198539 github.tar, small hash log, zstdcli, 129874 github.tar, small chain log, zstdcli, 41673 @@ -251,9 +251,9 @@ silesia, level 13, advanced silesia, level 16, advanced one pass, 4356799 silesia, level 19, advanced one pass, 4265851 silesia, no source size, advanced one pass, 4832054 -silesia, long distance mode, advanced one pass, 4826152 +silesia, long distance mode, advanced one pass, 4823264 silesia, multithreaded, advanced one pass, 4833065 -silesia, multithreaded long distance mode, advanced one pass, 4827151 +silesia, multithreaded long distance mode, advanced one pass, 4824293 silesia, small window log, advanced one pass, 7094480 silesia, small hash log, advanced one pass, 6525510 silesia, small chain log, advanced one pass, 4912248 @@ -285,9 +285,9 @@ silesia.tar, level 13, advanced silesia.tar, level 16, advanced one pass, 4355572 silesia.tar, level 19, advanced one pass, 4257629 silesia.tar, no source size, advanced one pass, 4829268 -silesia.tar, long distance mode, advanced one pass, 4819329 +silesia.tar, long distance mode, advanced one pass, 4815868 silesia.tar, multithreaded, advanced one pass, 4836000 -silesia.tar, multithreaded long distance mode, advanced one pass, 4831411 +silesia.tar, multithreaded long distance mode, advanced one pass, 4827826 silesia.tar, small window log, advanced one pass, 7100064 silesia.tar, small hash log, advanced one pass, 6530222 silesia.tar, small chain log, advanced one pass, 4915689 @@ -535,9 +535,9 @@ github.tar, level 19 with dict copy, advanced github.tar, level 19 with dict load, advanced one pass, 32428 github.tar, no source size, advanced one pass, 38884 github.tar, no source size with dict, advanced one pass, 37995 -github.tar, long distance mode, advanced one pass, 38771 +github.tar, long distance mode, advanced one pass, 40156 github.tar, multithreaded, advanced one pass, 38884 -github.tar, multithreaded long distance mode, advanced one pass, 38771 +github.tar, multithreaded long distance mode, advanced one pass, 40139 github.tar, small window log, advanced one pass, 198535 github.tar, small hash log, advanced one pass, 129870 github.tar, small chain log, advanced one pass, 41669 @@ -569,9 +569,9 @@ silesia, level 13, advanced silesia, level 16, advanced one pass small out, 4356799 silesia, level 19, advanced one pass small out, 4265851 silesia, no source size, advanced one pass small out, 4832054 -silesia, long distance mode, advanced one pass small out, 4826152 +silesia, long distance mode, advanced one pass small out, 4823264 silesia, multithreaded, advanced one pass small out, 4833065 -silesia, multithreaded long distance mode, advanced one pass small out, 4827151 +silesia, multithreaded long distance mode, advanced one pass small out, 4824293 silesia, small window log, advanced one pass small out, 7094480 silesia, small hash log, advanced one pass small out, 6525510 silesia, small chain log, advanced one pass small out, 4912248 @@ -603,9 +603,9 @@ silesia.tar, level 13, advanced silesia.tar, level 16, advanced one pass small out, 4355572 silesia.tar, level 19, advanced one pass small out, 4257629 silesia.tar, no source size, advanced one pass small out, 4829268 -silesia.tar, long distance mode, advanced one pass small out, 4819329 +silesia.tar, long distance mode, advanced one pass small out, 4815868 silesia.tar, multithreaded, advanced one pass small out, 4836000 -silesia.tar, multithreaded long distance mode, advanced one pass small out, 4831411 +silesia.tar, multithreaded long distance mode, advanced one pass small out, 4827826 silesia.tar, small window log, advanced one pass small out, 7100064 silesia.tar, small hash log, advanced one pass small out, 6530222 silesia.tar, small chain log, advanced one pass small out, 4915689 @@ -853,9 +853,9 @@ github.tar, level 19 with dict copy, advanced github.tar, level 19 with dict load, advanced one pass small out, 32428 github.tar, no source size, advanced one pass small out, 38884 github.tar, no source size with dict, advanced one pass small out, 37995 -github.tar, long distance mode, advanced one pass small out, 38771 +github.tar, long distance mode, advanced one pass small out, 40156 github.tar, multithreaded, advanced one pass small out, 38884 -github.tar, multithreaded long distance mode, advanced one pass small out, 38771 +github.tar, multithreaded long distance mode, advanced one pass small out, 40139 github.tar, small window log, advanced one pass small out, 198535 github.tar, small hash log, advanced one pass small out, 129870 github.tar, small chain log, advanced one pass small out, 41669 @@ -887,9 +887,9 @@ silesia, level 13, advanced silesia, level 16, advanced streaming, 4358094 silesia, level 19, advanced streaming, 4265908 silesia, no source size, advanced streaming, 4835768 -silesia, long distance mode, advanced streaming, 4829919 +silesia, long distance mode, advanced streaming, 4827032 silesia, multithreaded, advanced streaming, 4833065 -silesia, multithreaded long distance mode, advanced streaming, 4827151 +silesia, multithreaded long distance mode, advanced streaming, 4824293 silesia, small window log, advanced streaming, 7110591 silesia, small hash log, advanced streaming, 6525259 silesia, small chain log, advanced streaming, 4911577 @@ -921,9 +921,9 @@ silesia.tar, level 13, advanced silesia.tar, level 16, advanced streaming, 4358029 silesia.tar, level 19, advanced streaming, 4258228 silesia.tar, no source size, advanced streaming, 4846779 -silesia.tar, long distance mode, advanced streaming, 4829404 +silesia.tar, long distance mode, advanced streaming, 4825842 silesia.tar, multithreaded, advanced streaming, 4836000 -silesia.tar, multithreaded long distance mode, advanced streaming, 4831411 +silesia.tar, multithreaded long distance mode, advanced streaming, 4827826 silesia.tar, small window log, advanced streaming, 7117024 silesia.tar, small hash log, advanced streaming, 6529503 silesia.tar, small chain log, advanced streaming, 4915956 @@ -1171,9 +1171,9 @@ github.tar, level 19 with dict copy, advanced github.tar, level 19 with dict load, advanced streaming, 32428 github.tar, no source size, advanced streaming, 38881 github.tar, no source size with dict, advanced streaming, 38111 -github.tar, long distance mode, advanced streaming, 38771 +github.tar, long distance mode, advanced streaming, 40156 github.tar, multithreaded, advanced streaming, 38884 -github.tar, multithreaded long distance mode, advanced streaming, 38771 +github.tar, multithreaded long distance mode, advanced streaming, 40139 github.tar, small window log, advanced streaming, 199553 github.tar, small hash log, advanced streaming, 129870 github.tar, small chain log, advanced streaming, 41669 From 67fad95f7971b97085fb838e3aa47cd37e9d7908 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 9 Feb 2025 11:11:49 -0800 Subject: [PATCH 09/10] derive hashratelog from hashlog when only hashlog is set --- lib/compress/zstd_ldm.c | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c index 981287d85..6aa6197e9 100644 --- a/lib/compress/zstd_ldm.c +++ b/lib/compress/zstd_ldm.c @@ -133,18 +133,26 @@ done: } void ZSTD_ldm_adjustParameters(ldmParams_t* params, - ZSTD_compressionParameters const* cParams) + const ZSTD_compressionParameters* cParams) { params->windowLog = cParams->windowLog; ZSTD_STATIC_ASSERT(LDM_BUCKET_SIZE_LOG <= ZSTD_LDM_BUCKETSIZELOG_MAX); DEBUGLOG(4, "ZSTD_ldm_adjustParameters"); if (params->hashRateLog == 0) { - assert(1 <= (int)cParams->strategy && (int)cParams->strategy <= 9); - /* mapping: strat1 -> rate8 ... strat9 -> rate4*/ - params->hashRateLog = 7 - (cParams->strategy/3); + if (params->hashLog > 0) { + /* if params->hashLog is set, derive hashRateLog from it */ + assert(params->hashLog <= ZSTD_HASHLOG_MAX); + if (params->windowLog > params->hashLog) { + params->hashRateLog = params->windowLog - params->hashLog; + } + } else { + assert(1 <= (int)cParams->strategy && (int)cParams->strategy <= 9); + /* mapping: strat1 -> rate8 ... strat9 -> rate4*/ + params->hashRateLog = 7 - (cParams->strategy/3); + } } if (params->hashLog == 0) { - params->hashLog = MIN(MAX(ZSTD_HASHLOG_MIN, params->windowLog - params->hashRateLog), ZSTD_HASHLOG_MAX); + params->hashLog = BOUNDED(ZSTD_HASHLOG_MIN, params->windowLog - params->hashRateLog, ZSTD_HASHLOG_MAX); } if (params->minMatchLength == 0) { params->minMatchLength = LDM_MIN_MATCH_LENGTH; @@ -152,11 +160,8 @@ void ZSTD_ldm_adjustParameters(ldmParams_t* params, params->minMatchLength /= 2; } if (params->bucketSizeLog==0) { - params->bucketSizeLog = LDM_BUCKET_SIZE_LOG; - if (cParams->strategy > ZSTD_lazy) { - params->bucketSizeLog += (U32)cParams->strategy - (U32)ZSTD_lazy; - } - params->bucketSizeLog = MIN(params->bucketSizeLog, ZSTD_LDM_BUCKETSIZELOG_MAX); + assert(1 <= (int)cParams->strategy && (int)cParams->strategy <= 9); + params->bucketSizeLog = BOUNDED(LDM_BUCKET_SIZE_LOG, (U32)cParams->strategy, ZSTD_LDM_BUCKETSIZELOG_MAX); } params->bucketSizeLog = MIN(params->bucketSizeLog, params->hashLog); } From d2c562b803a4c49dbd5afb1717db095aae1557b1 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 10 Feb 2025 10:48:56 -0800 Subject: [PATCH 10/10] update hrlog comment --- lib/compress/zstd_ldm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c index 6aa6197e9..070551cad 100644 --- a/lib/compress/zstd_ldm.c +++ b/lib/compress/zstd_ldm.c @@ -147,7 +147,7 @@ void ZSTD_ldm_adjustParameters(ldmParams_t* params, } } else { assert(1 <= (int)cParams->strategy && (int)cParams->strategy <= 9); - /* mapping: strat1 -> rate8 ... strat9 -> rate4*/ + /* mapping from [fast, rate7] to [btultra2, rate4] */ params->hashRateLog = 7 - (cParams->strategy/3); } }