From af0c9501d120ff83a7e0871a0b18ad7237813a8c Mon Sep 17 00:00:00 2001 From: Nick Magerko Date: Thu, 15 Aug 2019 23:57:55 -0700 Subject: [PATCH 01/10] Add --stream-size=# command --- programs/fileio.c | 21 +++++++++++++++++++-- programs/fileio.h | 1 + programs/zstdcli.c | 6 +++++- tests/playTests.sh | 22 +++++++++++++++++++++- 4 files changed, 46 insertions(+), 4 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 569a410c1..82d70075b 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -304,6 +304,7 @@ struct FIO_prefs_s { int ldmMinMatch; int ldmBucketSizeLog; int ldmHashRateLog; + size_t streamSrcSize; size_t targetCBlockSize; ZSTD_literalCompressionMode_e literalCompressionMode; @@ -349,6 +350,7 @@ FIO_prefs_t* FIO_createPreferences(void) ret->ldmMinMatch = 0; ret->ldmBucketSizeLog = FIO_LDM_PARAM_NOTSET; ret->ldmHashRateLog = FIO_LDM_PARAM_NOTSET; + ret->streamSrcSize = 0; ret->targetCBlockSize = 0; ret->literalCompressionMode = ZSTD_lcm_auto; return ret; @@ -418,6 +420,10 @@ void FIO_setRsyncable(FIO_prefs_t* const prefs, int rsyncable) { prefs->rsyncable = rsyncable; } +void FIO_setStreamSrcSize(FIO_prefs_t* const prefs, size_t streamSrcSize) { + prefs->streamSrcSize = streamSrcSize; +} + void FIO_setTargetCBlockSize(FIO_prefs_t* const prefs, size_t targetCBlockSize) { prefs->targetCBlockSize = targetCBlockSize; } @@ -698,9 +704,20 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs, CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_rsyncable, prefs->rsyncable) ); #endif /* dictionary */ - CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, srcSize) ); /* set the value temporarily for dictionary loading, to adapt compression parameters */ + /* set the pledged size for dictionary loading, to adapt compression parameters */ + if (srcSize == ZSTD_CONTENTSIZE_UNKNOWN && prefs->streamSrcSize > 0) { + /* unknown source size; use the declared stream size and disable writing this size to frame during compression */ + CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, prefs->streamSrcSize) ); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_contentSizeFlag, 0) ); + } else { + /* use the known source size for adaption */ + CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, srcSize) ); + } CHECK( ZSTD_CCtx_loadDictionary(ress.cctx, dictBuffer, dictBuffSize) ); - CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, ZSTD_CONTENTSIZE_UNKNOWN) ); /* reset */ + if (srcSize != ZSTD_CONTENTSIZE_UNKNOWN || prefs->streamSrcSize == 0) { + /* reset pledge when src size is known or stream size is declared */ + CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, ZSTD_CONTENTSIZE_UNKNOWN) ); + } free(dictBuffer); } diff --git a/programs/fileio.h b/programs/fileio.h index 311f8c0e1..13f6f1d05 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -71,6 +71,7 @@ void FIO_setOverlapLog(FIO_prefs_t* const prefs, int overlapLog); void FIO_setRemoveSrcFile(FIO_prefs_t* const prefs, unsigned flag); void FIO_setSparseWrite(FIO_prefs_t* const prefs, unsigned sparse); /**< 0: no sparse; 1: disable on stdout; 2: always enabled */ void FIO_setRsyncable(FIO_prefs_t* const prefs, int rsyncable); +void FIO_setStreamSrcSize(FIO_prefs_t* const prefs, size_t streamSrcSize); void FIO_setTargetCBlockSize(FIO_prefs_t* const prefs, size_t targetCBlockSize); void FIO_setLiteralCompressionMode( FIO_prefs_t* const prefs, diff --git a/programs/zstdcli.c b/programs/zstdcli.c index de286cdf2..401e1ee2c 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -141,6 +141,7 @@ static int usage_advanced(const char* programName) DISPLAY( "--long[=#]: enable long distance matching with given window log (default: %u)\n", g_defaultMaxWindowLog); DISPLAY( "--fast[=#]: switch to ultra fast compression level (default: %u)\n", 1); DISPLAY( "--adapt : dynamically adapt compression level to I/O conditions \n"); + DISPLAY( "--stream-size=# : optimize compression parameters for streaming input of given number of bytes \n"); DISPLAY( "--target-compressed-block-size=# : make compressed block near targeted size \n"); #ifdef ZSTD_MULTITHREAD DISPLAY( " -T# : spawns # compression threads (default: 1, 0==# cores) \n"); @@ -588,6 +589,7 @@ int main(int argCount, const char* argv[]) const char* suffix = ZSTD_EXTENSION; unsigned maxDictSize = g_defaultMaxDictSize; unsigned dictID = 0; + size_t streamSrcSize = 0; size_t targetCBlockSize = 0; int dictCLevel = g_defaultDictCLevel; unsigned dictSelect = g_defaultSelectivityLevel; @@ -745,6 +747,7 @@ int main(int argCount, const char* argv[]) if (longCommandWArg(&argument, "--maxdict=")) { maxDictSize = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "--dictID=")) { dictID = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "--zstd=")) { if (!parseCompressionParameters(argument, &compressionParams)) CLEAN_RETURN(badusage(programName)); continue; } + if (longCommandWArg(&argument, "--stream-size=")) { streamSrcSize = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "--target-compressed-block-size=")) { targetCBlockSize = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "--long")) { unsigned ldmWindowLog = 0; @@ -1150,6 +1153,7 @@ int main(int argCount, const char* argv[]) FIO_setAdaptMin(prefs, adaptMin); FIO_setAdaptMax(prefs, adaptMax); FIO_setRsyncable(prefs, rsyncable); + FIO_setStreamSrcSize(prefs, streamSrcSize); FIO_setTargetCBlockSize(prefs, targetCBlockSize); FIO_setLiteralCompressionMode(prefs, literalCompressionMode); if (adaptMin > cLevel) cLevel = adaptMin; @@ -1160,7 +1164,7 @@ int main(int argCount, const char* argv[]) else operationResult = FIO_compressMultipleFilenames(prefs, filenameTable, filenameIdx, outFileName, suffix, dictFileName, cLevel, compressionParams); #else - (void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; (void)targetCBlockSize; /* not used when ZSTD_NOCOMPRESS set */ + (void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; (void)streamSrcSize; (void)targetCBlockSize; /* not used when ZSTD_NOCOMPRESS set */ DISPLAY("Compression not supported \n"); #endif } else { /* decompression or test */ diff --git a/tests/playTests.sh b/tests/playTests.sh index 69387321f..431a53a18 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -108,7 +108,6 @@ else fi - println "\n===> simple tests " ./datagen > tmp @@ -1020,4 +1019,25 @@ test -f dictionary rm -f tmp* dictionary +println "\n===> stream-size mode" + +./datagen -g11000 > tmp +println "test : basic file compression vs sized streaming compression" +$ZSTD -14 -f tmp -o tmp.zst |& tee file.out +cat tmp | $ZSTD -14 -f -o tmp.zst --stream-size=11000 |& tee stream_sized.out + +file_ratio=$(cat file.out | awk '{print $4}' | sed 's/%//g') +stream_sized_ratio=$(cat stream_sized.out | awk '{print $4}' | sed 's/%//g') +rm file.out stream_sized.out + +ratio_diff=$(echo $file_ratio - $stream_sized_ratio | bc) +if [ $(echo "(100 * $ratio_diff) > 5" | bc -l) == 1 ] +then + die "greater than 0.05% difference between file and sized-streaming compression" +fi + +println "test : incorrect stream size" +cat tmp | $ZSTD -14 -f -o tmp.zst --stream-size=11001 && die "should fail with incorrect stream size" + + rm -f tmp* From 85d07c6c474abc8f16b069332fbaba3054819d14 Mon Sep 17 00:00:00 2001 From: Nick Magerko Date: Fri, 16 Aug 2019 12:49:21 -0700 Subject: [PATCH 02/10] Tweak stdout, stderr redirection in new playTests --- tests/playTests.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/playTests.sh b/tests/playTests.sh index 431a53a18..c516aa712 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -1023,8 +1023,8 @@ println "\n===> stream-size mode" ./datagen -g11000 > tmp println "test : basic file compression vs sized streaming compression" -$ZSTD -14 -f tmp -o tmp.zst |& tee file.out -cat tmp | $ZSTD -14 -f -o tmp.zst --stream-size=11000 |& tee stream_sized.out +$ZSTD -14 -f tmp -o tmp.zst 2>&1 | tee file.out +cat tmp | $ZSTD -14 -f -o tmp.zst --stream-size=11000 2>&1 | tee stream_sized.out file_ratio=$(cat file.out | awk '{print $4}' | sed 's/%//g') stream_sized_ratio=$(cat stream_sized.out | awk '{print $4}' | sed 's/%//g') @@ -1035,7 +1035,10 @@ if [ $(echo "(100 * $ratio_diff) > 5" | bc -l) == 1 ] then die "greater than 0.05% difference between file and sized-streaming compression" fi - +println "test : sized streaming compression and decompression" +cat tmp | $ZSTD -14 -f tmp -o --stream-size=11000 tmp.zst +$ZSTD -df tmp.zst -o tmp_decompress +cmp tmp tmp_decompress || die "difference between original and decompressed file" println "test : incorrect stream size" cat tmp | $ZSTD -14 -f -o tmp.zst --stream-size=11001 && die "should fail with incorrect stream size" From c403b12f9dbab9ba3ab30fca6e1ce3c31d2f8923 Mon Sep 17 00:00:00 2001 From: Nick Magerko Date: Mon, 19 Aug 2019 09:01:31 -0700 Subject: [PATCH 03/10] Set pledged size just before compression --- programs/fileio.c | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 82d70075b..75b271a8f 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -704,21 +704,7 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs, CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_rsyncable, prefs->rsyncable) ); #endif /* dictionary */ - /* set the pledged size for dictionary loading, to adapt compression parameters */ - if (srcSize == ZSTD_CONTENTSIZE_UNKNOWN && prefs->streamSrcSize > 0) { - /* unknown source size; use the declared stream size and disable writing this size to frame during compression */ - CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, prefs->streamSrcSize) ); - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_contentSizeFlag, 0) ); - } else { - /* use the known source size for adaption */ - CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, srcSize) ); - } CHECK( ZSTD_CCtx_loadDictionary(ress.cctx, dictBuffer, dictBuffSize) ); - if (srcSize != ZSTD_CONTENTSIZE_UNKNOWN || prefs->streamSrcSize == 0) { - /* reset pledge when src size is known or stream size is declared */ - CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, ZSTD_CONTENTSIZE_UNKNOWN) ); - } - free(dictBuffer); } @@ -1020,6 +1006,10 @@ FIO_compressZstdFrame(FIO_prefs_t* const prefs, /* init */ if (fileSize != UTIL_FILESIZE_UNKNOWN) { CHECK(ZSTD_CCtx_setPledgedSrcSize(ress.cctx, fileSize)); + } else if (prefs->streamSrcSize > 0) { + /* unknown source size; use the declared stream size and disable writing this size to frame during compression */ + CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, prefs->streamSrcSize) ); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_contentSizeFlag, 0) ); } (void)srcFileName; From bbd83c2ab34dc498a63ff10e869e26fbe1e221a4 Mon Sep 17 00:00:00 2001 From: Nick Magerko Date: Mon, 19 Aug 2019 09:11:22 -0700 Subject: [PATCH 04/10] Update man page --- programs/zstd.1.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/programs/zstd.1.md b/programs/zstd.1.md index 3ab2667a0..e3f729282 100644 --- a/programs/zstd.1.md +++ b/programs/zstd.1.md @@ -152,6 +152,12 @@ the last one takes effect. This feature does not work with `--single-thread`. You probably don't want to use it with long range mode, since it will decrease the effectiveness of the synchronization points, but your milage may vary. +* `--stream-size` : + When handling input from a stream, `zstd` must guess how large the source size + will be when optimizing compression parameters. If the stream size is relatively + small, this guess may be a poor one, resulting in a higher compression ratio than + expected. This feature will set the source size of a stream. Note that it must + be exact; incorrect stream sizes will cause an error. * `-D file`: use `file` as Dictionary to compress or decompress FILE(s) * `--no-dictID`: From f781cf672bfd23dd467a6dcf08ba094180032c76 Mon Sep 17 00:00:00 2001 From: Nick Magerko Date: Mon, 19 Aug 2019 11:07:43 -0700 Subject: [PATCH 05/10] Remove extraneous parameter --- programs/fileio.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 75b271a8f..f5ecf7296 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -639,7 +639,6 @@ typedef struct { static cRess_t FIO_createCResources(FIO_prefs_t* const prefs, const char* dictFileName, int cLevel, - U64 srcSize, ZSTD_compressionParameters comprParams) { cRess_t ress; memset(&ress, 0, sizeof(ress)); @@ -1371,7 +1370,7 @@ int FIO_compressFilename(FIO_prefs_t* const prefs, U64 const fileSize = UTIL_getFileSize(srcFileName); U64 const srcSize = (fileSize == UTIL_FILESIZE_UNKNOWN) ? ZSTD_CONTENTSIZE_UNKNOWN : fileSize; - cRess_t const ress = FIO_createCResources(prefs, dictFileName, compressionLevel, srcSize, comprParams); + cRess_t const ress = FIO_createCResources(prefs, dictFileName, compressionLevel, comprParams); int const result = FIO_compressFilename_srcFile(prefs, ress, dstFileName, srcFileName, compressionLevel); @@ -1424,8 +1423,7 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, int error = 0; U64 const firstFileSize = UTIL_getFileSize(inFileNamesTable[0]); U64 const firstSrcSize = (firstFileSize == UTIL_FILESIZE_UNKNOWN) ? ZSTD_CONTENTSIZE_UNKNOWN : firstFileSize; - U64 const srcSize = (nbFiles != 1) ? ZSTD_CONTENTSIZE_UNKNOWN : firstSrcSize ; - cRess_t ress = FIO_createCResources(prefs, dictFileName, compressionLevel, srcSize, comprParams); + cRess_t ress = FIO_createCResources(prefs, dictFileName, compressionLevel, comprParams); /* init */ assert(outFileName != NULL || suffix != NULL); From a24dc3a935a43ef8088711afadece9b94ae59a27 Mon Sep 17 00:00:00 2001 From: Nick Magerko Date: Mon, 19 Aug 2019 11:14:56 -0700 Subject: [PATCH 06/10] Remove extraneous variables --- programs/fileio.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index f5ecf7296..5492f9448 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1367,9 +1367,6 @@ int FIO_compressFilename(FIO_prefs_t* const prefs, const char* dictFileName, int compressionLevel, ZSTD_compressionParameters comprParams) { - U64 const fileSize = UTIL_getFileSize(srcFileName); - U64 const srcSize = (fileSize == UTIL_FILESIZE_UNKNOWN) ? ZSTD_CONTENTSIZE_UNKNOWN : fileSize; - cRess_t const ress = FIO_createCResources(prefs, dictFileName, compressionLevel, comprParams); int const result = FIO_compressFilename_srcFile(prefs, ress, dstFileName, srcFileName, compressionLevel); @@ -1421,8 +1418,6 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, ZSTD_compressionParameters comprParams) { int error = 0; - U64 const firstFileSize = UTIL_getFileSize(inFileNamesTable[0]); - U64 const firstSrcSize = (firstFileSize == UTIL_FILESIZE_UNKNOWN) ? ZSTD_CONTENTSIZE_UNKNOWN : firstFileSize; cRess_t ress = FIO_createCResources(prefs, dictFileName, compressionLevel, comprParams); /* init */ From 30bfa228e84500855e80e792f7de796257d99ab3 Mon Sep 17 00:00:00 2001 From: Nick Magerko Date: Mon, 19 Aug 2019 11:20:28 -0700 Subject: [PATCH 07/10] Keep content size flag set in stream size mode --- programs/fileio.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 5492f9448..873013a51 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1006,9 +1006,8 @@ FIO_compressZstdFrame(FIO_prefs_t* const prefs, if (fileSize != UTIL_FILESIZE_UNKNOWN) { CHECK(ZSTD_CCtx_setPledgedSrcSize(ress.cctx, fileSize)); } else if (prefs->streamSrcSize > 0) { - /* unknown source size; use the declared stream size and disable writing this size to frame during compression */ + /* unknown source size; use the declared stream size */ CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, prefs->streamSrcSize) ); - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_contentSizeFlag, 0) ); } (void)srcFileName; From b3540507f5239f9d0810c3b9e2b920601687738d Mon Sep 17 00:00:00 2001 From: Nick Magerko Date: Wed, 21 Aug 2019 10:27:54 -0700 Subject: [PATCH 08/10] Remove bc from play tests --- tests/playTests.sh | 41 +++++++++++++++++------------------------ 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/tests/playTests.sh b/tests/playTests.sh index c516aa712..b74076763 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -408,6 +408,23 @@ println "compress multiple files including a missing one (notHere) : " $ZSTD -f tmp1 notHere tmp2 && die "missing file not detected!" +println "\n===> stream-size mode" + +./datagen -g11000 > tmp +println "test : basic file compression vs sized streaming compression" +file_size=$($ZSTD -14 -f tmp -o tmp.zst && wc -c < tmp.zst) +stream_size=$(cat tmp | $ZSTD -14 --stream-size=11000 | wc -c) +if [ "$stream_size" -gt "$file_size" ]; then + die "hinted compression larger than expected" +fi +println "test : sized streaming compression and decompression" +cat tmp | $ZSTD -14 -f tmp -o --stream-size=11000 tmp.zst +$ZSTD -df tmp.zst -o tmp_decompress +cmp tmp tmp_decompress || die "difference between original and decompressed file" +println "test : incorrect stream size" +cat tmp | $ZSTD -14 -f -o tmp.zst --stream-size=11001 && die "should fail with incorrect stream size" + + println "\n===> dictionary tests " println "- test with raw dict (content only) " @@ -1019,28 +1036,4 @@ test -f dictionary rm -f tmp* dictionary -println "\n===> stream-size mode" - -./datagen -g11000 > tmp -println "test : basic file compression vs sized streaming compression" -$ZSTD -14 -f tmp -o tmp.zst 2>&1 | tee file.out -cat tmp | $ZSTD -14 -f -o tmp.zst --stream-size=11000 2>&1 | tee stream_sized.out - -file_ratio=$(cat file.out | awk '{print $4}' | sed 's/%//g') -stream_sized_ratio=$(cat stream_sized.out | awk '{print $4}' | sed 's/%//g') -rm file.out stream_sized.out - -ratio_diff=$(echo $file_ratio - $stream_sized_ratio | bc) -if [ $(echo "(100 * $ratio_diff) > 5" | bc -l) == 1 ] -then - die "greater than 0.05% difference between file and sized-streaming compression" -fi -println "test : sized streaming compression and decompression" -cat tmp | $ZSTD -14 -f tmp -o --stream-size=11000 tmp.zst -$ZSTD -df tmp.zst -o tmp_decompress -cmp tmp tmp_decompress || die "difference between original and decompressed file" -println "test : incorrect stream size" -cat tmp | $ZSTD -14 -f -o tmp.zst --stream-size=11001 && die "should fail with incorrect stream size" - - rm -f tmp* From 2cdda8b3c43c23981941cdd97c1a517981d2339e Mon Sep 17 00:00:00 2001 From: Nick Magerko Date: Thu, 22 Aug 2019 09:13:28 -0700 Subject: [PATCH 09/10] Minor documentation update --- programs/zstd.1.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/programs/zstd.1.md b/programs/zstd.1.md index e3f729282..ee5db9d63 100644 --- a/programs/zstd.1.md +++ b/programs/zstd.1.md @@ -144,6 +144,11 @@ the last one takes effect. Due to the chaotic nature of dynamic adaptation, compressed result is not reproducible. _note_ : at the time of this writing, `--adapt` can remain stuck at low speed when combined with multiple worker threads (>=2). +* `--stream-size=#` : + When handling input from a stream, `zstd` must guess how large the source size + will be when optimizing compression parameters. This option sets the pledged source + size of a stream to eliminate that guesswork. Note that the pledged size must be exact; + incorrect stream sizes will cause an error. * `--rsyncable` : `zstd` will periodically synchronize the compression state to make the compressed file more rsync-friendly. There is a negligible impact to @@ -152,12 +157,6 @@ the last one takes effect. This feature does not work with `--single-thread`. You probably don't want to use it with long range mode, since it will decrease the effectiveness of the synchronization points, but your milage may vary. -* `--stream-size` : - When handling input from a stream, `zstd` must guess how large the source size - will be when optimizing compression parameters. If the stream size is relatively - small, this guess may be a poor one, resulting in a higher compression ratio than - expected. This feature will set the source size of a stream. Note that it must - be exact; incorrect stream sizes will cause an error. * `-D file`: use `file` as Dictionary to compress or decompress FILE(s) * `--no-dictID`: From fd486a846abcbf6c92496dd4915d6a46bd4790c2 Mon Sep 17 00:00:00 2001 From: Nick Magerko Date: Thu, 22 Aug 2019 09:37:47 -0700 Subject: [PATCH 10/10] Differentiate --stream-size from --size-hint --- programs/zstd.1.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/programs/zstd.1.md b/programs/zstd.1.md index ee5db9d63..1bdc42654 100644 --- a/programs/zstd.1.md +++ b/programs/zstd.1.md @@ -145,10 +145,10 @@ the last one takes effect. _note_ : at the time of this writing, `--adapt` can remain stuck at low speed when combined with multiple worker threads (>=2). * `--stream-size=#` : - When handling input from a stream, `zstd` must guess how large the source size - will be when optimizing compression parameters. This option sets the pledged source - size of a stream to eliminate that guesswork. Note that the pledged size must be exact; - incorrect stream sizes will cause an error. + Sets the pledged source size of input coming from a stream. This value must be exact, as it + will be included in the produced frame header. Incorrect stream sizes will cause an error. + This information will be used to better optimize compression parameters, resulting in + better and potentially faster compression, especially for smaller source sizes. * `--rsyncable` : `zstd` will periodically synchronize the compression state to make the compressed file more rsync-friendly. There is a negligible impact to