From fe9239891867c5fd1ca25f06bdb39eddc6a04c45 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 31 Oct 2016 15:48:58 -0700 Subject: [PATCH 1/3] updated README, mentioning .zst suffix --- README.md | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 0ab1b667b..c1b2bda4f 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,8 @@ - **Zstd**, short for Zstandard, is a fast lossless compression algorithm, + __Zstandard__, or `zstd` as short version, is a fast lossless compression algorithm, targeting real-time compression scenarios at zlib-level and better compression ratios. -It is provided as an open-source BSD-licensed **C** library. +It is provided as an open-source BSD-licensed **C** library, +and a command line utility producing and decoding `.zst` compressed files. For other programming languages, you can consult a list of known ports on [Zstandard homepage](http://www.zstd.net/#other-languages). @@ -16,17 +17,17 @@ As a reference, several fast compression algorithms were tested and compared on [Silesia compression corpus]: http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia -|Name | Ratio | C.speed | D.speed | -|-----------------|-------|--------:|--------:| -| | | MB/s | MB/s | -|**zstd 0.8.2 -1**|**2.877**|**330**| **940** | -| [zlib] 1.2.8 -1 | 2.730 | 95 | 360 | -| brotli 0.4 -0 | 2.708 | 320 | 375 | -| QuickLZ 1.5 | 2.237 | 510 | 605 | -| LZO 2.09 | 2.106 | 610 | 870 | -| [LZ4] r131 | 2.101 | 620 | 3100 | -| Snappy 1.1.3 | 2.091 | 480 | 1600 | -| LZF 3.6 | 2.077 | 375 | 790 | +| Name | Ratio | C.speed | D.speed | +|-------------------------|-------|--------:|--------:| +| | | MB/s | MB/s | +|**zstd 0.8.2 -1** |**2.877**| **330** | **940** | +| [zlib] 1.2.8 deflate -1 | 2.730 | 95 | 360 | +| brotli 0.4 -0 | 2.708 | 320 | 375 | +| QuickLZ 1.5 | 2.237 | 510 | 605 | +| LZO 2.09 | 2.106 | 610 | 870 | +| [LZ4] r131 | 2.101 | 620 | 3100 | +| Snappy 1.1.3 | 2.091 | 480 | 1600 | +| LZF 3.6 | 2.077 | 375 | 790 | [zlib]:http://www.zlib.net/ [LZ4]: http://www.lz4.org/ From 3d197d4c8c66671fc8ef7468abc4d983b03fbdc3 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 31 Oct 2016 18:10:32 -0700 Subject: [PATCH 2/3] updated man page --- README.md | 2 +- programs/zstd.1 | 75 +++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 67 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index c1b2bda4f..694bfd53f 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ As a reference, several fast compression algorithms were tested and compared on | Name | Ratio | C.speed | D.speed | |-------------------------|-------|--------:|--------:| | | | MB/s | MB/s | -|**zstd 0.8.2 -1** |**2.877**| **330** | **940** | +| **zstd 0.8.2 -1** |**2.877**| **330** | **940** | | [zlib] 1.2.8 deflate -1 | 2.730 | 95 | 360 | | brotli 0.4 -0 | 2.708 | 320 | 375 | | QuickLZ 1.5 | 2.237 | 510 | 605 | diff --git a/programs/zstd.1 b/programs/zstd.1 index 812883717..9d91b5100 100644 --- a/programs/zstd.1 +++ b/programs/zstd.1 @@ -27,16 +27,70 @@ is equivalent to .SH DESCRIPTION .PP -\fBzstd\fR is a fast lossless compression algorithm. +\fBzstd\fR is a fast lossless compression algorithm +and data compression tool, +with command line syntax similar to \fB gzip (1) \fR and \fB xz (1) \fR . It is based on the \fBLZ77\fR family, with further FSE & huff0 entropy stages. -\fBzstd\fR offers configurable compression speed, with fast modes at > 200 MB/s per core. -It also features a very fast decoder, with speed > 500 MB/s per core. +\fBzstd\fR offers highly configurable compression speed, +with fast modes at > 200 MB/s per core, +and strong modes nearing lzma compression ratios. +It also features a very fast decoder, with speeds > 500 MB/s per core. -\fBzstd\fR command line is generally similar to gzip, but features the following differences : - - Source files are preserved by default - It's possible to remove them automatically by using \fB--rm\fR command +\fBzstd\fR command line syntax is generally similar to gzip, +but features the following differences : + - Source files are preserved by default. + It's possible to remove them automatically by using \fB--rm\fR command. - When compressing a single file, \fBzstd\fR displays progress notifications and result summary by default. - Use \fB-q\fR to turn them off + Use \fB-q\fR to turn them off + +.PP +.B zstd +compresses or decompresses each +.I file +according to the selected operation mode. +If no +.I files +are given or +.I file +is +.BR \- , +.B zstd +reads from standard input and writes the processed data +to standard output. +.B zstd +will refuse (display an error and skip the +.IR file ) +to write compressed data to standard output if it is a terminal. +Similarly, +.B zstd +will refuse to read compressed data +from standard input if it is a terminal. + +.PP +Unless +.B \-\-stdout +is specified, +.I files +are written to a new file whose name is derived from the source +.I file +name: +.IP \(bu 3 +When compressing, the suffix +.B .zst +is appended to the source filename to get the target filename. +.IP \(bu 3 +When decompressing, the +.B .zst +suffix is removed from the filename to get the target filename. + +.SS "Concatenation with .zst files" +It is possible to concatenate +.B .zst +files as is. +.B zstd +will decompress such files as if they were a single +.B .zst +file. @@ -112,7 +166,8 @@ No files are created or removed. .TP .BR \--no-dictID do not store dictionary ID within frame header (dictionary compression). - the decoder will have to rely on implicit knowledge about which dictionary to use, it won't be able to check if it's correct. + The decoder will have to rely on implicit knowledge about which dictionary to use, +it won't be able to check if it's correct. .TP .B \-o file save result into `file` (only possible with a single INPUT-FILE) @@ -125,6 +180,8 @@ No files are created or removed. .TP .BR \--[no-]sparse enable / disable sparse FS support, to make files with many zeroes smaller on disk. + Creating sparse files may save disk space and speed up the decompression +by reducing the amount of disk I/O. default : enabled when output is into a file, and disabled when output is stdout. This setting overrides default and can force sparse mode over stdout. .TP @@ -162,7 +219,7 @@ No files are created or removed. All arguments after -- are treated as files -.SH DICTIONARY +.SH DICTIONARY BUILDER .PP \fBzstd\fR offers \fIdictionary\fR compression, useful for very small files and messages. It's possible to train \fBzstd\fR with some samples, the result of which is saved into a file called `dictionary`. From 4c0b44f8ced84c4c8edfa07b564d31e4fa3e8885 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 1 Nov 2016 11:13:22 -0700 Subject: [PATCH 3/3] minor display improvement in fuzzer when nb of tests is very large --- NEWS | 7 ++++--- tests/fuzzer.c | 2 +- tests/zstreamtest.c | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/NEWS b/NEWS index 1b0c3d6c1..7710a07aa 100644 --- a/NEWS +++ b/NEWS @@ -2,9 +2,10 @@ v1.1.1 New : command -M#, --memory=, --memlimit=, --memlimit-decompress= to limit allowed memory consumption New : doc/zstd_manual.html, by Przemyslaw Skibinski Improved : slightly better compression ratio at --ultra levels (>= 20) -Added : ZSTD_initCStream_usingCDict(), ZSTD_initDStream_usingDDict() (experimental section) -Added : example/multiple_streaming_compression -Changed : zstd_errors.h is now part of include installation +Improved : better memory usage when using streaming compression API, thanks to @Rogier-5 report +Added : API : ZSTD_initCStream_usingCDict(), ZSTD_initDStream_usingDDict() (experimental section) +Added : example/multiple_streaming_compression.c +Changed : zstd_errors.h is now installed within /include (and replaces errors_public.h) Updated man page Fixed : zstd-small, zstd-compress and zstd-decompress compilation targets diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 22f034d13..f8110498d 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -523,7 +523,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD /* notification */ if (nbTests >= testNb) { DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); } - else { DISPLAYUPDATE(2, "\r%6u ", testNb); } + else { DISPLAYUPDATE(2, "\r%6u ", testNb); } FUZ_rand(&coreSeed); { U32 const prime1 = 2654435761U; lseed = coreSeed ^ prime1; } diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index 38b6a76cc..aa119e636 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -452,8 +452,8 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres size_t maxTestSize; /* init */ - DISPLAYUPDATE(2, "\r%6u", testNb); - if (nbTests >= testNb) DISPLAYUPDATE(2, "/%6u ", nbTests); + if (nbTests >= testNb) { DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); } + else { DISPLAYUPDATE(2, "\r%6u ", testNb); } FUZ_rand(&coreSeed); lseed = coreSeed ^ prime1;