ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1069 from facebook/statusLine

Browse Source 
restored simple status line during zstd compression
This commit is contained in:
Yann Collet
2018-03-22 18:37:16 -07:00
committed by GitHub
parent 29b021f9a0 184e1c8b8c
commit 6aee5417a2
6 changed files with 30 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+11 -10
View File
@@ -21,24 +21,25 @@ Development branch status : [![Build Status][travisDevBadge]][travisLink] [![B
### Benchmarks
For reference, several fast compression algorithms were tested and compared
on a server running Linux Debian (`Linux version 4.8.0-1-amd64`),
on a server running Linux Debian (`Linux version 4.14.0-3-amd64`),
with a Core i7-6700K CPU @ 4.0GHz,
using [lzbench], an open-source in-memory benchmark by @inikep
compiled with GCC 6.3.0,
compiled with [gcc] 7.3.0,
on the [Silesia compression corpus].
[lzbench]: https://github.com/inikep/lzbench
[Silesia compression corpus]: http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia
[gcc]: https://gcc.gnu.org/
| Compressor name | Ratio | Compression| Decompress.|
| --------------- | ------| -----------| ---------- |
| **zstd 1.1.3 -1** | 2.877 | 430 MB/s | 1110 MB/s |
| zlib 1.2.8 -1 | 2.743 | 110 MB/s | 400 MB/s |
| brotli 0.5.2 -0 | 2.708 | 400 MB/s | 430 MB/s |
| **zstd 1.3.4 -1** | 2.877 | 470 MB/s | 1380 MB/s |
| zlib 1.2.11 -1 | 2.743 | 110 MB/s | 400 MB/s |
| brotli 1.0.2 -0 | 2.701 | 410 MB/s | 430 MB/s |
| quicklz 1.5.0 -1 | 2.238 | 550 MB/s | 710 MB/s |
| lzo1x 2.09 -1 | 2.108 | 650 MB/s | 830 MB/s |
| lz4 1.7.5 | 2.101 | 720 MB/s | 3600 MB/s |
| snappy 1.1.3 | 2.091 | 500 MB/s | 1650 MB/s |
| lz4 1.8.1 | 2.101 | 750 MB/s | 3700 MB/s |
| snappy 1.1.4 | 2.091 | 530 MB/s | 1800 MB/s |
| lzf 3.6 -1 | 2.077 | 400 MB/s | 860 MB/s |
[zlib]:http://www.zlib.net/
@@ -50,15 +51,15 @@ Decompression speed is preserved and remains roughly the same at all settings,
a property shared by most LZ compression algorithms, such as [zlib] or lzma.
The following tests were run
on a server running Linux Debian (`Linux version 4.8.0-1-amd64`)
on a server running Linux Debian (`Linux version 4.14.0-3-amd64`)
with a Core i7-6700K CPU @ 4.0GHz,
using [lzbench], an open-source in-memory benchmark by @inikep
compiled with GCC 6.3.0,
compiled with [gcc] 7.3.0,
on the [Silesia compression corpus].
Compression Speed vs Ratio | Decompression Speed
---------------------------|--------------------
![Compression Speed vs Ratio](doc/images/Cspeed4.png "Compression Speed vs Ratio") | ![Decompression Speed](doc/images/Dspeed4.png "Decompression Speed")
![Compression Speed vs Ratio](doc/images/CSpeed2.png "Compression Speed vs Ratio") | ![Decompression Speed](doc/images/DSpeed3.png "Decompression Speed")
A few other algorithms can produce higher compression ratios at slower speeds, falling outside of the graph.
For a larger picture including slow modes, [click on this link](doc/images/DCspeed5.png).
doc/images/CSpeed2.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 72 KiB

doc/images/Cspeed4.png
BIN
View File
Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 70 KiB

doc/images/DSpeed3.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 26 KiB

doc/images/Dspeed4.png
BIN
View File
Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 24 KiB

programs/fileio.c
+19 -8
View File
@@ -88,10 +88,12 @@ static const U64 g_refreshRate = SEC_TO_MICRO / 6;
static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
#define READY_FOR_UPDATE() (UTIL_clockSpanMicro(g_displayClock) > g_refreshRate)
#define DELAY_NEXT_UPDATE() { g_displayClock = UTIL_getTime(); }
#define DISPLAYUPDATE(l, ...) { \
if (g_displayLevel>=l) { \
if (READY_FOR_UPDATE() || (g_displayLevel>=4)) { \
g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \
if (READY_FOR_UPDATE() || (g_displayLevel>=4)) { \
DELAY_NEXT_UPDATE(); \
DISPLAY(__VA_ARGS__); \
if (g_displayLevel>=4) fflush(stderr); \
} } }
@@ -401,9 +403,9 @@ static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName)
#ifndef ZSTD_NOCOMPRESS
/*-**********************************************************************
* Compression
************************************************************************/
/* **********************************************************************
* Compression
************************************************************************/
typedef struct {
FILE* srcFile;
FILE* dstFile;
@@ -782,12 +784,21 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
}
if (READY_FOR_UPDATE()) {
ZSTD_frameProgression const zfp = ZSTD_getFrameProgression(ress.cctx);
DISPLAYUPDATE(2, "\r(%i) Read :%6u MB - Consumed :%6u MB - Compressed :%6u MB => %.2f%%",
double const cShare = (double)zfp.produced / (zfp.consumed + !zfp.consumed/*avoid div0*/) * 100;
if (g_displayLevel >= 3) {
DISPLAYUPDATE(3, "\r(L%i) Buffered :%4u MB - Consumed :%4u MB - Compressed :%4u MB => %.2f%%",
compressionLevel,
(U32)(zfp.ingested >> 20),
(U32)((zfp.ingested - zfp.consumed) >> 20),
(U32)(zfp.consumed >> 20),
(U32)(zfp.produced >> 20),
(double)zfp.produced / (zfp.consumed + !zfp.consumed/*avoid div0*/) * 100 );
cShare );
} else { /* g_displayLevel == 2 */
DISPLAYLEVEL(2, "\rRead : %u ", (U32)(zfp.consumed >> 20));
if (fileSize != UTIL_FILESIZE_UNKNOWN)
DISPLAYLEVEL(2, "/ %u ", (U32)(fileSize >> 20));
DISPLAYLEVEL(2, "MB ==> %2.f%% ", cShare);
DELAY_NEXT_UPDATE();
}
}
}
} while (directive != ZSTD_e_end);