From 3b6ae77e1562d4f5ef9beeb76845b36b98f6592a Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 8 Jul 2016 23:42:22 +0200 Subject: [PATCH 01/42] comment clarification --- lib/common/zstd.h | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/lib/common/zstd.h b/lib/common/zstd.h index 304edd364..186628fb8 100644 --- a/lib/common/zstd.h +++ b/lib/common/zstd.h @@ -336,7 +336,7 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci Then, consume your input using ZSTD_compressContinue(). There are some important considerations to keep in mind when using this advanced function : - ZSTD_compressContinue() has no internal buffer. It uses externally provided buffer only. - - Interface is synchronous : input will be entirely consumed and produce 1+ compressed blocks. + - Interface is synchronous : input is consumed entirely and produce 1 (or more) compressed blocks. - Caller must ensure there is enough space in `dst` to store compressed data under worst case scenario. Worst case evaluation is provided by ZSTD_compressBound(). ZSTD_compressContinue() doesn't guarantee recover after a failed compression. @@ -392,15 +392,23 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ds Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively. ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue(). ZSTD_decompressContinue() requires this exact amount of bytes, or it will fail. - ZSTD_decompressContinue() needs previous data blocks during decompression, up to `windowSize`. - They should preferably be located contiguously, prior to current block. Alternatively, a round buffer is also possible. @result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity). It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header. + ZSTD_decompressContinue() needs previous data blocks during decompression, up to `windowSize`. + They should preferably be located contiguously, prior to current block. + Alternatively, a round buffer of sufficient size is also possible. Sufficient size is determined by frame parameters. + ZSTD_decompressContinue() is very sensitive to contiguity, + if 2 blocks don't follow each other, make sure that either the compressor breaks contiguity at the same place, + or that previous contiguous segment is large enough to properly handle maximum back-reference. + A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero. Context can then be reset to start a new decompression. + + == Special case : skippable frames == + Skippable frames allow the integration of user-defined data into a flow of concatenated frames. Skippable frames will be ignored (skipped) by a decompressor. The format of skippable frame is following: a) Skippable frame ID - 4 Bytes, Little endian format, any value from 0x184D2A50 to 0x184D2A5F From 0809a8863da6209676dde6539cea7d5d3aa2dfd9 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 9 Jul 2016 18:25:10 +0200 Subject: [PATCH 02/42] added simple examples --- examples/README.md | 9 ++- examples/dictionary_decompression.c | 49 +++++++++--- examples/simple_compression.c | 112 ++++++++++++++++++++++++++ examples/simple_decompression.c | 120 ++++++++++++++++++++++++++++ 4 files changed, 277 insertions(+), 13 deletions(-) create mode 100644 examples/simple_compression.c create mode 100644 examples/simple_decompression.c diff --git a/examples/README.md b/examples/README.md index a3d59315e..594e2eaf8 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,7 +1,14 @@ Zstandard library : usage examples ================================== +- [Simple compression](simple_compression.c) + Compress a single file. + Introduces usage of : `ZSTD_compress()` + +- [Simple decompression](simple_decompression.c) + Decompress a single file compressed by zstd. + Introduces usage of : `ZSTD_decompress()` + - [Dictionary decompression](dictionary_decompression.c) Decompress multiple files using the same dictionary. - Compatible with Legacy modes. Introduces usage of : `ZSTD_createDDict()` and `ZSTD_decompress_usingDDict()` diff --git a/examples/dictionary_decompression.c b/examples/dictionary_decompression.c index e307bea5b..c873fa7c3 100644 --- a/examples/dictionary_decompression.c +++ b/examples/dictionary_decompression.c @@ -1,12 +1,37 @@ -#include // exit +/* + Dictionary decompression + Educational program using zstd library + Copyright (C) Yann Collet 2016 + + GPL v2 License + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - zstd homepage : http://www.zstd.net/ +*/ + +#include // malloc, exit #include // printf #include // strerror #include // errno #include // stat -#include +#include // presumes zstd library is installed -static off_t fsizeX(const char *filename) +static off_t fsize_X(const char *filename) { struct stat st; if (stat(filename, &st) == 0) return st.st_size; @@ -15,7 +40,7 @@ static off_t fsizeX(const char *filename) exit(1); } -static FILE* fopenX(const char *filename, const char *instruction) +static FILE* fopen_X(const char *filename, const char *instruction) { FILE* const inFile = fopen(filename, instruction); if (inFile) return inFile; @@ -24,7 +49,7 @@ static FILE* fopenX(const char *filename, const char *instruction) exit(2); } -static void* mallocX(size_t size) +static void* malloc_X(size_t size) { void* const buff = malloc(size); if (buff) return buff; @@ -33,11 +58,11 @@ static void* mallocX(size_t size) exit(3); } -static void* loadFileX(const char* fileName, size_t* size) +static void* loadFile_X(const char* fileName, size_t* size) { - off_t const buffSize = fsizeX(fileName); - FILE* const inFile = fopenX(fileName, "rb"); - void* const buffer = mallocX(buffSize); + off_t const buffSize = fsize_X(fileName); + FILE* const inFile = fopen_X(fileName, "rb"); + void* const buffer = malloc_X(buffSize); size_t const readSize = fread(buffer, 1, buffSize, inFile); if (readSize != (size_t)buffSize) { printf("fread: %s : %s \n", fileName, strerror(errno)); @@ -52,7 +77,7 @@ static void* loadFileX(const char* fileName, size_t* size) static const ZSTD_DDict* createDict(const char* dictFileName) { size_t dictSize; - void* const dictBuffer = loadFileX(dictFileName, &dictSize); + void* const dictBuffer = loadFile_X(dictFileName, &dictSize); const ZSTD_DDict* const ddict = ZSTD_createDDict(dictBuffer, dictSize); free(dictBuffer); return ddict; @@ -65,13 +90,13 @@ unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); static void decompress(const char* fname, const ZSTD_DDict* ddict) { size_t cSize; - void* const cBuff = loadFileX(fname, &cSize); + void* const cBuff = loadFile_X(fname, &cSize); unsigned long long const rSize = ZSTD_getDecompressedSize(cBuff, cSize); if (rSize==0) { printf("%s : original size unknown \n", fname); exit(5); } - void* const rBuff = mallocX(rSize); + void* const rBuff = malloc_X(rSize); ZSTD_DCtx* const dctx = ZSTD_createDCtx(); size_t const dSize = ZSTD_decompress_usingDDict(dctx, rBuff, rSize, cBuff, cSize, ddict); diff --git a/examples/simple_compression.c b/examples/simple_compression.c new file mode 100644 index 000000000..1b6ef429e --- /dev/null +++ b/examples/simple_compression.c @@ -0,0 +1,112 @@ +/* + Simple compression + Educational program using zstd library + Copyright (C) Yann Collet 2016 + + GPL v2 License + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - zstd homepage : http://www.zstd.net/ +*/ + +#include // malloc, exit +#include // printf +#include // strerror +#include // errno +#include // stat +#include // presumes zstd library is installed + + +static off_t fsize_X(const char *filename) +{ + struct stat st; + if (stat(filename, &st) == 0) return st.st_size; + /* error */ + printf("stat: %s : %s \n", filename, strerror(errno)); + exit(1); +} + +static FILE* fopen_X(const char *filename, const char *instruction) +{ + FILE* const inFile = fopen(filename, instruction); + if (inFile) return inFile; + /* error */ + printf("fopen: %s : %s \n", filename, strerror(errno)); + exit(2); +} + +static void* malloc_X(size_t size) +{ + void* const buff = malloc(size); + if (buff) return buff; + /* error */ + printf("malloc: %s \n", strerror(errno)); + exit(3); +} + +static void* loadFile_X(const char* fileName, size_t* size) +{ + off_t const buffSize = fsize_X(fileName); + FILE* const inFile = fopen_X(fileName, "rb"); + void* const buffer = malloc_X(buffSize); + size_t const readSize = fread(buffer, 1, buffSize, inFile); + if (readSize != (size_t)buffSize) { + printf("fread: %s : %s \n", fileName, strerror(errno)); + exit(4); + } + fclose(inFile); + *size = buffSize; + return buffer; +} + + +static void compress(const char* fname) +{ + size_t fSize; + void* const fBuff = loadFile_X(fname, &fSize); + size_t const cBuffSize = ZSTD_compressBound(fSize); + void* const cBuff = malloc_X(cBuffSize); + + size_t const cSize = ZSTD_compress(cBuff, cBuffSize, fBuff, fSize, 1); + if (ZSTD_isError(cSize)) { + printf("error compressing %s : %s \n", fname, ZSTD_getErrorName(cSize)); + exit(7); + } + + /* success */ + printf("%25s : %6u -> %7u \n", fname, (unsigned)fSize, (unsigned)cSize); + + free(fBuff); + free(cBuff); +} + + +int main(int argc, const char** argv) +{ + const char* const exeName = argv[0]; + + if (argc!=2) { + printf("wrong arguments\n"); + printf("usage:\n"); + printf("%s FILE\n", exeName); + return 1; + } + + compress(argv[1]); + + printf("%s compressed. \n", argv[1]); +} diff --git a/examples/simple_decompression.c b/examples/simple_decompression.c new file mode 100644 index 000000000..da4515027 --- /dev/null +++ b/examples/simple_decompression.c @@ -0,0 +1,120 @@ +/* + Simple decompression + Educational program using zstd library + Copyright (C) Yann Collet 2016 + + GPL v2 License + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - zstd homepage : http://www.zstd.net/ +*/ + +#include // malloc, exit +#include // printf +#include // strerror +#include // errno +#include // stat +#include // presumes zstd library is installed + + +static off_t fsize_X(const char *filename) +{ + struct stat st; + if (stat(filename, &st) == 0) return st.st_size; + /* error */ + printf("stat: %s : %s \n", filename, strerror(errno)); + exit(1); +} + +static FILE* fopen_X(const char *filename, const char *instruction) +{ + FILE* const inFile = fopen(filename, instruction); + if (inFile) return inFile; + /* error */ + printf("fopen: %s : %s \n", filename, strerror(errno)); + exit(2); +} + +static void* malloc_X(size_t size) +{ + void* const buff = malloc(size); + if (buff) return buff; + /* error */ + printf("malloc: %s \n", strerror(errno)); + exit(3); +} + +static void* loadFile_X(const char* fileName, size_t* size) +{ + off_t const buffSize = fsize_X(fileName); + FILE* const inFile = fopen_X(fileName, "rb"); + void* const buffer = malloc_X(buffSize); + size_t const readSize = fread(buffer, 1, buffSize, inFile); + if (readSize != (size_t)buffSize) { + printf("fread: %s : %s \n", fileName, strerror(errno)); + exit(4); + } + fclose(inFile); + *size = buffSize; + return buffer; +} + + +/* prototype declared here, as it is currently part of experimental section */ +unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); + +static void decompress(const char* fname) +{ + size_t cSize; + void* const cBuff = loadFile_X(fname, &cSize); + unsigned long long const rSize = ZSTD_getDecompressedSize(cBuff, cSize); + if (rSize==0) { + printf("%s : original size unknown \n", fname); + exit(5); + } + void* const rBuff = malloc_X(rSize); + + size_t const dSize = ZSTD_decompress(rBuff, rSize, cBuff, cSize); + + if (dSize != rSize) { + printf("error decoding %s : %s \n", fname, ZSTD_getErrorName(dSize)); + exit(7); + } + + /* success */ + printf("%25s : %6u -> %7u \n", fname, (unsigned)cSize, (unsigned)rSize); + + free(rBuff); + free(cBuff); +} + + +int main(int argc, const char** argv) +{ + const char* const exeName = argv[0]; + + if (argc!=2) { + printf("wrong arguments\n"); + printf("usage:\n"); + printf("%s FILE\n", exeName); + return 1; + } + + decompress(argv[1]); + + printf("%s decoded. \n", argv[1]); +} From e66708daf7f42dd5cd4f1c54a38a912661e4e633 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 9 Jul 2016 22:56:12 +0200 Subject: [PATCH 03/42] updated doc with zstd homepage --- README.md | 7 +++++-- examples/dictionary_decompression.c | 3 ++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 7b58e5e72..120cc96f1 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,9 @@ - **Zstd**, short for Zstandard, is a fast lossless compression algorithm, targeting real-time compression scenarios at zlib-level and better compression ratios. + **Zstd**, short for Zstandard, is a fast lossless compression algorithm, + targeting real-time compression scenarios at zlib-level and better compression ratios. -It is provided as a BSD-license package, hosted on Github. +It is provided as an open-source C library BSD-licensed. +If you're looking for a different programming language, +you can consult a list of known ports on [Zstandard homepage](httP://www.zstd.net). |Branch |Status | |------------|---------| diff --git a/examples/dictionary_decompression.c b/examples/dictionary_decompression.c index c873fa7c3..39a8189fc 100644 --- a/examples/dictionary_decompression.c +++ b/examples/dictionary_decompression.c @@ -73,7 +73,8 @@ static void* loadFile_X(const char* fileName, size_t* size) return buffer; } - +/* createDict() : + `dictFileName` is supposed to have been created using `zstd --train` */ static const ZSTD_DDict* createDict(const char* dictFileName) { size_t dictSize; From 25c506601c47c19228814252350a8d306308f660 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 10 Jul 2016 01:45:34 +0200 Subject: [PATCH 04/42] promote ZSTD_getDecompressedSize() to stable API --- examples/dictionary_decompression.c | 3 --- examples/simple_decompression.c | 3 --- lib/common/zstd.h | 18 +++++++----------- 3 files changed, 7 insertions(+), 17 deletions(-) diff --git a/examples/dictionary_decompression.c b/examples/dictionary_decompression.c index 39a8189fc..797075dee 100644 --- a/examples/dictionary_decompression.c +++ b/examples/dictionary_decompression.c @@ -85,9 +85,6 @@ static const ZSTD_DDict* createDict(const char* dictFileName) } -/* prototype declared here, as it currently is part of experimental section */ -unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); - static void decompress(const char* fname, const ZSTD_DDict* ddict) { size_t cSize; diff --git a/examples/simple_decompression.c b/examples/simple_decompression.c index da4515027..1b58e552d 100644 --- a/examples/simple_decompression.c +++ b/examples/simple_decompression.c @@ -74,9 +74,6 @@ static void* loadFile_X(const char* fileName, size_t* size) } -/* prototype declared here, as it is currently part of experimental section */ -unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); - static void decompress(const char* fname) { size_t cSize; diff --git a/lib/common/zstd.h b/lib/common/zstd.h index 186628fb8..146219d29 100644 --- a/lib/common/zstd.h +++ b/lib/common/zstd.h @@ -85,9 +85,14 @@ ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel); +/** ZSTD_getDecompressedSize() : +* @return : decompressed size if known, 0 otherwise. + note : to know precise reason why result is `0`, follow up with ZSTD_getFrameParams() */ +unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); + /*! ZSTD_decompress() : - `compressedSize` : is the _exact_ size of the compressed blob, otherwise decompression will fail. - `dstCapacity` must be large enough, equal or larger than originalSize. + `compressedSize` : is the _exact_ size of compressed input, otherwise decompression will fail. + `dstCapacity` must be equal or larger than originalSize. @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), or an errorCode if it fails (which can be tested using ZSTD_isError()) */ ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity, @@ -298,15 +303,6 @@ ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx, /*--- Advanced Decompression functions ---*/ -/** ZSTD_getDecompressedSize() : -* compatible with legacy mode -* @return : decompressed size if known, 0 otherwise - note : 0 can mean any of the following : - - decompressed size is not provided within frame header - - frame header unknown / not supported - - frame header not completely provided (`srcSize` too small) */ -unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); - /*! ZSTD_createDCtx_advanced() : * Create a ZSTD decompression context using external alloc and free functions */ ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); From 677ed26aa70677380b2d27adc247f0534c1ab8d8 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 10 Jul 2016 14:23:30 +0200 Subject: [PATCH 05/42] Added examples/Makefile --- Makefile | 2 +- examples/.gitignore | 9 ++++++ examples/Makefile | 54 +++++++++++++++++++++++++++++++++ examples/simple_compression.c | 39 +++++++++++++++++++++--- examples/simple_decompression.c | 4 ++- 5 files changed, 102 insertions(+), 6 deletions(-) create mode 100644 examples/.gitignore create mode 100644 examples/Makefile diff --git a/Makefile b/Makefile index 12a401207..b11fc5774 100644 --- a/Makefile +++ b/Makefile @@ -170,7 +170,7 @@ bmi32test: clean CFLAGS="-O3 -mbmi -m32 -Werror" $(MAKE) -C $(PRGDIR) test staticAnalyze: clean - CPPFLAGS=-g scan-build --status-bugs -v $(MAKE) all + CPPFLAGS=-g scan-build --status-bugs -v $(MAKE) all endif diff --git a/examples/.gitignore b/examples/.gitignore new file mode 100644 index 000000000..5c9836d3a --- /dev/null +++ b/examples/.gitignore @@ -0,0 +1,9 @@ +#build +simple_compression +simple_decompression +dictionary_decompression + +#test artefact +tmp* +test* +*.zst diff --git a/examples/Makefile b/examples/Makefile new file mode 100644 index 000000000..b20d14a74 --- /dev/null +++ b/examples/Makefile @@ -0,0 +1,54 @@ +# ########################################################################## +# ZSTD educational examples - Makefile +# Copyright (C) Yann Collet 2016 +# +# GPL v2 License +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# You can contact the author at : +# - zstd homepage : http://www.zstd.net/ +# ########################################################################## + +# This Makefile presumes libzstd is installed, using `sudo make install` + +LDFLAGS+= -lzstd + +.PHONY: default all clean test + +default: all + +all: simple_compression simple_decompression dictionary_decompression + +simple_compression : simple_compression.c + $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ + +simple_decompression : simple_decompression.c + $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ + +dictionary_decompression : dictionary_decompression.c + $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ + +clean: + @rm -f core *.o tmp* result* *.zst \ + simple_compression simple_decompression dictionary_decompression + @echo Cleaning completed + +test: all + cp README.md tmp + ./simple_compression tmp + @echo starting simple_decompression + ./simple_decompression tmp.zst + @echo tests completed diff --git a/examples/simple_compression.c b/examples/simple_compression.c index 1b6ef429e..08c6c9f51 100644 --- a/examples/simple_compression.c +++ b/examples/simple_compression.c @@ -74,7 +74,23 @@ static void* loadFile_X(const char* fileName, size_t* size) } -static void compress(const char* fname) +static void saveFile_X(const char* fileName, const void* buff, size_t buffSize) +{ + FILE* const oFile = fopen_X(fileName, "wb"); + size_t const wSize = fwrite(buff, 1, buffSize, oFile); + if (wSize != (size_t)buffSize) { + printf("fwrite: %s : %s \n", fileName, strerror(errno)); + exit(5); + } + size_t const closeError = fclose(oFile); + if (closeError) { + printf("fclose: %s : %s \n", fileName, strerror(errno)); + exit(6); + } +} + + +static void compress(const char* fname, const char* oname) { size_t fSize; void* const fBuff = loadFile_X(fname, &fSize); @@ -87,17 +103,31 @@ static void compress(const char* fname) exit(7); } + saveFile_X(oname, cBuff, cSize); + /* success */ - printf("%25s : %6u -> %7u \n", fname, (unsigned)fSize, (unsigned)cSize); + printf("%25s : %6u -> %7u - %s \n", fname, (unsigned)fSize, (unsigned)cSize, oname); free(fBuff); free(cBuff); } +static const char* createOutFilename(const char* filename) +{ + size_t const inL = strlen(filename); + size_t const outL = inL + 5; + void* outSpace = malloc_X(outL); + memset(outSpace, 0, outL); + strcat(outSpace, filename); + strcat(outSpace, ".zst"); + return (const char*)outSpace; +} + int main(int argc, const char** argv) { const char* const exeName = argv[0]; + const char* const inFilename = argv[1]; if (argc!=2) { printf("wrong arguments\n"); @@ -106,7 +136,8 @@ int main(int argc, const char** argv) return 1; } - compress(argv[1]); + const char* const outFilename = createOutFilename(inFilename); + compress(inFilename, outFilename); - printf("%s compressed. \n", argv[1]); + return 0; } diff --git a/examples/simple_decompression.c b/examples/simple_decompression.c index 1b58e552d..b907afa19 100644 --- a/examples/simple_decompression.c +++ b/examples/simple_decompression.c @@ -113,5 +113,7 @@ int main(int argc, const char** argv) decompress(argv[1]); - printf("%s decoded. \n", argv[1]); + printf("%s correctly decoded (in memory). \n", argv[1]); + + return 0; } From 3ae543ce751afa493eb39f4c2958c521295e6ea5 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 11 Jul 2016 03:12:17 +0200 Subject: [PATCH 06/42] added ZSTD_estimateCCtxSize() --- lib/common/zstd.h | 7 ++++++- lib/compress/zstd_compress.c | 35 +++++++++++++++++++++++--------- lib/decompress/zstd_decompress.c | 4 ++-- programs/paramgrill.c | 10 ++++----- 4 files changed, 37 insertions(+), 19 deletions(-) diff --git a/lib/common/zstd.h b/lib/common/zstd.h index 146219d29..e03a457b1 100644 --- a/lib/common/zstd.h +++ b/lib/common/zstd.h @@ -61,7 +61,7 @@ extern "C" { ***************************************/ #define ZSTD_VERSION_MAJOR 0 #define ZSTD_VERSION_MINOR 7 -#define ZSTD_VERSION_RELEASE 3 +#define ZSTD_VERSION_RELEASE 4 #define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE #define ZSTD_QUOTE(str) #str @@ -262,6 +262,11 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v /*-************************************* * Advanced compression functions ***************************************/ +/*! ZSTD_estimateCCtxSize() : + * Gives the amount of memory allocated for a ZSTD_CCtx given a set of compression parameters. + * `frameContentSize` is an optional parameter, provide `0` if unknown */ +size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams, unsigned long long frameContentSize); + /*! ZSTD_createCCtx_advanced() : * Create a ZSTD compression context using external alloc and free functions */ ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem); diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index cd55c7ae7..70e0a7409 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -175,6 +175,11 @@ size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx) return 0; /* reserved as a potential error code in the future */ } +size_t ZSTD_sizeofCCtx(const ZSTD_CCtx* cctx) +{ + return sizeof(*cctx) + cctx->workSpaceSize; +} + const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) /* hidden interface */ { return &(ctx->seqStore); @@ -244,17 +249,27 @@ ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, u } -size_t ZSTD_sizeofCCtx(ZSTD_compressionParameters cParams) /* hidden interface, for paramagrill */ +size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams, unsigned long long frameContentSize) { - ZSTD_CCtx* const zc = ZSTD_createCCtx(); - ZSTD_parameters params; - memset(¶ms, 0, sizeof(params)); - params.cParams = cParams; - params.fParams.contentSizeFlag = 1; - ZSTD_compressBegin_advanced(zc, NULL, 0, params, 0); - { size_t const ccsize = sizeof(*zc) + zc->workSpaceSize; - ZSTD_freeCCtx(zc); - return ccsize; } + const size_t blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog); + const U32 divider = (cParams.searchLength==3) ? 3 : 4; + const size_t maxNbSeq = blockSize / divider; + const size_t tokenSpace = blockSize + 11*maxNbSeq; + + const size_t chainSize = (cParams.strategy == ZSTD_fast) ? 0 : (1 << cParams.chainLog); + const size_t hSize = ((size_t)1) << cParams.hashLog; + const U32 hashLog3 = (cParams.searchLength>3) ? 0 : + ( (!frameContentSize || frameContentSize >= 8192) ? ZSTD_HASHLOG3_MAX : + ((frameContentSize >= 2048) ? ZSTD_HASHLOG3_MIN + 1 : ZSTD_HASHLOG3_MIN) ); + const size_t h3Size = ((size_t)1) << hashLog3; + const size_t tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); + + size_t const optSpace = ((MaxML+1) + (MaxLL+1) + (MaxOff+1) + (1< dstCapacity) return ERROR(dstSize_tooSmall); memset(dst, byte, length); @@ -1001,7 +1001,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, decodedSize = ZSTD_copyRawBlock(op, oend-op, ip, cBlockSize); break; case bt_rle : - decodedSize = ZSTD_generateNxByte(op, oend-op, *ip, blockProperties.origSize); + decodedSize = ZSTD_generateNxBytes(op, oend-op, *ip, blockProperties.origSize); break; case bt_end : /* end of frame */ diff --git a/programs/paramgrill.c b/programs/paramgrill.c index 6cf4ccd89..3078748f2 100644 --- a/programs/paramgrill.c +++ b/programs/paramgrill.c @@ -48,7 +48,7 @@ #endif #include "mem.h" -#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters */ +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters, ZSTD_estimateCCtxSize */ #include "zstd.h" #include "datagen.h" #include "xxhash.h" @@ -274,6 +274,7 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr, const int startTime =BMK_GetMilliStart(); DISPLAY("\r%79s\r", ""); + memset(¶ms, 0, sizeof(params)); params.cParams = cParams; params.fParams.contentSizeFlag = 0; for (loopNb = 1; loopNb <= g_nbIterations; loopNb++) { @@ -407,8 +408,6 @@ static void BMK_printWinners(FILE* f, const winnerInfo_t* winners, size_t srcSiz BMK_printWinners2(stdout, winners, srcSize); } -size_t ZSTD_sizeofCCtx(ZSTD_compressionParameters params); /* hidden interface, declared here */ - static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters params, const void* srcBuffer, size_t srcSize, ZSTD_CCtx* ctx) @@ -442,8 +441,8 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para double W_DMemUsed_note = W_ratioNote * ( 40 + 9*cLevel) - log((double)W_DMemUsed); double O_DMemUsed_note = O_ratioNote * ( 40 + 9*cLevel) - log((double)O_DMemUsed); - size_t W_CMemUsed = (1 << params.windowLog) + ZSTD_sizeofCCtx(params); - size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + ZSTD_sizeofCCtx(winners[cLevel].params); + size_t W_CMemUsed = (1 << params.windowLog) + ZSTD_estimateCCtxSize(params, srcSize); + size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + ZSTD_estimateCCtxSize(winners[cLevel].params, srcSize); double W_CMemUsed_note = W_ratioNote * ( 50 + 13*cLevel) - log((double)W_CMemUsed); double O_CMemUsed_note = O_ratioNote * ( 50 + 13*cLevel) - log((double)O_CMemUsed); @@ -453,7 +452,6 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para double W_DSpeed_note = W_ratioNote * ( 20 + 2*cLevel) + log((double)testResult.dSpeed); double O_DSpeed_note = O_ratioNote * ( 20 + 2*cLevel) + log((double)winners[cLevel].result.dSpeed); - if (W_DMemUsed_note < O_DMemUsed_note) { /* uses too much Decompression memory for too little benefit */ if (W_ratio > O_ratio) From 8e0ee681b8b2dd0ff7b5f42f79e0902465c15eea Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 11 Jul 2016 13:09:52 +0200 Subject: [PATCH 07/42] added ZSTD_sizeofDCtx() --- lib/common/zstd.h | 8 ++++++++ lib/decompress/zstd_decompress.c | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/common/zstd.h b/lib/common/zstd.h index e03a457b1..adb011d7d 100644 --- a/lib/common/zstd.h +++ b/lib/common/zstd.h @@ -276,6 +276,10 @@ ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem); ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, ZSTD_parameters params, ZSTD_customMem customMem); +/*! ZSTD_sizeofCCtx() : + * Gives the amount of memory used by a given ZSTD_CCtx */ +size_t ZSTD_sizeofCCtx(const ZSTD_CCtx* cctx); + ZSTDLIB_API unsigned ZSTD_maxCLevel (void); /*! ZSTD_getParams() : @@ -312,6 +316,10 @@ ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx, * Create a ZSTD decompression context using external alloc and free functions */ ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); +/*! ZSTD_sizeofCCtx() : + * Gives the amount of memory used by a given ZSTD_CCtx */ +size_t ZSTD_sizeofDCtx(const ZSTD_DCtx* dctx); + /* **************************************************************** * Streaming functions (direct mode - synchronous and buffer-less) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 761d85203..50a263a78 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -135,7 +135,7 @@ struct ZSTD_DCtx_s BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; }; /* typedef'd to ZSTD_DCtx within "zstd_static.h" */ -size_t ZSTD_sizeofDCtx (void) { return sizeof(ZSTD_DCtx); } /* non published interface */ +size_t ZSTD_sizeofDCtx (const ZSTD_DCtx* dctx) { return sizeof(*dctx); } size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) { From d158c35e9fc81dd7cd204df77b66344b03aa8d4e Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 11 Jul 2016 13:46:25 +0200 Subject: [PATCH 08/42] added ZSTD_estimateDCtxSize() --- lib/common/zstd.h | 14 +++++++++----- lib/decompress/zstd_decompress.c | 2 ++ 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/lib/common/zstd.h b/lib/common/zstd.h index adb011d7d..0819bf20f 100644 --- a/lib/common/zstd.h +++ b/lib/common/zstd.h @@ -265,7 +265,7 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v /*! ZSTD_estimateCCtxSize() : * Gives the amount of memory allocated for a ZSTD_CCtx given a set of compression parameters. * `frameContentSize` is an optional parameter, provide `0` if unknown */ -size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams, unsigned long long frameContentSize); +ZSTDLIB_API size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams, unsigned long long frameContentSize); /*! ZSTD_createCCtx_advanced() : * Create a ZSTD compression context using external alloc and free functions */ @@ -278,7 +278,7 @@ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictS /*! ZSTD_sizeofCCtx() : * Gives the amount of memory used by a given ZSTD_CCtx */ -size_t ZSTD_sizeofCCtx(const ZSTD_CCtx* cctx); +ZSTDLIB_API size_t ZSTD_sizeofCCtx(const ZSTD_CCtx* cctx); ZSTDLIB_API unsigned ZSTD_maxCLevel (void); @@ -312,13 +312,17 @@ ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx, /*--- Advanced Decompression functions ---*/ +/*! ZSTD_estimateDCtxSize() : + * Gives the potential amount of memory allocated to create a ZSTD_DCtx */ +ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void); + /*! ZSTD_createDCtx_advanced() : * Create a ZSTD decompression context using external alloc and free functions */ ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); -/*! ZSTD_sizeofCCtx() : - * Gives the amount of memory used by a given ZSTD_CCtx */ -size_t ZSTD_sizeofDCtx(const ZSTD_DCtx* dctx); +/*! ZSTD_sizeofDCtx() : + * Gives the amount of memory used by a given ZSTD_DCtx */ +ZSTDLIB_API size_t ZSTD_sizeofDCtx(const ZSTD_DCtx* dctx); /* **************************************************************** diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 50a263a78..a48c9abdf 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -137,6 +137,8 @@ struct ZSTD_DCtx_s size_t ZSTD_sizeofDCtx (const ZSTD_DCtx* dctx) { return sizeof(*dctx); } +size_t ZSTD_estimateDCtxSize(void) { return sizeof(ZSTD_DCtx); } + size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) { dctx->expected = ZSTD_frameHeaderSize_min; From 45dc35628c9c2917079c58cd7f75e90b325f0cb1 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 12 Jul 2016 09:47:31 +0200 Subject: [PATCH 09/42] first version of doubleFast --- lib/common/zstd.h | 6 +- lib/compress/zstd_compress.c | 301 ++++++++++++++++++++++++++++++++++- programs/fuzzer.c | 2 +- 3 files changed, 298 insertions(+), 11 deletions(-) diff --git a/lib/common/zstd.h b/lib/common/zstd.h index 0819bf20f..f30e76cc3 100644 --- a/lib/common/zstd.h +++ b/lib/common/zstd.h @@ -230,7 +230,7 @@ static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable f /*--- Types ---*/ -typedef enum { ZSTD_fast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt } ZSTD_strategy; /*< from faster to stronger */ +typedef enum { ZSTD_fast, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt } ZSTD_strategy; /*< from faster to stronger */ typedef struct { unsigned windowLog; /*< largest match distance : larger == more compression, more memory needed during decompression */ @@ -325,9 +325,9 @@ ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); ZSTDLIB_API size_t ZSTD_sizeofDCtx(const ZSTD_DCtx* dctx); -/* **************************************************************** +/* ****************************************************************** * Streaming functions (direct mode - synchronous and buffer-less) -******************************************************************/ +********************************************************************/ ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 70e0a7409..133899ef9 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1083,6 +1083,11 @@ static const U64 prime7bytes = 58295818150454627ULL; static size_t ZSTD_hash7(U64 u, U32 h) { return (size_t)(((u << (64-56)) * prime7bytes) >> (64-h)) ; } static size_t ZSTD_hash7Ptr(const void* p, U32 h) { return ZSTD_hash7(MEM_readLE64(p), h); } +//static const U64 prime8bytes = 58295818150454627ULL; +static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL; +static size_t ZSTD_hash8(U64 u, U32 h) { return (size_t)(((u) * prime8bytes) >> (64-h)) ; } +static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(MEM_readLE64(p), h); } + static size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls) { switch(mls) @@ -1092,6 +1097,7 @@ static size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls) case 5: return ZSTD_hash5Ptr(p, hBits); case 6: return ZSTD_hash6Ptr(p, hBits); case 7: return ZSTD_hash7Ptr(p, hBits); + case 8: return ZSTD_hash8Ptr(p, hBits); } } @@ -1151,7 +1157,7 @@ void ZSTD_compressBlock_fast_generic(ZSTD_CCtx* cctx, hashTable[h] = current; /* update hash table */ if ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1))) { /* note : by construction, offset_1 <= current */ - mLength = ZSTD_count(ip+1+EQUAL_READ32, ip+1+EQUAL_READ32-offset_1, iend) + EQUAL_READ32; + mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4; ip++; ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, 0, mLength-MINMATCH); } else { @@ -1160,7 +1166,7 @@ void ZSTD_compressBlock_fast_generic(ZSTD_CCtx* cctx, ip += ((ip-anchor) >> g_searchStrength) + 1; continue; } - mLength = ZSTD_count(ip+EQUAL_READ32, match+EQUAL_READ32, iend) + EQUAL_READ32; + mLength = ZSTD_count(ip+4, match+4, iend) + 4; offset = (U32)(ip-match); while (((ip>anchor) & (match>lowest)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ offset_2 = offset_1; @@ -1182,7 +1188,7 @@ void ZSTD_compressBlock_fast_generic(ZSTD_CCtx* cctx, && ( (offset_2>0) & (MEM_read32(ip) == MEM_read32(ip - offset_2)) )) { /* store sequence */ - size_t const rLength = ZSTD_count(ip+EQUAL_READ32, ip+EQUAL_READ32-offset_2, iend) + EQUAL_READ32; + size_t const rLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4; { U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; } /* swap offset_2 <=> offset_1 */ hashTable[ZSTD_hashPtr(ip, hBits, mls)] = (U32)(ip-base); ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, rLength-MINMATCH); @@ -1336,6 +1342,283 @@ static void ZSTD_compressBlock_fast_extDict(ZSTD_CCtx* ctx, } +/*-************************************* +* Double Fast +***************************************/ +static void ZSTD_fillDoubleHashTable (ZSTD_CCtx* cctx, const void* end, const U32 mls) +{ + U32* const hashLarge = cctx->hashTable; + const U32 hBitsL = cctx->params.cParams.hashLog; + U32* const hashSmall = cctx->chainTable; + const U32 hBitsS = cctx->params.cParams.chainLog; + const BYTE* const base = cctx->base; + const BYTE* ip = base + cctx->nextToUpdate; + const BYTE* const iend = ((const BYTE*)end) - 8; + const size_t fastHashFillStep = 3; + + while(ip <= iend) { + hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip - base); + hashLarge[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip - base); + ip += fastHashFillStep; + } +} + + +FORCE_INLINE +void ZSTD_compressBlock_doubleFast_generic(ZSTD_CCtx* cctx, + const void* src, size_t srcSize, + const U32 mls) +{ + U32* const hashLong = cctx->hashTable; + const U32 hBitsL = cctx->params.cParams.hashLog; + U32* const hashSmall = cctx->chainTable; + const U32 hBitsS = cctx->params.cParams.chainLog; + seqStore_t* seqStorePtr = &(cctx->seqStore); + const BYTE* const base = cctx->base; + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const U32 lowestIndex = cctx->dictLimit; + const BYTE* const lowest = base + lowestIndex; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - 8; + U32 offset_1=cctx->rep[0], offset_2=cctx->rep[1]; + U32 offsetSaved = 0; + + /* init */ + ip += (ip==lowest); + { U32 const maxRep = (U32)(ip-lowest); + if (offset_2 > maxRep) offsetSaved = offset_2, offset_2 = 0; + if (offset_1 > maxRep) offsetSaved = offset_1, offset_1 = 0; + } + + /* Main Search Loop */ + while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */ + size_t mLength; + size_t const h2 = ZSTD_hashPtr(ip, hBitsL, 8); + size_t const h = ZSTD_hashPtr(ip, hBitsS, mls); + U32 const current = (U32)(ip-base); + U32 const matchIndexL = hashLong[h2]; + U32 const matchIndexS = hashSmall[h]; + const BYTE* matchLong = base + matchIndexL; + const BYTE* match = base + matchIndexS; + hashLong[h2] = hashSmall[h] = current; /* update hash tables */ + + if ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1))) { /* note : by construction, offset_1 <= current */ + mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4; + ip++; + ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, 0, mLength-MINMATCH); + } else { + size_t offset; + if ( (matchIndexL > lowestIndex) && (MEM_read64(matchLong) == MEM_read64(ip)) ) { + mLength = ZSTD_count(ip+8, matchLong+8, iend) + 8; + offset = ip-matchLong; + while (((ip>anchor) & (matchLong>lowest)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */ + } else if ( (matchIndexS > lowestIndex) && (MEM_read32(match) == MEM_read32(ip)) ) { + mLength = ZSTD_count(ip+4, match+4, iend) + 4; + offset = ip-match; + while (((ip>anchor) & (match>lowest)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ + } else { + ip += ((ip-anchor) >> g_searchStrength) + 1; + continue; + } + + offset_2 = offset_1; + offset_1 = offset; + + ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + } + + /* match found */ + ip += mLength; + anchor = ip; + + if (ip <= ilimit) { + /* Fill Table */ + hashLong[ZSTD_hashPtr(base+current+2, hBitsL, 8)] = + hashSmall[ZSTD_hashPtr(base+current+2, hBitsS, mls)] = current+2; /* here because current+2 could be > iend-8 */ + hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = + hashSmall[ZSTD_hashPtr(ip-2, hBitsS, mls)] = (U32)(ip-2-base); + + /* check immediate repcode */ + while ( (ip <= ilimit) + && ( (offset_2>0) + & (MEM_read32(ip) == MEM_read32(ip - offset_2)) )) { + /* store sequence */ + size_t const rLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4; + { size_t const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; } /* swap offset_2 <=> offset_1 */ + hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip-base); + hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip-base); + ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, rLength-MINMATCH); + ip += rLength; + anchor = ip; + continue; /* faster when present ... (?) */ + } } } + + /* save reps for next block */ + cctx->savedRep[0] = offset_1 ? offset_1 : offsetSaved; + cctx->savedRep[1] = offset_2 ? offset_2 : offsetSaved; + + /* Last Literals */ + { size_t const lastLLSize = iend - anchor; + memcpy(seqStorePtr->lit, anchor, lastLLSize); + seqStorePtr->lit += lastLLSize; + } +} + + +static void ZSTD_compressBlock_doubleFast(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +{ + const U32 mls = ctx->params.cParams.searchLength; + switch(mls) + { + default: + case 4 : + ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 4); return; + case 5 : + ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 5); return; + case 6 : + ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 6); return; + case 7 : + ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 7); return; + } +} + + +static void ZSTD_compressBlock_doubleFast_extDict_generic(ZSTD_CCtx* ctx, + const void* src, size_t srcSize, + const U32 mls) +{ + U32* const hashLong = ctx->hashTable; + const U32 hBitsL = ctx->params.cParams.hashLog; + U32* const hashSmall = ctx->chainTable; + const U32 hBitsS = ctx->params.cParams.chainLog; + seqStore_t* seqStorePtr = &(ctx->seqStore); + const BYTE* const base = ctx->base; + const BYTE* const dictBase = ctx->dictBase; + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const U32 lowestIndex = ctx->lowLimit; + const BYTE* const dictStart = dictBase + lowestIndex; + const U32 dictLimit = ctx->dictLimit; + const BYTE* const lowPrefixPtr = base + dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - 8; + U32 offset_1=ctx->rep[0], offset_2=ctx->rep[1]; + + /* Search Loop */ + while (ip < ilimit) { /* < instead of <=, because (ip+1) */ + const size_t hSmall = ZSTD_hashPtr(ip, hBitsS, mls); + const U32 matchIndex = hashSmall[hSmall]; + const BYTE* matchBase = matchIndex < dictLimit ? dictBase : base; + const BYTE* match = matchBase + matchIndex; + + const size_t hLong = ZSTD_hashPtr(ip, hBitsL, 8); + const U32 matchLongIndex = hashLong[hLong]; + const BYTE* matchLongBase = matchLongIndex < dictLimit ? dictBase : base; + const BYTE* matchLong = matchLongBase + matchLongIndex; + + const U32 current = (U32)(ip-base); + const U32 repIndex = current + 1 - offset_1; /* offset_1 expected <= current +1 */ + const BYTE* repBase = repIndex < dictLimit ? dictBase : base; + const BYTE* repMatch = repBase + repIndex; + size_t mLength; + hashSmall[hSmall] = hashLong[hLong] = current; /* update hash table */ + + if ( (((U32)((dictLimit-1) - repIndex) >= 3) /* intentional underflow */ & (repIndex > lowestIndex)) + && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { + const BYTE* repMatchEnd = repIndex < dictLimit ? dictEnd : iend; + mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, lowPrefixPtr) + 4; + ip++; + ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, 0, mLength-MINMATCH); + } else { + if ((matchLongIndex > lowestIndex) && (MEM_read64(matchLong) == MEM_read64(ip))) { + const BYTE* matchEnd = matchLongIndex < dictLimit ? dictEnd : iend; + const BYTE* lowMatchPtr = matchLongIndex < dictLimit ? dictStart : lowPrefixPtr; + U32 offset; + mLength = ZSTD_count_2segments(ip+8, matchLong+8, iend, matchEnd, lowPrefixPtr) + 8; + offset = current - matchLongIndex; + while (((ip>anchor) & (matchLong>lowMatchPtr)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */ + offset_2 = offset_1; + offset_1 = offset; + ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + } else if ((matchIndex > lowestIndex) && (MEM_read32(match) != MEM_read32(ip))) { + const BYTE* matchEnd = matchIndex < dictLimit ? dictEnd : iend; + const BYTE* lowMatchPtr = matchIndex < dictLimit ? dictStart : lowPrefixPtr; + U32 offset; + mLength = ZSTD_count_2segments(ip+4, match+4, iend, matchEnd, lowPrefixPtr) + 4; + while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ + offset = current - matchIndex; + offset_2 = offset_1; + offset_1 = offset; + ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + } else { + ip += ((ip-anchor) >> g_searchStrength) + 1; + continue; + } } + + /* found a match : store it */ + ip += mLength; + anchor = ip; + + if (ip <= ilimit) { + /* Fill Table */ + hashSmall[ZSTD_hashPtr(base+current+2, hBitsS, mls)] = current+2; + hashLong[ZSTD_hashPtr(base+current+2, hBitsL, 8)] = current+2; + hashSmall[ZSTD_hashPtr(ip-2, hBitsS, mls)] = (U32)(ip-2-base); + hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base); + /* check immediate repcode */ + while (ip <= ilimit) { + U32 const current2 = (U32)(ip-base); + U32 const repIndex2 = current2 - offset_2; + const BYTE* repMatch2 = repIndex2 < dictLimit ? dictBase + repIndex2 : base + repIndex2; + if ( (((U32)((dictLimit-1) - repIndex2) >= 3) & (repIndex2 > lowestIndex)) /* intentional overflow */ + && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { + const BYTE* const repEnd2 = repIndex2 < dictLimit ? dictEnd : iend; + size_t const repLength2 = ZSTD_count_2segments(ip+EQUAL_READ32, repMatch2+EQUAL_READ32, iend, repEnd2, lowPrefixPtr) + EQUAL_READ32; + U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ + ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, repLength2-MINMATCH); + hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2; + hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2; + ip += repLength2; + anchor = ip; + continue; + } + break; + } } } + + /* save reps for next block */ + ctx->savedRep[0] = offset_1; ctx->savedRep[1] = offset_2; + + /* Last Literals */ + { size_t const lastLLSize = iend - anchor; + memcpy(seqStorePtr->lit, anchor, lastLLSize); + seqStorePtr->lit += lastLLSize; + } +} + + +static void ZSTD_compressBlock_doubleFast_extDict(ZSTD_CCtx* ctx, + const void* src, size_t srcSize) +{ + const U32 mls = ctx->params.cParams.searchLength; + switch(mls) + { + default: + case 4 : + ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 4); return; + case 5 : + ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 5); return; + case 6 : + ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 6); return; + case 7 : + ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 7); return; + } +} + + /*-************************************* * Binary Tree search ***************************************/ @@ -2095,9 +2378,9 @@ typedef void (*ZSTD_blockCompressor) (ZSTD_CCtx* ctx, const void* src, size_t sr static ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int extDict) { - static const ZSTD_blockCompressor blockCompressor[2][6] = { - { ZSTD_compressBlock_fast, ZSTD_compressBlock_greedy, ZSTD_compressBlock_lazy, ZSTD_compressBlock_lazy2, ZSTD_compressBlock_btlazy2, ZSTD_compressBlock_btopt }, - { ZSTD_compressBlock_fast_extDict, ZSTD_compressBlock_greedy_extDict, ZSTD_compressBlock_lazy_extDict,ZSTD_compressBlock_lazy2_extDict, ZSTD_compressBlock_btlazy2_extDict, ZSTD_compressBlock_btopt_extDict } + static const ZSTD_blockCompressor blockCompressor[2][7] = { + { ZSTD_compressBlock_fast, ZSTD_compressBlock_doubleFast, ZSTD_compressBlock_greedy, ZSTD_compressBlock_lazy, ZSTD_compressBlock_lazy2, ZSTD_compressBlock_btlazy2, ZSTD_compressBlock_btopt }, + { ZSTD_compressBlock_fast_extDict, ZSTD_compressBlock_doubleFast_extDict, ZSTD_compressBlock_greedy_extDict, ZSTD_compressBlock_lazy_extDict,ZSTD_compressBlock_lazy2_extDict, ZSTD_compressBlock_btlazy2_extDict, ZSTD_compressBlock_btopt_extDict } }; return blockCompressor[extDict][(U32)strat]; @@ -2308,6 +2591,10 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t ZSTD_fillHashTable (zc, iend, zc->params.cParams.searchLength); break; + case ZSTD_dfast: + ZSTD_fillDoubleHashTable (zc, iend, zc->params.cParams.searchLength); + break; + case ZSTD_greedy: case ZSTD_lazy: case ZSTD_lazy2: @@ -2655,7 +2942,7 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV { 0, 0, 0, 0, 0, 0, ZSTD_fast }, /* level 0 - never used */ { 19, 13, 14, 1, 7, 4, ZSTD_fast }, /* level 1 */ { 19, 15, 16, 1, 6, 4, ZSTD_fast }, /* level 2 */ - { 20, 18, 20, 1, 6, 4, ZSTD_fast }, /* level 3 */ + { 20, 16, 17, 1, 6, 4, ZSTD_dfast }, /* level 3 */ { 20, 13, 17, 2, 5, 4, ZSTD_greedy }, /* level 4.*/ { 20, 15, 18, 3, 5, 4, ZSTD_greedy }, /* level 5 */ { 21, 16, 19, 2, 5, 4, ZSTD_lazy }, /* level 6 */ diff --git a/programs/fuzzer.c b/programs/fuzzer.c index f4ffdad04..e56d6e1d6 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -711,7 +711,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD while (totalCSize < cSize) { size_t const inSize = ZSTD_nextSrcSizeToDecompress(dctx); size_t const genSize = ZSTD_decompressContinue(dctx, dstBuffer+totalGenSize, dstBufferSize-totalGenSize, cBuffer+totalCSize, inSize); - CHECK (ZSTD_isError(genSize), "streaming decompression error : %s", ZSTD_getErrorName(genSize)); + CHECK (ZSTD_isError(genSize), "ZSTD_decompressContinue error : %s", ZSTD_getErrorName(genSize)); totalGenSize += genSize; totalCSize += inSize; } From fc0eafbe84bd7df94a446c0a9cc852237b355990 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 12 Jul 2016 09:54:42 +0200 Subject: [PATCH 10/42] minor readme mod --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 120cc96f1..b87e35381 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ **Zstd**, short for Zstandard, is a fast lossless compression algorithm, targeting real-time compression scenarios at zlib-level and better compression ratios. -It is provided as an open-source C library BSD-licensed. -If you're looking for a different programming language, -you can consult a list of known ports on [Zstandard homepage](httP://www.zstd.net). +It is provided as an open-source BSD-licensed **C** library. +For other programming languages, +you can consult a list of known ports on [Zstandard homepage](http://www.zstd.net/#other-languages). |Branch |Status | |------------|---------| From 73d74a05b9cc1f2e8655d48da8d6c79cd57efa42 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 12 Jul 2016 13:03:48 +0200 Subject: [PATCH 11/42] fixed dfast strategy --- lib/compress/zstd_compress.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 133899ef9..70f17b537 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1544,7 +1544,7 @@ static void ZSTD_compressBlock_doubleFast_extDict_generic(ZSTD_CCtx* ctx, offset_2 = offset_1; offset_1 = offset; ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); - } else if ((matchIndex > lowestIndex) && (MEM_read32(match) != MEM_read32(ip))) { + } else if ((matchIndex > lowestIndex) && (MEM_read32(match) == MEM_read32(ip))) { const BYTE* matchEnd = matchIndex < dictLimit ? dictEnd : iend; const BYTE* lowMatchPtr = matchIndex < dictLimit ? dictStart : lowPrefixPtr; U32 offset; @@ -2410,7 +2410,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, BYTE* op = ostart; const U32 maxDist = 1 << cctx->params.cParams.windowLog; ZSTD_stats_t* stats = &cctx->seqStore.stats; - ZSTD_statsInit(stats); + ZSTD_statsInit(stats); /* debug only */ if (cctx->params.fParams.checksumFlag) XXH64_update(&cctx->xxhState, src, srcSize); From a43a854cdbb08dd72dee12785023a6c5e3a459f9 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 12 Jul 2016 13:42:10 +0200 Subject: [PATCH 12/42] updated paramgrill --- lib/compress/zstd_compress.c | 10 ++-- programs/paramgrill.c | 96 +++++++++++++++++------------------- 2 files changed, 49 insertions(+), 57 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 70f17b537..08157c1e1 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2939,7 +2939,7 @@ unsigned ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; } static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEVEL+1] = { { /* "default" */ /* W, C, H, S, L, TL, strat */ - { 0, 0, 0, 0, 0, 0, ZSTD_fast }, /* level 0 - never used */ + { 18, 12, 12, 1, 7, 4, ZSTD_fast }, /* level 0 - not used */ { 19, 13, 14, 1, 7, 4, ZSTD_fast }, /* level 1 */ { 19, 15, 16, 1, 6, 4, ZSTD_fast }, /* level 2 */ { 20, 16, 17, 1, 6, 4, ZSTD_dfast }, /* level 3 */ @@ -2965,7 +2965,7 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV }, { /* for srcSize <= 256 KB */ /* W, C, H, S, L, T, strat */ - { 0, 0, 0, 0, 0, 0, ZSTD_fast }, /* level 0 */ + { 18, 12, 12, 1, 7, 4, ZSTD_fast }, /* level 0 - not used */ { 18, 13, 14, 1, 6, 4, ZSTD_fast }, /* level 1 */ { 18, 15, 17, 1, 5, 4, ZSTD_fast }, /* level 2 */ { 18, 13, 15, 1, 5, 4, ZSTD_greedy }, /* level 3.*/ @@ -2991,7 +2991,7 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV }, { /* for srcSize <= 128 KB */ /* W, C, H, S, L, T, strat */ - { 0, 0, 0, 0, 0, 0, ZSTD_fast }, /* level 0 - never used */ + { 17, 12, 12, 1, 7, 4, ZSTD_fast }, /* level 0 - not used */ { 17, 12, 13, 1, 6, 4, ZSTD_fast }, /* level 1 */ { 17, 13, 16, 1, 5, 4, ZSTD_fast }, /* level 2 */ { 17, 13, 14, 2, 5, 4, ZSTD_greedy }, /* level 3 */ @@ -3017,7 +3017,7 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV }, { /* for srcSize <= 16 KB */ /* W, C, H, S, L, T, strat */ - { 0, 0, 0, 0, 0, 0, ZSTD_fast }, /* level 0 -- never used */ + { 14, 12, 12, 1, 7, 4, ZSTD_fast }, /* level 0 - not used */ { 14, 14, 14, 1, 4, 4, ZSTD_fast }, /* level 1 */ { 14, 14, 15, 1, 4, 4, ZSTD_fast }, /* level 2 */ { 14, 14, 14, 4, 4, 4, ZSTD_greedy }, /* level 3.*/ @@ -3065,7 +3065,7 @@ ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long l } /*! ZSTD_getParams() : -* same as ZSTD_getCParams(), but @return a `ZSTD_parameters` object instead of a `ZSTD_compressionParameters`. +* same as ZSTD_getCParams(), but @return a `ZSTD_parameters` object (instead of `ZSTD_compressionParameters`). * All fields of `ZSTD_frameParameters` are set to default (0) */ ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSize, size_t dictSize) { ZSTD_parameters params; diff --git a/programs/paramgrill.c b/programs/paramgrill.c index 3078748f2..ffa0b3e0f 100644 --- a/programs/paramgrill.c +++ b/programs/paramgrill.c @@ -363,6 +363,7 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr, const char* g_stratName[] = { "ZSTD_fast ", + "ZSTD_dfast ", "ZSTD_greedy ", "ZSTD_lazy ", "ZSTD_lazy2 ", @@ -635,13 +636,12 @@ static void BMK_selectRandomStart( static void BMK_benchMem(void* srcBuffer, size_t srcSize) { - ZSTD_CCtx* ctx = ZSTD_createCCtx(); + ZSTD_CCtx* const ctx = ZSTD_createCCtx(); ZSTD_compressionParameters params; winnerInfo_t winners[NB_LEVELS_TRACKED]; - int i; unsigned u; const char* rfName = "grillResults.txt"; - FILE* f; + FILE* const f = fopen(rfName, "w"); const size_t blockSize = g_blockSize ? g_blockSize : srcSize; if (g_singleRun) { @@ -653,8 +653,8 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize) } /* init */ + if (ctx==NULL) { DISPLAY("ZSTD_createCCtx() failed \n"); exit(1); } memset(winners, 0, sizeof(winners)); - f = fopen(rfName, "w"); if (f==NULL) { DISPLAY("error opening %s \n", rfName); exit(1); } if (g_target) @@ -672,18 +672,16 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize) g_cSpeedTarget[u] = (g_cSpeedTarget[u-1] * 25) >> 5; /* populate initial solution */ - { - const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); - for (i=1; i<=maxSeeds; i++) { + { const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); + int i; + for (i=0; i<=maxSeeds; i++) { params = ZSTD_getCParams(i, blockSize, 0); BMK_seed(winners, params, srcBuffer, srcSize, ctx); - } - } + } } BMK_printWinners(f, winners, srcSize); /* start tests */ - { - const int milliStart = BMK_GetMilliStart(); + { const int milliStart = BMK_GetMilliStart(); do { BMK_selectRandomStart(f, winners, srcBuffer, srcSize, ctx); } while (BMK_GetMilliSpan(milliStart) < g_grillDuration); @@ -702,8 +700,8 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize) static int benchSample(void) { void* origBuff; - size_t benchedSize = sampleSize; - const char* name = "Sample 10MiB"; + size_t const benchedSize = sampleSize; + const char* const name = "Sample 10MiB"; /* Allocation */ origBuff = malloc(benchedSize); @@ -722,37 +720,31 @@ static int benchSample(void) } -int benchFiles(char** fileNamesTable, int nbFiles) +int benchFiles(const char** fileNamesTable, int nbFiles) { int fileIdx=0; /* Loop for each file */ while (fileIdx inFileSize) benchedSize = (size_t)inFileSize; if (benchedSize < inFileSize) DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20)); - - /* Alloc */ - origBuff = (char*) malloc((size_t)benchedSize); - if(!origBuff) { + origBuff = malloc(benchedSize); + if (origBuff==NULL) { DISPLAY("\nError: not enough memory!\n"); fclose(inFile); return 12; @@ -760,26 +752,28 @@ int benchFiles(char** fileNamesTable, int nbFiles) /* Fill input buffer */ DISPLAY("Loading %s... \r", inFileName); - readSize = fread(origBuff, 1, benchedSize, inFile); - fclose(inFile); - - if(readSize != benchedSize) { - DISPLAY("\nError: problem reading file '%s' !! \n", inFileName); - free(origBuff); - return 13; - } + { size_t const readSize = fread(origBuff, 1, benchedSize, inFile); + fclose(inFile); + if(readSize != benchedSize) { + DISPLAY("\nError: problem reading file '%s' !! \n", inFileName); + free(origBuff); + return 13; + } } /* bench */ DISPLAY("\r%79s\r", ""); DISPLAY("using %s : \n", inFileName); BMK_benchMem(origBuff, benchedSize); + + /* clean */ + free(origBuff); } return 0; } -int optimizeForSize(char* inFileName) +int optimizeForSize(const char* inFileName) { FILE* inFile; U64 inFileSize; @@ -824,8 +818,7 @@ int optimizeForSize(char* inFileName) DISPLAY("\r%79s\r", ""); DISPLAY("optimizing for %s : \n", inFileName); - { - ZSTD_CCtx* ctx = ZSTD_createCCtx(); + { ZSTD_CCtx* ctx = ZSTD_createCCtx(); ZSTD_compressionParameters params; winnerInfo_t winner; BMK_result_t candidate; @@ -837,8 +830,7 @@ int optimizeForSize(char* inFileName) winner.result.cSize = (size_t)(-1); /* find best solution from default params */ - { - const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); + { const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); for (i=1; i<=maxSeeds; i++) { params = ZSTD_getCParams(i, blockSize, 0); BMK_benchParam(&candidate, origBuff, benchedSize, ctx, params); @@ -853,8 +845,7 @@ int optimizeForSize(char* inFileName) BMK_printWinner(stdout, 99, winner.result, winner.params, benchedSize); /* start tests */ - { - const int milliStart = BMK_GetMilliStart(); + { const int milliStart = BMK_GetMilliStart(); do { params = winner.params; paramVariation(¶ms); @@ -889,7 +880,7 @@ int optimizeForSize(char* inFileName) } -static int usage(char* exename) +static int usage(const char* exename) { DISPLAY( "Usage :\n"); DISPLAY( " %s [arg] file\n", exename); @@ -902,27 +893,28 @@ static int usage(char* exename) static int usage_advanced(void) { DISPLAY( "\nAdvanced options :\n"); - DISPLAY( " -i# : iteration loops [1-9](default : %i)\n", NBLOOPS); - DISPLAY( " -B# : cut input into blocks of size # (default : single block)\n"); - DISPLAY( " -P# : generated sample compressibility (default : %.1f%%)\n", COMPRESSIBILITY_DEFAULT * 100); - DISPLAY( " -S : Single run\n"); + DISPLAY( " -T# : set level 1 speed objective \n"); + DISPLAY( " -B# : cut input into blocks of size # (default : single block) \n"); + DISPLAY( " -i# : iteration loops [1-9](default : %i) \n", NBLOOPS); + DISPLAY( " -S : Single run \n"); + DISPLAY( " -P# : generated sample compressibility (default : %.1f%%) \n", COMPRESSIBILITY_DEFAULT * 100); return 0; } -static int badusage(char* exename) +static int badusage(const char* exename) { DISPLAY("Wrong parameters\n"); usage(exename); return 1; } -int main(int argc, char** argv) +int main(int argc, const char** argv) { int i, filenamesStart=0, result; - char* exename=argv[0]; - char* input_filename=0; + const char* exename=argv[0]; + const char* input_filename=0; U32 optimizer = 0; U32 main_pause = 0; @@ -938,7 +930,7 @@ int main(int argc, char** argv) if (argc<1) { badusage(exename); return 1; } for(i=1; i Date: Tue, 12 Jul 2016 15:11:40 +0200 Subject: [PATCH 13/42] fixed conversion warning --- lib/compress/zstd_compress.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 08157c1e1..2f995169c 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1409,14 +1409,14 @@ void ZSTD_compressBlock_doubleFast_generic(ZSTD_CCtx* cctx, ip++; ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, 0, mLength-MINMATCH); } else { - size_t offset; + U32 offset; if ( (matchIndexL > lowestIndex) && (MEM_read64(matchLong) == MEM_read64(ip)) ) { mLength = ZSTD_count(ip+8, matchLong+8, iend) + 8; - offset = ip-matchLong; + offset = (U32)(ip-matchLong); while (((ip>anchor) & (matchLong>lowest)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */ } else if ( (matchIndexS > lowestIndex) && (MEM_read32(match) == MEM_read32(ip)) ) { mLength = ZSTD_count(ip+4, match+4, iend) + 4; - offset = ip-match; + offset = (U32)(ip-match); while (((ip>anchor) & (match>lowest)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ } else { ip += ((ip-anchor) >> g_searchStrength) + 1; @@ -1446,7 +1446,7 @@ void ZSTD_compressBlock_doubleFast_generic(ZSTD_CCtx* cctx, & (MEM_read32(ip) == MEM_read32(ip - offset_2)) )) { /* store sequence */ size_t const rLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4; - { size_t const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; } /* swap offset_2 <=> offset_1 */ + { U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; } /* swap offset_2 <=> offset_1 */ hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip-base); hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip-base); ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, rLength-MINMATCH); From 650a8778c1e677ce90b2d9b98904d6588209ec40 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 13 Jul 2016 11:49:05 +0200 Subject: [PATCH 14/42] minor filter improvement --- programs/paramgrill.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/programs/paramgrill.c b/programs/paramgrill.c index ffa0b3e0f..ad37205f4 100644 --- a/programs/paramgrill.c +++ b/programs/paramgrill.c @@ -506,6 +506,8 @@ static ZSTD_compressionParameters* sanitizeParams(ZSTD_compressionParameters par g_params = params; if (params.strategy == ZSTD_fast) g_params.chainLog = 0, g_params.searchLog = 0; + if (params.strategy == ZSTD_dfast) + g_params.searchLog = 0; if (params.strategy != ZSTD_btopt ) g_params.targetLength = 0; return &g_params; From 696c4d7ef599fdb03e66746a4836ee59fa57895d Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 13 Jul 2016 13:11:08 +0200 Subject: [PATCH 15/42] new paramgrill mode : `-O#` : find optimal parameters for a given sample and a given target speed --- programs/paramgrill.c | 72 +++++++++++++++++++++++-------------------- 1 file changed, 38 insertions(+), 34 deletions(-) diff --git a/programs/paramgrill.c b/programs/paramgrill.c index ad37205f4..f990e5fc1 100644 --- a/programs/paramgrill.c +++ b/programs/paramgrill.c @@ -775,30 +775,23 @@ int benchFiles(const char** fileNamesTable, int nbFiles) } -int optimizeForSize(const char* inFileName) +int optimizeForSize(const char* inFileName, U32 targetSpeed) { - FILE* inFile; - U64 inFileSize; - size_t benchedSize; - size_t readSize; - char* origBuff; + FILE* const inFile = fopen( inFileName, "rb" ); + U64 const inFileSize = UTIL_getFileSize(inFileName); + size_t benchedSize = BMK_findMaxMem(inFileSize*3) / 3; + void* origBuff; - /* Check file existence */ - inFile = fopen( inFileName, "rb" ); - if (inFile==NULL) { - DISPLAY( "Pb opening %s\n", inFileName); - return 11; - } + /* Init */ + if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; } /* Memory allocation & restrictions */ - inFileSize = UTIL_getFileSize(inFileName); - benchedSize = (size_t) BMK_findMaxMem(inFileSize*3) / 3; if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize; if (benchedSize < inFileSize) DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20)); /* Alloc */ - origBuff = (char*) malloc((size_t)benchedSize); + origBuff = malloc(benchedSize); if(!origBuff) { DISPLAY("\nError: not enough memory!\n"); fclose(inFile); @@ -807,37 +800,40 @@ int optimizeForSize(const char* inFileName) /* Fill input buffer */ DISPLAY("Loading %s... \r", inFileName); - readSize = fread(origBuff, 1, benchedSize, inFile); - fclose(inFile); - - if(readSize != benchedSize) { - DISPLAY("\nError: problem reading file '%s' !! \n", inFileName); - free(origBuff); - return 13; - } + { size_t const readSize = fread(origBuff, 1, benchedSize, inFile); + fclose(inFile); + if(readSize != benchedSize) { + DISPLAY("\nError: problem reading file '%s' !! \n", inFileName); + free(origBuff); + return 13; + } } /* bench */ DISPLAY("\r%79s\r", ""); - DISPLAY("optimizing for %s : \n", inFileName); + DISPLAY("optimizing for %s - limit speed %u MB/s \n", inFileName, targetSpeed); + targetSpeed *= 1000; - { ZSTD_CCtx* ctx = ZSTD_createCCtx(); + { ZSTD_CCtx* const ctx = ZSTD_createCCtx(); ZSTD_compressionParameters params; winnerInfo_t winner; BMK_result_t candidate; const size_t blockSize = g_blockSize ? g_blockSize : benchedSize; - int i; /* init */ + if (ctx==NULL) { DISPLAY("\n ZSTD_createCCtx error \n"); free(origBuff); return 14;} memset(&winner, 0, sizeof(winner)); winner.result.cSize = (size_t)(-1); /* find best solution from default params */ { const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); + int i; for (i=1; i<=maxSeeds; i++) { params = ZSTD_getCParams(i, blockSize, 0); BMK_benchParam(&candidate, origBuff, benchedSize, ctx, params); + if (candidate.cSpeed < targetSpeed) + break; if ( (candidate.cSize < winner.result.cSize) - ||((candidate.cSize == winner.result.cSize) && (candidate.cSpeed > winner.result.cSpeed)) ) + | ((candidate.cSize == winner.result.cSize) & (candidate.cSpeed > winner.result.cSpeed)) ) { winner.params = params; winner.result = candidate; @@ -851,7 +847,7 @@ int optimizeForSize(const char* inFileName) do { params = winner.params; paramVariation(¶ms); - if ((FUZ_rand(&g_rand) & 15) == 1) params = randomParams(); + if ((FUZ_rand(&g_rand) & 15) == 3) params = randomParams(); /* exclude faster if already played set of params */ if (FUZ_rand(&g_rand) & ((1 << NB_TESTS_PLAYED(params))-1)) continue; @@ -861,8 +857,10 @@ int optimizeForSize(const char* inFileName) BMK_benchParam(&candidate, origBuff, benchedSize, ctx, params); /* improvement found => new winner */ - if ( (candidate.cSize < winner.result.cSize) - ||((candidate.cSize == winner.result.cSize) && (candidate.cSpeed > winner.result.cSpeed)) ) { + if ( (candidate.cSpeed > targetSpeed) + & ( (candidate.cSize < winner.result.cSize) + | ((candidate.cSize == winner.result.cSize) & (candidate.cSpeed > winner.result.cSpeed)) ) ) + { winner.params = params; winner.result = candidate; BMK_printWinner(stdout, 99, winner.result, winner.params, benchedSize); @@ -878,6 +876,7 @@ int optimizeForSize(const char* inFileName) ZSTD_freeCCtx(ctx); } + free(origBuff); return 0; } @@ -898,6 +897,7 @@ static int usage_advanced(void) DISPLAY( " -T# : set level 1 speed objective \n"); DISPLAY( " -B# : cut input into blocks of size # (default : single block) \n"); DISPLAY( " -i# : iteration loops [1-9](default : %i) \n", NBLOOPS); + DISPLAY( " -O# : find Optimized parameters for # target speed (default : 0) \n"); DISPLAY( " -S : Single run \n"); DISPLAY( " -P# : generated sample compressibility (default : %.1f%%) \n", COMPRESSIBILITY_DEFAULT * 100); return 0; @@ -919,6 +919,7 @@ int main(int argc, const char** argv) const char* input_filename=0; U32 optimizer = 0; U32 main_pause = 0; + U32 targetSpeed = 0; /* checks */ if (NB_LEVELS_TRACKED <= ZSTD_maxCLevel()) { @@ -956,7 +957,7 @@ int main(int argc, const char** argv) /* Modify Nb Iterations */ case 'i': argument++; - if ((argument[0] >='0') && (argument[0] <='9')) + if ((argument[0] >='0') & (argument[0] <='9')) g_nbIterations = *argument++ - '0'; break; @@ -964,7 +965,7 @@ int main(int argc, const char** argv) case 'P': argument++; { U32 proba32 = 0; - while ((argument[0]>= '0') && (argument[0]<= '9')) + while ((argument[0]>= '0') & (argument[0]<= '9')) proba32 = (proba32*10) + (*argument++ - '0'); g_compressibility = (double)proba32 / 100.; } @@ -973,6 +974,9 @@ int main(int argc, const char** argv) case 'O': argument++; optimizer=1; + targetSpeed = 0; + while ((*argument >= '0') & (*argument <= '9')) + targetSpeed = (targetSpeed*10) + (*argument++ - '0'); break; /* Run Single conf */ @@ -1050,7 +1054,7 @@ int main(int argc, const char** argv) case 'B': g_blockSize = 0; argument++; - while ((*argument >='0') && (*argument <='9')) + while ((*argument >='0') & (*argument <='9')) g_blockSize = (g_blockSize*10) + (*argument++ - '0'); if (*argument=='K') g_blockSize<<=10, argument++; /* allows using KB notation */ if (*argument=='M') g_blockSize<<=20, argument++; @@ -1073,7 +1077,7 @@ int main(int argc, const char** argv) result = benchSample(); else { if (optimizer) - result = optimizeForSize(input_filename); + result = optimizeForSize(input_filename, targetSpeed); else result = benchFiles(argv+filenamesStart, argc-filenamesStart); } From fbc69f8649cb4c58e6aa50172306a95c768ee794 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 13 Jul 2016 13:52:58 +0200 Subject: [PATCH 16/42] changed for #245 --- lib/dictBuilder/zdict.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index f1af41962..4e7d5b168 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -284,7 +284,7 @@ static dictItem ZDICT_analyzePos( U32 refinedEnd = end; DISPLAYLEVEL(4, "\n"); - DISPLAYLEVEL(4, "found %3u matches of length >= %u at pos %7u ", (U32)(end-start), MINMATCHLENGTH, (U32)pos); + DISPLAYLEVEL(4, "found %3u matches of length >= %i at pos %7u ", (U32)(end-start), MINMATCHLENGTH, (U32)pos); DISPLAYLEVEL(4, "\n"); for (searchLength = MINMATCHLENGTH ; ; searchLength++) { From 2cac5b30b96c2581cf57088a06f6134d25a483a7 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 13 Jul 2016 14:15:08 +0200 Subject: [PATCH 17/42] changed default compression level to 3 (can be modified with macro ZSTDCLI_DEFAULT_CLEVEL) --- .gitignore | 3 +++ programs/fileio.c | 9 +++++---- programs/zstdcli.c | 12 ++++++++++-- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 181652401..f8024e024 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,6 @@ _zstdbench/ # CMake projects/cmake/ + +# Test artefacts +tmp* diff --git a/programs/fileio.c b/programs/fileio.c index 3eb8d881e..f805545b5 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -330,7 +330,6 @@ static int FIO_compressFilename_internal(cRess_t ress, } } /* Main compression loop */ - readsize = 0; while (1) { /* Fill input Buffer */ size_t const inSize = fread(ress.srcBuffer, (size_t)1, ress.srcBufferSize, srcFile); @@ -338,8 +337,8 @@ static int FIO_compressFilename_internal(cRess_t ress, readsize += inSize; DISPLAYUPDATE(2, "\rRead : %u MB ", (U32)(readsize>>20)); - { /* Compress using buffered streaming */ - size_t usedInSize = inSize; + /* Compress using buffered streaming */ + { size_t usedInSize = inSize; size_t cSize = ress.dstBufferSize; { size_t const result = ZBUFF_compressContinue(ress.ctx, ress.dstBuffer, &cSize, ress.srcBuffer, &usedInSize); if (ZBUFF_isError(result)) EXM_THROW(23, "Compression error : %s ", ZBUFF_getErrorName(result)); } @@ -366,8 +365,10 @@ static int FIO_compressFilename_internal(cRess_t ress, } /* Status */ + { size_t const len = strlen(srcFileName); + if (len > 20) srcFileName += len-20; } DISPLAYLEVEL(2, "\r%79s\r", ""); - DISPLAYLEVEL(2,"%-20.20s :%6.2f%% (%6llu =>%6llu bytes, %s) \n", srcFileName, + DISPLAYLEVEL(2,"%-20.20s :%6.2f%% (%6llu => %6llu bytes, %s) \n", srcFileName, (double)compressedfilesize/readsize*100, (unsigned long long)readsize, (unsigned long long) compressedfilesize, dstFileName); diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 24fc33b7d..6669f5a27 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -28,6 +28,14 @@ */ +/*-************************************ +* Tuning parameters +**************************************/ +#ifndef ZSTDCLI_DEFAULT_CLEVEL +# define ZSTDCLI_DEFAULT_CLEVEL 3 +#endif + + /*-************************************ * Includes **************************************/ @@ -207,7 +215,7 @@ int main(int argCount, const char** argv) nextArgumentIsMaxDict=0, nextArgumentIsDictID=0, nextArgumentIsFile=0; - unsigned cLevel = 1; + unsigned cLevel = ZSTDCLI_DEFAULT_CLEVEL; unsigned cLevelLast = 1; unsigned recursive = 0; const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); /* argCount >= 1 */ @@ -331,7 +339,7 @@ int main(int argCount, const char** argv) /* test compressed file */ case 't': decode=1; outFileName=nulmark; argument++; break; - /* dictionary name */ + /* destination file name */ case 'o': nextArgumentIsOutFileName=1; argument++; break; /* recursive */ From 3c242e79d3eb93d8bdda76556d193fae76313f9d Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 13 Jul 2016 14:56:24 +0200 Subject: [PATCH 18/42] updated compression levels table --- lib/compress/zstd_compress.c | 52 ++++++++++++++++++------------------ programs/bench.c | 13 ++++----- programs/fileio.c | 3 +-- 3 files changed, 34 insertions(+), 34 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 2f995169c..905eafd31 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2939,24 +2939,24 @@ unsigned ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; } static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEVEL+1] = { { /* "default" */ /* W, C, H, S, L, TL, strat */ - { 18, 12, 12, 1, 7, 4, ZSTD_fast }, /* level 0 - not used */ - { 19, 13, 14, 1, 7, 4, ZSTD_fast }, /* level 1 */ - { 19, 15, 16, 1, 6, 4, ZSTD_fast }, /* level 2 */ - { 20, 16, 17, 1, 6, 4, ZSTD_dfast }, /* level 3 */ - { 20, 13, 17, 2, 5, 4, ZSTD_greedy }, /* level 4.*/ - { 20, 15, 18, 3, 5, 4, ZSTD_greedy }, /* level 5 */ - { 21, 16, 19, 2, 5, 4, ZSTD_lazy }, /* level 6 */ - { 21, 17, 20, 3, 5, 4, ZSTD_lazy }, /* level 7 */ - { 21, 18, 20, 3, 5, 4, ZSTD_lazy2 }, /* level 8.*/ - { 21, 20, 20, 3, 5, 4, ZSTD_lazy2 }, /* level 9 */ - { 21, 19, 21, 4, 5, 4, ZSTD_lazy2 }, /* level 10 */ - { 22, 20, 22, 4, 5, 4, ZSTD_lazy2 }, /* level 11 */ - { 22, 20, 22, 5, 5, 4, ZSTD_lazy2 }, /* level 12 */ - { 22, 21, 22, 5, 5, 4, ZSTD_lazy2 }, /* level 13 */ - { 22, 21, 22, 6, 5, 4, ZSTD_lazy2 }, /* level 14 */ - { 22, 21, 21, 5, 5, 4, ZSTD_btlazy2 }, /* level 15 */ - { 23, 22, 22, 5, 5, 4, ZSTD_btlazy2 }, /* level 16 */ - { 23, 23, 22, 5, 5, 4, ZSTD_btlazy2 }, /* level 17.*/ + { 18, 12, 12, 1, 7, 16, ZSTD_fast }, /* level 0 - not used */ + { 19, 13, 14, 1, 7, 16, ZSTD_fast }, /* level 1 */ + { 19, 15, 16, 1, 6, 16, ZSTD_fast }, /* level 2 */ + { 20, 16, 18, 1, 5, 16, ZSTD_dfast }, /* level 3 */ + { 20, 13, 17, 2, 5, 16, ZSTD_greedy }, /* level 4.*/ + { 20, 15, 18, 3, 5, 16, ZSTD_greedy }, /* level 5 */ + { 21, 16, 19, 2, 5, 16, ZSTD_lazy }, /* level 6 */ + { 21, 17, 20, 3, 5, 16, ZSTD_lazy }, /* level 7 */ + { 21, 18, 20, 3, 5, 16, ZSTD_lazy2 }, /* level 8.*/ + { 21, 20, 20, 3, 5, 16, ZSTD_lazy2 }, /* level 9 */ + { 21, 19, 21, 4, 5, 16, ZSTD_lazy2 }, /* level 10 */ + { 22, 20, 22, 4, 5, 16, ZSTD_lazy2 }, /* level 11 */ + { 22, 20, 22, 5, 5, 16, ZSTD_lazy2 }, /* level 12 */ + { 22, 21, 22, 5, 5, 16, ZSTD_lazy2 }, /* level 13 */ + { 22, 21, 22, 6, 5, 16, ZSTD_lazy2 }, /* level 14 */ + { 22, 21, 21, 5, 5, 16, ZSTD_btlazy2 }, /* level 15 */ + { 23, 22, 22, 5, 5, 16, ZSTD_btlazy2 }, /* level 16 */ + { 23, 23, 22, 5, 5, 16, ZSTD_btlazy2 }, /* level 17.*/ { 23, 23, 22, 6, 5, 24, ZSTD_btopt }, /* level 18.*/ { 23, 23, 22, 6, 3, 48, ZSTD_btopt }, /* level 19.*/ { 25, 26, 23, 7, 3, 64, ZSTD_btopt }, /* level 20.*/ @@ -3018,14 +3018,14 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV { /* for srcSize <= 16 KB */ /* W, C, H, S, L, T, strat */ { 14, 12, 12, 1, 7, 4, ZSTD_fast }, /* level 0 - not used */ - { 14, 14, 14, 1, 4, 4, ZSTD_fast }, /* level 1 */ - { 14, 14, 15, 1, 4, 4, ZSTD_fast }, /* level 2 */ - { 14, 14, 14, 4, 4, 4, ZSTD_greedy }, /* level 3.*/ - { 14, 14, 14, 3, 4, 4, ZSTD_lazy }, /* level 4.*/ - { 14, 14, 14, 4, 4, 4, ZSTD_lazy2 }, /* level 5 */ - { 14, 14, 14, 5, 4, 4, ZSTD_lazy2 }, /* level 6 */ - { 14, 14, 14, 6, 4, 4, ZSTD_lazy2 }, /* level 7.*/ - { 14, 14, 14, 7, 4, 4, ZSTD_lazy2 }, /* level 8.*/ + { 14, 14, 14, 1, 7, 4, ZSTD_fast }, /* level 1 */ + { 14, 14, 14, 1, 4, 4, ZSTD_fast }, /* level 2 */ + { 14, 14, 14, 1, 4, 4, ZSTD_dfast }, /* level 3.*/ + { 14, 14, 14, 4, 4, 4, ZSTD_greedy }, /* level 4.*/ + { 14, 14, 14, 3, 4, 4, ZSTD_lazy }, /* level 5.*/ + { 14, 14, 14, 4, 4, 4, ZSTD_lazy2 }, /* level 6 */ + { 14, 14, 14, 5, 4, 4, ZSTD_lazy2 }, /* level 7 */ + { 14, 14, 14, 6, 4, 4, ZSTD_lazy2 }, /* level 8.*/ { 14, 15, 14, 6, 4, 4, ZSTD_btlazy2 }, /* level 9.*/ { 14, 15, 14, 3, 3, 6, ZSTD_btopt }, /* level 10.*/ { 14, 15, 14, 6, 3, 8, ZSTD_btopt }, /* level 11.*/ diff --git a/programs/bench.c b/programs/bench.c index 3fe3f5a34..a463576b7 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -142,14 +142,14 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, const size_t* fileSizes, U32 nbFiles, const void* dictBuffer, size_t dictBufferSize, benchResult_t *result) { - size_t const blockSize = (g_blockSize>=32 ? g_blockSize : srcSize) + (!srcSize); /* avoid div by 0 */ + size_t const blockSize = (g_blockSize>=32 ? g_blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ; U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles; blockParam_t* const blockTable = (blockParam_t*) malloc(maxNbBlocks * sizeof(blockParam_t)); size_t const maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); /* add some room for safety */ void* const compressedBuffer = malloc(maxCompressedSize); void* const resultBuffer = malloc(srcSize); - ZSTD_CCtx* ctx = ZSTD_createCCtx(); - ZSTD_DCtx* dctx = ZSTD_createDCtx(); + ZSTD_CCtx* const ctx = ZSTD_createCCtx(); + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); U32 nbBlocks; UTIL_time_t ticksPerSecond; @@ -215,12 +215,13 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, UTIL_waitForNextTick(ticksPerSecond); UTIL_getTime(&clockStart); - { size_t const refSrcSize = (nbBlocks == 1) ? srcSize : 0; - ZSTD_parameters const zparams = ZSTD_getParams(cLevel, refSrcSize, dictBufferSize); + { //size_t const refSrcSize = (nbBlocks == 1) ? srcSize : 0; + //ZSTD_parameters const zparams = ZSTD_getParams(cLevel, refSrcSize, dictBufferSize); + ZSTD_parameters const zparams = ZSTD_getParams(cLevel, blockSize, dictBufferSize); ZSTD_customMem const cmem = { NULL, NULL, NULL }; U32 nbLoops = 0; ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, zparams, cmem); - if (cdict==NULL) EXM_THROW(1, "ZSTD_createCDict() allocation failure"); + if (cdict==NULL) EXM_THROW(1, "ZSTD_createCDict_advanced() allocation failure"); do { U32 blockNb; for (blockNb=0; blockNb 20) srcFileName += len-20; } + if (strlen(srcFileName) > 20) srcFileName += strlen(srcFileName)-20; /* display last 20 characters */ DISPLAYLEVEL(2, "\r%79s\r", ""); DISPLAYLEVEL(2,"%-20.20s :%6.2f%% (%6llu => %6llu bytes, %s) \n", srcFileName, (double)compressedfilesize/readsize*100, (unsigned long long)readsize, (unsigned long long) compressedfilesize, From 2b1a3638e66e770922406ea4c966179b317f5303 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 13 Jul 2016 15:16:00 +0200 Subject: [PATCH 19/42] changed macro name to ZSTDCLI_CLEVEL_DEFAULT --- lib/compress/zstd_compress.c | 20 ++++++++++---------- programs/zstdcli.c | 6 +++--- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 905eafd31..15a79d705 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -3017,16 +3017,16 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV }, { /* for srcSize <= 16 KB */ /* W, C, H, S, L, T, strat */ - { 14, 12, 12, 1, 7, 4, ZSTD_fast }, /* level 0 - not used */ - { 14, 14, 14, 1, 7, 4, ZSTD_fast }, /* level 1 */ - { 14, 14, 14, 1, 4, 4, ZSTD_fast }, /* level 2 */ - { 14, 14, 14, 1, 4, 4, ZSTD_dfast }, /* level 3.*/ - { 14, 14, 14, 4, 4, 4, ZSTD_greedy }, /* level 4.*/ - { 14, 14, 14, 3, 4, 4, ZSTD_lazy }, /* level 5.*/ - { 14, 14, 14, 4, 4, 4, ZSTD_lazy2 }, /* level 6 */ - { 14, 14, 14, 5, 4, 4, ZSTD_lazy2 }, /* level 7 */ - { 14, 14, 14, 6, 4, 4, ZSTD_lazy2 }, /* level 8.*/ - { 14, 15, 14, 6, 4, 4, ZSTD_btlazy2 }, /* level 9.*/ + { 14, 12, 12, 1, 7, 6, ZSTD_fast }, /* level 0 - not used */ + { 14, 14, 14, 1, 7, 6, ZSTD_fast }, /* level 1 */ + { 14, 14, 14, 1, 4, 6, ZSTD_fast }, /* level 2 */ + { 14, 14, 14, 1, 4, 6, ZSTD_dfast }, /* level 3.*/ + { 14, 14, 14, 4, 4, 6, ZSTD_greedy }, /* level 4.*/ + { 14, 14, 14, 3, 4, 6, ZSTD_lazy }, /* level 5.*/ + { 14, 14, 14, 4, 4, 6, ZSTD_lazy2 }, /* level 6 */ + { 14, 14, 14, 5, 4, 6, ZSTD_lazy2 }, /* level 7 */ + { 14, 14, 14, 6, 4, 6, ZSTD_lazy2 }, /* level 8.*/ + { 14, 15, 14, 6, 4, 6, ZSTD_btlazy2 }, /* level 9.*/ { 14, 15, 14, 3, 3, 6, ZSTD_btopt }, /* level 10.*/ { 14, 15, 14, 6, 3, 8, ZSTD_btopt }, /* level 11.*/ { 14, 15, 14, 6, 3, 16, ZSTD_btopt }, /* level 12.*/ diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 6669f5a27..edc87a00b 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -31,8 +31,8 @@ /*-************************************ * Tuning parameters **************************************/ -#ifndef ZSTDCLI_DEFAULT_CLEVEL -# define ZSTDCLI_DEFAULT_CLEVEL 3 +#ifndef ZSTDCLI_CLEVEL_DEFAULT +# define ZSTDCLI_CLEVEL_DEFAULT 3 #endif @@ -215,7 +215,7 @@ int main(int argCount, const char** argv) nextArgumentIsMaxDict=0, nextArgumentIsDictID=0, nextArgumentIsFile=0; - unsigned cLevel = ZSTDCLI_DEFAULT_CLEVEL; + unsigned cLevel = ZSTDCLI_CLEVEL_DEFAULT; unsigned cLevelLast = 1; unsigned recursive = 0; const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); /* argCount >= 1 */ From 158e7703bbf9aeea908f2fe2158a732423d91b98 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 13 Jul 2016 16:45:24 +0200 Subject: [PATCH 20/42] reduced paramgrill dependency to C standard lib only --- programs/paramgrill.c | 184 +++++++++++++++--------------------------- 1 file changed, 67 insertions(+), 117 deletions(-) diff --git a/programs/paramgrill.c b/programs/paramgrill.c index f990e5fc1..da49cf36f 100644 --- a/programs/paramgrill.c +++ b/programs/paramgrill.c @@ -22,30 +22,16 @@ - zstd homepage : http://www.zstd.net/ */ -/*-************************************ -* Compiler Options -**************************************/ -/* gettimeofday() are not supported by MSVC */ -#if defined(_MSC_VER) || defined(_WIN32) -# define BMK_LEGACY_TIMER 1 -#endif - /*-************************************ * Dependencies **************************************/ -#include "util.h" /* Compiler options, UTIL_GetFileSize */ -#include /* malloc */ -#include /* fprintf, fopen, ftello64 */ -#include /* strcmp */ -#include /* log */ - -/* Use ftime() if gettimeofday() is not available on your target */ -#if defined(BMK_LEGACY_TIMER) -# include /* timeb, ftime */ -#else -# include /* gettimeofday */ -#endif +#include "util.h" /* Compiler options, UTIL_GetFileSize */ +#include /* malloc */ +#include /* fprintf, fopen, ftello64 */ +#include /* strcmp */ +#include /* log */ +#include /* clock_t */ #include "mem.h" #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters, ZSTD_estimateCCtxSize */ @@ -67,7 +53,7 @@ #define GB *(1ULL<<30) #define NBLOOPS 2 -#define TIMELOOP 2000 +#define TIMELOOP (2 * CLOCKS_PER_SEC) #define NB_LEVELS_TRACKED 30 @@ -76,9 +62,9 @@ static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t #define COMPRESSIBILITY_DEFAULT 0.50 static const size_t sampleSize = 10000000; -static const int g_grillDuration = 50000000; /* about 13 hours */ -static const int g_maxParamTime = 15000; /* 15 sec */ -static const int g_maxVariationTime = 60000; /* 60 sec */ +static const U32 g_grillDuration_s = 60000; /* about 16 hours */ +static const clock_t g_maxParamTime = 15 * CLOCKS_PER_SEC; +static const clock_t g_maxVariationTime = 60 * CLOCKS_PER_SEC; static const int g_maxNbVariations = 64; @@ -111,49 +97,15 @@ void BMK_SetNbIterations(int nbLoops) * Private functions *********************************************************/ -#if defined(BMK_LEGACY_TIMER) +static clock_t BMK_clockSpan(clock_t cStart) { return clock() - cStart; } /* works even if overflow ; max span ~ 30 mn */ -static int BMK_GetMilliStart(void) -{ - /* Based on Legacy ftime() - * Rolls over every ~ 12.1 days (0x100000/24/60/60) - * Use GetMilliSpan to correct for rollover */ - struct timeb tb; - int nCount; - ftime( &tb ); - nCount = (int) (tb.millitm + (tb.time & 0xfffff) * 1000); - return nCount; -} - -#else - -static int BMK_GetMilliStart(void) -{ - /* Based on newer gettimeofday() - * Use GetMilliSpan to correct for rollover */ - struct timeval tv; - int nCount; - gettimeofday(&tv, NULL); - nCount = (int) (tv.tv_usec/1000 + (tv.tv_sec & 0xfffff) * 1000); - return nCount; -} - -#endif - - -static int BMK_GetMilliSpan( int nTimeStart ) -{ - int nSpan = BMK_GetMilliStart() - nTimeStart; - if ( nSpan < 0 ) - nSpan += 0x100000 * 1000; - return nSpan; -} +static U32 BMK_timeSpan(time_t tStart) { return (U32)difftime(time(NULL), tStart); } /* accuracy in seconds only, span can be multiple years */ static size_t BMK_findMaxMem(U64 requiredMem) { - size_t step = 64 MB; - BYTE* testmem=NULL; + size_t const step = 64 MB; + void* testmem = NULL; requiredMem = (((requiredMem >> 26) + 1) << 26); if (requiredMem > maxMemory) requiredMem = maxMemory; @@ -161,7 +113,7 @@ static size_t BMK_findMaxMem(U64 requiredMem) requiredMem += 2*step; while (!testmem) { requiredMem -= step; - testmem = (BYTE*) malloc ((size_t)requiredMem); + testmem = malloc ((size_t)requiredMem); } free (testmem); @@ -188,8 +140,8 @@ U32 FUZ_rand(U32* src) *********************************************************/ typedef struct { size_t cSize; - U32 cSpeed; - U32 dSpeed; + double cSpeed; + double dSpeed; } BMK_result_t; typedef struct @@ -265,36 +217,33 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr, RDG_genBuffer(compressedBuffer, maxCompressedSize, 0.10, 0.10, 1); /* Bench */ - { - U32 loopNb; + { U32 loopNb; size_t cSize = 0; double fastestC = 100000000., fastestD = 100000000.; double ratio = 0.; U64 crcCheck = 0; - const int startTime =BMK_GetMilliStart(); + time_t const benchStart = clock(); DISPLAY("\r%79s\r", ""); memset(¶ms, 0, sizeof(params)); params.cParams = cParams; - params.fParams.contentSizeFlag = 0; for (loopNb = 1; loopNb <= g_nbIterations; loopNb++) { int nbLoops; - int milliTime; U32 blockNb; - const int totalTime = BMK_GetMilliSpan(startTime); + clock_t roundStart, roundClock; - /* early break (slow params) */ - if (totalTime > g_maxParamTime) break; + { clock_t const benchTime = BMK_clockSpan(benchStart); + if (benchTime > g_maxParamTime) break; } /* Compression */ DISPLAY("\r%1u-%s : %9u ->", loopNb, name, (U32)srcSize); memset(compressedBuffer, 0xE5, maxCompressedSize); nbLoops = 0; - milliTime = BMK_GetMilliStart(); - while (BMK_GetMilliStart() == milliTime); - milliTime = BMK_GetMilliStart(); - while (BMK_GetMilliSpan(milliTime) < TIMELOOP) { + roundStart = clock(); + while (clock() == roundStart); + roundStart = clock(); + while (BMK_clockSpan(roundStart) < TIMELOOP) { for (blockNb=0; blockNb", loopNb, name, (U32)srcSize); - DISPLAY(" %9u (%4.3f),%7.1f MB/s", (U32)cSize, ratio, (double)srcSize / fastestC / 1000.); + DISPLAY(" %9u (%4.3f),%7.1f MB/s", (U32)cSize, ratio, (double)srcSize / fastestC / 1000000.); resultPtr->cSize = cSize; - resultPtr->cSpeed = (U32)((double)srcSize / fastestC); + resultPtr->cSpeed = (double)srcSize / fastestC; #if 1 /* Decompression */ memset(resultBuffer, 0xD6, srcSize); nbLoops = 0; - milliTime = BMK_GetMilliStart(); - while (BMK_GetMilliStart() == milliTime); - milliTime = BMK_GetMilliStart(); - for ( ; BMK_GetMilliSpan(milliTime) < TIMELOOP; nbLoops++) { + roundStart = clock(); + while (clock() == roundStart); + roundStart = clock(); + for ( ; BMK_clockSpan(roundStart) < TIMELOOP; nbLoops++) { for (blockNb=0; blockNb ", loopNb, name, (U32)srcSize); - DISPLAY("%9u (%4.3f),%7.1f MB/s, ", (U32)cSize, ratio, (double)srcSize / fastestC / 1000.); - DISPLAY("%7.1f MB/s", (double)srcSize / fastestD / 1000.); - resultPtr->dSpeed = (U32)((double)srcSize / fastestD); + DISPLAY("%9u (%4.3f),%7.1f MB/s, ", (U32)cSize, ratio, (double)srcSize / fastestC / 1000000.); + DISPLAY("%7.1f MB/s", (double)srcSize / fastestD / 1000000.); + resultPtr->dSpeed = (double)srcSize / fastestD; /* CRC Checking */ crcCheck = XXH64(resultBuffer, srcSize, 0); @@ -378,11 +327,11 @@ static void BMK_printWinner(FILE* f, U32 cLevel, BMK_result_t result, ZSTD_compr params.targetLength, g_stratName[(U32)(params.strategy)]); fprintf(f, "/* level %2u */ /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", - cLevel, (double)srcSize / result.cSize, (double)result.cSpeed / 1000., (double)result.dSpeed / 1000.); + cLevel, (double)srcSize / result.cSize, result.cSpeed / 1000000., result.dSpeed / 1000000.); } -static U32 g_cSpeedTarget[NB_LEVELS_TRACKED] = { 0 }; /* NB_LEVELS_TRACKED : checked at main() */ +static double g_cSpeedTarget[NB_LEVELS_TRACKED] = { 0. }; /* NB_LEVELS_TRACKED : checked at main() */ typedef struct { BMK_result_t result; @@ -447,11 +396,11 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para double W_CMemUsed_note = W_ratioNote * ( 50 + 13*cLevel) - log((double)W_CMemUsed); double O_CMemUsed_note = O_ratioNote * ( 50 + 13*cLevel) - log((double)O_CMemUsed); - double W_CSpeed_note = W_ratioNote * ( 30 + 10*cLevel) + log((double)testResult.cSpeed); - double O_CSpeed_note = O_ratioNote * ( 30 + 10*cLevel) + log((double)winners[cLevel].result.cSpeed); + double W_CSpeed_note = W_ratioNote * ( 30 + 10*cLevel) + log(testResult.cSpeed); + double O_CSpeed_note = O_ratioNote * ( 30 + 10*cLevel) + log(winners[cLevel].result.cSpeed); - double W_DSpeed_note = W_ratioNote * ( 20 + 2*cLevel) + log((double)testResult.dSpeed); - double O_DSpeed_note = O_ratioNote * ( 20 + 2*cLevel) + log((double)winners[cLevel].result.dSpeed); + double W_DSpeed_note = W_ratioNote * ( 20 + 2*cLevel) + log(testResult.dSpeed); + double O_DSpeed_note = O_ratioNote * ( 20 + 2*cLevel) + log(winners[cLevel].result.dSpeed); if (W_DMemUsed_note < O_DMemUsed_note) { /* uses too much Decompression memory for too little benefit */ @@ -473,16 +422,16 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para /* too large compression speed difference for the compression benefit */ if (W_ratio > O_ratio) DISPLAY ("Compression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n", - W_ratio, (double)(testResult.cSpeed) / 1000., - O_ratio, (double)(winners[cLevel].result.cSpeed) / 1000., cLevel); + W_ratio, testResult.cSpeed / 1000000, + O_ratio, winners[cLevel].result.cSpeed / 1000000., cLevel); continue; } if (W_DSpeed_note < O_DSpeed_note ) { /* too large decompression speed difference for the compression benefit */ if (W_ratio > O_ratio) DISPLAY ("Decompression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n", - W_ratio, (double)(testResult.dSpeed) / 1000., - O_ratio, (double)(winners[cLevel].result.dSpeed) / 1000., cLevel); + W_ratio, testResult.dSpeed / 1000000., + O_ratio, winners[cLevel].result.dSpeed / 1000000., cLevel); continue; } @@ -578,9 +527,9 @@ static void playAround(FILE* f, winnerInfo_t* winners, ZSTD_CCtx* ctx) { int nbVariations = 0; - const int startTime = BMK_GetMilliStart(); + clock_t const clockStart = clock(); - while (BMK_GetMilliSpan(startTime) < g_maxVariationTime) { + while (BMK_clockSpan(clockStart) < g_maxVariationTime) { ZSTD_compressionParameters p = params; if (nbVariations++ > g_maxNbVariations) break; @@ -641,11 +590,15 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize) ZSTD_CCtx* const ctx = ZSTD_createCCtx(); ZSTD_compressionParameters params; winnerInfo_t winners[NB_LEVELS_TRACKED]; - unsigned u; - const char* rfName = "grillResults.txt"; + const char* const rfName = "grillResults.txt"; FILE* const f = fopen(rfName, "w"); const size_t blockSize = g_blockSize ? g_blockSize : srcSize; + /* init */ + if (ctx==NULL) { DISPLAY("ZSTD_createCCtx() failed \n"); exit(1); } + memset(winners, 0, sizeof(winners)); + if (f==NULL) { DISPLAY("error opening %s \n", rfName); exit(1); } + if (g_singleRun) { BMK_result_t testResult; g_params = ZSTD_adjustCParams(g_params, srcSize, 0); @@ -654,24 +607,21 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize) return; } - /* init */ - if (ctx==NULL) { DISPLAY("ZSTD_createCCtx() failed \n"); exit(1); } - memset(winners, 0, sizeof(winners)); - if (f==NULL) { DISPLAY("error opening %s \n", rfName); exit(1); } - if (g_target) - g_cSpeedTarget[1] = g_target * 1000; + g_cSpeedTarget[1] = g_target * 1000000; else { /* baseline config for level 1 */ BMK_result_t testResult; params = ZSTD_getCParams(1, blockSize, 0); BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, params); - g_cSpeedTarget[1] = (testResult.cSpeed * 31) >> 5; + g_cSpeedTarget[1] = (testResult.cSpeed * 31) / 32; } /* establish speed objectives (relative to level 1) */ - for (u=2; u<=ZSTD_maxCLevel(); u++) - g_cSpeedTarget[u] = (g_cSpeedTarget[u-1] * 25) >> 5; + { unsigned u; + for (u=2; u<=ZSTD_maxCLevel(); u++) + g_cSpeedTarget[u] = (g_cSpeedTarget[u-1] * 25) / 32; + } /* populate initial solution */ { const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); @@ -683,10 +633,10 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize) BMK_printWinners(f, winners, srcSize); /* start tests */ - { const int milliStart = BMK_GetMilliStart(); + { const time_t grillStart = time(NULL); do { BMK_selectRandomStart(f, winners, srcBuffer, srcSize, ctx); - } while (BMK_GetMilliSpan(milliStart) < g_grillDuration); + } while (BMK_timeSpan(grillStart) < g_grillDuration_s); } /* end summary */ @@ -843,7 +793,7 @@ int optimizeForSize(const char* inFileName, U32 targetSpeed) BMK_printWinner(stdout, 99, winner.result, winner.params, benchedSize); /* start tests */ - { const int milliStart = BMK_GetMilliStart(); + { time_t const grillStart = time(NULL); do { params = winner.params; paramVariation(¶ms); @@ -865,7 +815,7 @@ int optimizeForSize(const char* inFileName, U32 targetSpeed) winner.result = candidate; BMK_printWinner(stdout, 99, winner.result, winner.params, benchedSize); } - } while (BMK_GetMilliSpan(milliStart) < g_grillDuration); + } while (BMK_timeSpan(grillStart) < g_grillDuration_s); } /* end summary */ From 3c174f4da9f95d41557fe03dcf20924d9abbe3ad Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 13 Jul 2016 17:19:57 +0200 Subject: [PATCH 21/42] fixed minor coverity warning --- lib/dictBuilder/zdict.c | 2 +- lib/legacy/zstd_v04.c | 15 ++++++--------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index 4e7d5b168..5ae49456b 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -972,7 +972,7 @@ size_t ZDICT_trainFromBuffer_unsafe( for (u=1; upos; u++) { U32 l = dictList[u].length; ptr -= l; - if (ptr<(BYTE*)dictBuffer) return ERROR(GENERIC); /* should not happen */ + if (ptr<(BYTE*)dictBuffer) { free(dictList); return ERROR(GENERIC); } /* should not happen */ memcpy(ptr, (const char*)samplesBuffer+dictList[u].pos, l); } } diff --git a/lib/legacy/zstd_v04.c b/lib/legacy/zstd_v04.c index 35469048a..c5bfa1e7c 100644 --- a/lib/legacy/zstd_v04.c +++ b/lib/legacy/zstd_v04.c @@ -3864,11 +3864,9 @@ static size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDs case ZBUFFds_readHeader : /* read header from src */ - { - size_t headerSize = ZSTD_getFrameParams(&(zbc->params), src, *srcSizePtr); + { size_t const headerSize = ZSTD_getFrameParams(&(zbc->params), src, *srcSizePtr); if (ZSTD_isError(headerSize)) return headerSize; - if (headerSize) - { + if (headerSize) { /* not enough input to decode header : tell how many bytes would be necessary */ memcpy(zbc->headerBuffer+zbc->hPos, src, *srcSizePtr); zbc->hPos += *srcSizePtr; @@ -3882,8 +3880,7 @@ static size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDs case ZBUFFds_loadHeader: /* complete header from src */ - { - size_t headerSize = ZBUFF_limitCopy( + { size_t headerSize = ZBUFF_limitCopy( zbc->headerBuffer + zbc->hPos, ZSTD_frameHeaderSize_max - zbc->hPos, src, *srcSizePtr); zbc->hPos += headerSize; @@ -3895,12 +3892,12 @@ static size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDs *maxDstSizePtr = 0; return headerSize - zbc->hPos; } } + /* intentional fallthrough */ case ZBUFFds_decodeHeader: /* apply header to create / resize buffers */ - { - size_t neededOutSize = (size_t)1 << zbc->params.windowLog; - size_t neededInSize = BLOCKSIZE; /* a block is never > BLOCKSIZE */ + { size_t const neededOutSize = (size_t)1 << zbc->params.windowLog; + size_t const neededInSize = BLOCKSIZE; /* a block is never > BLOCKSIZE */ if (zbc->inBuffSize < neededInSize) { free(zbc->inBuff); zbc->inBuffSize = neededInSize; From f0bc673b26d7e76b0c9b6b1a3d58e84f237a89cc Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 13 Jul 2016 17:30:21 +0200 Subject: [PATCH 22/42] minor spec wording --- zstd_compression_format.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/zstd_compression_format.md b/zstd_compression_format.md index 9f2227406..d432f1166 100644 --- a/zstd_compression_format.md +++ b/zstd_compression_format.md @@ -189,10 +189,10 @@ depending on local limitations. __Unused bit__ -The value of this bit is unimportant -and not interpreted by a decoder compliant with this specification version. -It may be used in a future revision, -to signal a property which is not required to properly decode the frame. +The value of this bit should be set to zero. +A decoder compliant with this specification version should not interpret it. +It might be used in a future version, +to signal a property which is not mandatory to properly decode the frame. __Reserved bit__ From 5e80dd3261bbcba9c8b942fd1f7aa0266b0060c0 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 13 Jul 2016 17:38:39 +0200 Subject: [PATCH 23/42] fixed minor coverity warnings --- NEWS | 3 +++ examples/dictionary_decompression.c | 1 + lib/compress/huf_compress.c | 2 +- lib/dictBuilder/zdict.h | 9 +------- lib/legacy/zstd_v04.c | 36 +++++++++++------------------ lib/legacy/zstd_v05.c | 28 ++++++++-------------- programs/datagen.c | 14 ++++------- programs/fileio.c | 9 ++++---- programs/zstdcli.c | 1 + 9 files changed, 38 insertions(+), 65 deletions(-) diff --git a/NEWS b/NEWS index 6a27ae2cf..c34f3175a 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,6 @@ +v0.7.4 +Modified : default compression level for CLI is 3 + v0.7.3 New : compression format specification New : `--` separator, stating that all following arguments are file names. Suggested by Chip Turner. diff --git a/examples/dictionary_decompression.c b/examples/dictionary_decompression.c index 797075dee..2d51f5e34 100644 --- a/examples/dictionary_decompression.c +++ b/examples/dictionary_decompression.c @@ -78,6 +78,7 @@ static void* loadFile_X(const char* fileName, size_t* size) static const ZSTD_DDict* createDict(const char* dictFileName) { size_t dictSize; + printf("loading dictionary %s \n", dictFileName); void* const dictBuffer = loadFile_X(dictFileName, &dictSize); const ZSTD_DDict* const ddict = ZSTD_createDDict(dictBuffer, dictSize); free(dictBuffer); diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c index 3533bb613..301b5f912 100644 --- a/lib/compress/huf_compress.c +++ b/lib/compress/huf_compress.c @@ -239,7 +239,7 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits) /* repay normalized cost */ { U32 const noSymbol = 0xF0F0F0F0; - U32 rankLast[HUF_TABLELOG_MAX+1]; + U32 rankLast[HUF_TABLELOG_MAX+2]; int pos; /* Get pos of last (smallest) symbol per rank */ diff --git a/lib/dictBuilder/zdict.h b/lib/dictBuilder/zdict.h index 39acdf852..b96b828f7 100644 --- a/lib/dictBuilder/zdict.h +++ b/lib/dictBuilder/zdict.h @@ -84,10 +84,6 @@ const char* ZDICT_getErrorName(size_t errorCode); * Use them only in association with static linking. * ==================================================================================== */ - -/*-************************************* -* Public type -***************************************/ typedef struct { unsigned selectivityLevel; /* 0 means default; larger => bigger selection => larger dictionary */ unsigned compressionLevel; /* 0 means default; target a specific zstd compression level */ @@ -97,9 +93,6 @@ typedef struct { } ZDICT_params_t; -/*-************************************* -* Public functions -***************************************/ /*! ZDICT_trainFromBuffer_advanced() : Same as ZDICT_trainFromBuffer() with control over more parameters. `parameters` is optional and can be provided with values set to 0 to mean "default". @@ -117,4 +110,4 @@ size_t ZDICT_trainFromBuffer_advanced(void* dictBuffer, size_t dictBufferCapacit } #endif -#endif +#endif /* DICTBUILDER_H_001 */ diff --git a/lib/legacy/zstd_v04.c b/lib/legacy/zstd_v04.c index c5bfa1e7c..23ed133ef 100644 --- a/lib/legacy/zstd_v04.c +++ b/lib/legacy/zstd_v04.c @@ -3620,36 +3620,26 @@ static size_t ZSTD_decompressContinue(ZSTD_DCtx* ctx, void* dst, size_t maxDstSi switch (ctx->stage) { case ZSTDds_getFrameHeaderSize : - { - /* get frame header size */ - if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */ - ctx->headerSize = ZSTD_decodeFrameHeader_Part1(ctx, src, ZSTD_frameHeaderSize_min); - if (ZSTD_isError(ctx->headerSize)) return ctx->headerSize; - memcpy(ctx->headerBuffer, src, ZSTD_frameHeaderSize_min); - if (ctx->headerSize > ZSTD_frameHeaderSize_min) - { - ctx->expected = ctx->headerSize - ZSTD_frameHeaderSize_min; - ctx->stage = ZSTDds_decodeFrameHeader; - return 0; - } - ctx->expected = 0; /* not necessary to copy more */ - } + /* get frame header size */ + if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */ + ctx->headerSize = ZSTD_decodeFrameHeader_Part1(ctx, src, ZSTD_frameHeaderSize_min); + if (ZSTD_isError(ctx->headerSize)) return ctx->headerSize; + memcpy(ctx->headerBuffer, src, ZSTD_frameHeaderSize_min); + if (ctx->headerSize > ZSTD_frameHeaderSize_min) return ERROR(GENERIC); /* impossible */ + ctx->expected = 0; /* not necessary to copy more */ + /* fallthrough */ case ZSTDds_decodeFrameHeader: - { - /* get frame header */ - size_t result; - memcpy(ctx->headerBuffer + ZSTD_frameHeaderSize_min, src, ctx->expected); - result = ZSTD_decodeFrameHeader_Part2(ctx, ctx->headerBuffer, ctx->headerSize); + /* get frame header */ + { size_t const result = ZSTD_decodeFrameHeader_Part2(ctx, ctx->headerBuffer, ctx->headerSize); if (ZSTD_isError(result)) return result; ctx->expected = ZSTD_blockHeaderSize; ctx->stage = ZSTDds_decodeBlockHeader; return 0; } case ZSTDds_decodeBlockHeader: - { - /* Decode block header */ - blockProperties_t bp; - size_t blockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp); + /* Decode block header */ + { blockProperties_t bp; + size_t const blockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp); if (ZSTD_isError(blockSize)) return blockSize; if (bp.blockType == bt_end) { diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c index 9c57d18fc..f3c720fd2 100644 --- a/lib/legacy/zstd_v05.c +++ b/lib/legacy/zstd_v05.c @@ -3872,25 +3872,17 @@ size_t ZSTDv05_decompressContinue(ZSTDv05_DCtx* dctx, void* dst, size_t maxDstSi switch (dctx->stage) { case ZSTDv05ds_getFrameHeaderSize : - { - /* get frame header size */ - if (srcSize != ZSTDv05_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */ - dctx->headerSize = ZSTDv05_decodeFrameHeader_Part1(dctx, src, ZSTDv05_frameHeaderSize_min); - if (ZSTDv05_isError(dctx->headerSize)) return dctx->headerSize; - memcpy(dctx->headerBuffer, src, ZSTDv05_frameHeaderSize_min); - if (dctx->headerSize > ZSTDv05_frameHeaderSize_min) { - dctx->expected = dctx->headerSize - ZSTDv05_frameHeaderSize_min; - dctx->stage = ZSTDv05ds_decodeFrameHeader; - return 0; - } - dctx->expected = 0; /* not necessary to copy more */ - } + /* get frame header size */ + if (srcSize != ZSTDv05_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */ + dctx->headerSize = ZSTDv05_decodeFrameHeader_Part1(dctx, src, ZSTDv05_frameHeaderSize_min); + if (ZSTDv05_isError(dctx->headerSize)) return dctx->headerSize; + memcpy(dctx->headerBuffer, src, ZSTDv05_frameHeaderSize_min); + if (dctx->headerSize > ZSTDv05_frameHeaderSize_min) return ERROR(GENERIC); /* should never happen */ + dctx->expected = 0; /* not necessary to copy more */ + /* fallthrough */ case ZSTDv05ds_decodeFrameHeader: - { - /* get frame header */ - size_t result; - memcpy(dctx->headerBuffer + ZSTDv05_frameHeaderSize_min, src, dctx->expected); - result = ZSTDv05_decodeFrameHeader_Part2(dctx, dctx->headerBuffer, dctx->headerSize); + /* get frame header */ + { size_t const result = ZSTDv05_decodeFrameHeader_Part2(dctx, dctx->headerBuffer, dctx->headerSize); if (ZSTDv05_isError(result)) return result; dctx->expected = ZSTDv05_blockHeaderSize; dctx->stage = ZSTDv05ds_decodeBlockHeader; diff --git a/programs/datagen.c b/programs/datagen.c index 0b3dce2e2..6cb5111ff 100644 --- a/programs/datagen.c +++ b/programs/datagen.c @@ -30,7 +30,7 @@ /*-************************************ -* Includes +* Dependencies **************************************/ #include /* malloc */ #include /* FILE, fwrite, fprintf */ @@ -94,12 +94,10 @@ static void RDG_fillLiteralDistrib(BYTE* ldt, double ld) U32 u; if (ld<=0.0) ld = 0.0; - //TRACE(" percent:%5.2f%% \n", ld*100.); - //TRACE(" start:(%c)[%02X] ", character, character); for (u=0; u lastChar) character = firstChar; } @@ -109,8 +107,6 @@ static void RDG_fillLiteralDistrib(BYTE* ldt, double ld) static BYTE RDG_genChar(U32* seed, const BYTE* ldt) { U32 const id = RDG_rand(seed) & LTMASK; - //TRACE(" %u : \n", id); - //TRACE(" %4u [%4u] ; val : %4u \n", id, id&255, ldt[id]); return ldt[id]; /* memory-sanitizer fails here, stating "uninitialized value" when table initialized with P==0.0. Checked : table is fully initialized */ } @@ -162,7 +158,6 @@ void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double match U32 const randOffset = RDG_rand15Bits(seedPtr) + 1; U32 const offset = repeatOffset ? prevOffset : (U32) MIN(randOffset , pos); size_t match = pos - offset; - //TRACE("pos : %u; offset: %u ; length : %u \n", (U32)pos, offset, length); while (pos < d) buffPtr[pos++] = buffPtr[match++]; /* correctly manages overlaps */ prevOffset = offset; } else { @@ -177,9 +172,8 @@ void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double match void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba, unsigned seed) { BYTE ldt[LTSIZE]; - memset(ldt, '0', sizeof(ldt)); + memset(ldt, '0', sizeof(ldt)); /* yes, character '0', this is intentional */ if (litProba<=0.0) litProba = matchProba / 4.5; - //TRACE(" percent:%5.2f%% \n", litProba*100.); RDG_fillLiteralDistrib(ldt, litProba); RDG_genBlock(buffer, size, 0, matchProba, ldt, &seed); } @@ -196,7 +190,7 @@ void RDG_genStdout(unsigned long long size, double matchProba, double litProba, /* init */ if (buff==NULL) { fprintf(stderr, "datagen: error: %s \n", strerror(errno)); exit(1); } if (litProba<=0.0) litProba = matchProba / 4.5; - memset(ldt, '0', sizeof(ldt)); + memset(ldt, '0', sizeof(ldt)); /* yes, character '0', this is intentional */ RDG_fillLiteralDistrib(ldt, litProba); SET_BINARY_MODE(stdout); diff --git a/programs/fileio.c b/programs/fileio.c index fb2dda7a9..492dc9143 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -273,11 +273,10 @@ typedef struct { static cRess_t FIO_createCResources(const char* dictFileName) { cRess_t ress; + memset(&ress, 0, sizeof(ress)); ress.ctx = ZBUFF_createCCtx(); if (ress.ctx == NULL) EXM_THROW(30, "zstd: allocation error : can't create ZBUFF context"); - - /* Allocate Memory */ ress.srcBufferSize = ZBUFF_recommendedCInSize(); ress.srcBuffer = malloc(ress.srcBufferSize); ress.dstBufferSize = ZBUFF_recommendedCOutSize(); @@ -502,12 +501,11 @@ typedef struct { static dRess_t FIO_createDResources(const char* dictFileName) { dRess_t ress; + memset(&ress, 0, sizeof(ress)); - /* init */ + /* Allocation */ ress.dctx = ZBUFF_createDCtx(); if (ress.dctx==NULL) EXM_THROW(60, "Can't create ZBUFF decompression context"); - - /* Allocate Memory */ ress.srcBufferSize = ZBUFF_recommendedDInSize(); ress.srcBuffer = malloc(ress.srcBufferSize); ress.dstBufferSize = ZBUFF_recommendedDOutSize(); @@ -710,6 +708,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName) return FIO_passThrough(dstFile, srcFile, ress.srcBuffer, ress.srcBufferSize); else { DISPLAYLEVEL(1, "zstd: %s: not in zstd format \n", srcFileName); + fclose(srcFile); return 1; } } } filesize += FIO_decompressFrame(ress, dstFile, srcFile, toRead); diff --git a/programs/zstdcli.c b/programs/zstdcli.c index edc87a00b..7b41865d1 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -462,6 +462,7 @@ int main(int argCount, const char** argv) if (dictBuild) { #ifndef ZSTD_NODICT ZDICT_params_t dictParams; + memset(&dictParams, 0, sizeof(dictParams)); dictParams.compressionLevel = dictCLevel; dictParams.selectivityLevel = dictSelect; dictParams.notificationLevel = displayLevel; From 44f684ded39647005e5cd24aba7d1b61fde5f989 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 13 Jul 2016 19:30:40 +0200 Subject: [PATCH 24/42] fixed minor coverity warning --- programs/fileio.c | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 492dc9143..32a1e1b3e 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -367,8 +367,9 @@ static int FIO_compressFilename_internal(cRess_t ress, if (strlen(srcFileName) > 20) srcFileName += strlen(srcFileName)-20; /* display last 20 characters */ DISPLAYLEVEL(2, "\r%79s\r", ""); DISPLAYLEVEL(2,"%-20.20s :%6.2f%% (%6llu => %6llu bytes, %s) \n", srcFileName, - (double)compressedfilesize/readsize*100, (unsigned long long)readsize, (unsigned long long) compressedfilesize, - dstFileName); + (double)compressedfilesize/(readsize+(!readsize) /* avoid div by zero */ )*100, + (unsigned long long)readsize, (unsigned long long) compressedfilesize, + dstFileName); return 0; } @@ -396,7 +397,7 @@ static int FIO_compressFilename_srcFile(cRess_t ress, result = FIO_compressFilename_internal(ress, dstFileName, srcFileName, cLevel); fclose(ress.srcFile); - if ((g_removeSrcFile) && (!result)) remove(srcFileName); + if ((g_removeSrcFile) && (!result)) { if (remove(srcFileName)) EXM_THROW(1, "zstd: %s: %s", srcFileName, strerror(errno)); } return result; } @@ -417,7 +418,7 @@ static int FIO_compressFilename_dstFile(cRess_t ress, result = FIO_compressFilename_srcFile(ress, dstFileName, srcFileName, cLevel); if (fclose(ress.dstFile)) { DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno)); result=1; } - if (result!=0) remove(dstFileName); /* remove operation artefact */ + if (result!=0) { if (remove(dstFileName)) EXM_THROW(1, "zstd: %s: %s", dstFileName, strerror(errno)); } /* remove operation artefact */ return result; } @@ -443,13 +444,14 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile const char* dictFileName, int compressionLevel) { int missed_files = 0; - char* dstFileName = (char*)malloc(FNSPACE); size_t dfnSize = FNSPACE; + char* dstFileName = (char*)malloc(FNSPACE); size_t const suffixSize = suffix ? strlen(suffix) : 0; - cRess_t ress; + cRess_t ress = FIO_createCResources(dictFileName); /* init */ - ress = FIO_createCResources(dictFileName); + if (dstFileName==NULL) EXM_THROW(27, "FIO_compressMultipleFilenames : allocation error for dstFileName"); + if (suffix == NULL) EXM_THROW(28, "FIO_compressMultipleFilenames : dst unknown"); /* should never happen */ /* loop on each file */ if (!strcmp(suffix, stdoutmark)) { @@ -719,8 +721,8 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName) DISPLAYLEVEL(2, "%-20.20s: %llu bytes \n", srcFileName, filesize); /* Close */ - fclose(srcFile); - if (g_removeSrcFile) remove(srcFileName); + if (fclose(srcFile)) EXM_THROW(32, "zstd: %s close error", srcFileName); /* error should never happen */ + if (g_removeSrcFile) { if (remove(srcFileName)) EXM_THROW(32, "zstd: %s: %s", srcFileName, strerror(errno)); }; return 0; } @@ -767,19 +769,21 @@ int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles int missingFiles = 0; dRess_t ress = FIO_createDResources(dictFileName); + if (suffix==NULL) EXM_THROW(70, "zstd: decompression: unknown dst"); /* should never happen */ + if (!strcmp(suffix, stdoutmark) || !strcmp(suffix, nulmark)) { unsigned u; ress.dstFile = FIO_openDstFile(suffix); if (ress.dstFile == 0) EXM_THROW(71, "cannot open %s", suffix); for (u=0; u Date: Thu, 14 Jul 2016 16:52:45 +0200 Subject: [PATCH 25/42] fixed conversion warning --- lib/compress/zstd_compress.c | 19 +++++++++++++------ lib/dictBuilder/zdict.c | 3 +-- programs/paramgrill.c | 2 +- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 15a79d705..a53c172f6 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -152,7 +152,7 @@ ZSTD_CCtx* ZSTD_createCCtx(void) ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) { - ZSTD_CCtx* ctx; + ZSTD_CCtx* cctx; if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem; @@ -160,11 +160,11 @@ ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) if (!customMem.customAlloc || !customMem.customFree) return NULL; - ctx = (ZSTD_CCtx*) customMem.customAlloc(customMem.opaque, sizeof(ZSTD_CCtx)); - if (!ctx) return NULL; - memset(ctx, 0, sizeof(ZSTD_CCtx)); - memcpy(&ctx->customMem, &customMem, sizeof(ZSTD_customMem)); - return ctx; + cctx = (ZSTD_CCtx*) customMem.customAlloc(customMem.opaque, sizeof(ZSTD_CCtx)); + if (!cctx) return NULL; + memset(cctx, 0, sizeof(ZSTD_CCtx)); + memcpy(&(cctx->customMem), &customMem, sizeof(ZSTD_customMem)); + return cctx; } size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx) @@ -2321,8 +2321,15 @@ _storeSequence: /* Save reps for next block */ ctx->savedRep[0] = offset_1; ctx->savedRep[1] = offset_2; + static unsigned nbBlocks = 0; + printf("nbBlocks : %u \n", ++nbBlocks); + if (nbBlocks == 185) + printf("@"); + /* Last Literals */ { size_t const lastLLSize = iend - anchor; + if (lastLLSize == 4181) + printf("~"); memcpy(seqStorePtr->lit, anchor, lastLLSize); seqStorePtr->lit += lastLLSize; } diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index 5ae49456b..44dca8d2d 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -710,8 +710,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, goto _cleanup; } if (compressionLevel==0) compressionLevel=g_compressionLevel_default; - params.cParams = ZSTD_getCParams(compressionLevel, averageSampleSize, dictBufferSize); - params.fParams.contentSizeFlag = 0; + params = ZSTD_getParams(compressionLevel, averageSampleSize, dictBufferSize); { size_t const beginResult = ZSTD_compressBegin_advanced(esr.ref, dictBuffer, dictBufferSize, params, 0); if (ZSTD_isError(beginResult)) { eSize = ERROR(GENERIC); diff --git a/programs/paramgrill.c b/programs/paramgrill.c index da49cf36f..3e5d4ba7b 100644 --- a/programs/paramgrill.c +++ b/programs/paramgrill.c @@ -222,7 +222,7 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr, double fastestC = 100000000., fastestD = 100000000.; double ratio = 0.; U64 crcCheck = 0; - time_t const benchStart = clock(); + clock_t const benchStart = clock(); DISPLAY("\r%79s\r", ""); memset(¶ms, 0, sizeof(params)); From 8847238cac146b8642a5c4e944de8221c025b739 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 14 Jul 2016 17:05:38 +0200 Subject: [PATCH 26/42] simplified ZSTD_estimateCCtxSize() --- lib/common/zstd.h | 4 ++-- lib/compress/zstd_compress.c | 10 +++------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/lib/common/zstd.h b/lib/common/zstd.h index f30e76cc3..b52b1de64 100644 --- a/lib/common/zstd.h +++ b/lib/common/zstd.h @@ -215,7 +215,7 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, #define ZSTD_HASHLOG_MAX ZSTD_WINDOWLOG_MAX #define ZSTD_HASHLOG_MIN 12 #define ZSTD_HASHLOG3_MAX 17 -#define ZSTD_HASHLOG3_MIN 15 +//#define ZSTD_HASHLOG3_MIN 15 #define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1) #define ZSTD_SEARCHLOG_MIN 1 #define ZSTD_SEARCHLENGTH_MAX 7 @@ -265,7 +265,7 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v /*! ZSTD_estimateCCtxSize() : * Gives the amount of memory allocated for a ZSTD_CCtx given a set of compression parameters. * `frameContentSize` is an optional parameter, provide `0` if unknown */ -ZSTDLIB_API size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams, unsigned long long frameContentSize); +ZSTDLIB_API size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams); /*! ZSTD_createCCtx_advanced() : * Create a ZSTD compression context using external alloc and free functions */ diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index a53c172f6..c42f56ebc 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -249,7 +249,7 @@ ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, u } -size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams, unsigned long long frameContentSize) +size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams) { const size_t blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog); const U32 divider = (cParams.searchLength==3) ? 3 : 4; @@ -258,9 +258,7 @@ size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams, unsigned long l const size_t chainSize = (cParams.strategy == ZSTD_fast) ? 0 : (1 << cParams.chainLog); const size_t hSize = ((size_t)1) << cParams.hashLog; - const U32 hashLog3 = (cParams.searchLength>3) ? 0 : - ( (!frameContentSize || frameContentSize >= 8192) ? ZSTD_HASHLOG3_MAX : - ((frameContentSize >= 2048) ? ZSTD_HASHLOG3_MIN + 1 : ZSTD_HASHLOG3_MIN) ); + const U32 hashLog3 = (cParams.searchLength>3) ? 0 : MIN(ZSTD_HASHLOG3_MAX, cParams.windowLog); const size_t h3Size = ((size_t)1) << hashLog3; const size_t tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); @@ -283,9 +281,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, const size_t tokenSpace = blockSize + 11*maxNbSeq; const size_t chainSize = (params.cParams.strategy == ZSTD_fast) ? 0 : (1 << params.cParams.chainLog); const size_t hSize = ((size_t)1) << params.cParams.hashLog; - const U32 hashLog3 = (params.cParams.searchLength>3) ? 0 : - ( (!frameContentSize || frameContentSize >= 8192) ? ZSTD_HASHLOG3_MAX : - ((frameContentSize >= 2048) ? ZSTD_HASHLOG3_MIN + 1 : ZSTD_HASHLOG3_MIN) ); + const U32 hashLog3 = (params.cParams.searchLength>3) ? 0 : MIN(ZSTD_HASHLOG3_MAX, params.cParams.windowLog); const size_t h3Size = ((size_t)1) << hashLog3; const size_t tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); From 17508f1a167bdce4c26c77e8cc7f78509f976970 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 14 Jul 2016 17:18:20 +0200 Subject: [PATCH 27/42] fixed a few minor coverity warnings --- lib/dictBuilder/zdict.c | 7 ++++--- programs/fileio.c | 14 ++++++++------ 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index 44dca8d2d..aee654773 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -514,8 +514,9 @@ static size_t ZDICT_trainBuffer(dictItem* dictList, U32 dictListSize, /* sort */ DISPLAYLEVEL(2, "sorting %u files of total size %u MB ...\n", nbFiles, (U32)(bufferSize>>20)); - { int const divSuftSortResult = divsufsort((const unsigned char*)buffer, suffix, (int)bufferSize, 0); - if (divSuftSortResult != 0) { result = ERROR(GENERIC); goto _cleanup; } } + { int const divSuftSortResult = divsufsort((const unsigned char*)buffer, suffix, (int)bufferSize, 0); + if (divSuftSortResult != 0) { result = ERROR(GENERIC); goto _cleanup; } + } suffix[bufferSize] = (int)bufferSize; /* leads into noise */ suffix0[0] = (int)bufferSize; /* leads into noise */ /* build reverse suffix sort */ @@ -933,7 +934,7 @@ size_t ZDICT_trainFromBuffer_unsafe( /* init */ { unsigned u; for (u=0, sBuffSize=0; u Date: Thu, 14 Jul 2016 17:46:38 +0200 Subject: [PATCH 28/42] removed debugging traces --- lib/compress/zstd_compress.c | 7 ------- 1 file changed, 7 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index c42f56ebc..e58383344 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2317,15 +2317,8 @@ _storeSequence: /* Save reps for next block */ ctx->savedRep[0] = offset_1; ctx->savedRep[1] = offset_2; - static unsigned nbBlocks = 0; - printf("nbBlocks : %u \n", ++nbBlocks); - if (nbBlocks == 185) - printf("@"); - /* Last Literals */ { size_t const lastLLSize = iend - anchor; - if (lastLLSize == 4181) - printf("~"); memcpy(seqStorePtr->lit, anchor, lastLLSize); seqStorePtr->lit += lastLLSize; } From e20d5cf1176d97df3e33ae05f443290b61710978 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 14 Jul 2016 20:46:24 +0200 Subject: [PATCH 29/42] fixed paramgrill --- programs/paramgrill.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/programs/paramgrill.c b/programs/paramgrill.c index 3e5d4ba7b..04a55c876 100644 --- a/programs/paramgrill.c +++ b/programs/paramgrill.c @@ -391,8 +391,8 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para double W_DMemUsed_note = W_ratioNote * ( 40 + 9*cLevel) - log((double)W_DMemUsed); double O_DMemUsed_note = O_ratioNote * ( 40 + 9*cLevel) - log((double)O_DMemUsed); - size_t W_CMemUsed = (1 << params.windowLog) + ZSTD_estimateCCtxSize(params, srcSize); - size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + ZSTD_estimateCCtxSize(winners[cLevel].params, srcSize); + size_t W_CMemUsed = (1 << params.windowLog) + ZSTD_estimateCCtxSize(params); + size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + ZSTD_estimateCCtxSize(winners[cLevel].params); double W_CMemUsed_note = W_ratioNote * ( 50 + 13*cLevel) - log((double)W_CMemUsed); double O_CMemUsed_note = O_ratioNote * ( 50 + 13*cLevel) - log((double)O_CMemUsed); From e9ed5cdc9460e9a894de5735cbb837a5818c6bfc Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 14 Jul 2016 21:02:57 +0200 Subject: [PATCH 30/42] fixed minor coverity warning --- lib/dictBuilder/zdict.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index aee654773..27e95d76a 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -929,8 +929,8 @@ size_t ZDICT_trainFromBuffer_unsafe( size_t dictSize = 0; /* checks */ - if (maxDictSize <= g_provision_entropySize + g_min_fast_dictContent) return ERROR(dstSize_tooSmall); if (!dictList) return ERROR(memory_allocation); + if (maxDictSize <= g_provision_entropySize + g_min_fast_dictContent) { free(dictList); return ERROR(dstSize_tooSmall); } /* init */ { unsigned u; for (u=0, sBuffSize=0; u Date: Thu, 14 Jul 2016 22:43:12 +0200 Subject: [PATCH 31/42] fixed issue with small dictionary --- .coverity.yml | 5 +++++ NEWS | 3 +++ lib/compress/zstd_compress.c | 9 +++++---- 3 files changed, 13 insertions(+), 4 deletions(-) create mode 100644 .coverity.yml diff --git a/.coverity.yml b/.coverity.yml new file mode 100644 index 000000000..907f09601 --- /dev/null +++ b/.coverity.yml @@ -0,0 +1,5 @@ +configurationVersion: 1 + +filters: + # third-party embedded + - filePath: lib/dictBuilder/divsufsort.c diff --git a/NEWS b/NEWS index c34f3175a..cecc8f498 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,9 @@ v0.7.4 +Added : new examples +Fixed : segfault when using small dictionaries Modified : default compression level for CLI is 3 + v0.7.3 New : compression format specification New : `--` separator, stating that all following arguments are file names. Suggested by Chip Turner. diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index e58383344..ae182a973 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -273,9 +273,10 @@ size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams) /*! ZSTD_resetCCtx_advanced() : note : 'params' is expected to be validated */ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, - ZSTD_parameters params, U64 frameContentSize, U32 reset) + ZSTD_parameters params, U64 frameContentSize, + U32 reset, U32 fullBlockSize) { /* note : params considered validated here */ - const size_t blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << params.cParams.windowLog); + const size_t blockSize = fullBlockSize ? ZSTD_BLOCKSIZE_MAX : MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << params.cParams.windowLog); const U32 divider = (params.cParams.searchLength==3) ? 3 : 4; const size_t maxNbSeq = blockSize / divider; const size_t tokenSpace = blockSize + 11*maxNbSeq; @@ -357,7 +358,7 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx) if (srcCCtx->stage!=1) return ERROR(stage_wrong); memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem)); - ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params, srcCCtx->frameContentSize, 0); + ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params, srcCCtx->frameContentSize, 0, 1); dstCCtx->params.fParams.contentSizeFlag = 0; /* content size different from the one set during srcCCtx init */ /* copy tables */ @@ -2694,7 +2695,7 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* zc, const void* dict, size_t dictSize, ZSTD_parameters params, U64 pledgedSrcSize) { - size_t const resetError = ZSTD_resetCCtx_advanced(zc, params, pledgedSrcSize, 1); + size_t const resetError = ZSTD_resetCCtx_advanced(zc, params, pledgedSrcSize, 1, 0); if (ZSTD_isError(resetError)) return resetError; return ZSTD_compress_insertDictionary(zc, dict, dictSize); From 00bbb6bfef0974640e13ad3014a5c3245c6bcdf3 Mon Sep 17 00:00:00 2001 From: Cade Daniel Date: Thu, 14 Jul 2016 16:42:50 -0700 Subject: [PATCH 32/42] Adding brew formula for easy install --- zstd.rb | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 zstd.rb diff --git a/zstd.rb b/zstd.rb new file mode 100644 index 000000000..5f2f6529e --- /dev/null +++ b/zstd.rb @@ -0,0 +1,18 @@ +class Zstd < Formula + desc "Zstandard - Fast real-time compression algorithm" + homepage "http://www.zstd.net/" + url "https://github.com/Cyan4973/zstd/archive/v0.7.3.tar.gz" + sha256 "767da2a321b70d57a0f0776c39192a6c235c8f1fd7f1268eafde94a8869c3c71" + + def install + system "make", "install", "PREFIX=#{prefix}" + end + + test do + (testpath/"input.txt").write("Hello, world." * 10) + system "#{bin}/zstd", "input.txt", "-o", "compressed.zst" + system "#{bin}/zstd", "--test", "compressed.zst" + system "#{bin}/zstd", "-d", "compressed.zst", "-o", "decompressed.txt" + system "cmp", "input.txt", "decompressed.txt" + end +end From 227cc39e15ca914ff8b9633c96fd9b6c301a5fc7 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 15 Jul 2016 11:27:09 +0200 Subject: [PATCH 33/42] improved efficiency for large messages with small dictionaries --- NEWS | 2 +- lib/compress/zstd_compress.c | 2 +- programs/fuzzer.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/NEWS b/NEWS index cecc8f498..8c2808e0e 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,6 @@ v0.7.4 Added : new examples -Fixed : segfault when using small dictionaries +Fixed : segfault when using small dictionaries, reported by Felix Handte Modified : default compression level for CLI is 3 diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index ae182a973..98239b5d1 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2695,7 +2695,7 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* zc, const void* dict, size_t dictSize, ZSTD_parameters params, U64 pledgedSrcSize) { - size_t const resetError = ZSTD_resetCCtx_advanced(zc, params, pledgedSrcSize, 1, 0); + size_t const resetError = ZSTD_resetCCtx_advanced(zc, params, pledgedSrcSize, 1, (pledgedSrcSize==0) ); if (ZSTD_isError(resetError)) return resetError; return ZSTD_compress_insertDictionary(zc, dict, dictSize); diff --git a/programs/fuzzer.c b/programs/fuzzer.c index e56d6e1d6..b95e930d0 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -209,7 +209,7 @@ static int basicUnitTests(U32 seed, double compressibility) cSize += r); CHECKPLUS(r, ZSTD_compressEnd(ctxDuplicated, (char*)compressedBuffer+cSize, ZSTD_compressBound(CNBuffSize)-cSize), cSize += r); - if (cSize != cSizeOrig) goto _output_error; /* should be identical ==> have same size */ + if (cSize != cSizeOrig) goto _output_error; /* should be identical ==> same size */ } DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100); From 961b6a0e348bc36cedb158b3208de639f1fd12ae Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 15 Jul 2016 11:56:53 +0200 Subject: [PATCH 34/42] ZSTD_compressBlock() limits block size depending on windowLog parameter --- lib/common/zstd.h | 2 +- lib/compress/zstd_compress.c | 11 ++++++----- lib/dictBuilder/zdict.c | 7 ++++--- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/lib/common/zstd.h b/lib/common/zstd.h index b52b1de64..47bae1157 100644 --- a/lib/common/zstd.h +++ b/lib/common/zstd.h @@ -441,7 +441,7 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ds User will have to take in charge required information to regenerate data, such as compressed and content sizes. A few rules to respect : - - Uncompressed block size must be <= ZSTD_BLOCKSIZE_MAX (128 KB) + - Uncompressed block size must be <= MIN (128 KB, 1 << windowLog) + If you need to compress more, cut data into multiple blocks + Consider using the regular ZSTD_compress() instead, as frame metadata costs become negligible when source size is large. - Compressing and decompressing require a context structure diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 98239b5d1..52dc72dd7 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -274,9 +274,9 @@ size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams) note : 'params' is expected to be validated */ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, ZSTD_parameters params, U64 frameContentSize, - U32 reset, U32 fullBlockSize) + U32 reset) { /* note : params considered validated here */ - const size_t blockSize = fullBlockSize ? ZSTD_BLOCKSIZE_MAX : MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << params.cParams.windowLog); + const size_t blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << params.cParams.windowLog); const U32 divider = (params.cParams.searchLength==3) ? 3 : 4; const size_t maxNbSeq = blockSize / divider; const size_t tokenSpace = blockSize + 11*maxNbSeq; @@ -358,7 +358,7 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx) if (srcCCtx->stage!=1) return ERROR(stage_wrong); memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem)); - ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params, srcCCtx->frameContentSize, 0, 1); + ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params, srcCCtx->frameContentSize, 0); dstCCtx->params.fParams.contentSizeFlag = 0; /* content size different from the one set during srcCCtx init */ /* copy tables */ @@ -2560,7 +2560,8 @@ size_t ZSTD_compressContinue (ZSTD_CCtx* zc, size_t ZSTD_compressBlock(ZSTD_CCtx* zc, void* dst, size_t dstCapacity, const void* src, size_t srcSize) { - if (srcSize > ZSTD_BLOCKSIZE_MAX) return ERROR(srcSize_wrong); + size_t const blockSizeMax = MIN (ZSTD_BLOCKSIZE_MAX, 1 << zc->params.cParams.windowLog); + if (srcSize > blockSizeMax) return ERROR(srcSize_wrong); ZSTD_LOG_BLOCK("%p: ZSTD_compressBlock searchLength=%d\n", zc->base, zc->params.cParams.searchLength); return ZSTD_compressContinue_internal(zc, dst, dstCapacity, src, srcSize, 0); } @@ -2695,7 +2696,7 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* zc, const void* dict, size_t dictSize, ZSTD_parameters params, U64 pledgedSrcSize) { - size_t const resetError = ZSTD_resetCCtx_advanced(zc, params, pledgedSrcSize, 1, (pledgedSrcSize==0) ); + size_t const resetError = ZSTD_resetCCtx_advanced(zc, params, pledgedSrcSize, 1); if (ZSTD_isError(resetError)) return resetError; return ZSTD_compress_insertDictionary(zc, dict, dictSize); diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index 27e95d76a..0378a313a 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -581,13 +581,14 @@ typedef struct #define MAXREPOFFSET 1024 -static void ZDICT_countEStats(EStats_ress_t esr, +static void ZDICT_countEStats(EStats_ress_t esr, ZSTD_parameters params, U32* countLit, U32* offsetcodeCount, U32* matchlengthCount, U32* litlengthCount, U32* repOffsets, const void* src, size_t srcSize) { + size_t const blockSizeMax = MIN (ZSTD_BLOCKSIZE_MAX, 1 << params.cParams.windowLog); size_t cSize; - if (srcSize > ZSTD_BLOCKSIZE_MAX) srcSize = ZSTD_BLOCKSIZE_MAX; /* protection vs large samples */ + if (srcSize > blockSizeMax) srcSize = blockSizeMax; /* protection vs large samples */ { size_t const errorCode = ZSTD_copyCCtx(esr.zc, esr.ref); if (ZSTD_isError(errorCode)) { DISPLAYLEVEL(1, "warning : ZSTD_copyCCtx failed \n"); return; } } @@ -721,7 +722,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, /* collect stats on all files */ for (u=0; u Date: Fri, 15 Jul 2016 12:20:26 +0200 Subject: [PATCH 35/42] adapted fuzzer test to new blockSizeMax rule for ZSTD_compressBlock() --- programs/fuzzer.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/programs/fuzzer.c b/programs/fuzzer.c index b95e930d0..77a711869 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -321,8 +321,8 @@ static int basicUnitTests(U32 seed, double compressibility) /* block API tests */ { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); ZSTD_DCtx* const dctx = ZSTD_createDCtx(); - static const size_t blockSize = 100 KB; - static const size_t dictSize = 16 KB; + static const size_t dictSize = 65 KB; + static const size_t blockSize = 100 KB; /* won't cause pb with small dict size */ size_t cSize2; /* basic block compression */ From 98c8884999397d324a1457dca25f10eeec624769 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 15 Jul 2016 16:12:38 +0200 Subject: [PATCH 36/42] added target zstd in root Makefile --- Makefile | 4 +++- examples/simple_compression.c | 19 +++++++++---------- lib/compress/huf_compress.c | 1 + lib/compress/zstd_compress.c | 2 +- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index b11fc5774..9f5e1ebfa 100644 --- a/Makefile +++ b/Makefile @@ -41,7 +41,7 @@ else VOID = /dev/null endif -.PHONY: default all zlibwrapper zstdprogram clean install uninstall travis-install test clangtest gpptest armtest usan asan uasan +.PHONY: default all zlibwrapper zstdprogram zstd clean install uninstall travis-install test clangtest gpptest armtest usan asan uasan default: zstdprogram @@ -53,6 +53,8 @@ zstdprogram: $(MAKE) -C $(PRGDIR) cp $(PRGDIR)/zstd . +zstd: zstdprogram + zlibwrapper: $(MAKE) -C $(ZSTDDIR) all $(MAKE) -C $(ZWRAPDIR) all diff --git a/examples/simple_compression.c b/examples/simple_compression.c index 08c6c9f51..71a40c271 100644 --- a/examples/simple_compression.c +++ b/examples/simple_compression.c @@ -24,7 +24,7 @@ */ #include // malloc, exit -#include // printf +#include // fprintf, perror #include // strerror #include // errno #include // stat @@ -36,7 +36,7 @@ static off_t fsize_X(const char *filename) struct stat st; if (stat(filename, &st) == 0) return st.st_size; /* error */ - printf("stat: %s : %s \n", filename, strerror(errno)); + perror(filename); exit(1); } @@ -45,7 +45,7 @@ static FILE* fopen_X(const char *filename, const char *instruction) FILE* const inFile = fopen(filename, instruction); if (inFile) return inFile; /* error */ - printf("fopen: %s : %s \n", filename, strerror(errno)); + perror(filename); exit(2); } @@ -54,7 +54,7 @@ static void* malloc_X(size_t size) void* const buff = malloc(size); if (buff) return buff; /* error */ - printf("malloc: %s \n", strerror(errno)); + perror(NULL); exit(3); } @@ -65,7 +65,7 @@ static void* loadFile_X(const char* fileName, size_t* size) void* const buffer = malloc_X(buffSize); size_t const readSize = fread(buffer, 1, buffSize, inFile); if (readSize != (size_t)buffSize) { - printf("fread: %s : %s \n", fileName, strerror(errno)); + fprintf(stderr, "fread: %s : %s \n", fileName, strerror(errno)); exit(4); } fclose(inFile); @@ -79,12 +79,11 @@ static void saveFile_X(const char* fileName, const void* buff, size_t buffSize) FILE* const oFile = fopen_X(fileName, "wb"); size_t const wSize = fwrite(buff, 1, buffSize, oFile); if (wSize != (size_t)buffSize) { - printf("fwrite: %s : %s \n", fileName, strerror(errno)); + fprintf(stderr, "fwrite: %s : %s \n", fileName, strerror(errno)); exit(5); } - size_t const closeError = fclose(oFile); - if (closeError) { - printf("fclose: %s : %s \n", fileName, strerror(errno)); + if (fclose(oFile)) { + perror(fileName); exit(6); } } @@ -99,7 +98,7 @@ static void compress(const char* fname, const char* oname) size_t const cSize = ZSTD_compress(cBuff, cBuffSize, fBuff, fSize, 1); if (ZSTD_isError(cSize)) { - printf("error compressing %s : %s \n", fname, ZSTD_getErrorName(cSize)); + fprintf(stderr, "error compressing %s : %s \n", fname, ZSTD_getErrorName(cSize)); exit(7); } diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c index 301b5f912..b5b0eb440 100644 --- a/lib/compress/huf_compress.c +++ b/lib/compress/huf_compress.c @@ -535,6 +535,7 @@ static size_t HUF_compress_internal ( { size_t const hSize = HUF_writeCTable (op, dstSize, CTable, maxSymbolValue, huffLog); if (HUF_isError(hSize)) return hSize; if (hSize + 12 >= srcSize) return 0; /* not useful to try compression */ + //static U64 totalHSize = 0; static U32 nbHSize = 0; totalHSize += hSize; nbHSize++; if ((nbHSize & 63) == 1) printf("average : %6.3f \n", (double)totalHSize / nbHSize); op += hSize; } diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 52dc72dd7..60af2e71c 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -657,7 +657,7 @@ static size_t ZSTD_compressLiterals (ZSTD_CCtx* zc, : HUF_compress2 (ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 12); } - if ((cLitSize==0) || (cLitSize >= srcSize - minGain)) + if ((cLitSize==0) | (cLitSize >= srcSize - minGain)) return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); if (cLitSize==1) return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize); From f6ff53cd4ed1158925885d722a8538822b59f072 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 15 Jul 2016 17:03:38 +0200 Subject: [PATCH 37/42] implemented dictID reserved ranges --- lib/dictBuilder/zdict.c | 4 +++- zstd_compression_format.md | 25 +++++++++++++++++++++---- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index 0378a313a..c8c8ae301 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -894,7 +894,8 @@ size_t ZDICT_addEntropyTablesFromBuffer_advanced(void* dictBuffer, size_t dictCo /* dictionary header */ MEM_writeLE32(dictBuffer, ZSTD_DICT_MAGIC); { U64 const randomID = XXH64((char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize, 0); - U32 const dictID = params.dictID ? params.dictID : (U32)(randomID>>11); + U32 const compliantID = (randomID % ((1U<<31)-32768)) + 32768; + U32 const dictID = params.dictID ? params.dictID : compliantID; MEM_writeLE32((char*)dictBuffer+4, dictID); } hSize = 8; @@ -912,6 +913,7 @@ size_t ZDICT_addEntropyTablesFromBuffer_advanced(void* dictBuffer, size_t dictCo return MIN(dictBufferCapacity, hSize+dictContentSize); } + #define DIB_MINSAMPLESSIZE (DIB_FASTSEGMENTSIZE*3) /*! ZDICT_trainFromBuffer_unsafe() : * `samplesBuffer` must be followed by noisy guard band. diff --git a/zstd_compression_format.md b/zstd_compression_format.md index d432f1166..c6afeab1c 100644 --- a/zstd_compression_format.md +++ b/zstd_compression_format.md @@ -16,7 +16,7 @@ Distribution of this document is unlimited. ### Version -0.1.0 (08/07/16) +0.1.1 (15/07/16) Introduction @@ -258,9 +258,9 @@ depending on local limitations. __Dictionary ID__ -This is a variable size field, which contains an ID. -It checks if the correct dictionary is used for decoding. -Note that this field is optional. If it's not present, +This is a variable size field, which contains +the ID of the dictionary required to properly decode the frame. +Note that this field is optional. When it's not present, it's up to the caller to make sure it uses the correct dictionary. Field size depends on __Dictionary ID flag__. @@ -271,6 +271,15 @@ Field size depends on __Dictionary ID flag__. It's allowed to represent a small ID (for example `13`) with a large 4-bytes dictionary ID, losing some compacity in the process. +_Reserved ranges :_ +If the frame is going to be distributed in a private environment, +any dictionary ID can be used. +However, for public distribution of compressed frames using a dictionary, +some ranges are reserved for future use : +- low : 1 - 32767 : reserved +- high : >= (2^31) : reserved + + __Frame Content Size__ This is the original (uncompressed) size. @@ -1136,6 +1145,13 @@ __Header__ : 4 bytes ID, value 0xEC30A437, Little Endian format __Dict_ID__ : 4 bytes, stored in Little Endian format. DictID can be any value, except 0 (which means no DictID). It's used by decoders to check if they use the correct dictionary. + _Reserved ranges :_ + If the frame is going to be distributed in a private environment, + any dictionary ID can be used. + However, for public distribution of compressed frames, + some ranges are reserved for future use : + - low : 1 - 32767 : reserved + - high : >= (2^31) : reserved __Stats__ : Entropy tables, following the same format as a [compressed blocks]. They are stored in following order : @@ -1152,4 +1168,5 @@ __Content__ : Where the actual dictionary content is. Version changes --------------- +0.1.1 reserved dictID ranges 0.1.0 initial release From b21e9cbe8a30a25e22902905f7a5646d4890d516 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 15 Jul 2016 17:31:13 +0200 Subject: [PATCH 38/42] minor specification clarifications, suggested by @ebiggers --- zstd_compression_format.md | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/zstd_compression_format.md b/zstd_compression_format.md index c6afeab1c..95742386f 100644 --- a/zstd_compression_format.md +++ b/zstd_compression_format.md @@ -506,7 +506,7 @@ Compressed and regenerated size fields follow big endian convention. This section is only present when literals block type is `Compressed` (`0`). Prefix coding represents symbols from an a priori known alphabet -by bit sequences (codes), one code for each symbol, +by bit sequences (codewords), one codeword for each symbol, in a manner such that different symbols may be represented by bit sequences of different lengths, but a parser can always parse an encoded string @@ -515,14 +515,13 @@ unambiguously symbol-by-symbol. Given an alphabet with known symbol frequencies, the Huffman algorithm allows the construction of an optimal prefix code using the fewest bits of any possible prefix codes for that alphabet. -Such a code is called a Huffman code. Prefix code must not exceed a maximum code length. More bits improve accuracy but cost more header size, and require more memory for decoding operations. The current format limits the maximum depth to 15 bits. -The reference decoder goes further, by limiting it to 11 bits. +The reference decoder goes further, by limiting it to 12 bits. It is recommended to remain compatible with reference decoder. @@ -618,20 +617,19 @@ When both states have overflowed the bitstream, end is reached. ##### Conversion from weights to huffman prefix codes All present symbols shall now have a `weight` value. -A `weight` directly represents a `range` of prefix codes, -following the formulae : `range = weight ? 1 << (weight-1) : 0 ;` Symbols are sorted by weight. +Symbols with a weight of zero are removed. Within same weight, symbols keep natural order. Starting from lowest weight, -symbols are being allocated to a range of prefix codes. -Symbols with a weight of zero are not present. - -It is then possible to transform weights into nbBits : +symbols are being allocated to a `range`. +A `weight` directly represents a `range`, +following the formulae : `range = weight ? 1 << (weight-1) : 0 ;` +Similarly, it is possible to transform weights into nbBits : `nbBits = nbBits ? maxBits + 1 - weight : 0;` . __Example__ : -Let's presume the following huffman tree has been decoded : +Let's presume the following list of weights has been decoded : | Literal | 0 | 1 | 2 | 3 | 4 | 5 | | ------- | --- | --- | --- | --- | --- | --- | @@ -644,8 +642,9 @@ it gives the following distribution : | ------------ | --- | --- | --- | --- | --- | ---- | | weight | 0 | 1 | 1 | 2 | 3 | 4 | | range | 0 | 1 | 1 | 2 | 4 | 8 | -| prefix codes | N/A | 0 | 1 | 2-3 | 4-7 | 8-15 | +| table entries| N/A | 0 | 1 | 2-3 | 4-7 | 8-15 | | nb bits | 0 | 4 | 4 | 3 | 2 | 1 | +| prefix codes | N/A | 0000| 0001| 001 | 01 | 1 | #### Literals bitstreams @@ -696,12 +695,12 @@ it's possible to read the bitstream in a little-endian fashion, keeping track of already used bits. Reading the last `maxBits` bits, -it's then possible to compare extracted value to the prefix codes table, +it's then possible to compare extracted value to decoding table, determining the symbol to decode and number of bits to discard. The process continues up to reading the required number of symbols per stream. If a bitstream is not entirely and exactly consumed, -hence reaching exactly its beginning position with all bits consumed, +hence reaching exactly its beginning position with _all_ bits consumed, the decoding process is considered faulty. @@ -713,7 +712,7 @@ A literal copy command specifies a length. It is the number of bytes to be copied (or extracted) from the literal section. A match copy command specifies an offset and a length. The offset gives the position to copy from, -which can stand within a previous block. +which can be within a previous block. There are 3 symbol types, `literalLength`, `matchLength` and `offset`, which are encoded together, interleaved in a single _bitstream_. From 6cacd34d4455e5c8bdd7ab411e13537274564c3e Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 15 Jul 2016 17:58:13 +0200 Subject: [PATCH 39/42] minor formatting changes --- lib/compress/zstd_compress.c | 4 ++-- zstd_compression_format.md | 9 +++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 60af2e71c..26b6d6e33 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -653,8 +653,8 @@ static size_t ZSTD_compressLiterals (ZSTD_CCtx* zc, singleStream = 1; cLitSize = HUF_compress1X_usingCTable(ostart+lhSize, dstCapacity-lhSize, src, srcSize, zc->hufTable); } else { - cLitSize = singleStream ? HUF_compress1X(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 12) - : HUF_compress2 (ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 12); + cLitSize = singleStream ? HUF_compress1X(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11) + : HUF_compress2 (ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11); } if ((cLitSize==0) | (cLitSize >= srcSize - minGain)) diff --git a/zstd_compression_format.md b/zstd_compression_format.md index 95742386f..13c4ace18 100644 --- a/zstd_compression_format.md +++ b/zstd_compression_format.md @@ -1149,8 +1149,9 @@ __Dict_ID__ : 4 bytes, stored in Little Endian format. any dictionary ID can be used. However, for public distribution of compressed frames, some ranges are reserved for future use : - - low : 1 - 32767 : reserved - - high : >= (2^31) : reserved + + - low range : 1 - 32767 : reserved + - high range : >= (2^31) : reserved __Stats__ : Entropy tables, following the same format as a [compressed blocks]. They are stored in following order : @@ -1167,5 +1168,5 @@ __Content__ : Where the actual dictionary content is. Version changes --------------- -0.1.1 reserved dictID ranges -0.1.0 initial release +- 0.1.1 reserved dictID ranges +- 0.1.0 initial release From cadd7cd54fa5df69bdaa8ca8bfde95a940644b7a Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 15 Jul 2016 18:52:37 +0200 Subject: [PATCH 40/42] added dictionary_compression.c example --- examples/.gitignore | 1 + examples/Makefile | 9 +- examples/dictionary_compression.c | 163 ++++++++++++++++++++++++++++ examples/dictionary_decompression.c | 2 +- 4 files changed, 172 insertions(+), 3 deletions(-) create mode 100644 examples/dictionary_compression.c diff --git a/examples/.gitignore b/examples/.gitignore index 5c9836d3a..9d241dba6 100644 --- a/examples/.gitignore +++ b/examples/.gitignore @@ -1,6 +1,7 @@ #build simple_compression simple_decompression +dictionary_compression dictionary_decompression #test artefact diff --git a/examples/Makefile b/examples/Makefile index b20d14a74..5e3f0e17f 100644 --- a/examples/Makefile +++ b/examples/Makefile @@ -30,7 +30,8 @@ LDFLAGS+= -lzstd default: all -all: simple_compression simple_decompression dictionary_decompression +all: simple_compression simple_decompression \ + dictionary_compression dictionary_decompression simple_compression : simple_compression.c $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ @@ -38,12 +39,16 @@ simple_compression : simple_compression.c simple_decompression : simple_decompression.c $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ +dictionary_compression : dictionary_compression.c + $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ + dictionary_decompression : dictionary_decompression.c $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ clean: @rm -f core *.o tmp* result* *.zst \ - simple_compression simple_decompression dictionary_decompression + simple_compression simple_decompression \ + dictionary_compression dictionary_decompression @echo Cleaning completed test: all diff --git a/examples/dictionary_compression.c b/examples/dictionary_compression.c new file mode 100644 index 000000000..fc176a3db --- /dev/null +++ b/examples/dictionary_compression.c @@ -0,0 +1,163 @@ +/* + Dictionary decompression + Educational program using zstd library + Copyright (C) Yann Collet 2016 + + GPL v2 License + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - zstd homepage : http://www.zstd.net/ +*/ + +#include // malloc, exit +#include // printf +#include // strerror +#include // errno +#include // stat +#include // presumes zstd library is installed + + +static off_t fsize_X(const char *filename) +{ + struct stat st; + if (stat(filename, &st) == 0) return st.st_size; + /* error */ + perror(filename); + exit(1); +} + +static FILE* fopen_X(const char *filename, const char *instruction) +{ + FILE* const inFile = fopen(filename, instruction); + if (inFile) return inFile; + /* error */ + perror(filename); + exit(2); +} + +static void* malloc_X(size_t size) +{ + void* const buff = malloc(size); + if (buff) return buff; + /* error */ + perror(NULL); + exit(3); +} + +static void* loadFile_X(const char* fileName, size_t* size) +{ + off_t const buffSize = fsize_X(fileName); + FILE* const inFile = fopen_X(fileName, "rb"); + void* const buffer = malloc_X(buffSize); + size_t const readSize = fread(buffer, 1, buffSize, inFile); + if (readSize != (size_t)buffSize) { + fprintf(stderr, "fread: %s : %s \n", fileName, strerror(errno)); + exit(4); + } + fclose(inFile); + *size = buffSize; + return buffer; +} + +static void saveFile_X(const char* fileName, const void* buff, size_t buffSize) +{ + FILE* const oFile = fopen_X(fileName, "wb"); + size_t const wSize = fwrite(buff, 1, buffSize, oFile); + if (wSize != (size_t)buffSize) { + fprintf(stderr, "fwrite: %s : %s \n", fileName, strerror(errno)); + exit(5); + } + if (fclose(oFile)) { + perror(fileName); + exit(6); + } +} + +/* createDict() : + `dictFileName` is supposed to have been created using `zstd --train` */ +static const ZSTD_CDict* createDict(const char* dictFileName) +{ + size_t dictSize; + printf("loading dictionary %s \n", dictFileName); + void* const dictBuffer = loadFile_X(dictFileName, &dictSize); + const ZSTD_CDict* const ddict = ZSTD_createCDict(dictBuffer, dictSize, 3); + free(dictBuffer); + return ddict; +} + + +static void compress(const char* fname, const char* oname, const ZSTD_CDict* cdict) +{ + size_t fSize; + void* const fBuff = loadFile_X(fname, &fSize); + size_t const cBuffSize = ZSTD_compressBound(fSize); + void* const cBuff = malloc_X(cBuffSize); + + ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + size_t const cSize = ZSTD_compress_usingCDict(cctx, cBuff, cBuffSize, fBuff, fSize, cdict); + if (ZSTD_isError(cSize)) { + fprintf(stderr, "error compressing %s : %s \n", fname, ZSTD_getErrorName(cSize)); + exit(7); + } + + saveFile_X(oname, cBuff, cSize); + + /* success */ + printf("%25s : %6u -> %7u - %s \n", fname, (unsigned)fSize, (unsigned)cSize, oname); + + ZSTD_freeCCtx(cctx); + free(fBuff); + free(cBuff); +} + + +static char* createOutFilename(const char* filename) +{ + size_t const inL = strlen(filename); + size_t const outL = inL + 5; + void* outSpace = malloc_X(outL); + memset(outSpace, 0, outL); + strcat(outSpace, filename); + strcat(outSpace, ".zst"); + return (char*)outSpace; +} + +int main(int argc, const char** argv) +{ + const char* const exeName = argv[0]; + + if (argc<3) { + fprintf(stderr, "wrong arguments\n"); + fprintf(stderr, "usage:\n"); + fprintf(stderr, "%s [FILES] dictionary\n", exeName); + return 1; + } + + /* load dictionary only once */ + const char* const dictName = argv[argc-1]; + const ZSTD_CDict* const dictPtr = createDict(dictName); + + int u; + for (u=1; u Date: Fri, 15 Jul 2016 18:56:07 +0200 Subject: [PATCH 41/42] updated doc --- examples/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/examples/README.md b/examples/README.md index 594e2eaf8..2f4603881 100644 --- a/examples/README.md +++ b/examples/README.md @@ -9,6 +9,10 @@ Zstandard library : usage examples Decompress a single file compressed by zstd. Introduces usage of : `ZSTD_decompress()` +- [Dictionary compression](dictionary_compression.c) + Compress multiple files using the same dictionary. + Introduces usage of : `ZSTD_createCDict()` and `ZSTD_compress_usingCDict()` + - [Dictionary decompression](dictionary_decompression.c) Decompress multiple files using the same dictionary. Introduces usage of : `ZSTD_createDDict()` and `ZSTD_decompress_usingDDict()` From 988bcf360a41408e1c86c61dc1fd0a03672ac95a Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 15 Jul 2016 19:43:30 +0200 Subject: [PATCH 42/42] -v and --verbose increase display level by 1 --- programs/zstdcli.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 7b41865d1..4fa802698 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -264,7 +264,7 @@ int main(int argCount, const char** argv) if (!strcmp(argument, "--force")) { FIO_overwriteMode(); continue; } if (!strcmp(argument, "--version")) { displayOut=stdout; DISPLAY(WELCOME_MESSAGE); CLEAN_RETURN(0); } if (!strcmp(argument, "--help")) { displayOut=stdout; CLEAN_RETURN(usage_advanced(programName)); } - if (!strcmp(argument, "--verbose")) { displayLevel=4; continue; } + if (!strcmp(argument, "--verbose")) { displayLevel++; continue; } if (!strcmp(argument, "--quiet")) { displayLevel--; continue; } if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; displayLevel-=(displayLevel==2); continue; } if (!strcmp(argument, "--ultra")) { FIO_setMaxWLog(0); continue; } @@ -325,7 +325,7 @@ int main(int argCount, const char** argv) case 'f': FIO_overwriteMode(); forceStdout=1; argument++; break; /* Verbose mode */ - case 'v': displayLevel=4; argument++; break; + case 'v': displayLevel++; argument++; break; /* Quiet mode */ case 'q': displayLevel--; argument++; break;