From c6e845398a52cbed8af8623351b4ace286b59a70 Mon Sep 17 00:00:00 2001 From: Christophe Chevalier Date: Mon, 7 Dec 2015 17:44:09 +0100 Subject: [PATCH 01/22] Add ZSTDLIB_API macro to prefix all exported methods with__declspec(dllexport) when building a DLL (on windows) --- lib/zstd.h | 42 +++++++++++++++++++++++++++----------- lib/zstd_buffered.h | 32 ++++++++++++++--------------- lib/zstd_buffered_static.h | 2 +- lib/zstd_static.h | 36 ++++++++++++++++---------------- 4 files changed, 65 insertions(+), 47 deletions(-) diff --git a/lib/zstd.h b/lib/zstd.h index e4d441497..0795cbb08 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -43,6 +43,24 @@ extern "C" { #include /* size_t */ +/* *************************************************************** +* Tuning parameters +*****************************************************************/ +/*! +* ZSTD_DLL_EXPORT : +* Enable exporting of functions when building a DLL +*/ +#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# if _WIN32 +# define ZSTDLIB_API __declspec(dllexport) +# else +# define ZSTDLIB_API extern +# endif +#else +# define ZSTDLIB_API +#endif + + /* ************************************* * Version ***************************************/ @@ -50,18 +68,18 @@ extern "C" { #define ZSTD_VERSION_MINOR 4 /* for new (non-breaking) interface capabilities */ #define ZSTD_VERSION_RELEASE 3 /* for tweaks, bug-fixes, or development */ #define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) -unsigned ZSTD_versionNumber (void); +ZSTDLIB_API unsigned ZSTD_versionNumber (void); /* ************************************* * Simple functions ***************************************/ -size_t ZSTD_compress( void* dst, size_t maxDstSize, - const void* src, size_t srcSize, - int compressionLevel); +ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t maxDstSize, + const void* src, size_t srcSize, + int compressionLevel); -size_t ZSTD_decompress( void* dst, size_t maxOriginalSize, - const void* src, size_t compressedSize); +ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t maxOriginalSize, + const void* src, size_t compressedSize); /** ZSTD_compress() : @@ -83,25 +101,25 @@ ZSTD_decompress() : /* ************************************* * Tool functions ***************************************/ -size_t ZSTD_compressBound(size_t srcSize); /** maximum compressed size (worst case scenario) */ +ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /** maximum compressed size (worst case scenario) */ /* Error Management */ -unsigned ZSTD_isError(size_t code); /** tells if a return value is an error code */ -const char* ZSTD_getErrorName(size_t code); /** provides error code string */ +ZSTDLIB_API unsigned ZSTD_isError(size_t code); /** tells if a return value is an error code */ +ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /** provides error code string */ /* ************************************* * Advanced functions ***************************************/ typedef struct ZSTD_CCtx_s ZSTD_CCtx; /* incomplete type */ -ZSTD_CCtx* ZSTD_createCCtx(void); -size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); +ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void); +ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); /** ZSTD_compressCCtx() : Same as ZSTD_compress(), but requires a ZSTD_CCtx working space already allocated */ -size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel); +ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel); #if defined (__cplusplus) diff --git a/lib/zstd_buffered.h b/lib/zstd_buffered.h index 80ba819db..80e6e599d 100644 --- a/lib/zstd_buffered.h +++ b/lib/zstd_buffered.h @@ -51,13 +51,13 @@ extern "C" { * Streaming functions ***************************************/ typedef struct ZBUFF_CCtx_s ZBUFF_CCtx; -ZBUFF_CCtx* ZBUFF_createCCtx(void); -size_t ZBUFF_freeCCtx(ZBUFF_CCtx* cctx); +ZSTDLIB_API ZBUFF_CCtx* ZBUFF_createCCtx(void); +ZSTDLIB_API size_t ZBUFF_freeCCtx(ZBUFF_CCtx* cctx); -size_t ZBUFF_compressInit(ZBUFF_CCtx* cctx, int compressionLevel); -size_t ZBUFF_compressContinue(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr, const void* src, size_t* srcSizePtr); -size_t ZBUFF_compressFlush(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr); -size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr); +ZSTDLIB_API size_t ZBUFF_compressInit(ZBUFF_CCtx* cctx, int compressionLevel); +ZSTDLIB_API size_t ZBUFF_compressContinue(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr, const void* src, size_t* srcSizePtr); +ZSTDLIB_API size_t ZBUFF_compressFlush(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr); +ZSTDLIB_API size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr); /** ************************************************ * Streaming compression @@ -97,11 +97,11 @@ size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr); typedef struct ZBUFF_DCtx_s ZBUFF_DCtx; -ZBUFF_DCtx* ZBUFF_createDCtx(void); -size_t ZBUFF_freeDCtx(ZBUFF_DCtx* dctx); +ZSTDLIB_API ZBUFF_DCtx* ZBUFF_createDCtx(void); +ZSTDLIB_API size_t ZBUFF_freeDCtx(ZBUFF_DCtx* dctx); -size_t ZBUFF_decompressInit(ZBUFF_DCtx* dctx); -size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx, void* dst, size_t* maxDstSizePtr, const void* src, size_t* srcSizePtr); +ZSTDLIB_API size_t ZBUFF_decompressInit(ZBUFF_DCtx* dctx); +ZSTDLIB_API size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx, void* dst, size_t* maxDstSizePtr, const void* src, size_t* srcSizePtr); /** ************************************************ * Streaming decompression @@ -129,15 +129,15 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx, void* dst, size_t* maxDstSizeP /* ************************************* * Tool functions ***************************************/ -unsigned ZBUFF_isError(size_t errorCode); -const char* ZBUFF_getErrorName(size_t errorCode); +ZSTDLIB_API unsigned ZBUFF_isError(size_t errorCode); +ZSTDLIB_API const char* ZBUFF_getErrorName(size_t errorCode); /** The below functions provide recommended buffer sizes for Compression or Decompression operations. * These sizes are not compulsory, they just tend to offer better latency */ -size_t ZBUFF_recommendedCInSize(void); -size_t ZBUFF_recommendedCOutSize(void); -size_t ZBUFF_recommendedDInSize(void); -size_t ZBUFF_recommendedDOutSize(void); +ZSTDLIB_API size_t ZBUFF_recommendedCInSize(void); +ZSTDLIB_API size_t ZBUFF_recommendedCOutSize(void); +ZSTDLIB_API size_t ZBUFF_recommendedDInSize(void); +ZSTDLIB_API size_t ZBUFF_recommendedDOutSize(void); #if defined (__cplusplus) diff --git a/lib/zstd_buffered_static.h b/lib/zstd_buffered_static.h index cba1d6fca..7d9ee27d2 100644 --- a/lib/zstd_buffered_static.h +++ b/lib/zstd_buffered_static.h @@ -52,7 +52,7 @@ extern "C" { /* ************************************* * Advanced Streaming functions ***************************************/ -size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* cctx, ZSTD_parameters params); +ZSTDLIB_API size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* cctx, ZSTD_parameters params); #if defined (__cplusplus) diff --git a/lib/zstd_static.h b/lib/zstd_static.h index 5c3156248..6a987026a 100644 --- a/lib/zstd_static.h +++ b/lib/zstd_static.h @@ -85,29 +85,29 @@ typedef struct /** ZSTD_getParams * return ZSTD_parameters structure for a selected compression level and srcSize. * srcSizeHint value is optional, select 0 if not known */ -ZSTD_parameters ZSTD_getParams(int compressionLevel, U64 srcSizeHint); +ZSTDLIB_API ZSTD_parameters ZSTD_getParams(int compressionLevel, U64 srcSizeHint); /** ZSTD_validateParams * correct params value to remain within authorized range */ -void ZSTD_validateParams(ZSTD_parameters* params); +ZSTDLIB_API void ZSTD_validateParams(ZSTD_parameters* params); /** ZSTD_compress_advanced * Same as ZSTD_compressCCtx(), with fine-tune control of each compression parameter */ -size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx, - void* dst, size_t maxDstSize, - const void* src, size_t srcSize, - ZSTD_parameters params); +ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx, + void* dst, size_t maxDstSize, + const void* src, size_t srcSize, + ZSTD_parameters params); /* ************************************** * Streaming functions (bufferless mode) ****************************************/ -size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize, int compressionLevel); -size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, ZSTD_parameters params); -size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* ctx, const void* src, size_t srcSize); +ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize, int compressionLevel); +ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, ZSTD_parameters params); +ZSTDLIB_API size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* ctx, const void* src, size_t srcSize); -size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); -size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize); +ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); +ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize); /** Streaming compression, bufferless mode @@ -136,15 +136,15 @@ size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize); typedef struct ZSTD_DCtx_s ZSTD_DCtx; -ZSTD_DCtx* ZSTD_createDCtx(void); -size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); +ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void); +ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); -size_t ZSTD_resetDCtx(ZSTD_DCtx* dctx); -size_t ZSTD_getFrameParams(ZSTD_parameters* params, const void* src, size_t srcSize); -void ZSTD_decompress_insertDictionary(ZSTD_DCtx* ctx, const void* src, size_t srcSize); +ZSTDLIB_API size_t ZSTD_resetDCtx(ZSTD_DCtx* dctx); +ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_parameters* params, const void* src, size_t srcSize); +ZSTDLIB_API void ZSTD_decompress_insertDictionary(ZSTD_DCtx* ctx, const void* src, size_t srcSize); -size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx); -size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); +ZSTDLIB_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx); +ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); /** Streaming decompression, bufferless mode From d2199e72501737faaf87c4bdc4a33eb06eddb998 Mon Sep 17 00:00:00 2001 From: Christophe Chevalier Date: Mon, 7 Dec 2015 19:01:41 +0100 Subject: [PATCH 02/22] Added ZSTDLIB_API definition to zstd_buffered.h which does not seem to include zstd.h --- lib/zstd_buffered.h | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/lib/zstd_buffered.h b/lib/zstd_buffered.h index 80e6e599d..fffa70a05 100644 --- a/lib/zstd_buffered.h +++ b/lib/zstd_buffered.h @@ -47,6 +47,24 @@ extern "C" { #include /* size_t */ +/* *************************************************************** +* Tuning parameters +*****************************************************************/ +/*! +* ZSTD_DLL_EXPORT : +* Enable exporting of functions when building a DLL +*/ +#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# if _WIN32 +# define ZSTDLIB_API __declspec(dllexport) +# else +# define ZSTDLIB_API extern +# endif +#else +# define ZSTDLIB_API +#endif + + /* ************************************* * Streaming functions ***************************************/ From 0cde77bc5ec1f75e09c28b6b794c2ea73e5e4e8e Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 8 Dec 2015 14:47:46 +0100 Subject: [PATCH 03/22] fixed comment (reported by @annulen) --- lib/zstd_static.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/zstd_static.h b/lib/zstd_static.h index 5c3156248..2f2352f65 100644 --- a/lib/zstd_static.h +++ b/lib/zstd_static.h @@ -120,7 +120,7 @@ size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize); Use ZSTD_compressBegin(). You may also prefer the advanced derivative ZSTD_compressBegin_advanced(), for finer parameter control. - It's then possible to add a dictionary with ZSTD_compressDictionary() + It's then possible to add a dictionary with ZSTD_compress_insertDictionary() Note that dictionary presence is a "hidden" information, the decoder needs to be aware that it is required for proper decoding, or decoding will fail. From d66db2ff04e45dca5c3c0cf6f098a4bd1c70a70e Mon Sep 17 00:00:00 2001 From: Konstantin Tokarev Date: Tue, 8 Dec 2015 18:11:10 +0300 Subject: [PATCH 04/22] Move tests from zstd-playTests target to separate shell script. This patch allows me to run tests on MIPS board which lacks make. --- programs/Makefile | 72 +------------------------------------------ programs/playTests.sh | 62 +++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 71 deletions(-) create mode 100755 programs/playTests.sh diff --git a/programs/Makefile b/programs/Makefile index 6e9d6b56f..fb2f3524f 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -169,77 +169,7 @@ test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zbuff32 test-all: test test32 valgrindTest zstd-playTests: datagen - @echo "\n**** frame concatenation **** " - @echo "hello " > hello.tmp - @echo "world!" > world.tmp - @cat hello.tmp world.tmp > helloworld.tmp - $(ZSTD) hello.tmp > hello.zstd - $(ZSTD) world.tmp > world.zstd - @cat hello.zstd world.zstd > helloworld.zstd - $(ZSTD) -df helloworld.zstd > result.tmp - cat result.tmp - sdiff helloworld.tmp result.tmp - @rm *.tmp *.zstd - @echo frame concatenation test completed - @echo "**** flush write error test **** " - echo foo | $(ZSTD) > /dev/full; if [ $$? -eq 0 ] ; then echo "write error not detected!"; false; fi - echo foo | $(ZSTD) | $(ZSTD) -d > /dev/full; if [ $$? -eq 0 ] ; then echo "write error not detected!"; false; fi - @echo "**** zstd round-trip tests **** " - @./datagen | md5sum > tmp1 - ./datagen | $(ZSTD) -v | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen | $(ZSTD) -6 -v | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - @./datagen -g270000000 | md5sum > tmp1 - ./datagen -g270000000 | $(ZSTD) -v | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g270000000 | $(ZSTD) -v2 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g270000000 | $(ZSTD) -v3 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - @./datagen -g140000000 -P60| md5sum > tmp1 - ./datagen -g140000000 -P60 | $(ZSTD) -v4 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g140000000 -P60 | $(ZSTD) -v5 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g140000000 -P60 | $(ZSTD) -v6 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - @./datagen -g70000000 -P70 | md5sum > tmp1 - ./datagen -g70000000 -P70 | $(ZSTD) -v7 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g70000000 -P70 | $(ZSTD) -v8 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g70000000 -P70 | $(ZSTD) -v9 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - @./datagen -g35000000 -P75 | md5sum > tmp1 - ./datagen -g35000000 -P75 | $(ZSTD) -v10 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g35000000 -P75 | $(ZSTD) -v11 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g35000000 -P75 | $(ZSTD) -v12 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - @./datagen -g18000000 -P80 | md5sum > tmp1 - ./datagen -g18000000 -P80 | $(ZSTD) -v13 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g18000000 -P80 | $(ZSTD) -v14 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g18000000 -P80 | $(ZSTD) -v15 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g18000000 -P80 | $(ZSTD) -v16 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g18000000 -P80 | $(ZSTD) -v17 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - @./datagen -g50000000 -P94 | md5sum > tmp1 - ./datagen -g50000000 -P94 | $(ZSTD) -v18 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g50000000 -P94 | $(ZSTD) -v19 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - @./datagen -g99000000 -P99 | md5sum > tmp1 - ./datagen -g99000000 -P99 | $(ZSTD) -v20 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g6000000000 -P99| md5sum > tmp1 - ./datagen -g6000000000 -P99| $(ZSTD) -vq | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 + ZSTD=$(ZSTD) ./playTests.sh test-zstd: ZSTD = ./zstd test-zstd: zstd zstd-playTests diff --git a/programs/playTests.sh b/programs/playTests.sh new file mode 100755 index 000000000..08090f6bc --- /dev/null +++ b/programs/playTests.sh @@ -0,0 +1,62 @@ +#!/bin/sh -e + +die() { + echo "$@" 1>&2 + exit 1 +} + +echo "\n**** frame concatenation **** " + +echo "hello " > hello.tmp +echo "world!" > world.tmp +cat hello.tmp world.tmp > helloworld.tmp +$ZSTD hello.tmp > hello.zstd +$ZSTD world.tmp > world.zstd +cat hello.zstd world.zstd > helloworld.zstd +$ZSTD -df helloworld.zstd > result.tmp +cat result.tmp +sdiff helloworld.tmp result.tmp +rm *.tmp *.zstd + +echo frame concatenation test completed + +echo "**** flush write error test **** " + +echo foo | $ZSTD > /dev/full && die "write error not detected!" +echo foo | $ZSTD | $ZSTD -d > /dev/full && die "write error not detected!" + +echo "**** zstd round-trip tests **** " +./datagen | md5sum > tmp1 +./datagen | $ZSTD -v | $ZSTD -d | md5sum > tmp2 +diff tmp1 tmp2 +./datagen | $ZSTD -6 -v | $ZSTD -d | md5sum > tmp2 +diff tmp1 tmp2 +./datagen -g270000000 | md5sum > tmp1 +./datagen -g270000000 | $ZSTD -v | $ZSTD -d | md5sum > tmp2 +diff tmp1 tmp2 +./datagen -g270000000 | $ZSTD -v2 | $ZSTD -d | md5sum > tmp2 +diff tmp1 tmp2 +./datagen -g270000000 | $ZSTD -v3 | $ZSTD -d | md5sum > tmp2 +diff tmp1 tmp2 +./datagen -g140000000 -P60| md5sum > tmp1 +./datagen -g140000000 -P60 | $ZSTD -v4 | $ZSTD -d | md5sum > tmp2 +diff tmp1 tmp2 +./datagen -g140000000 -P60 | $ZSTD -v5 | $ZSTD -d | md5sum > tmp2 +diff tmp1 tmp2 +./datagen -g140000000 -P60 | $ZSTD -v6 | $ZSTD -d | md5sum > tmp2 +diff tmp1 tmp2 +./datagen -g70000000 -P70 | md5sum > tmp1 +./datagen -g70000000 -P70 | $ZSTD -v7 | $ZSTD -d | md5sum > tmp2 +diff tmp1 tmp2 +./datagen -g70000000 -P70 | $ZSTD -v8 | $ZSTD -d | md5sum > tmp2 +diff tmp1 tmp2 +./datagen -g70000000 -P70 | $ZSTD -v9 | $ZSTD -d | md5sum > tmp2 +diff tmp1 tmp2 +./datagen -g35000000 -P75 | md5sum > tmp1 +./datagen -g35000000 -P75 | $ZSTD -v10 | $ZSTD -d | md5sum > tmp2 +diff tmp1 tmp2 +./datagen -g35000000 -P75 | $ZSTD -v11 | $ZSTD -d | md5sum > tmp2 +diff tmp1 tmp2 +./datagen -g35000000 -P75 | $ZSTD -v12 | $ZSTD -d | md5sum > tmp2 +diff tmp1 tmp2 + From 76be3785131181a392491297e82a646bb0b7f61b Mon Sep 17 00:00:00 2001 From: Konstantin Tokarev Date: Tue, 8 Dec 2015 18:36:37 +0300 Subject: [PATCH 05/22] playTests.sh: refactored round-trip tests --- programs/playTests.sh | 73 ++++++++++++++++++++++--------------------- 1 file changed, 38 insertions(+), 35 deletions(-) diff --git a/programs/playTests.sh b/programs/playTests.sh index 08090f6bc..d29260aa8 100755 --- a/programs/playTests.sh +++ b/programs/playTests.sh @@ -5,7 +5,24 @@ die() { exit 1 } -echo "\n**** frame concatenation **** " +roundTripTest() { + if [ -n "$3" ]; then + local c="$3" + local p="$2" + else + local c="$2" + fi + + rm -f tmp1 tmp2 + echo "roundTripTest: ./datagen $1 $p | $ZSTD -v$c | $ZSTD -d" + ./datagen $1 $p | md5sum > tmp1 + ./datagen $1 $p | $ZSTD -v$c | $ZSTD -d | md5sum > tmp2 + diff -q tmp1 tmp2 +} + +[ -n "$ZSTD" ] || die "ZSTD variable must be defined!" + +printf "\n**** frame concatenation **** " echo "hello " > hello.tmp echo "world!" > world.tmp @@ -16,7 +33,7 @@ cat hello.zstd world.zstd > helloworld.zstd $ZSTD -df helloworld.zstd > result.tmp cat result.tmp sdiff helloworld.tmp result.tmp -rm *.tmp *.zstd +rm ./*.tmp ./*.zstd echo frame concatenation test completed @@ -26,37 +43,23 @@ echo foo | $ZSTD > /dev/full && die "write error not detected!" echo foo | $ZSTD | $ZSTD -d > /dev/full && die "write error not detected!" echo "**** zstd round-trip tests **** " -./datagen | md5sum > tmp1 -./datagen | $ZSTD -v | $ZSTD -d | md5sum > tmp2 -diff tmp1 tmp2 -./datagen | $ZSTD -6 -v | $ZSTD -d | md5sum > tmp2 -diff tmp1 tmp2 -./datagen -g270000000 | md5sum > tmp1 -./datagen -g270000000 | $ZSTD -v | $ZSTD -d | md5sum > tmp2 -diff tmp1 tmp2 -./datagen -g270000000 | $ZSTD -v2 | $ZSTD -d | md5sum > tmp2 -diff tmp1 tmp2 -./datagen -g270000000 | $ZSTD -v3 | $ZSTD -d | md5sum > tmp2 -diff tmp1 tmp2 -./datagen -g140000000 -P60| md5sum > tmp1 -./datagen -g140000000 -P60 | $ZSTD -v4 | $ZSTD -d | md5sum > tmp2 -diff tmp1 tmp2 -./datagen -g140000000 -P60 | $ZSTD -v5 | $ZSTD -d | md5sum > tmp2 -diff tmp1 tmp2 -./datagen -g140000000 -P60 | $ZSTD -v6 | $ZSTD -d | md5sum > tmp2 -diff tmp1 tmp2 -./datagen -g70000000 -P70 | md5sum > tmp1 -./datagen -g70000000 -P70 | $ZSTD -v7 | $ZSTD -d | md5sum > tmp2 -diff tmp1 tmp2 -./datagen -g70000000 -P70 | $ZSTD -v8 | $ZSTD -d | md5sum > tmp2 -diff tmp1 tmp2 -./datagen -g70000000 -P70 | $ZSTD -v9 | $ZSTD -d | md5sum > tmp2 -diff tmp1 tmp2 -./datagen -g35000000 -P75 | md5sum > tmp1 -./datagen -g35000000 -P75 | $ZSTD -v10 | $ZSTD -d | md5sum > tmp2 -diff tmp1 tmp2 -./datagen -g35000000 -P75 | $ZSTD -v11 | $ZSTD -d | md5sum > tmp2 -diff tmp1 tmp2 -./datagen -g35000000 -P75 | $ZSTD -v12 | $ZSTD -d | md5sum > tmp2 -diff tmp1 tmp2 + +roundTripTest +roundTripTest '' 6 + +roundTripTest -g270000000 1 +roundTripTest -g270000000 2 +roundTripTest -g270000000 3 + +roundTripTest -g140000000 -P60 4 +roundTripTest -g140000000 -P60 5 +roundTripTest -g140000000 -P60 6 + +roundTripTest -g70000000 -P70 7 +roundTripTest -g70000000 -P70 8 +roundTripTest -g70000000 -P70 9 + +roundTripTest -g35000000 -P75 10 +roundTripTest -g35000000 -P75 11 +roundTripTest -g35000000 -P75 12 From 0b570b59d2d5189d72cf35a81a355e5bb6357b97 Mon Sep 17 00:00:00 2001 From: Konstantin Tokarev Date: Tue, 8 Dec 2015 18:47:43 +0300 Subject: [PATCH 06/22] playTests.sh: Added --test-large-data switch. --- programs/Makefile | 2 +- programs/playTests.sh | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/programs/Makefile b/programs/Makefile index fb2f3524f..49061f945 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -169,7 +169,7 @@ test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zbuff32 test-all: test test32 valgrindTest zstd-playTests: datagen - ZSTD=$(ZSTD) ./playTests.sh + ZSTD=$(ZSTD) ./playTests.sh --test-large-data test-zstd: ZSTD = ./zstd test-zstd: zstd zstd-playTests diff --git a/programs/playTests.sh b/programs/playTests.sh index d29260aa8..e97f93db8 100755 --- a/programs/playTests.sh +++ b/programs/playTests.sh @@ -47,6 +47,11 @@ echo "**** zstd round-trip tests **** " roundTripTest roundTripTest '' 6 +if [ "$1" != "--test-large-data" ]; then + echo "Skipping large data tests" + exit 0 +fi + roundTripTest -g270000000 1 roundTripTest -g270000000 2 roundTripTest -g270000000 3 From 2b465840a89e1e8254165b436168492712dd294c Mon Sep 17 00:00:00 2001 From: Konstantin Tokarev Date: Tue, 8 Dec 2015 19:36:42 +0300 Subject: [PATCH 07/22] Re-added accidentally lost round-trip tests for compression levels > 12. --- programs/playTests.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/programs/playTests.sh b/programs/playTests.sh index e97f93db8..56327ff70 100755 --- a/programs/playTests.sh +++ b/programs/playTests.sh @@ -68,3 +68,14 @@ roundTripTest -g35000000 -P75 10 roundTripTest -g35000000 -P75 11 roundTripTest -g35000000 -P75 12 +roundTripTest -g18000000 -P80 13 +roundTripTest -g18000000 -P80 14 +roundTripTest -g18000000 -P80 15 +roundTripTest -g18000000 -P80 16 +roundTripTest -g18000000 -P80 17 + +roundTripTest -g50000000 -P94 18 +roundTripTest -g50000000 -P94 19 + +roundTripTest -g99000000 -P99 20 +roundTripTest -g6000000000 -P99 q From d608088ca37fd7172261cbe8aa9d98770d9b8c54 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 9 Dec 2015 09:05:22 +0100 Subject: [PATCH 08/22] added : ZSTD_maxCLevel() added : 256KB blocks mode --- lib/zstd_compress.c | 7 ++++--- lib/zstd_static.h | 27 ++++++++++++++++++++++++++- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index c18d8f86c..046c03541 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -65,6 +65,7 @@ /* ************************************* * Constants ***************************************/ +unsigned ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; } static const U32 g_searchStrength = 8; @@ -599,7 +600,7 @@ static unsigned ZSTD_NbCommonBytes (register size_t val) # if defined(_MSC_VER) && defined(_WIN64) unsigned long r = 0; _BitScanForward64( &r, (U64)val ); - return (int)(r>>3); + return (unsigned)(r>>3); # elif defined(__GNUC__) && (__GNUC__ >= 3) return (__builtin_ctzll((U64)val) >> 3); # else @@ -612,7 +613,7 @@ static unsigned ZSTD_NbCommonBytes (register size_t val) # if defined(_MSC_VER) unsigned long r=0; _BitScanForward( &r, (U32)val ); - return (int)(r>>3); + return (unsigned)(r>>3); # elif defined(__GNUC__) && (__GNUC__ >= 3) return (__builtin_ctz((U32)val) >> 3); # else @@ -2097,7 +2098,7 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* ctx, ZSTD_parameters ZSTD_getParams(int compressionLevel, U64 srcSizeHint) { ZSTD_parameters result; - int tableID = ((srcSizeHint-1) <= 128 KB) + ((srcSizeHint-1) <= 16 KB); /* intentional underflow for srcSizeHint == 0 */ + int tableID = ((srcSizeHint-1) <= 256 KB) + ((srcSizeHint-1) <= 128 KB) + ((srcSizeHint-1) <= 16 KB); /* intentional underflow for srcSizeHint == 0 */ if (compressionLevel<=0) compressionLevel = 1; if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL; result = ZSTD_defaultParameters[tableID][compressionLevel]; diff --git a/lib/zstd_static.h b/lib/zstd_static.h index 2f2352f65..c8aae4c13 100644 --- a/lib/zstd_static.h +++ b/lib/zstd_static.h @@ -180,7 +180,8 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, co * Pre-defined compression levels ***************************************/ #define ZSTD_MAX_CLEVEL 20 -static const ZSTD_parameters ZSTD_defaultParameters[3][ZSTD_MAX_CLEVEL+1] = { +unsigned ZSTD_maxCLevel (void); +static const ZSTD_parameters ZSTD_defaultParameters[4][ZSTD_MAX_CLEVEL+1] = { { /* "default" */ /* W, C, H, S, L, strat */ { 0, 18, 12, 12, 1, 4, ZSTD_fast }, /* level 0 - never used */ @@ -205,6 +206,30 @@ static const ZSTD_parameters ZSTD_defaultParameters[3][ZSTD_MAX_CLEVEL+1] = { { 0, 25, 26, 23, 5, 5, ZSTD_btlazy2 }, /* level 19 */ { 0, 26, 27, 25, 9, 5, ZSTD_btlazy2 }, /* level 20 */ }, +{ /* for srcSize <= 256 KB */ + /* W, C, H, S, L, strat */ + { 0, 0, 0, 0, 0, 0, ZSTD_fast }, /* level 0 - never used */ + { 0, 18, 16, 15, 1, 7, ZSTD_fast }, /* level 1 */ + { 0, 18, 16, 16, 1, 7, ZSTD_fast }, /* level 2 */ + { 0, 18, 18, 18, 1, 7, ZSTD_fast }, /* level 3 */ + { 0, 18, 14, 15, 4, 6, ZSTD_greedy }, /* level 4 */ + { 0, 18, 16, 16, 1, 6, ZSTD_lazy }, /* level 5 */ + { 0, 18, 15, 15, 3, 6, ZSTD_lazy }, /* level 6 */ + { 0, 18, 15, 15, 4, 6, ZSTD_lazy }, /* level 7 */ + { 0, 18, 16, 18, 4, 6, ZSTD_lazy }, /* level 8 */ + { 0, 18, 18, 18, 4, 6, ZSTD_lazy }, /* level 9 */ + { 0, 18, 18, 18, 5, 6, ZSTD_lazy }, /* level 10 */ + { 0, 18, 18, 19, 6, 6, ZSTD_lazy }, /* level 11 */ + { 0, 18, 18, 19, 7, 6, ZSTD_lazy }, /* level 12 */ + { 0, 18, 19, 15, 7, 5, ZSTD_btlazy2 }, /* level 13 */ + { 0, 18, 19, 16, 8, 5, ZSTD_btlazy2 }, /* level 14 */ + { 0, 18, 19, 17, 9, 5, ZSTD_btlazy2 }, /* level 15 */ + { 0, 18, 19, 17, 10, 5, ZSTD_btlazy2 }, /* level 16 */ + { 0, 18, 19, 17, 11, 5, ZSTD_btlazy2 }, /* level 17 */ + { 0, 18, 19, 17, 12, 5, ZSTD_btlazy2 }, /* level 18 */ + { 0, 18, 19, 17, 13, 5, ZSTD_btlazy2 }, /* level 19 */ + { 0, 18, 19, 17, 14, 5, ZSTD_btlazy2 }, /* level 20 */ +}, { /* for srcSize <= 128 KB */ /* W, C, H, S, L, strat */ { 0, 17, 12, 12, 1, 4, ZSTD_fast }, /* level 0 - never used */ From 7b05324a712dabf68545a00b4b91fb6579ab67a1 Mon Sep 17 00:00:00 2001 From: Christophe Chevalier Date: Wed, 9 Dec 2015 15:48:22 +0100 Subject: [PATCH 09/22] Only define ZSTDLIB_API when building a Windows DLL --- lib/zstd.h | 10 +++------- lib/zstd_buffered.h | 10 +++------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/lib/zstd.h b/lib/zstd.h index 0795cbb08..b0c841f3a 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -48,14 +48,10 @@ extern "C" { *****************************************************************/ /*! * ZSTD_DLL_EXPORT : -* Enable exporting of functions when building a DLL +* Enable exporting of functions when building a Windows DLL */ -#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) -# if _WIN32 -# define ZSTDLIB_API __declspec(dllexport) -# else -# define ZSTDLIB_API extern -# endif +#if defined(_WIN32) && defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZSTDLIB_API __declspec(dllexport) #else # define ZSTDLIB_API #endif diff --git a/lib/zstd_buffered.h b/lib/zstd_buffered.h index fffa70a05..755728b1b 100644 --- a/lib/zstd_buffered.h +++ b/lib/zstd_buffered.h @@ -52,14 +52,10 @@ extern "C" { *****************************************************************/ /*! * ZSTD_DLL_EXPORT : -* Enable exporting of functions when building a DLL +* Enable exporting of functions when building a Windows DLL */ -#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) -# if _WIN32 -# define ZSTDLIB_API __declspec(dllexport) -# else -# define ZSTDLIB_API extern -# endif +#if defined(_WIN32) && defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZSTDLIB_API __declspec(dllexport) #else # define ZSTDLIB_API #endif From 2abb04d90de5db42ad9f27c911169061aecf25b3 Mon Sep 17 00:00:00 2001 From: Christophe Chevalier Date: Wed, 9 Dec 2015 23:55:23 +0100 Subject: [PATCH 10/22] Added ZSTDLIB_API to new ZSTD_maxCLevel method --- lib/zstd_static.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/zstd_static.h b/lib/zstd_static.h index 212cd8371..ebb8c83cf 100644 --- a/lib/zstd_static.h +++ b/lib/zstd_static.h @@ -180,7 +180,7 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ma * Pre-defined compression levels ***************************************/ #define ZSTD_MAX_CLEVEL 20 -unsigned ZSTD_maxCLevel (void); +ZSTDLIB_API unsigned ZSTD_maxCLevel (void); static const ZSTD_parameters ZSTD_defaultParameters[4][ZSTD_MAX_CLEVEL+1] = { { /* "default" */ /* W, C, H, S, L, strat */ From 695a6cb46387ed11e956866f9df610585c36b63f Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 10 Dec 2015 15:51:38 +0100 Subject: [PATCH 11/22] fixed #94 --- programs/fullbench.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/programs/fullbench.c b/programs/fullbench.c index cb4822022..0eeb88e85 100644 --- a/programs/fullbench.c +++ b/programs/fullbench.c @@ -235,7 +235,7 @@ size_t local_ZSTD_decodeLiteralsBlock(void* dst, size_t dstSize, void* buff2, co size_t local_ZSTD_decodeSeqHeaders(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize) { - U32 DTableML[1<<11], DTableLL[1<<10], DTableOffb[1<<9]; + U32 DTableML[FSE_DTABLE_SIZE_U32(10)], DTableLL[FSE_DTABLE_SIZE_U32(10)], DTableOffb[FSE_DTABLE_SIZE_U32(9)]; /* MLFSELog, LLFSELog and OffFSELog are not public values */ const BYTE* dumps; size_t length; int nbSeq; @@ -245,7 +245,6 @@ size_t local_ZSTD_decodeSeqHeaders(void* dst, size_t dstSize, void* buff2, const - /********************************************************* * Bench functions *********************************************************/ @@ -429,7 +428,7 @@ int benchFiles(char** fileNamesTable, int nbFiles, U32 benchNb) return 11; } - // Memory allocation & restrictions + /* Memory allocation & restrictions */ inFileSize = BMK_GetFileSize(inFileName); benchedSize = (size_t) BMK_findMaxMem(inFileSize*3) / 3; if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize; From fa9df079a42996485d3e625e89c61d2ca94894fd Mon Sep 17 00:00:00 2001 From: Christophe Chevalier Date: Thu, 10 Dec 2015 16:23:48 +0100 Subject: [PATCH 12/22] Add a Visual Studio project for building zstdlib.dll on Windows - Exports all methods marked with ZSTDLIB_API (ZSTD_DLL_EXPORT=1) - Build without support for legacy version (ZSTD_LEGACY_SUPPORT=0) --- .gitignore | 18 ++ visual/2012/fullbench/fullbench.vcxproj | 14 +- visual/2012/fuzzer/fuzzer.vcxproj | 14 +- visual/2012/zstd.sln | 14 +- visual/2012/zstd/zstd.vcxproj | 10 +- visual/2012/zstdlib/resource.h | Bin 0 -> 812 bytes visual/2012/zstdlib/zstdlib.rc | Bin 0 -> 4610 bytes visual/2012/zstdlib/zstdlib.vcxproj | 216 ++++++++++++++++++++ visual/2012/zstdlib/zstdlib.vcxproj.filters | 80 ++++++++ 9 files changed, 346 insertions(+), 20 deletions(-) create mode 100644 visual/2012/zstdlib/resource.h create mode 100644 visual/2012/zstdlib/zstdlib.rc create mode 100644 visual/2012/zstdlib/zstdlib.vcxproj create mode 100644 visual/2012/zstdlib/zstdlib.vcxproj.filters diff --git a/.gitignore b/.gitignore index c4c9282bd..072fe39c1 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,21 @@ # Visual solution files *.suo *.user + +# Build results +[Dd]ebug/ +[Rr]elease/ +[Rr]eleases/ +x64/ +x86/ +[Bb]in/ +[Oo]bj/ + +# Visual C++ cache files +ipch/ +*.aps +*.ncb +*.opendb +*.opensdf +*.sdf +*.cachefile \ No newline at end of file diff --git a/visual/2012/fullbench/fullbench.vcxproj b/visual/2012/fullbench/fullbench.vcxproj index 9a36e5b62..c0d737682 100644 --- a/visual/2012/fullbench/fullbench.vcxproj +++ b/visual/2012/fullbench/fullbench.vcxproj @@ -27,26 +27,26 @@ Application true - v110 + v120 Unicode Application true - v110 + v120 Unicode Application false - v110 + v120 true Unicode Application false - v110 + v120 true Unicode @@ -111,7 +111,7 @@ WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) true true - /analyze:stacksize19000 %(AdditionalOptions) + /analyze:stacksize25000 %(AdditionalOptions) Console @@ -129,7 +129,7 @@ WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) true true - /analyze:stacksize19000 %(AdditionalOptions) + /analyze:stacksize25000 %(AdditionalOptions) Console @@ -149,7 +149,7 @@ WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) true true - /analyze:stacksize19000 %(AdditionalOptions) + /analyze:stacksize25000 %(AdditionalOptions) Console diff --git a/visual/2012/fuzzer/fuzzer.vcxproj b/visual/2012/fuzzer/fuzzer.vcxproj index 63ab47205..0844efe74 100644 --- a/visual/2012/fuzzer/fuzzer.vcxproj +++ b/visual/2012/fuzzer/fuzzer.vcxproj @@ -27,26 +27,26 @@ Application true - v110 + v120 Unicode Application true - v110 + v120 Unicode Application false - v110 + v120 true Unicode Application false - v110 + v120 true Unicode @@ -111,7 +111,7 @@ WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) true true - /analyze:stacksize19000 %(AdditionalOptions) + /analyze:stacksize25000 %(AdditionalOptions) Console @@ -129,7 +129,7 @@ WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) true true - /analyze:stacksize19000 %(AdditionalOptions) + /analyze:stacksize25000 %(AdditionalOptions) Console @@ -149,7 +149,7 @@ WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) true true - /analyze:stacksize19000 %(AdditionalOptions) + /analyze:stacksize25000 %(AdditionalOptions) Console diff --git a/visual/2012/zstd.sln b/visual/2012/zstd.sln index 2b5eb9895..4f2447fbb 100644 --- a/visual/2012/zstd.sln +++ b/visual/2012/zstd.sln @@ -1,12 +1,16 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Express 2012 for Windows Desktop +# Visual Studio 14 +VisualStudioVersion = 14.0.24720.0 +MinimumVisualStudioVersion = 10.0.40219.1 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zstd", "zstd\zstd.vcxproj", "{4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}" EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fuzzer", "fuzzer\fuzzer.vcxproj", "{6FD4352B-346C-4703-96EA-D4A8B9A6976E}" EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fullbench", "fullbench\fullbench.vcxproj", "{61ABD629-1CC8-4FD7-9281-6B8DBB9D3DF8}" EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zstdlib", "zstdlib\zstdlib.vcxproj", "{8BFD8150-94D5-4BF9-8A50-7BD9929A0850}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Win32 = Debug|Win32 @@ -39,6 +43,14 @@ Global {61ABD629-1CC8-4FD7-9281-6B8DBB9D3DF8}.Release|Win32.Build.0 = Release|Win32 {61ABD629-1CC8-4FD7-9281-6B8DBB9D3DF8}.Release|x64.ActiveCfg = Release|x64 {61ABD629-1CC8-4FD7-9281-6B8DBB9D3DF8}.Release|x64.Build.0 = Release|x64 + {8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Debug|Win32.ActiveCfg = Debug|Win32 + {8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Debug|Win32.Build.0 = Debug|Win32 + {8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Debug|x64.ActiveCfg = Debug|x64 + {8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Debug|x64.Build.0 = Debug|x64 + {8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Release|Win32.ActiveCfg = Release|Win32 + {8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Release|Win32.Build.0 = Release|Win32 + {8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Release|x64.ActiveCfg = Release|x64 + {8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/visual/2012/zstd/zstd.vcxproj b/visual/2012/zstd/zstd.vcxproj index 524cf6276..2ab12858b 100644 --- a/visual/2012/zstd/zstd.vcxproj +++ b/visual/2012/zstd/zstd.vcxproj @@ -63,27 +63,27 @@ Application true - v110 Unicode + v120 Application true - v110 + v120 Unicode Application false - v110 true + v120 Unicode Application false - v110 true + v120 Unicode @@ -104,7 +104,7 @@ true - $(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\legacy;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); true diff --git a/visual/2012/zstdlib/resource.h b/visual/2012/zstdlib/resource.h new file mode 100644 index 0000000000000000000000000000000000000000..db984ef4d6c17963c0691479aa0061d0c8000ffe GIT binary patch literal 812 zcmezWPoF`bp_-wZ!H>b8A)dj7!IdF^!Ii-e%<^M!X7FTiWe8@dWvFG~W#D1}sa9a{ zWyoYmW+-CFXDDXKXGmixVNhTQW5{GEW+-JyWXNGqU~p#8X3%C(U`S_3WyoVlWhi1u zWGG=sWk_LAV8~?1V@PJmVJKxtVMt|AU`S)gWXNGiWza*iA(WvQsyd0Gl0ku?ilLaH zgdv3?har<8i9wH{h#?uS8sshzhPcg-A(f#5>>`lqX$*-Br3^V>ei_(pr3|SI#b8_W z8HyMbz^*H2P+-VsNMguhNM%T7C}Ajuo2>w5DKlg;q=8+V#-PC9$PmB~%n-s5%HYD_ z2@W++20w-{27d-`23H0bBprEp^aL@uGB`50F!(e0G59bd zi9sQz%uvdZ2Tn<83}d02IYb<27d-`23H0bunRz8H#jlKKOmREXaxrBc^M`T zqCwb@0hdYzu#cS>Tp8TKok3lvMlr8ZF5 zfiOtN;KVT3!Dvto530wD88R928S)rl;vgD?!x(}Y;u*piTp5BGg26SdA45Dib-6M4 zgWCtlGNAki5(nu7l~kS#J`ApCx)m4<7<3p+7<3qn8FUy7z&0u{1TX|K_%paLgfciY zgb<~}jR9i18@SAHX9#BSWpHE&X7FYJ*-*h?%#g<512!F$D$}4PmjbveRLn;CF21x2bwGk){p@xJ%Lon2JCJY8(Xad!Pa7741BtrnW ztTbS#U@*dvhm@13s$e!_&-y*d2Khzzd77(as14{#-a6nF7l!rh7xW%5rP|A?ZP=b+qqrf2yNl}RmMGPq< zg)XYuH57ayu0`5uXF%&c8fP0k4c@IcIsgxWU-*#yEMTL&iwxej;w&SS_2*HxH3VNeSVwTA>U2>|{fBg+5) literal 0 HcmV?d00001 diff --git a/visual/2012/zstdlib/zstdlib.vcxproj b/visual/2012/zstdlib/zstdlib.vcxproj new file mode 100644 index 000000000..124595556 --- /dev/null +++ b/visual/2012/zstdlib/zstdlib.vcxproj @@ -0,0 +1,216 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + {8BFD8150-94D5-4BF9-8A50-7BD9929A0850} + Win32Proj + zstdlib + + + + DynamicLibrary + true + Unicode + v120 + + + DynamicLibrary + true + v120 + Unicode + + + DynamicLibrary + false + true + v120 + Unicode + + + DynamicLibrary + false + true + v120 + Unicode + + + + + + + + + + + + + + + + + + + true + zstdlib_x86 + $(Platform)\$(Configuration)\ + $(SolutionDir)$(Platform)\$(Configuration)\ + $(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + true + + + true + zstdlib_x64 + $(Platform)\$(Configuration)\ + $(SolutionDir)$(Platform)\$(Configuration)\ + $(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + true + + + false + zstdlib_x86 + $(SolutionDir)$(Platform)\$(Configuration)\ + $(Platform)\$(Configuration)\ + $(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + false + + + false + zstdlib_x64 + $(SolutionDir)$(Platform)\$(Configuration)\ + $(Platform)\$(Configuration)\ + $(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + false + + + + + + Level4 + Disabled + ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + true + EnableFastChecks + MultiThreadedDebugDLL + EditAndContinue + true + true + /analyze:stacksize25000 %(AdditionalOptions) + + + Console + true + MachineX86 + + + + + + + Level4 + Disabled + ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + false + EnableFastChecks + MultiThreadedDebugDLL + ProgramDatabase + true + /analyze:stacksize25000 %(AdditionalOptions) + + + Console + true + + + + + Level4 + + + MaxSpeed + true + true + ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + true + /analyze:stacksize25000 %(AdditionalOptions) + MultiThreadedDLL + ProgramDatabase + All + + + Console + true + true + true + MachineX86 + + + + + Level4 + + + MaxSpeed + true + true + ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + false + false + MultiThreadedDLL + ProgramDatabase + true + true + All + + + Console + true + true + true + + + + + + \ No newline at end of file diff --git a/visual/2012/zstdlib/zstdlib.vcxproj.filters b/visual/2012/zstdlib/zstdlib.vcxproj.filters new file mode 100644 index 000000000..9e930d7b1 --- /dev/null +++ b/visual/2012/zstdlib/zstdlib.vcxproj.filters @@ -0,0 +1,80 @@ + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hh;hpp;hxx;hm;inl;inc;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + + + Resource Files + + + \ No newline at end of file From 7b7027f21e1f11c9bb40af832262408882a867af Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 10 Dec 2015 23:53:21 +0100 Subject: [PATCH 13/22] windows project are now VS2013 --- NEWS | 3 +++ visual/{2012 => 2013}/fullbench/fullbench.vcxproj | 0 .../fullbench/fullbench.vcxproj.filters | 0 visual/{2012 => 2013}/fuzzer/fuzzer.vcxproj | 0 visual/{2012 => 2013}/fuzzer/fuzzer.vcxproj.filters | 0 visual/{2012 => 2013}/zstd.sln | 0 visual/{2012 => 2013}/zstd/zstd.vcxproj | 0 visual/{2012 => 2013}/zstd/zstd.vcxproj.filters | 0 visual/{2012 => 2013}/zstdlib/resource.h | Bin visual/{2012 => 2013}/zstdlib/zstdlib.rc | Bin visual/{2012 => 2013}/zstdlib/zstdlib.vcxproj | 0 .../{2012 => 2013}/zstdlib/zstdlib.vcxproj.filters | 0 12 files changed, 3 insertions(+) rename visual/{2012 => 2013}/fullbench/fullbench.vcxproj (100%) rename visual/{2012 => 2013}/fullbench/fullbench.vcxproj.filters (100%) rename visual/{2012 => 2013}/fuzzer/fuzzer.vcxproj (100%) rename visual/{2012 => 2013}/fuzzer/fuzzer.vcxproj.filters (100%) rename visual/{2012 => 2013}/zstd.sln (100%) rename visual/{2012 => 2013}/zstd/zstd.vcxproj (100%) rename visual/{2012 => 2013}/zstd/zstd.vcxproj.filters (100%) rename visual/{2012 => 2013}/zstdlib/resource.h (100%) rename visual/{2012 => 2013}/zstdlib/zstdlib.rc (100%) rename visual/{2012 => 2013}/zstdlib/zstdlib.vcxproj (100%) rename visual/{2012 => 2013}/zstdlib/zstdlib.vcxproj.filters (100%) diff --git a/NEWS b/NEWS index ad8a737f9..e7785b51c 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,6 @@ +v0.4.4 +new : windows DLL project, thanks to Christophe Chevalier + v0.4.3 : new : external dictionary API new : zstd-frugal diff --git a/visual/2012/fullbench/fullbench.vcxproj b/visual/2013/fullbench/fullbench.vcxproj similarity index 100% rename from visual/2012/fullbench/fullbench.vcxproj rename to visual/2013/fullbench/fullbench.vcxproj diff --git a/visual/2012/fullbench/fullbench.vcxproj.filters b/visual/2013/fullbench/fullbench.vcxproj.filters similarity index 100% rename from visual/2012/fullbench/fullbench.vcxproj.filters rename to visual/2013/fullbench/fullbench.vcxproj.filters diff --git a/visual/2012/fuzzer/fuzzer.vcxproj b/visual/2013/fuzzer/fuzzer.vcxproj similarity index 100% rename from visual/2012/fuzzer/fuzzer.vcxproj rename to visual/2013/fuzzer/fuzzer.vcxproj diff --git a/visual/2012/fuzzer/fuzzer.vcxproj.filters b/visual/2013/fuzzer/fuzzer.vcxproj.filters similarity index 100% rename from visual/2012/fuzzer/fuzzer.vcxproj.filters rename to visual/2013/fuzzer/fuzzer.vcxproj.filters diff --git a/visual/2012/zstd.sln b/visual/2013/zstd.sln similarity index 100% rename from visual/2012/zstd.sln rename to visual/2013/zstd.sln diff --git a/visual/2012/zstd/zstd.vcxproj b/visual/2013/zstd/zstd.vcxproj similarity index 100% rename from visual/2012/zstd/zstd.vcxproj rename to visual/2013/zstd/zstd.vcxproj diff --git a/visual/2012/zstd/zstd.vcxproj.filters b/visual/2013/zstd/zstd.vcxproj.filters similarity index 100% rename from visual/2012/zstd/zstd.vcxproj.filters rename to visual/2013/zstd/zstd.vcxproj.filters diff --git a/visual/2012/zstdlib/resource.h b/visual/2013/zstdlib/resource.h similarity index 100% rename from visual/2012/zstdlib/resource.h rename to visual/2013/zstdlib/resource.h diff --git a/visual/2012/zstdlib/zstdlib.rc b/visual/2013/zstdlib/zstdlib.rc similarity index 100% rename from visual/2012/zstdlib/zstdlib.rc rename to visual/2013/zstdlib/zstdlib.rc diff --git a/visual/2012/zstdlib/zstdlib.vcxproj b/visual/2013/zstdlib/zstdlib.vcxproj similarity index 100% rename from visual/2012/zstdlib/zstdlib.vcxproj rename to visual/2013/zstdlib/zstdlib.vcxproj diff --git a/visual/2012/zstdlib/zstdlib.vcxproj.filters b/visual/2013/zstdlib/zstdlib.vcxproj.filters similarity index 100% rename from visual/2012/zstdlib/zstdlib.vcxproj.filters rename to visual/2013/zstdlib/zstdlib.vcxproj.filters From 9f5ab1a4f5d735b93f8ab881808a863d7f147ce8 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 11 Dec 2015 00:27:41 +0100 Subject: [PATCH 14/22] fix 32-bits windows decoding on high-compression archives --- lib/zstd_decompress.c | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/lib/zstd_decompress.c b/lib/zstd_decompress.c index d1e01c3e5..e1c30a8d6 100644 --- a/lib/zstd_decompress.c +++ b/lib/zstd_decompress.c @@ -527,14 +527,11 @@ FORCE_INLINE size_t ZSTD_execSequence(BYTE* op, *litPtr = litEnd; /* update for next sequence */ /* copy Match */ - /* check */ - //if (match > oLitEnd) return ERROR(corruption_detected); /* address space overflow test (is clang optimizer wrongly removing this test ?) */ - if (sequence.offset > (size_t)oLitEnd) return ERROR(corruption_detected); /* address space overflow test (this test seems preserved by clang optimizer) */ - - if (match < base) + if (sequence.offset > (size_t)(oLitEnd - base)) { /* offset beyond prefix */ - if (match < vBase) return ERROR(corruption_detected); + if (sequence.offset > (size_t)(oLitEnd - vBase)) + return ERROR(corruption_detected); match = dictEnd - (base-match); if (match + sequence.matchLength <= dictEnd) { @@ -773,8 +770,6 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* ctx, void* dst, size_t maxDstSize, con if (srcSize != ctx->expected) return ERROR(srcSize_wrong); if (dst != ctx->previousDstEnd) /* not contiguous */ { - if ((dst > ctx->base) && (dst < ctx->previousDstEnd)) /* rolling buffer : new segment into dictionary */ - ctx->base = (char*)dst; /* temporary affectation, for vBase calculation */ ctx->dictEnd = ctx->previousDstEnd; ctx->vBase = (const char*)dst - ((const char*)(ctx->previousDstEnd) - (const char*)(ctx->base)); ctx->base = dst; From 6c3e2e7855e9efbc56b95739b76d99ed797b4b2b Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 11 Dec 2015 10:44:07 +0100 Subject: [PATCH 15/22] fixed : high compression modes for Windows 32 bits --- .travis.yml | 1 + Makefile | 5 ++++- NEWS | 1 + lib/zstd_compress.c | 23 +++++++++++++---------- programs/Makefile | 2 +- programs/zstdcli.c | 2 +- 6 files changed, 21 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index 846f12175..2d5a73a6e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,6 +19,7 @@ env: - ZSTD_TRAVIS_CI_ENV="-C programs test-zstd_nolegacy" - ZSTD_TRAVIS_CI_ENV=usan - ZSTD_TRAVIS_CI_ENV=asan + - ZSTD_TRAVIS_CI_ENV=asan32 - ZSTD_TRAVIS_CI_ENV="-C programs valgrindTest" matrix: diff --git a/Makefile b/Makefile index 9e7b70ee0..abec33f86 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ # ################################################################ # Version number -export VERSION := 0.4.3 +export VERSION := 0.4.4 PRGDIR = programs ZSTDDIR = lib @@ -96,6 +96,9 @@ usan: clean asan: clean $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address" +asan32: clean + $(MAKE) -C $(PRGDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address" + uasan: clean $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address -fsanitize=undefined" diff --git a/NEWS b/NEWS index e7785b51c..50b26fb95 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,5 @@ v0.4.4 +Fixed : high compression modes for Windows 32 bits new : windows DLL project, thanks to Christophe Chevalier v0.4.3 : diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index 046c03541..11e613882 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -1026,7 +1026,7 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co size_t commonLengthSmaller=0, commonLengthLarger=0; const BYTE* const base = zc->base; const BYTE* match = base + matchIndex; - U32 current = (U32)(ip-base); + const U32 current = (U32)(ip-base); const U32 btLow = btMask >= current ? 0 : current - btMask; U32* smallerPtr = bt + 2*(current&btMask); U32* largerPtr = bt + 2*(current&btMask) + 1; @@ -1040,7 +1040,7 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co return (U32)(rleLength - mls); } - hashTable[h] = (U32)(ip - base); /* Update Hash Table */ + hashTable[h] = current; /* Update Hash Table */ while (nbCompares-- && (matchIndex > windowLow)) { @@ -1103,7 +1103,7 @@ size_t ZSTD_insertBtAndFindBestMatch ( size_t bestLength = 0; U32 dummy32; /* to be nullified at the end */ - hashTable[h] = (U32)(ip-base); /* Update Hash Table */ + hashTable[h] = current; /* Update Hash Table */ while (nbCompares-- && (matchIndex > windowLow)) { @@ -1213,7 +1213,7 @@ static U32 ZSTD_insertBt1_extDict(ZSTD_CCtx* zc, const BYTE* const ip, const U32 const BYTE* const dictEnd = dictBase + dictLimit; const BYTE* const prefixStart = base + dictLimit; const BYTE* match = base + matchIndex; - U32 current = (U32)(ip-base); + const U32 current = (U32)(ip-base); const U32 btLow = btMask >= current ? 0 : current - btMask; U32* smallerPtr = bt + 2*(current&btMask); U32* largerPtr = bt + 2*(current&btMask) + 1; @@ -1227,7 +1227,7 @@ static U32 ZSTD_insertBt1_extDict(ZSTD_CCtx* zc, const BYTE* const ip, const U32 return (U32)(rleLength - mls); } - hashTable[h] = (U32)(ip - base); /* Update Hash Table */ + hashTable[h] = current; /* Update Hash Table */ while (nbCompares-- && (matchIndex > windowLow)) { @@ -1318,7 +1318,7 @@ size_t ZSTD_insertBtAndFindBestMatch_extDict ( size_t bestLength = 0; U32 dummy32; /* to be nullified at the end */ - hashTable[h] = (U32)(ip-base); /* Update Hash Table */ + hashTable[h] = current; /* Update Hash Table */ while (nbCompares-- && (matchIndex > windowLow)) { @@ -1726,7 +1726,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx, /* init */ ZSTD_resetSeqStore(seqStorePtr); - if (((ip-base) - dictLimit) < REPCODE_STARTVALUE) ip += REPCODE_STARTVALUE; + if ((ip - prefixStart) < REPCODE_STARTVALUE) ip += REPCODE_STARTVALUE; /* Match Loop */ while (ip < ilimit) @@ -2008,13 +2008,16 @@ size_t ZSTD_compressContinue (ZSTD_CCtx* zc, } /* preemptive overflow correction */ - if ((zc->base > ip) || (zc->lowLimit > (1<<30) )) + if (zc->lowLimit > (1<<30)) { - U32 correction = zc->lowLimit-1; + U32 btplus = (zc->params.strategy == ZSTD_btlazy2); + U32 contentMask = (1 << (zc->params.contentLog - btplus)) - 1; + U32 newLowLimit = zc->lowLimit & contentMask; /* preserve position % contentSize */ + U32 correction = zc->lowLimit - newLowLimit; ZSTD_reduceIndex(zc, correction); zc->base += correction; zc->dictBase += correction; - zc->lowLimit -= correction; + zc->lowLimit = newLowLimit; zc->dictLimit -= correction; if (zc->nextToUpdate < correction) zc->nextToUpdate = 0; else zc->nextToUpdate -= correction; diff --git a/programs/Makefile b/programs/Makefile index 49061f945..b116fc6b0 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -30,7 +30,7 @@ # fullbench32: Same as fullbench, but forced to compile in 32-bits mode # ########################################################################## -VERSION?= 0.4.3 +VERSION?= 0.4.4 DESTDIR?= PREFIX ?= /usr/local diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 6f23c09a8..ef30f8132 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -70,7 +70,7 @@ **************************************/ #define COMPRESSOR_NAME "zstd command line interface" #ifndef ZSTD_VERSION -# define ZSTD_VERSION "v0.4.2" +# define ZSTD_VERSION "v0.4.4" #endif #define AUTHOR "Yann Collet" #define WELCOME_MESSAGE "*** %s %i-bits %s, by %s (%s) ***\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), ZSTD_VERSION, AUTHOR, __DATE__ From 3a061a4c9fe2bcd634a4915b3992501986771a9c Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 12 Dec 2015 11:17:42 +0100 Subject: [PATCH 16/22] extended static dictionary to buffered mode --- NEWS | 1 + lib/zstd_buffered.c | 20 ++++++++++++++++++-- lib/zstd_buffered.h | 10 ++++++++++ programs/zbufftest.c | 17 +++++++++++++++-- 4 files changed, 44 insertions(+), 4 deletions(-) diff --git a/NEWS b/NEWS index 50b26fb95..939c1ef9a 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,6 @@ v0.4.4 Fixed : high compression modes for Windows 32 bits +new : external dictionary API extended to buffered mode new : windows DLL project, thanks to Christophe Chevalier v0.4.3 : diff --git a/lib/zstd_buffered.c b/lib/zstd_buffered.c index 844c50405..c726f0de0 100644 --- a/lib/zstd_buffered.c +++ b/lib/zstd_buffered.c @@ -160,6 +160,12 @@ size_t ZBUFF_compressInit(ZBUFF_CCtx* zbc, int compressionLevel) } +ZSTDLIB_API size_t ZBUFF_compressWithDictionary(ZBUFF_CCtx* zbc, const void* src, size_t srcSize) +{ + ZSTD_compress_insertDictionary(zbc->zc, src, srcSize); + return 0; +} + /* *** Compression *** */ @@ -327,6 +333,8 @@ struct ZBUFF_DCtx_s { size_t outStart; size_t outEnd; size_t hPos; + const char* dict; + size_t dictSize; ZBUFF_dStage stage; unsigned char headerBuffer[ZSTD_frameHeaderSize_max]; }; /* typedef'd to ZBUFF_DCtx within "zstd_buffered.h" */ @@ -353,17 +361,23 @@ size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbc) } - /* *** Initialization *** */ size_t ZBUFF_decompressInit(ZBUFF_DCtx* zbc) { zbc->stage = ZBUFFds_readHeader; - zbc->hPos = zbc->inPos = zbc->outStart = zbc->outEnd = 0; + zbc->hPos = zbc->inPos = zbc->outStart = zbc->outEnd = zbc->dictSize = 0; return ZSTD_resetDCtx(zbc->zc); } +size_t ZBUFF_decompressWithDictionary(ZBUFF_DCtx* zbc, const void* src, size_t srcSize) +{ + zbc->dict = src; + zbc->dictSize = srcSize; + return 0; +} + /* *** Decompression *** */ @@ -442,6 +456,8 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDstSizePt if (zbc->outBuff == NULL) return ERROR(memory_allocation); } } + if (zbc->dictSize) + ZSTD_decompress_insertDictionary(zbc->zc, zbc->dict, zbc->dictSize); if (zbc->hPos) { /* some data already loaded into headerBuffer : transfer into inBuff */ diff --git a/lib/zstd_buffered.h b/lib/zstd_buffered.h index 755728b1b..d2316a82b 100644 --- a/lib/zstd_buffered.h +++ b/lib/zstd_buffered.h @@ -69,6 +69,7 @@ ZSTDLIB_API ZBUFF_CCtx* ZBUFF_createCCtx(void); ZSTDLIB_API size_t ZBUFF_freeCCtx(ZBUFF_CCtx* cctx); ZSTDLIB_API size_t ZBUFF_compressInit(ZBUFF_CCtx* cctx, int compressionLevel); +ZSTDLIB_API size_t ZBUFF_compressWithDictionary(ZBUFF_CCtx* cctx, const void* src, size_t srcSize); ZSTDLIB_API size_t ZBUFF_compressContinue(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr, const void* src, size_t* srcSizePtr); ZSTDLIB_API size_t ZBUFF_compressFlush(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr); ZSTDLIB_API size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr); @@ -81,6 +82,9 @@ ZSTDLIB_API size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* maxDst * Use ZBUFF_compressInit() to start a new compression operation. * ZBUFF_CCtx objects can be reused multiple times. * +* Optionally, a reference to a static dictionary can be created with ZBUFF_compressWithDictionary() +* Note that the dictionary content must remain accessible during the compression process. +* * Use ZBUFF_compressContinue() repetitively to consume input stream. * *srcSizePtr and *maxDstSizePtr can be any size. * The function will report how many bytes were read or written within *srcSizePtr and *maxDstSizePtr. @@ -115,6 +119,8 @@ ZSTDLIB_API ZBUFF_DCtx* ZBUFF_createDCtx(void); ZSTDLIB_API size_t ZBUFF_freeDCtx(ZBUFF_DCtx* dctx); ZSTDLIB_API size_t ZBUFF_decompressInit(ZBUFF_DCtx* dctx); +ZSTDLIB_API size_t ZBUFF_decompressWithDictionary(ZBUFF_DCtx* dctx, const void* src, size_t srcSize); + ZSTDLIB_API size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx, void* dst, size_t* maxDstSizePtr, const void* src, size_t* srcSizePtr); /** ************************************************ @@ -125,6 +131,10 @@ ZSTDLIB_API size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx, void* dst, size_t* * Use ZBUFF_decompressInit() to start a new decompression operation. * ZBUFF_DCtx objects can be reused multiple times. * +* Optionally, a reference to a static dictionary can be set, using ZBUFF_decompressWithDictionary() +* It must be the same content as the one set during compression phase. +* Dictionary content must remain accessible during the decompression process. +* * Use ZBUFF_decompressContinue() repetitively to consume your input. * *srcSizePtr and *maxDstSizePtr can be any size. * The function will report how many bytes were read or written by modifying *srcSizePtr and *maxDstSizePtr. diff --git a/programs/zbufftest.c b/programs/zbufftest.c index 590393561..ab8aa3486 100644 --- a/programs/zbufftest.c +++ b/programs/zbufftest.c @@ -161,6 +161,7 @@ static int basicUnitTests(U32 seed, double compressibility) ZBUFF_compressInit(zc, 1); readSize = CNBufferSize; genSize = compressedBufferSize; + ZBUFF_compressWithDictionary(zc, CNBuffer, 128 KB); result = ZBUFF_compressContinue(zc, compressedBuffer, &genSize, CNBuffer, &readSize); if (ZBUFF_isError(result)) goto _output_error; if (readSize != CNBufferSize) goto _output_error; /* entire input should be consumed */ @@ -174,6 +175,7 @@ static int basicUnitTests(U32 seed, double compressibility) /* Basic decompression test */ DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); ZBUFF_decompressInit(zd); + ZBUFF_decompressWithDictionary(zd, CNBuffer, 128 KB); readSize = cSize; genSize = CNBufferSize; result = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, compressedBuffer, &readSize); @@ -244,7 +246,6 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit U32 coreSeed = seed, lseed = 0; ZBUFF_CCtx* zc; ZBUFF_DCtx* zd; - XXH64_state_t crc64; U32 startTime = FUZ_GetMilliStart(); /* allocation */ @@ -280,10 +281,12 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit for ( ; (testNb <= nbTests) || (FUZ_GetMilliSpan(startTime) < g_testTime); testNb++ ) { size_t sampleSize, sampleStart; - size_t cSize; + const BYTE* dict; + size_t cSize, dictSize; size_t maxTestSize, totalTestSize, readSize, totalCSize, genSize, totalGenSize; size_t errorCode; U32 sampleSizeLog, buffNb, n, nbChunks; + XXH64_state_t crc64; U64 crcOrig, crcDest; /* init */ @@ -316,6 +319,15 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit maxTestSize = (size_t)1 << sampleSizeLog; maxTestSize += FUZ_rand(&lseed) & (maxTestSize-1); ZBUFF_compressInit(zc, (FUZ_rand(&lseed) % (20 - (sampleSizeLog/3))) + 1); + + sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog; + sampleSize = (size_t)1 << sampleSizeLog; + sampleSize += FUZ_rand(&lseed) & (sampleSize-1); + sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize); + dict = srcBuffer + sampleStart; + dictSize = sampleSize; + ZBUFF_compressWithDictionary(zc, dict, dictSize); + totalTestSize = 0; cSize = 0; for (n=0; n Date: Sat, 12 Dec 2015 12:44:44 +0100 Subject: [PATCH 17/22] fixed g++ warning --- lib/zstd_buffered.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/zstd_buffered.c b/lib/zstd_buffered.c index c726f0de0..99148c297 100644 --- a/lib/zstd_buffered.c +++ b/lib/zstd_buffered.c @@ -373,7 +373,7 @@ size_t ZBUFF_decompressInit(ZBUFF_DCtx* zbc) size_t ZBUFF_decompressWithDictionary(ZBUFF_DCtx* zbc, const void* src, size_t srcSize) { - zbc->dict = src; + zbc->dict = (const char*)src; zbc->dictSize = srcSize; return 0; } From 0700585fb978cfb78699d2f0295fdbcac24e368c Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 12 Dec 2015 12:54:48 +0100 Subject: [PATCH 18/22] fixed asan warning --- lib/zstd_compress.c | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index 11e613882..83f85c770 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -1034,6 +1034,7 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co const U32 windowLow = zc->lowLimit; if ( (current-matchIndex == 1) /* RLE */ + && (matchIndex > windowLow) && (MEM_read64(match) == MEM_read64(ip)) ) { size_t rleLength = ZSTD_count(ip+8, match+8, iend) + 8; From f6f3d7526a46fa2b81503ce19672d2acfad20dc8 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 13 Dec 2015 13:35:21 +0100 Subject: [PATCH 19/22] external dictionary capability added to command line --- NEWS | 2 +- programs/Makefile | 4 +-- programs/fileio.c | 79 ++++++++++++++++++++++++++++++++++++++----- programs/fileio.h | 4 +-- programs/playTests.sh | 7 ++++ programs/zstdcli.c | 29 +++++++++++----- 6 files changed, 104 insertions(+), 21 deletions(-) diff --git a/NEWS b/NEWS index 939c1ef9a..079483a3c 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,6 @@ v0.4.4 Fixed : high compression modes for Windows 32 bits -new : external dictionary API extended to buffered mode +new : external dictionary API extended to buffered mode and accessible through command line new : windows DLL project, thanks to Christophe Chevalier v0.4.3 : diff --git a/programs/Makefile b/programs/Makefile index b116fc6b0..822f2d24b 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -94,8 +94,8 @@ zstd-noBench: $(ZSTD_FILES) $(ZSTDDIR)/zstd_buffered.c \ zstdcli.c fileio.c $(ZSTD_FILEIO_LEGACY) $(CC) $(FLAGS) -DZSTD_NOBENCH $^ -o zstd$(EXT) -zstd-frugal: clean - CFLAGS=-Os $(MAKE) zstd-noBench ZSTD_LEGACY_SUPPORT=0 +zstd-frugal: clean + $(MAKE) zstd-noBench ZSTD_LEGACY_SUPPORT=0 fullbench : $(ZSTD_FILES) \ datagen.c fullbench.c diff --git a/programs/fileio.c b/programs/fileio.c index 27c72c574..3ee5faeea 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -121,6 +121,7 @@ #define CACHELINE 64 +#define MAX_DICT_SIZE (512 KB) /* ************************************* * Macros @@ -235,12 +236,13 @@ static U64 FIO_getFileSize(const char* infilename) } -unsigned long long FIO_compressFilename(const char* output_filename, const char* input_filename, int cLevel) +unsigned long long FIO_compressFilename(const char* output_filename, const char* input_filename, + const char* dictFileName, int cLevel) { U64 filesize = 0; U64 compressedfilesize = 0; - BYTE* inBuff; - BYTE* outBuff; + U64 dictSize = 0; + BYTE* inBuff, *outBuff, *dictBuff=NULL; size_t inBuffSize = ZBUFF_recommendedCInSize(); size_t outBuffSize = ZBUFF_recommendedCOutSize(); FILE* finput; @@ -252,16 +254,43 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char* ctx = ZBUFF_createCCtx(); inBuff = (BYTE*)malloc(inBuffSize); outBuff = (BYTE*)malloc(outBuffSize); - if (!inBuff || !outBuff || !ctx) EXM_THROW(21, "Allocation error : not enough memory"); + if (!inBuff || !outBuff || !ctx) EXM_THROW(20, "Allocation error : not enough memory"); + + /* dictionary */ + if (dictFileName) + { + FILE* dictHandle; + size_t read; + DISPLAYLEVEL(4,"Using %s as dictionary \n", dictFileName); + dictHandle = fopen(dictFileName, "rb"); + if (dictHandle==0) EXM_THROW(21, "Error opening dictionary file %s", dictFileName); + dictSize = FIO_getFileSize(dictFileName); + if (dictSize > MAX_DICT_SIZE) + { + int seekResult; + if (dictSize > 1 GB) EXM_THROW(21, "Dictionary file %s is too large", dictFileName); /* avoid extreme cases */ + DISPLAYLEVEL(2,"Dictionary %s is too large : using last %u bytes only \n", dictFileName, MAX_DICT_SIZE); + seekResult = fseek(dictHandle, dictSize-MAX_DICT_SIZE, SEEK_SET); /* use end of file */ + if (seekResult != 0) EXM_THROW(21, "Error seeking into dictionary file %s", dictFileName); + dictSize = MAX_DICT_SIZE; + } + dictBuff = (BYTE*)malloc(dictSize); + if (dictBuff==NULL) EXM_THROW(20, "Allocation error : not enough memory for dictBuff"); + read = fread(dictBuff, 1, (size_t)dictSize, dictHandle); + if (read!=dictSize) EXM_THROW(21, "Error reading dictionary file %s", dictFileName); + fclose(dictHandle); + } /* init */ FIO_getFileHandles(&finput, &foutput, input_filename, output_filename); - filesize = FIO_getFileSize(input_filename); + filesize = FIO_getFileSize(input_filename) + dictSize; errorCode = ZBUFF_compressInit_advanced(ctx, ZSTD_getParams(cLevel, filesize)); if (ZBUFF_isError(errorCode)) EXM_THROW(22, "Error initializing compression"); - filesize = 0; + errorCode = ZBUFF_compressWithDictionary(ctx, dictBuff, dictSize); + if (ZBUFF_isError(errorCode)) EXM_THROW(22, "Error initializing dictionary"); /* Main compression loop */ + filesize = 0; while (1) { size_t inSize; @@ -311,6 +340,7 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char* /* clean */ free(inBuff); free(outBuff); + free(dictBuff); ZBUFF_freeCCtx(ctx); fclose(finput); if (fclose(foutput)) EXM_THROW(28, "Write error : cannot properly close %s", output_filename); @@ -322,6 +352,7 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char* unsigned long long FIO_decompressFrame(FILE* foutput, FILE* finput, BYTE* inBuff, size_t inBuffSize, size_t alreadyLoaded, BYTE* outBuff, size_t outBuffSize, + BYTE* dictBuff, size_t dictSize, ZBUFF_DCtx* dctx) { U64 frameSize = 0; @@ -329,6 +360,7 @@ unsigned long long FIO_decompressFrame(FILE* foutput, FILE* finput, /* Main decompression Loop */ ZBUFF_decompressInit(dctx); + ZBUFF_decompressWithDictionary(dctx, dictBuff, dictSize); while (1) { /* Decode */ @@ -359,16 +391,42 @@ unsigned long long FIO_decompressFrame(FILE* foutput, FILE* finput, } -unsigned long long FIO_decompressFilename(const char* output_filename, const char* input_filename) +unsigned long long FIO_decompressFilename(const char* output_filename, const char* input_filename, const char* dictFileName) { FILE* finput, *foutput; BYTE* inBuff=NULL; size_t inBuffSize = ZBUFF_recommendedDInSize(); BYTE* outBuff=NULL; size_t outBuffSize = ZBUFF_recommendedDOutSize(); + BYTE* dictBuff=NULL; + size_t dictSize = 0; U64 filesize = 0; size_t toRead; + /* dictionary */ + if (dictFileName) + { + FILE* dictHandle; + size_t read; + DISPLAYLEVEL(4,"Using %s as dictionary \n", dictFileName); + dictHandle = fopen(dictFileName, "rb"); + if (dictHandle==0) EXM_THROW(21, "Error opening dictionary file %s", dictFileName); + dictSize = FIO_getFileSize(dictFileName); + if (dictSize > MAX_DICT_SIZE) + { + int seekResult; + if (dictSize > 1 GB) EXM_THROW(21, "Dictionary file %s is too large", dictFileName); /* avoid extreme cases */ + DISPLAYLEVEL(2,"Dictionary %s is too large : using last %u bytes only \n", dictFileName, MAX_DICT_SIZE); + seekResult = fseek(dictHandle, dictSize-MAX_DICT_SIZE, SEEK_SET); /* use end of file */ + if (seekResult != 0) EXM_THROW(21, "Error seeking into dictionary file %s", dictFileName); + dictSize = MAX_DICT_SIZE; + } + dictBuff = (BYTE*)malloc(dictSize); + if (dictBuff==NULL) EXM_THROW(20, "Allocation error : not enough memory for dictBuff"); + read = fread(dictBuff, 1, (size_t)dictSize, dictHandle); + if (read!=dictSize) EXM_THROW(21, "Error reading dictionary file %s", dictFileName); + fclose(dictHandle); + } /* Init */ ZBUFF_DCtx* dctx = ZBUFF_createDCtx(); @@ -396,7 +454,11 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha } #endif /* ZSTD_LEGACY_SUPPORT */ - filesize += FIO_decompressFrame(foutput, finput, inBuff, inBuffSize, toRead, outBuff, outBuffSize, dctx); + filesize += FIO_decompressFrame(foutput, finput, + inBuff, inBuffSize, toRead, + outBuff, outBuffSize, + dictBuff, dictSize, + dctx); } DISPLAYLEVEL(2, "\r%79s\r", ""); @@ -405,6 +467,7 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha /* clean */ free(inBuff); free(outBuff); + free(dictBuff); ZBUFF_freeDCtx(dctx); fclose(finput); if (fclose(foutput)) EXM_THROW(38, "Write error : cannot properly close %s", output_filename); diff --git a/programs/fileio.h b/programs/fileio.h index 037c819ea..0f5087879 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -52,8 +52,8 @@ void FIO_setNotificationLevel(unsigned level); /* ************************************* * Stream/File functions ***************************************/ -unsigned long long FIO_compressFilename (const char* outfilename, const char* infilename, int compressionLevel); -unsigned long long FIO_decompressFilename (const char* outfilename, const char* infilename); +unsigned long long FIO_compressFilename (const char* outfilename, const char* infilename, const char* dictFileName, int compressionLevel); +unsigned long long FIO_decompressFilename (const char* outfilename, const char* infilename, const char* dictFileName); /** FIO_compressFilename : @result : size of compressed file diff --git a/programs/playTests.sh b/programs/playTests.sh index 56327ff70..74993739c 100755 --- a/programs/playTests.sh +++ b/programs/playTests.sh @@ -42,6 +42,13 @@ echo "**** flush write error test **** " echo foo | $ZSTD > /dev/full && die "write error not detected!" echo foo | $ZSTD | $ZSTD -d > /dev/full && die "write error not detected!" +echo "*** dictionary tests *** " + +./datagen > tmpDict +./datagen -g1M | md5sum > tmp1 +./datagen -g1M | ./zstd -D tmpDict | ./zstd -D tmpDict -dv | md5sum > tmp2 +diff -q tmp1 tmp2 + echo "**** zstd round-trip tests **** " roundTripTest diff --git a/programs/zstdcli.c b/programs/zstdcli.c index ef30f8132..e04e40fd0 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -118,8 +118,7 @@ static int usage(const char* programName) DISPLAY( "input : a filename\n"); DISPLAY( " with no FILE, or when FILE is - , read standard input\n"); DISPLAY( "Arguments :\n"); - DISPLAY( " -1 : Fast compression (default) \n"); - DISPLAY( " -19 : High compression \n"); + DISPLAY( " -# : # compression level (1-19, default:1) \n"); DISPLAY( " -d : decompression (default for %s extension)\n", ZSTD_EXTENSION); //DISPLAY( " -z : force compression\n"); DISPLAY( " -f : overwrite output without prompting \n"); @@ -137,6 +136,7 @@ static int usage_advanced(const char* programName) DISPLAY( " -v : verbose mode\n"); DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n"); DISPLAY( " -c : force write to standard output, even if it is the console\n"); + DISPLAY( " -D file: use file content as Dictionary \n"); //DISPLAY( " -t : test compressed file integrity\n"); #ifndef ZSTD_NOBENCH DISPLAY( "Benchmark arguments :\n"); @@ -171,11 +171,13 @@ int main(int argCount, const char** argv) bench=0, decode=0, forceStdout=0, - main_pause=0; + main_pause=0, + nextEntryIsDictionary=0; unsigned cLevel = 1; const char* programName = argv[0]; const char* inFileName = NULL; const char* outFileName = NULL; + const char* dictFileName = NULL; char* dynNameSpace = NULL; const char extension[] = ZSTD_EXTENSION; unsigned fileNameStart = 0; @@ -249,8 +251,11 @@ int main(int argCount, const char** argv) /* Force stdout, even if stdout==console */ case 'c': forceStdout=1; outFileName=stdoutmark; displayLevel=1; argument++; break; - // Test - //case 't': decode=1; LZ4IO_setOverwrite(1); output_filename=nulmark; break; + /* Use file content as dictionary */ + case 'D': nextEntryIsDictionary = 1; argument++; break; + + /* Test -- not implemented */ + /* case 't': decode=1; LZ4IO_setOverwrite(1); output_filename=nulmark; break; */ /* Overwrite */ case 'f': FIO_overwriteMode(); argument++; break; @@ -261,7 +266,7 @@ int main(int argCount, const char** argv) /* Quiet mode */ case 'q': displayLevel--; argument++; break; - /* keep source file (default anyway, so useless; only for xz/lzma compatibility) */ + /* keep source file (default anyway, so useless; for gzip/xz compatibility) */ case 'k': argument++; break; #ifndef ZSTD_NOBENCH @@ -310,6 +315,14 @@ int main(int argCount, const char** argv) continue; } + /* dictionary */ + if (nextEntryIsDictionary) + { + nextEntryIsDictionary = 0; + dictFileName = argument; + continue; + } + /* first provided filename is input */ if (!inFileName) { inFileName = argument; fileNameStart = i; nbFiles = argCount-i; continue; } @@ -381,9 +394,9 @@ int main(int argCount, const char** argv) /* IO Stream/File */ FIO_setNotificationLevel(displayLevel); if (decode) - FIO_decompressFilename(outFileName, inFileName); + FIO_decompressFilename(outFileName, inFileName, dictFileName); else - FIO_compressFilename(outFileName, inFileName, cLevel); + FIO_compressFilename(outFileName, inFileName, dictFileName, cLevel); _end: if (main_pause) waitEnter(); From 60348b91f31efc9bbfa08f4f7bf8188ccb27cf08 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 13 Dec 2015 13:44:23 +0100 Subject: [PATCH 20/22] fixed silent conversion warnings --- programs/fileio.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 3ee5faeea..38b204bca 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -270,11 +270,11 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char* int seekResult; if (dictSize > 1 GB) EXM_THROW(21, "Dictionary file %s is too large", dictFileName); /* avoid extreme cases */ DISPLAYLEVEL(2,"Dictionary %s is too large : using last %u bytes only \n", dictFileName, MAX_DICT_SIZE); - seekResult = fseek(dictHandle, dictSize-MAX_DICT_SIZE, SEEK_SET); /* use end of file */ + seekResult = fseek(dictHandle, (size_t)(dictSize-MAX_DICT_SIZE), SEEK_SET); /* use end of file */ if (seekResult != 0) EXM_THROW(21, "Error seeking into dictionary file %s", dictFileName); dictSize = MAX_DICT_SIZE; } - dictBuff = (BYTE*)malloc(dictSize); + dictBuff = (BYTE*)malloc((size_t)dictSize); if (dictBuff==NULL) EXM_THROW(20, "Allocation error : not enough memory for dictBuff"); read = fread(dictBuff, 1, (size_t)dictSize, dictHandle); if (read!=dictSize) EXM_THROW(21, "Error reading dictionary file %s", dictFileName); @@ -286,7 +286,7 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char* filesize = FIO_getFileSize(input_filename) + dictSize; errorCode = ZBUFF_compressInit_advanced(ctx, ZSTD_getParams(cLevel, filesize)); if (ZBUFF_isError(errorCode)) EXM_THROW(22, "Error initializing compression"); - errorCode = ZBUFF_compressWithDictionary(ctx, dictBuff, dictSize); + errorCode = ZBUFF_compressWithDictionary(ctx, dictBuff, (size_t)dictSize); if (ZBUFF_isError(errorCode)) EXM_THROW(22, "Error initializing dictionary"); /* Main compression loop */ @@ -411,7 +411,7 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha DISPLAYLEVEL(4,"Using %s as dictionary \n", dictFileName); dictHandle = fopen(dictFileName, "rb"); if (dictHandle==0) EXM_THROW(21, "Error opening dictionary file %s", dictFileName); - dictSize = FIO_getFileSize(dictFileName); + dictSize = (size_t)FIO_getFileSize(dictFileName); if (dictSize > MAX_DICT_SIZE) { int seekResult; From 5d4fa0ed9b8a3aebcb4420b910446799dea2f5a8 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 13 Dec 2015 13:58:51 +0100 Subject: [PATCH 21/22] fixed arm-specific shadow warning --- programs/fileio.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 38b204bca..81635b934 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -260,7 +260,7 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char* if (dictFileName) { FILE* dictHandle; - size_t read; + size_t readSize; DISPLAYLEVEL(4,"Using %s as dictionary \n", dictFileName); dictHandle = fopen(dictFileName, "rb"); if (dictHandle==0) EXM_THROW(21, "Error opening dictionary file %s", dictFileName); @@ -276,8 +276,8 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char* } dictBuff = (BYTE*)malloc((size_t)dictSize); if (dictBuff==NULL) EXM_THROW(20, "Allocation error : not enough memory for dictBuff"); - read = fread(dictBuff, 1, (size_t)dictSize, dictHandle); - if (read!=dictSize) EXM_THROW(21, "Error reading dictionary file %s", dictFileName); + readSize = fread(dictBuff, 1, (size_t)dictSize, dictHandle); + if (readSize!=dictSize) EXM_THROW(21, "Error reading dictionary file %s", dictFileName); fclose(dictHandle); } @@ -407,7 +407,7 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha if (dictFileName) { FILE* dictHandle; - size_t read; + size_t readSize; DISPLAYLEVEL(4,"Using %s as dictionary \n", dictFileName); dictHandle = fopen(dictFileName, "rb"); if (dictHandle==0) EXM_THROW(21, "Error opening dictionary file %s", dictFileName); @@ -423,8 +423,8 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha } dictBuff = (BYTE*)malloc(dictSize); if (dictBuff==NULL) EXM_THROW(20, "Allocation error : not enough memory for dictBuff"); - read = fread(dictBuff, 1, (size_t)dictSize, dictHandle); - if (read!=dictSize) EXM_THROW(21, "Error reading dictionary file %s", dictFileName); + readSize = fread(dictBuff, 1, (size_t)dictSize, dictHandle); + if (readSize!=dictSize) EXM_THROW(21, "Error reading dictionary file %s", dictFileName); fclose(dictHandle); } From 397f1ffd5202321c307a4d88075a7583d013b305 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 13 Dec 2015 14:05:00 +0100 Subject: [PATCH 22/22] fixed test script --- programs/playTests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/playTests.sh b/programs/playTests.sh index 74993739c..25c7fb23a 100755 --- a/programs/playTests.sh +++ b/programs/playTests.sh @@ -46,7 +46,7 @@ echo "*** dictionary tests *** " ./datagen > tmpDict ./datagen -g1M | md5sum > tmp1 -./datagen -g1M | ./zstd -D tmpDict | ./zstd -D tmpDict -dv | md5sum > tmp2 +./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dv | md5sum > tmp2 diff -q tmp1 tmp2 echo "**** zstd round-trip tests **** "