From 0cde77bc5ec1f75e09c28b6b794c2ea73e5e4e8e Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 8 Dec 2015 14:47:46 +0100 Subject: [PATCH 1/6] fixed comment (reported by @annulen) --- lib/zstd_static.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/zstd_static.h b/lib/zstd_static.h index 5c3156248..2f2352f65 100644 --- a/lib/zstd_static.h +++ b/lib/zstd_static.h @@ -120,7 +120,7 @@ size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize); Use ZSTD_compressBegin(). You may also prefer the advanced derivative ZSTD_compressBegin_advanced(), for finer parameter control. - It's then possible to add a dictionary with ZSTD_compressDictionary() + It's then possible to add a dictionary with ZSTD_compress_insertDictionary() Note that dictionary presence is a "hidden" information, the decoder needs to be aware that it is required for proper decoding, or decoding will fail. From d66db2ff04e45dca5c3c0cf6f098a4bd1c70a70e Mon Sep 17 00:00:00 2001 From: Konstantin Tokarev Date: Tue, 8 Dec 2015 18:11:10 +0300 Subject: [PATCH 2/6] Move tests from zstd-playTests target to separate shell script. This patch allows me to run tests on MIPS board which lacks make. --- programs/Makefile | 72 +------------------------------------------ programs/playTests.sh | 62 +++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 71 deletions(-) create mode 100755 programs/playTests.sh diff --git a/programs/Makefile b/programs/Makefile index 6e9d6b56f..fb2f3524f 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -169,77 +169,7 @@ test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zbuff32 test-all: test test32 valgrindTest zstd-playTests: datagen - @echo "\n**** frame concatenation **** " - @echo "hello " > hello.tmp - @echo "world!" > world.tmp - @cat hello.tmp world.tmp > helloworld.tmp - $(ZSTD) hello.tmp > hello.zstd - $(ZSTD) world.tmp > world.zstd - @cat hello.zstd world.zstd > helloworld.zstd - $(ZSTD) -df helloworld.zstd > result.tmp - cat result.tmp - sdiff helloworld.tmp result.tmp - @rm *.tmp *.zstd - @echo frame concatenation test completed - @echo "**** flush write error test **** " - echo foo | $(ZSTD) > /dev/full; if [ $$? -eq 0 ] ; then echo "write error not detected!"; false; fi - echo foo | $(ZSTD) | $(ZSTD) -d > /dev/full; if [ $$? -eq 0 ] ; then echo "write error not detected!"; false; fi - @echo "**** zstd round-trip tests **** " - @./datagen | md5sum > tmp1 - ./datagen | $(ZSTD) -v | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen | $(ZSTD) -6 -v | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - @./datagen -g270000000 | md5sum > tmp1 - ./datagen -g270000000 | $(ZSTD) -v | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g270000000 | $(ZSTD) -v2 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g270000000 | $(ZSTD) -v3 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - @./datagen -g140000000 -P60| md5sum > tmp1 - ./datagen -g140000000 -P60 | $(ZSTD) -v4 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g140000000 -P60 | $(ZSTD) -v5 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g140000000 -P60 | $(ZSTD) -v6 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - @./datagen -g70000000 -P70 | md5sum > tmp1 - ./datagen -g70000000 -P70 | $(ZSTD) -v7 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g70000000 -P70 | $(ZSTD) -v8 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g70000000 -P70 | $(ZSTD) -v9 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - @./datagen -g35000000 -P75 | md5sum > tmp1 - ./datagen -g35000000 -P75 | $(ZSTD) -v10 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g35000000 -P75 | $(ZSTD) -v11 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g35000000 -P75 | $(ZSTD) -v12 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - @./datagen -g18000000 -P80 | md5sum > tmp1 - ./datagen -g18000000 -P80 | $(ZSTD) -v13 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g18000000 -P80 | $(ZSTD) -v14 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g18000000 -P80 | $(ZSTD) -v15 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g18000000 -P80 | $(ZSTD) -v16 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g18000000 -P80 | $(ZSTD) -v17 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - @./datagen -g50000000 -P94 | md5sum > tmp1 - ./datagen -g50000000 -P94 | $(ZSTD) -v18 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g50000000 -P94 | $(ZSTD) -v19 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - @./datagen -g99000000 -P99 | md5sum > tmp1 - ./datagen -g99000000 -P99 | $(ZSTD) -v20 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g6000000000 -P99| md5sum > tmp1 - ./datagen -g6000000000 -P99| $(ZSTD) -vq | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 + ZSTD=$(ZSTD) ./playTests.sh test-zstd: ZSTD = ./zstd test-zstd: zstd zstd-playTests diff --git a/programs/playTests.sh b/programs/playTests.sh new file mode 100755 index 000000000..08090f6bc --- /dev/null +++ b/programs/playTests.sh @@ -0,0 +1,62 @@ +#!/bin/sh -e + +die() { + echo "$@" 1>&2 + exit 1 +} + +echo "\n**** frame concatenation **** " + +echo "hello " > hello.tmp +echo "world!" > world.tmp +cat hello.tmp world.tmp > helloworld.tmp +$ZSTD hello.tmp > hello.zstd +$ZSTD world.tmp > world.zstd +cat hello.zstd world.zstd > helloworld.zstd +$ZSTD -df helloworld.zstd > result.tmp +cat result.tmp +sdiff helloworld.tmp result.tmp +rm *.tmp *.zstd + +echo frame concatenation test completed + +echo "**** flush write error test **** " + +echo foo | $ZSTD > /dev/full && die "write error not detected!" +echo foo | $ZSTD | $ZSTD -d > /dev/full && die "write error not detected!" + +echo "**** zstd round-trip tests **** " +./datagen | md5sum > tmp1 +./datagen | $ZSTD -v | $ZSTD -d | md5sum > tmp2 +diff tmp1 tmp2 +./datagen | $ZSTD -6 -v | $ZSTD -d | md5sum > tmp2 +diff tmp1 tmp2 +./datagen -g270000000 | md5sum > tmp1 +./datagen -g270000000 | $ZSTD -v | $ZSTD -d | md5sum > tmp2 +diff tmp1 tmp2 +./datagen -g270000000 | $ZSTD -v2 | $ZSTD -d | md5sum > tmp2 +diff tmp1 tmp2 +./datagen -g270000000 | $ZSTD -v3 | $ZSTD -d | md5sum > tmp2 +diff tmp1 tmp2 +./datagen -g140000000 -P60| md5sum > tmp1 +./datagen -g140000000 -P60 | $ZSTD -v4 | $ZSTD -d | md5sum > tmp2 +diff tmp1 tmp2 +./datagen -g140000000 -P60 | $ZSTD -v5 | $ZSTD -d | md5sum > tmp2 +diff tmp1 tmp2 +./datagen -g140000000 -P60 | $ZSTD -v6 | $ZSTD -d | md5sum > tmp2 +diff tmp1 tmp2 +./datagen -g70000000 -P70 | md5sum > tmp1 +./datagen -g70000000 -P70 | $ZSTD -v7 | $ZSTD -d | md5sum > tmp2 +diff tmp1 tmp2 +./datagen -g70000000 -P70 | $ZSTD -v8 | $ZSTD -d | md5sum > tmp2 +diff tmp1 tmp2 +./datagen -g70000000 -P70 | $ZSTD -v9 | $ZSTD -d | md5sum > tmp2 +diff tmp1 tmp2 +./datagen -g35000000 -P75 | md5sum > tmp1 +./datagen -g35000000 -P75 | $ZSTD -v10 | $ZSTD -d | md5sum > tmp2 +diff tmp1 tmp2 +./datagen -g35000000 -P75 | $ZSTD -v11 | $ZSTD -d | md5sum > tmp2 +diff tmp1 tmp2 +./datagen -g35000000 -P75 | $ZSTD -v12 | $ZSTD -d | md5sum > tmp2 +diff tmp1 tmp2 + From 76be3785131181a392491297e82a646bb0b7f61b Mon Sep 17 00:00:00 2001 From: Konstantin Tokarev Date: Tue, 8 Dec 2015 18:36:37 +0300 Subject: [PATCH 3/6] playTests.sh: refactored round-trip tests --- programs/playTests.sh | 73 ++++++++++++++++++++++--------------------- 1 file changed, 38 insertions(+), 35 deletions(-) diff --git a/programs/playTests.sh b/programs/playTests.sh index 08090f6bc..d29260aa8 100755 --- a/programs/playTests.sh +++ b/programs/playTests.sh @@ -5,7 +5,24 @@ die() { exit 1 } -echo "\n**** frame concatenation **** " +roundTripTest() { + if [ -n "$3" ]; then + local c="$3" + local p="$2" + else + local c="$2" + fi + + rm -f tmp1 tmp2 + echo "roundTripTest: ./datagen $1 $p | $ZSTD -v$c | $ZSTD -d" + ./datagen $1 $p | md5sum > tmp1 + ./datagen $1 $p | $ZSTD -v$c | $ZSTD -d | md5sum > tmp2 + diff -q tmp1 tmp2 +} + +[ -n "$ZSTD" ] || die "ZSTD variable must be defined!" + +printf "\n**** frame concatenation **** " echo "hello " > hello.tmp echo "world!" > world.tmp @@ -16,7 +33,7 @@ cat hello.zstd world.zstd > helloworld.zstd $ZSTD -df helloworld.zstd > result.tmp cat result.tmp sdiff helloworld.tmp result.tmp -rm *.tmp *.zstd +rm ./*.tmp ./*.zstd echo frame concatenation test completed @@ -26,37 +43,23 @@ echo foo | $ZSTD > /dev/full && die "write error not detected!" echo foo | $ZSTD | $ZSTD -d > /dev/full && die "write error not detected!" echo "**** zstd round-trip tests **** " -./datagen | md5sum > tmp1 -./datagen | $ZSTD -v | $ZSTD -d | md5sum > tmp2 -diff tmp1 tmp2 -./datagen | $ZSTD -6 -v | $ZSTD -d | md5sum > tmp2 -diff tmp1 tmp2 -./datagen -g270000000 | md5sum > tmp1 -./datagen -g270000000 | $ZSTD -v | $ZSTD -d | md5sum > tmp2 -diff tmp1 tmp2 -./datagen -g270000000 | $ZSTD -v2 | $ZSTD -d | md5sum > tmp2 -diff tmp1 tmp2 -./datagen -g270000000 | $ZSTD -v3 | $ZSTD -d | md5sum > tmp2 -diff tmp1 tmp2 -./datagen -g140000000 -P60| md5sum > tmp1 -./datagen -g140000000 -P60 | $ZSTD -v4 | $ZSTD -d | md5sum > tmp2 -diff tmp1 tmp2 -./datagen -g140000000 -P60 | $ZSTD -v5 | $ZSTD -d | md5sum > tmp2 -diff tmp1 tmp2 -./datagen -g140000000 -P60 | $ZSTD -v6 | $ZSTD -d | md5sum > tmp2 -diff tmp1 tmp2 -./datagen -g70000000 -P70 | md5sum > tmp1 -./datagen -g70000000 -P70 | $ZSTD -v7 | $ZSTD -d | md5sum > tmp2 -diff tmp1 tmp2 -./datagen -g70000000 -P70 | $ZSTD -v8 | $ZSTD -d | md5sum > tmp2 -diff tmp1 tmp2 -./datagen -g70000000 -P70 | $ZSTD -v9 | $ZSTD -d | md5sum > tmp2 -diff tmp1 tmp2 -./datagen -g35000000 -P75 | md5sum > tmp1 -./datagen -g35000000 -P75 | $ZSTD -v10 | $ZSTD -d | md5sum > tmp2 -diff tmp1 tmp2 -./datagen -g35000000 -P75 | $ZSTD -v11 | $ZSTD -d | md5sum > tmp2 -diff tmp1 tmp2 -./datagen -g35000000 -P75 | $ZSTD -v12 | $ZSTD -d | md5sum > tmp2 -diff tmp1 tmp2 + +roundTripTest +roundTripTest '' 6 + +roundTripTest -g270000000 1 +roundTripTest -g270000000 2 +roundTripTest -g270000000 3 + +roundTripTest -g140000000 -P60 4 +roundTripTest -g140000000 -P60 5 +roundTripTest -g140000000 -P60 6 + +roundTripTest -g70000000 -P70 7 +roundTripTest -g70000000 -P70 8 +roundTripTest -g70000000 -P70 9 + +roundTripTest -g35000000 -P75 10 +roundTripTest -g35000000 -P75 11 +roundTripTest -g35000000 -P75 12 From 0b570b59d2d5189d72cf35a81a355e5bb6357b97 Mon Sep 17 00:00:00 2001 From: Konstantin Tokarev Date: Tue, 8 Dec 2015 18:47:43 +0300 Subject: [PATCH 4/6] playTests.sh: Added --test-large-data switch. --- programs/Makefile | 2 +- programs/playTests.sh | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/programs/Makefile b/programs/Makefile index fb2f3524f..49061f945 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -169,7 +169,7 @@ test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zbuff32 test-all: test test32 valgrindTest zstd-playTests: datagen - ZSTD=$(ZSTD) ./playTests.sh + ZSTD=$(ZSTD) ./playTests.sh --test-large-data test-zstd: ZSTD = ./zstd test-zstd: zstd zstd-playTests diff --git a/programs/playTests.sh b/programs/playTests.sh index d29260aa8..e97f93db8 100755 --- a/programs/playTests.sh +++ b/programs/playTests.sh @@ -47,6 +47,11 @@ echo "**** zstd round-trip tests **** " roundTripTest roundTripTest '' 6 +if [ "$1" != "--test-large-data" ]; then + echo "Skipping large data tests" + exit 0 +fi + roundTripTest -g270000000 1 roundTripTest -g270000000 2 roundTripTest -g270000000 3 From 2b465840a89e1e8254165b436168492712dd294c Mon Sep 17 00:00:00 2001 From: Konstantin Tokarev Date: Tue, 8 Dec 2015 19:36:42 +0300 Subject: [PATCH 5/6] Re-added accidentally lost round-trip tests for compression levels > 12. --- programs/playTests.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/programs/playTests.sh b/programs/playTests.sh index e97f93db8..56327ff70 100755 --- a/programs/playTests.sh +++ b/programs/playTests.sh @@ -68,3 +68,14 @@ roundTripTest -g35000000 -P75 10 roundTripTest -g35000000 -P75 11 roundTripTest -g35000000 -P75 12 +roundTripTest -g18000000 -P80 13 +roundTripTest -g18000000 -P80 14 +roundTripTest -g18000000 -P80 15 +roundTripTest -g18000000 -P80 16 +roundTripTest -g18000000 -P80 17 + +roundTripTest -g50000000 -P94 18 +roundTripTest -g50000000 -P94 19 + +roundTripTest -g99000000 -P99 20 +roundTripTest -g6000000000 -P99 q From d608088ca37fd7172261cbe8aa9d98770d9b8c54 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 9 Dec 2015 09:05:22 +0100 Subject: [PATCH 6/6] added : ZSTD_maxCLevel() added : 256KB blocks mode --- lib/zstd_compress.c | 7 ++++--- lib/zstd_static.h | 27 ++++++++++++++++++++++++++- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index c18d8f86c..046c03541 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -65,6 +65,7 @@ /* ************************************* * Constants ***************************************/ +unsigned ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; } static const U32 g_searchStrength = 8; @@ -599,7 +600,7 @@ static unsigned ZSTD_NbCommonBytes (register size_t val) # if defined(_MSC_VER) && defined(_WIN64) unsigned long r = 0; _BitScanForward64( &r, (U64)val ); - return (int)(r>>3); + return (unsigned)(r>>3); # elif defined(__GNUC__) && (__GNUC__ >= 3) return (__builtin_ctzll((U64)val) >> 3); # else @@ -612,7 +613,7 @@ static unsigned ZSTD_NbCommonBytes (register size_t val) # if defined(_MSC_VER) unsigned long r=0; _BitScanForward( &r, (U32)val ); - return (int)(r>>3); + return (unsigned)(r>>3); # elif defined(__GNUC__) && (__GNUC__ >= 3) return (__builtin_ctz((U32)val) >> 3); # else @@ -2097,7 +2098,7 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* ctx, ZSTD_parameters ZSTD_getParams(int compressionLevel, U64 srcSizeHint) { ZSTD_parameters result; - int tableID = ((srcSizeHint-1) <= 128 KB) + ((srcSizeHint-1) <= 16 KB); /* intentional underflow for srcSizeHint == 0 */ + int tableID = ((srcSizeHint-1) <= 256 KB) + ((srcSizeHint-1) <= 128 KB) + ((srcSizeHint-1) <= 16 KB); /* intentional underflow for srcSizeHint == 0 */ if (compressionLevel<=0) compressionLevel = 1; if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL; result = ZSTD_defaultParameters[tableID][compressionLevel]; diff --git a/lib/zstd_static.h b/lib/zstd_static.h index 2f2352f65..c8aae4c13 100644 --- a/lib/zstd_static.h +++ b/lib/zstd_static.h @@ -180,7 +180,8 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, co * Pre-defined compression levels ***************************************/ #define ZSTD_MAX_CLEVEL 20 -static const ZSTD_parameters ZSTD_defaultParameters[3][ZSTD_MAX_CLEVEL+1] = { +unsigned ZSTD_maxCLevel (void); +static const ZSTD_parameters ZSTD_defaultParameters[4][ZSTD_MAX_CLEVEL+1] = { { /* "default" */ /* W, C, H, S, L, strat */ { 0, 18, 12, 12, 1, 4, ZSTD_fast }, /* level 0 - never used */ @@ -205,6 +206,30 @@ static const ZSTD_parameters ZSTD_defaultParameters[3][ZSTD_MAX_CLEVEL+1] = { { 0, 25, 26, 23, 5, 5, ZSTD_btlazy2 }, /* level 19 */ { 0, 26, 27, 25, 9, 5, ZSTD_btlazy2 }, /* level 20 */ }, +{ /* for srcSize <= 256 KB */ + /* W, C, H, S, L, strat */ + { 0, 0, 0, 0, 0, 0, ZSTD_fast }, /* level 0 - never used */ + { 0, 18, 16, 15, 1, 7, ZSTD_fast }, /* level 1 */ + { 0, 18, 16, 16, 1, 7, ZSTD_fast }, /* level 2 */ + { 0, 18, 18, 18, 1, 7, ZSTD_fast }, /* level 3 */ + { 0, 18, 14, 15, 4, 6, ZSTD_greedy }, /* level 4 */ + { 0, 18, 16, 16, 1, 6, ZSTD_lazy }, /* level 5 */ + { 0, 18, 15, 15, 3, 6, ZSTD_lazy }, /* level 6 */ + { 0, 18, 15, 15, 4, 6, ZSTD_lazy }, /* level 7 */ + { 0, 18, 16, 18, 4, 6, ZSTD_lazy }, /* level 8 */ + { 0, 18, 18, 18, 4, 6, ZSTD_lazy }, /* level 9 */ + { 0, 18, 18, 18, 5, 6, ZSTD_lazy }, /* level 10 */ + { 0, 18, 18, 19, 6, 6, ZSTD_lazy }, /* level 11 */ + { 0, 18, 18, 19, 7, 6, ZSTD_lazy }, /* level 12 */ + { 0, 18, 19, 15, 7, 5, ZSTD_btlazy2 }, /* level 13 */ + { 0, 18, 19, 16, 8, 5, ZSTD_btlazy2 }, /* level 14 */ + { 0, 18, 19, 17, 9, 5, ZSTD_btlazy2 }, /* level 15 */ + { 0, 18, 19, 17, 10, 5, ZSTD_btlazy2 }, /* level 16 */ + { 0, 18, 19, 17, 11, 5, ZSTD_btlazy2 }, /* level 17 */ + { 0, 18, 19, 17, 12, 5, ZSTD_btlazy2 }, /* level 18 */ + { 0, 18, 19, 17, 13, 5, ZSTD_btlazy2 }, /* level 19 */ + { 0, 18, 19, 17, 14, 5, ZSTD_btlazy2 }, /* level 20 */ +}, { /* for srcSize <= 128 KB */ /* W, C, H, S, L, strat */ { 0, 17, 12, 12, 1, 4, ZSTD_fast }, /* level 0 - never used */