diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index c18d8f86c..046c03541 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -65,6 +65,7 @@ /* ************************************* * Constants ***************************************/ +unsigned ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; } static const U32 g_searchStrength = 8; @@ -599,7 +600,7 @@ static unsigned ZSTD_NbCommonBytes (register size_t val) # if defined(_MSC_VER) && defined(_WIN64) unsigned long r = 0; _BitScanForward64( &r, (U64)val ); - return (int)(r>>3); + return (unsigned)(r>>3); # elif defined(__GNUC__) && (__GNUC__ >= 3) return (__builtin_ctzll((U64)val) >> 3); # else @@ -612,7 +613,7 @@ static unsigned ZSTD_NbCommonBytes (register size_t val) # if defined(_MSC_VER) unsigned long r=0; _BitScanForward( &r, (U32)val ); - return (int)(r>>3); + return (unsigned)(r>>3); # elif defined(__GNUC__) && (__GNUC__ >= 3) return (__builtin_ctz((U32)val) >> 3); # else @@ -2097,7 +2098,7 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* ctx, ZSTD_parameters ZSTD_getParams(int compressionLevel, U64 srcSizeHint) { ZSTD_parameters result; - int tableID = ((srcSizeHint-1) <= 128 KB) + ((srcSizeHint-1) <= 16 KB); /* intentional underflow for srcSizeHint == 0 */ + int tableID = ((srcSizeHint-1) <= 256 KB) + ((srcSizeHint-1) <= 128 KB) + ((srcSizeHint-1) <= 16 KB); /* intentional underflow for srcSizeHint == 0 */ if (compressionLevel<=0) compressionLevel = 1; if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL; result = ZSTD_defaultParameters[tableID][compressionLevel]; diff --git a/lib/zstd_static.h b/lib/zstd_static.h index 6a987026a..212cd8371 100644 --- a/lib/zstd_static.h +++ b/lib/zstd_static.h @@ -120,7 +120,7 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t maxDstSiz Use ZSTD_compressBegin(). You may also prefer the advanced derivative ZSTD_compressBegin_advanced(), for finer parameter control. - It's then possible to add a dictionary with ZSTD_compressDictionary() + It's then possible to add a dictionary with ZSTD_compress_insertDictionary() Note that dictionary presence is a "hidden" information, the decoder needs to be aware that it is required for proper decoding, or decoding will fail. @@ -180,7 +180,8 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ma * Pre-defined compression levels ***************************************/ #define ZSTD_MAX_CLEVEL 20 -static const ZSTD_parameters ZSTD_defaultParameters[3][ZSTD_MAX_CLEVEL+1] = { +unsigned ZSTD_maxCLevel (void); +static const ZSTD_parameters ZSTD_defaultParameters[4][ZSTD_MAX_CLEVEL+1] = { { /* "default" */ /* W, C, H, S, L, strat */ { 0, 18, 12, 12, 1, 4, ZSTD_fast }, /* level 0 - never used */ @@ -205,6 +206,30 @@ static const ZSTD_parameters ZSTD_defaultParameters[3][ZSTD_MAX_CLEVEL+1] = { { 0, 25, 26, 23, 5, 5, ZSTD_btlazy2 }, /* level 19 */ { 0, 26, 27, 25, 9, 5, ZSTD_btlazy2 }, /* level 20 */ }, +{ /* for srcSize <= 256 KB */ + /* W, C, H, S, L, strat */ + { 0, 0, 0, 0, 0, 0, ZSTD_fast }, /* level 0 - never used */ + { 0, 18, 16, 15, 1, 7, ZSTD_fast }, /* level 1 */ + { 0, 18, 16, 16, 1, 7, ZSTD_fast }, /* level 2 */ + { 0, 18, 18, 18, 1, 7, ZSTD_fast }, /* level 3 */ + { 0, 18, 14, 15, 4, 6, ZSTD_greedy }, /* level 4 */ + { 0, 18, 16, 16, 1, 6, ZSTD_lazy }, /* level 5 */ + { 0, 18, 15, 15, 3, 6, ZSTD_lazy }, /* level 6 */ + { 0, 18, 15, 15, 4, 6, ZSTD_lazy }, /* level 7 */ + { 0, 18, 16, 18, 4, 6, ZSTD_lazy }, /* level 8 */ + { 0, 18, 18, 18, 4, 6, ZSTD_lazy }, /* level 9 */ + { 0, 18, 18, 18, 5, 6, ZSTD_lazy }, /* level 10 */ + { 0, 18, 18, 19, 6, 6, ZSTD_lazy }, /* level 11 */ + { 0, 18, 18, 19, 7, 6, ZSTD_lazy }, /* level 12 */ + { 0, 18, 19, 15, 7, 5, ZSTD_btlazy2 }, /* level 13 */ + { 0, 18, 19, 16, 8, 5, ZSTD_btlazy2 }, /* level 14 */ + { 0, 18, 19, 17, 9, 5, ZSTD_btlazy2 }, /* level 15 */ + { 0, 18, 19, 17, 10, 5, ZSTD_btlazy2 }, /* level 16 */ + { 0, 18, 19, 17, 11, 5, ZSTD_btlazy2 }, /* level 17 */ + { 0, 18, 19, 17, 12, 5, ZSTD_btlazy2 }, /* level 18 */ + { 0, 18, 19, 17, 13, 5, ZSTD_btlazy2 }, /* level 19 */ + { 0, 18, 19, 17, 14, 5, ZSTD_btlazy2 }, /* level 20 */ +}, { /* for srcSize <= 128 KB */ /* W, C, H, S, L, strat */ { 0, 17, 12, 12, 1, 4, ZSTD_fast }, /* level 0 - never used */ diff --git a/programs/Makefile b/programs/Makefile index 6e9d6b56f..49061f945 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -169,77 +169,7 @@ test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zbuff32 test-all: test test32 valgrindTest zstd-playTests: datagen - @echo "\n**** frame concatenation **** " - @echo "hello " > hello.tmp - @echo "world!" > world.tmp - @cat hello.tmp world.tmp > helloworld.tmp - $(ZSTD) hello.tmp > hello.zstd - $(ZSTD) world.tmp > world.zstd - @cat hello.zstd world.zstd > helloworld.zstd - $(ZSTD) -df helloworld.zstd > result.tmp - cat result.tmp - sdiff helloworld.tmp result.tmp - @rm *.tmp *.zstd - @echo frame concatenation test completed - @echo "**** flush write error test **** " - echo foo | $(ZSTD) > /dev/full; if [ $$? -eq 0 ] ; then echo "write error not detected!"; false; fi - echo foo | $(ZSTD) | $(ZSTD) -d > /dev/full; if [ $$? -eq 0 ] ; then echo "write error not detected!"; false; fi - @echo "**** zstd round-trip tests **** " - @./datagen | md5sum > tmp1 - ./datagen | $(ZSTD) -v | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen | $(ZSTD) -6 -v | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - @./datagen -g270000000 | md5sum > tmp1 - ./datagen -g270000000 | $(ZSTD) -v | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g270000000 | $(ZSTD) -v2 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g270000000 | $(ZSTD) -v3 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - @./datagen -g140000000 -P60| md5sum > tmp1 - ./datagen -g140000000 -P60 | $(ZSTD) -v4 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g140000000 -P60 | $(ZSTD) -v5 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g140000000 -P60 | $(ZSTD) -v6 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - @./datagen -g70000000 -P70 | md5sum > tmp1 - ./datagen -g70000000 -P70 | $(ZSTD) -v7 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g70000000 -P70 | $(ZSTD) -v8 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g70000000 -P70 | $(ZSTD) -v9 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - @./datagen -g35000000 -P75 | md5sum > tmp1 - ./datagen -g35000000 -P75 | $(ZSTD) -v10 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g35000000 -P75 | $(ZSTD) -v11 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g35000000 -P75 | $(ZSTD) -v12 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - @./datagen -g18000000 -P80 | md5sum > tmp1 - ./datagen -g18000000 -P80 | $(ZSTD) -v13 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g18000000 -P80 | $(ZSTD) -v14 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g18000000 -P80 | $(ZSTD) -v15 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g18000000 -P80 | $(ZSTD) -v16 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g18000000 -P80 | $(ZSTD) -v17 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - @./datagen -g50000000 -P94 | md5sum > tmp1 - ./datagen -g50000000 -P94 | $(ZSTD) -v18 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g50000000 -P94 | $(ZSTD) -v19 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - @./datagen -g99000000 -P99 | md5sum > tmp1 - ./datagen -g99000000 -P99 | $(ZSTD) -v20 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g6000000000 -P99| md5sum > tmp1 - ./datagen -g6000000000 -P99| $(ZSTD) -vq | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 + ZSTD=$(ZSTD) ./playTests.sh --test-large-data test-zstd: ZSTD = ./zstd test-zstd: zstd zstd-playTests diff --git a/programs/playTests.sh b/programs/playTests.sh new file mode 100755 index 000000000..56327ff70 --- /dev/null +++ b/programs/playTests.sh @@ -0,0 +1,81 @@ +#!/bin/sh -e + +die() { + echo "$@" 1>&2 + exit 1 +} + +roundTripTest() { + if [ -n "$3" ]; then + local c="$3" + local p="$2" + else + local c="$2" + fi + + rm -f tmp1 tmp2 + echo "roundTripTest: ./datagen $1 $p | $ZSTD -v$c | $ZSTD -d" + ./datagen $1 $p | md5sum > tmp1 + ./datagen $1 $p | $ZSTD -v$c | $ZSTD -d | md5sum > tmp2 + diff -q tmp1 tmp2 +} + +[ -n "$ZSTD" ] || die "ZSTD variable must be defined!" + +printf "\n**** frame concatenation **** " + +echo "hello " > hello.tmp +echo "world!" > world.tmp +cat hello.tmp world.tmp > helloworld.tmp +$ZSTD hello.tmp > hello.zstd +$ZSTD world.tmp > world.zstd +cat hello.zstd world.zstd > helloworld.zstd +$ZSTD -df helloworld.zstd > result.tmp +cat result.tmp +sdiff helloworld.tmp result.tmp +rm ./*.tmp ./*.zstd + +echo frame concatenation test completed + +echo "**** flush write error test **** " + +echo foo | $ZSTD > /dev/full && die "write error not detected!" +echo foo | $ZSTD | $ZSTD -d > /dev/full && die "write error not detected!" + +echo "**** zstd round-trip tests **** " + +roundTripTest +roundTripTest '' 6 + +if [ "$1" != "--test-large-data" ]; then + echo "Skipping large data tests" + exit 0 +fi + +roundTripTest -g270000000 1 +roundTripTest -g270000000 2 +roundTripTest -g270000000 3 + +roundTripTest -g140000000 -P60 4 +roundTripTest -g140000000 -P60 5 +roundTripTest -g140000000 -P60 6 + +roundTripTest -g70000000 -P70 7 +roundTripTest -g70000000 -P70 8 +roundTripTest -g70000000 -P70 9 + +roundTripTest -g35000000 -P75 10 +roundTripTest -g35000000 -P75 11 +roundTripTest -g35000000 -P75 12 + +roundTripTest -g18000000 -P80 13 +roundTripTest -g18000000 -P80 14 +roundTripTest -g18000000 -P80 15 +roundTripTest -g18000000 -P80 16 +roundTripTest -g18000000 -P80 17 + +roundTripTest -g50000000 -P94 18 +roundTripTest -g50000000 -P94 19 + +roundTripTest -g99000000 -P99 20 +roundTripTest -g6000000000 -P99 q