From f06e238d6f3d432d978df6708fd7c3669fcf03e9 Mon Sep 17 00:00:00 2001 From: Peter Harris Date: Tue, 1 Dec 2015 14:58:24 -0500 Subject: [PATCH 1/6] Fix compression on Big Endian CPUs --- lib/zstd_compress.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index 5ca4437f9..8c30b8a69 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -620,7 +620,7 @@ static unsigned ZSTD_NbCommonBytes (register size_t val) } else /* Big Endian CPU */ { - if (MEM_32bits()) + if (MEM_64bits()) { # if defined(_MSC_VER) && defined(_WIN64) unsigned long r = 0; From a9b34280a163f0ae59e80398f317aec6c77c4ac5 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 2 Dec 2015 13:00:25 +0100 Subject: [PATCH 2/6] minor cosmetic improvement for small blocks --- programs/paramgrill.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/programs/paramgrill.c b/programs/paramgrill.c index 346a9c34b..44d3ffdc7 100644 --- a/programs/paramgrill.c +++ b/programs/paramgrill.c @@ -681,6 +681,8 @@ static void BMK_selectRandomStart( /* totally random entry */ ZSTD_parameters p; potentialRandomParams(&p, 1); + p.srcSize = srcSize; + ZSTD_validateParams(&p); playAround(f, winners, p, srcBuffer, srcSize, ctx); } else @@ -734,6 +736,7 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize) for (i=1; i<=maxSeeds; i++) { params = ZSTD_getParams(i, blockSize); + ZSTD_validateParams(¶ms); BMK_seed(winners, params, srcBuffer, srcSize, ctx); } } From a308259afa218dfd1f351ef9147fe857d1938498 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 2 Dec 2015 13:38:48 +0100 Subject: [PATCH 3/6] new : parameters for <= 16KB --- lib/zstd_compress.c | 2 +- lib/zstd_static.h | 30 +++++++++++++++++++++++++++--- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index 8c30b8a69..e9ff1726b 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -2036,7 +2036,7 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* ctx, ZSTD_parameters ZSTD_getParams(int compressionLevel, U64 srcSizeHint) { ZSTD_parameters result; - int tableID = ((srcSizeHint-1) <= 128 KB); /* intentional underflow for srcSizeHint == 0 */ + int tableID = ((srcSizeHint-1) <= 128 KB) + ((srcSizeHint-1) <= 16 KB); /* intentional underflow for srcSizeHint == 0 */ if (compressionLevel<=0) compressionLevel = 1; if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL; result = ZSTD_defaultParameters[tableID][compressionLevel]; diff --git a/lib/zstd_static.h b/lib/zstd_static.h index db31c9fef..f698153ec 100644 --- a/lib/zstd_static.h +++ b/lib/zstd_static.h @@ -169,9 +169,9 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, co * Pre-defined compression levels ***************************************/ #define ZSTD_MAX_CLEVEL 20 -static const ZSTD_parameters ZSTD_defaultParameters[2][ZSTD_MAX_CLEVEL+1] = { +static const ZSTD_parameters ZSTD_defaultParameters[3][ZSTD_MAX_CLEVEL+1] = { { /* "default" */ - /* W, C, H, S, L, strat */ + /* W, C, H, S, L, strat */ { 0, 18, 12, 12, 1, 4, ZSTD_fast }, /* level 0 - never used */ { 0, 19, 13, 14, 1, 7, ZSTD_fast }, /* level 1 */ { 0, 19, 15, 16, 1, 6, ZSTD_fast }, /* level 2 */ @@ -195,7 +195,7 @@ static const ZSTD_parameters ZSTD_defaultParameters[2][ZSTD_MAX_CLEVEL+1] = { { 0, 26, 27, 25, 9, 5, ZSTD_btlazy2 }, /* level 20 */ }, { /* for srcSize <= 128 KB */ - /* W, C, H, S, L, strat */ + /* W, C, H, S, L, strat */ { 0, 17, 12, 12, 1, 4, ZSTD_fast }, /* level 0 - never used */ { 0, 17, 12, 13, 1, 6, ZSTD_fast }, /* level 1 */ { 0, 17, 15, 16, 1, 5, ZSTD_fast }, /* level 2 */ @@ -218,6 +218,30 @@ static const ZSTD_parameters ZSTD_defaultParameters[2][ZSTD_MAX_CLEVEL+1] = { { 0, 17, 18, 16, 10, 4, ZSTD_btlazy2 }, /* level 19 */ { 0, 17, 18, 18, 12, 4, ZSTD_btlazy2 }, /* level 20 */ }, +{ /* for srcSize <= 16 KB */ + /* W, C, H, S, L, strat */ + { 0, 0, 0, 0, 0, 0, ZSTD_fast }, /* level 0 - never used */ + { 0, 14, 14, 14, 1, 4, ZSTD_fast }, /* level 1 */ + { 0, 14, 14, 16, 1, 4, ZSTD_fast }, /* level 1 */ + { 0, 14, 14, 14, 5, 4, ZSTD_greedy }, /* level 3 */ + { 0, 14, 14, 14, 8, 4, ZSTD_greedy }, /* level 4 */ + { 0, 14, 11, 14, 6, 4, ZSTD_lazy }, /* level 5 */ + { 0, 14, 14, 13, 6, 5, ZSTD_lazy }, /* level 6 */ + { 0, 14, 14, 14, 7, 6, ZSTD_lazy }, /* level 7 */ + { 0, 14, 14, 14, 8, 4, ZSTD_lazy }, /* level 8 */ + { 0, 14, 14, 15, 9, 4, ZSTD_lazy }, /* level 9 */ + { 0, 14, 14, 15, 10, 4, ZSTD_lazy }, /* level 10 */ + { 0, 14, 15, 15, 6, 4, ZSTD_btlazy2 }, /* level 11 */ + { 0, 14, 15, 15, 7, 4, ZSTD_btlazy2 }, /* level 12 */ + { 0, 14, 15, 15, 8, 4, ZSTD_btlazy2 }, /* level 13 */ + { 0, 14, 15, 15, 9, 4, ZSTD_btlazy2 }, /* level 14 */ + { 0, 14, 15, 15, 10, 4, ZSTD_btlazy2 }, /* level 15 */ + { 0, 14, 15, 15, 11, 4, ZSTD_btlazy2 }, /* level 16 */ + { 0, 14, 15, 15, 12, 4, ZSTD_btlazy2 }, /* level 17 */ + { 0, 14, 15, 15, 13, 4, ZSTD_btlazy2 }, /* level 18 */ + { 0, 14, 15, 15, 14, 4, ZSTD_btlazy2 }, /* level 19 */ + { 0, 14, 15, 15, 15, 4, ZSTD_btlazy2 }, /* level 20 */ +}, }; From 120230b895a368afa8dde8e8476a8c10c72a62f4 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 2 Dec 2015 14:00:45 +0100 Subject: [PATCH 4/6] compress uses less memory for small blocks --- lib/zstd_compress.c | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index e9ff1726b..6304e2b6e 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -100,8 +100,6 @@ static void ZSTD_resetSeqStore(seqStore_t* ssPtr) /* ************************************* * Context memory management ***************************************/ -#define WORKPLACESIZE (BLOCKSIZE*3) - struct ZSTD_CCtx_s { const BYTE* nextSrc; /* next block here to continue on current prefix */ @@ -113,6 +111,7 @@ struct ZSTD_CCtx_s ZSTD_parameters params; void* workSpace; size_t workSpaceSize; + size_t blockSize; seqStore_t seqStore; /* sequences storage ptrs */ U32* hashTable; @@ -170,11 +169,14 @@ void ZSTD_validateParams(ZSTD_parameters* params) static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, ZSTD_parameters params) { + /* note : params considered validated here */ + const size_t blockSize = MIN(BLOCKSIZE, (size_t)1 << params.windowLog); + /* reserve table memory */ { const U32 contentLog = (params.strategy == ZSTD_fast) ? 1 : params.contentLog; const size_t tableSpace = ((1 << contentLog) + (1 << params.hashLog)) * sizeof(U32); - const size_t neededSpace = tableSpace + WORKPLACESIZE; + const size_t neededSpace = tableSpace + (3*blockSize); if (zc->workSpaceSize < neededSpace) { free(zc->workSpace); @@ -195,12 +197,13 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, zc->dictLimit = 0; zc->lowLimit = 0; zc->params = params; + zc->blockSize = blockSize; zc->seqStore.offsetStart = (U32*) (zc->seqStore.buffer); - zc->seqStore.offCodeStart = (BYTE*) (zc->seqStore.offsetStart + (BLOCKSIZE>>2)); - zc->seqStore.litStart = zc->seqStore.offCodeStart + (BLOCKSIZE>>2); - zc->seqStore.litLengthStart = zc->seqStore.litStart + BLOCKSIZE; - zc->seqStore.matchLengthStart = zc->seqStore.litLengthStart + (BLOCKSIZE>>2); - zc->seqStore.dumpsStart = zc->seqStore.matchLengthStart + (BLOCKSIZE>>2); + zc->seqStore.offCodeStart = (BYTE*) (zc->seqStore.offsetStart + (blockSize>>2)); + zc->seqStore.litStart = zc->seqStore.offCodeStart + (blockSize>>2); + zc->seqStore.litLengthStart = zc->seqStore.litStart + blockSize; + zc->seqStore.matchLengthStart = zc->seqStore.litLengthStart + (blockSize>>2); + zc->seqStore.dumpsStart = zc->seqStore.matchLengthStart + (blockSize>>2); return 0; } @@ -1919,7 +1922,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* ctxPtr, void* dst, size_t maxDstSize, const void* src, size_t srcSize) { - size_t blockSize = BLOCKSIZE; + size_t blockSize = ctxPtr->blockSize; size_t remaining = srcSize; const BYTE* ip = (const BYTE*)src; BYTE* const ostart = (BYTE*)dst; From aaf6f2099319d217c4a07b8e8e906ef86f9cb365 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 2 Dec 2015 15:03:11 +0100 Subject: [PATCH 5/6] update NEWS for 0.4.2 --- NEWS | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/NEWS b/NEWS index b76f578a1..00cfb8c2e 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,7 @@ +v0.4.2 : +Generic minor improvements for small blocks +Fixed : big-endian compatibility, by Peter Harris (#85) + v0.4.1 Fixed : ZSTD_LEGACY_SUPPORT=0 build mode (reported by Luben) removed `zstd.c` From 6a2f03295ccd153eb4bf6f4d5be2b4da83caaf46 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 2 Dec 2015 15:05:22 +0100 Subject: [PATCH 6/6] update version to 0.4.2 --- Makefile | 2 +- lib/zstd.h | 2 +- programs/Makefile | 2 +- programs/zstdcli.c | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index fbdb05c91..a1b6d1230 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ # ################################################################ # Version number -export VERSION := 0.4.0 +export VERSION := 0.4.2 PRGDIR = programs ZSTDDIR = lib diff --git a/lib/zstd.h b/lib/zstd.h index 96bb184f0..15fc62633 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -48,7 +48,7 @@ extern "C" { ***************************************/ #define ZSTD_VERSION_MAJOR 0 /* for breaking interface changes */ #define ZSTD_VERSION_MINOR 4 /* for new (non-breaking) interface capabilities */ -#define ZSTD_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */ +#define ZSTD_VERSION_RELEASE 2 /* for tweaks, bug-fixes, or development */ #define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) unsigned ZSTD_versionNumber (void); diff --git a/programs/Makefile b/programs/Makefile index c6a58939c..17fed5b0a 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -30,7 +30,7 @@ # fullbench32: Same as fullbench, but forced to compile in 32-bits mode # ########################################################################## -VERSION?= 0.4.0 +VERSION?= 0.4.2 DESTDIR?= PREFIX ?= /usr/local diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 235b56cf3..7904a616f 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -70,7 +70,7 @@ **************************************/ #define COMPRESSOR_NAME "zstd command line interface" #ifndef ZSTD_VERSION -# define ZSTD_VERSION "v0.4.0" +# define ZSTD_VERSION "v0.4.2" #endif #define AUTHOR "Yann Collet" #define WELCOME_MESSAGE "*** %s %i-bits %s, by %s (%s) ***\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), ZSTD_VERSION, AUTHOR, __DATE__