From 2cc72f1fd35b9778181f6e2fe554d6ca162450d0 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 6 Jun 2016 17:50:07 +0200 Subject: [PATCH 01/11] fixed initialization issue in bench --- lib/compress/zstd_compress.c | 2 +- programs/bench.c | 1 + programs/fileio.c | 1 - 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 48ca30e49..9c06f603c 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2148,7 +2148,7 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity, ZSTD_parameters params, U64 pledgedSrcSize, U32 dictID) { BYTE* const op = (BYTE*)dst; U32 const dictIDSizeCode = (dictID>0) + (dictID>=256) + (dictID>=65536); /* 0-3 */ - U32 const checksumFlag = params.fParams.checksumFlag; + U32 const checksumFlag = params.fParams.checksumFlag>0; U32 const windowSize = 1U << params.cParams.windowLog; U32 const directModeFlag = params.fParams.contentSizeFlag && (windowSize > (pledgedSrcSize-1)); BYTE const windowLogByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3); diff --git a/programs/bench.c b/programs/bench.c index 08a972c75..ecc609d83 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -219,6 +219,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, do { U32 blockNb; { ZSTD_parameters params; + memset(¶ms, 0, sizeof(params)); params.cParams = ZSTD_getCParams(cLevel, blockSize, dictBufferSize); params.fParams.contentSizeFlag = 1; { size_t const initResult = ZSTD_compressBegin_advanced(refCtx, dictBuffer, dictBufferSize, params, blockSize); diff --git a/programs/fileio.c b/programs/fileio.c index 37f6e7822..47302f9ca 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -622,7 +622,6 @@ unsigned long long FIO_decompressFrame(dRess_t ress, /* Main decompression Loop */ while (1) { - /* Decode */ size_t inSize=readSize, decodedSize=ress.dstBufferSize; size_t const toRead = ZBUFF_decompressContinue(ress.dctx, ress.dstBuffer, &decodedSize, ress.srcBuffer, &inSize); if (ZBUFF_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZBUFF_getErrorName(toRead)); From c28f9e271358d3c258236acb2748d90006031652 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 6 Jun 2016 18:00:00 +0200 Subject: [PATCH 02/11] attempt to re-enable msan tests --- .travis.yml | 3 +++ Makefile | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 879aff8a9..6a2e90309 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,6 +22,9 @@ matrix: - os: linux sudo: false env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=asan + - os: linux + sudo: false + env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=msan - os: linux sudo: false env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=zlibwrapper diff --git a/Makefile b/Makefile index 755b0c1a8..18db04245 100644 --- a/Makefile +++ b/Makefile @@ -114,7 +114,7 @@ ppctest-w-install: clean ppcinstall ppctest ppc64test: clean $(MAKE) -C $(PRGDIR) datagen # use native, faster - $(MAKE) -C $(PRGDIR) test CC=powerpc64le-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static" + $(MAKE) -C $(PRGDIR) test CC=powerpc64le-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static" ppc64install: clean sudo apt-get update -y -q @@ -130,7 +130,7 @@ asan: clean $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address" msan: clean - $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory" # datagen.c fails this test, for no obvious reason + $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory" # datagen.c used to fail this test for no obvious reason asan32: clean $(MAKE) -C $(PRGDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address" From f72dc6091b24141db660eeb6f56b482c9f722155 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 6 Jun 2016 19:42:47 +0200 Subject: [PATCH 03/11] removed msan tests, due to issues with `datagen.c` --- .travis.yml | 3 --- programs/datagen.c | 47 +++++++++++++++++++++++++++------------------- 2 files changed, 28 insertions(+), 22 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6a2e90309..879aff8a9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,9 +22,6 @@ matrix: - os: linux sudo: false env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=asan - - os: linux - sudo: false - env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=msan - os: linux sudo: false env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=zlibwrapper diff --git a/programs/datagen.c b/programs/datagen.c index fd9b2316f..ec118f5d1 100644 --- a/programs/datagen.c +++ b/programs/datagen.c @@ -27,9 +27,9 @@ * Includes **************************************/ #include /* malloc */ -#include /* FILE, fwrite */ +#include /* FILE, fwrite, fprintf */ #include /* memcpy */ -#include "mem.h" +#include "mem.h" /* U32 */ /*-************************************ @@ -92,7 +92,7 @@ static void RDG_fillLiteralDistrib(BYTE* ldt, double ld) for (u=0; u lastChar) character = firstChar; } @@ -108,43 +108,52 @@ static BYTE RDG_genChar(U32* seed, const BYTE* ldt) } -#define RDG_RAND15BITS ( RDG_rand(seed) & 0x7FFF ) -#define RDG_RANDLENGTH ( (RDG_rand(seed) & 7) ? (RDG_rand(seed) & 0xF) : (RDG_rand(seed) & 0x1FF) + 0xF) +static U32 RDG_rand15Bits (unsigned* seedPtr) +{ + return RDG_rand(seedPtr) & 0x7FFF; +} + +static U32 RDG_randLength(unsigned* seedPtr) +{ + if (RDG_rand(seedPtr) & 7) + return (RDG_rand(seedPtr) & 0xF); + return (RDG_rand(seedPtr) & 0x1FF) + 0xF; +} + void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double matchProba, const BYTE* ldt, unsigned* seedPtr) { - BYTE* buffPtr = (BYTE*)buffer; - const U32 matchProba32 = (U32)(32768 * matchProba); + BYTE* const buffPtr = (BYTE*)buffer; + U32 const matchProba32 = (U32)(32768 * matchProba); size_t pos = prefixSize; - U32* seed = seedPtr; U32 prevOffset = 1; /* special case : sparse content */ while (matchProba >= 1.0) { - size_t size0 = RDG_rand(seed) & 3; + size_t size0 = RDG_rand(seedPtr) & 3; size0 = (size_t)1 << (16 + size0 * 2); - size0 += RDG_rand(seed) & (size0-1); /* because size0 is power of 2*/ + size0 += RDG_rand(seedPtr) & (size0-1); /* because size0 is power of 2*/ if (buffSize < pos + size0) { memset(buffPtr+pos, 0, buffSize-pos); return; } memset(buffPtr+pos, 0, size0); pos += size0; - buffPtr[pos-1] = RDG_genChar(seed, ldt); + buffPtr[pos-1] = RDG_genChar(seedPtr, ldt); continue; } /* init */ - if (pos==0) buffPtr[0] = RDG_genChar(seed, ldt), pos=1; + if (pos==0) buffPtr[0] = RDG_genChar(seedPtr, ldt), pos=1; /* Generate compressible data */ while (pos < buffSize) { /* Select : Literal (char) or Match (within 32K) */ - if (RDG_RAND15BITS < matchProba32) { + if (RDG_rand15Bits(seedPtr) < matchProba32) { /* Copy (within 32K) */ - U32 const length = RDG_RANDLENGTH + 4; + U32 const length = RDG_randLength(seedPtr) + 4; U32 const d = (U32) MIN(pos + length , buffSize); - U32 const repeatOffset = (RDG_rand(seed) & 15) == 2; - U32 const randOffset = RDG_RAND15BITS + 1; + U32 const repeatOffset = (RDG_rand(seedPtr) & 15) == 2; + U32 const randOffset = RDG_rand15Bits(seedPtr) + 1; U32 const offset = repeatOffset ? prevOffset : (U32) MIN(randOffset , pos); size_t match = pos - offset; //TRACE("pos : %u; offset: %u ; length : %u \n", (U32)pos, offset, length); @@ -152,9 +161,9 @@ void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double match prevOffset = offset; } else { /* Literal (noise) */ - U32 const length = RDG_RANDLENGTH; + U32 const length = RDG_randLength(seedPtr); U32 const d = (U32) MIN(pos + length, buffSize); - while (pos < d) buffPtr[pos++] = RDG_genChar(seed, ldt); + while (pos < d) buffPtr[pos++] = RDG_genChar(seedPtr, ldt); } } } @@ -174,7 +183,7 @@ void RDG_genStdout(unsigned long long size, double matchProba, double litProba, { size_t const stdBlockSize = 128 KB; size_t const stdDictSize = 32 KB; - BYTE* buff = (BYTE*)malloc(stdDictSize + stdBlockSize); + BYTE* const buff = (BYTE*)malloc(stdDictSize + stdBlockSize); U64 total = 0; BYTE ldt[LTSIZE]; From 9d504ae85b6615dbfca90db8f33978a8df44aa33 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 6 Jun 2016 19:52:35 +0200 Subject: [PATCH 04/11] Added decoding of RLE blocks --- lib/decompress/zstd_decompress.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 4630eb8a3..9b7b1886b 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -53,7 +53,7 @@ /*-******************************************************* * Dependencies *********************************************************/ -#include /* memcpy, memmove */ +#include /* memcpy, memmove, memset */ #include /* debug only : printf */ #include "mem.h" /* low level memory routines */ #define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ @@ -938,6 +938,14 @@ size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, } +size_t ZSTD_generateNxByte(void* dst, size_t dstCapacity, BYTE byte, size_t length) +{ + if (length > dstCapacity) return ERROR(dstSize_tooSmall); + memset(dst, byte, length); + return length; +} + + /*! ZSTD_decompressFrame() : * `dctx` must be properly initialized */ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, @@ -982,7 +990,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, decodedSize = ZSTD_copyRawBlock(op, oend-op, ip, cBlockSize); break; case bt_rle : - return ERROR(GENERIC); /* not yet supported */ + decodedSize = ZSTD_generateNxByte(op, oend-op, *ip, blockProperties.origSize); break; case bt_end : /* end of frame */ From 81e13ef7cfceb5b79e11eb4c3042b5c4cbadd9c4 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 7 Jun 2016 00:51:51 +0200 Subject: [PATCH 05/11] first implementation of the new dictionary API (untested) --- lib/common/zstd.h | 125 ++++++++++++++++++++++--------- lib/compress/zstd_compress.c | 85 +++++++++++++++++++-- lib/decompress/zstd_decompress.c | 74 ++++++++++++++++++ 3 files changed, 242 insertions(+), 42 deletions(-) diff --git a/lib/common/zstd.h b/lib/common/zstd.h index b899656d1..ea0840b69 100644 --- a/lib/common/zstd.h +++ b/lib/common/zstd.h @@ -129,9 +129,47 @@ ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapa /*-*********************** * Dictionary API *************************/ +/*! ZSTD_createCDict() : +* Create a digested dictionary, ready to start compression operation without startup delay. +* `dict` can be released after creation */ +typedef struct ZSTD_CDict_s ZSTD_CDict; +ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel); +ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict); + +/*! ZSTD_compress_usingCDict() : +* Compression using a pre-digested Dictionary +* In contrast with older ZSTD_compress_usingDict(), use dictionary without significant overhead. +* Note that compression level is decided during dictionary creation */ +ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict); + +/*! ZSTD_createDDict() : +* Create a digested dictionary, ready to start decompression operation without startup delay. +* `dict` can be released after creation */ +typedef struct ZSTD_DDict_s ZSTD_DDict; +ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize); +ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict); + +/*! ZSTD_decompress_usingDDict() : +* Decompression using a pre-digested Dictionary +* In contrast with older ZSTD_decompress_usingDict(), use dictionary without significant overhead. */ +ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_DDict* ddict); + + +/*-*********************** +* Deprecated API +*************************/ /*! ZSTD_compress_usingDict() : +* *** Warning : this function will soon be declared deprecated *** * Compression using a pre-defined Dictionary content (see dictBuilder). -* Note : dict can be NULL, in which case, it's equivalent to ZSTD_compressCCtx() */ +* Note 1 : This function load the dictionary, resulting in a significant startup time. +* Note 2 : `dict` must remain valid and unmodified during compression operation. +* Note 3 : `dict` can be `NULL`, in which case, it's equivalent to ZSTD_compressCCtx() */ ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, @@ -139,9 +177,12 @@ ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, int compressionLevel); /*! ZSTD_decompress_usingDict() : +* *** Warning : this function will soon be declared deprecated *** * Decompression using a pre-defined Dictionary content (see dictBuilder). -* Dictionary must be identical to the one used during compression, otherwise regenerated data will be corrupted. -* Note : dict can be NULL, in which case, it's equivalent to ZSTD_decompressDCtx() */ +* Dictionary must be identical to the one used during compression. +* Note 1 : This function load the dictionary, resulting in a significant startup time +* Note 2 : `dict` must remain valid and unmodified during compression operation. +* Note 3 : `dict` can be `NULL`, in which case, it's equivalent to ZSTD_decompressDCtx() */ ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, @@ -158,7 +199,7 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, * ==================================================================================== */ /*--- Dependency ---*/ -#include "mem.h" +#include "mem.h" /* U32 */ /*--- Constants ---*/ @@ -187,22 +228,22 @@ static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable f /*--- Types ---*/ -typedef enum { ZSTD_fast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt } ZSTD_strategy; /* from faster to stronger */ +typedef enum { ZSTD_fast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt } ZSTD_strategy; /*< from faster to stronger */ typedef struct { - U32 windowLog; /* largest match distance : larger == more compression, more memory needed during decompression */ - U32 chainLog; /* fully searched segment : larger == more compression, slower, more memory (useless for fast) */ - U32 hashLog; /* dispatch table : larger == faster, more memory */ - U32 searchLog; /* nb of searches : larger == more compression, slower */ - U32 searchLength; /* match length searched : larger == faster decompression, sometimes less compression */ - U32 targetLength; /* acceptable match size for optimal parser (only) : larger == more compression, slower */ + U32 windowLog; /*< largest match distance : larger == more compression, more memory needed during decompression */ + U32 chainLog; /*< fully searched segment : larger == more compression, slower, more memory (useless for fast) */ + U32 hashLog; /*< dispatch table : larger == faster, more memory */ + U32 searchLog; /*< nb of searches : larger == more compression, slower */ + U32 searchLength; /*< match length searched : larger == faster decompression, sometimes less compression */ + U32 targetLength; /*< acceptable match size for optimal parser (only) : larger == more compression, slower */ ZSTD_strategy strategy; } ZSTD_compressionParameters; typedef struct { - U32 contentSizeFlag; /* 1: content size will be in frame header (if known). */ - U32 checksumFlag; /* 1: will generate a 22-bits checksum at end of frame, to be used for error detection by decompressor */ - U32 noDictIDFlag; /* 1: no dict ID will be saved into frame header (if dictionary compression) */ + U32 contentSizeFlag; /*< 1: content size will be in frame header (if known). */ + U32 checksumFlag; /*< 1: will generate a 22-bits checksum at end of frame, to be used for error detection by decompressor */ + U32 noDictIDFlag; /*< 1: no dict ID will be saved into frame header (if dictionary compression) */ } ZSTD_frameParameters; typedef struct { @@ -217,15 +258,16 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v /*-************************************* -* Advanced functions +* Advanced compression functions ***************************************/ /*! ZSTD_createCCtx_advanced() : * Create a ZSTD compression context using external alloc and free functions */ ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem); -/*! ZSTD_createDCtx_advanced() : - * Create a ZSTD decompression context using external alloc and free functions */ -ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); +/*! ZSTD_createCDict_advanced() : + * Create a ZSTD_CDict using external alloc and free, and customized compression parameters */ +ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, + ZSTD_parameters params, ZSTD_customMem customMem); ZSTDLIB_API unsigned ZSTD_maxCLevel (void); @@ -251,27 +293,11 @@ ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx, const void* dict,size_t dictSize, ZSTD_parameters params); -/*! ZSTD_compress_usingPreparedDCtx() : -* Same as ZSTD_compress_usingDict, but using a reference context `preparedCCtx`, where dictionary has been loaded. -* It avoids reloading the dictionary each time. -* `preparedCCtx` must have been properly initialized using ZSTD_compressBegin_usingDict() or ZSTD_compressBegin_advanced(). -* Requires 2 contexts : 1 for reference (preparedCCtx) which will not be modified, and 1 to run the compression operation (cctx) */ -ZSTDLIB_API size_t ZSTD_compress_usingPreparedCCtx( - ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize); - /*- Advanced Decompression functions -*/ -/*! ZSTD_decompress_usingPreparedDCtx() : -* Same as ZSTD_decompress_usingDict, but using a reference context `preparedDCtx`, where dictionary has been loaded. -* It avoids reloading the dictionary each time. -* `preparedDCtx` must have been properly initialized using ZSTD_decompressBegin_usingDict(). -* Requires 2 contexts : 1 for reference (preparedDCtx), which will not be modified, and 1 to run the decompression operation (dctx) */ -ZSTDLIB_API size_t ZSTD_decompress_usingPreparedDCtx( - ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize); +/*! ZSTD_createDCtx_advanced() : + * Create a ZSTD decompression context using external alloc and free functions */ +ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); /* ************************************** @@ -405,6 +431,31 @@ ZSTDLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult); ZSTDLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code); +/*-*********************** +* Deprecated API +*************************/ +/*! ZSTD_compress_usingPreparedCCtx() : +* *** Warning : this function will soon be deprecated ! *** +* Same as ZSTD_compress_usingDict, but using a reference context `preparedCCtx`, where dictionary has been loaded. +* It avoids reloading the dictionary each time. +* `preparedCCtx` must have been properly initialized using ZSTD_compressBegin_usingDict() or ZSTD_compressBegin_advanced(). +* Requires 2 contexts : 1 for reference (preparedCCtx) which will not be modified, and 1 to run the compression operation (cctx) */ +ZSTDLIB_API size_t ZSTD_compress_usingPreparedCCtx( + ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + +/*! ZSTD_decompress_usingPreparedDCtx() : +* *** Warning : this function will soon be deprecated ! *** +* Same as ZSTD_decompress_usingDict, but using a reference context `preparedDCtx`, where dictionary has been loaded. +* It avoids reloading the dictionary each time. +* `preparedDCtx` must have been properly initialized using ZSTD_decompressBegin_usingDict(). +* Requires 2 contexts : 1 for reference (preparedDCtx), which will not be modified, and 1 to run the decompression operation (dctx) */ +ZSTDLIB_API size_t ZSTD_decompress_usingPreparedDCtx( + ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + #endif /* ZSTD_STATIC_LINKING_ONLY */ #if defined (__cplusplus) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 9c06f603c..e57f38d90 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2385,7 +2385,7 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* zc, /*! ZSTD_compressBegin_advanced() : * @return : 0, or an error code */ -size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* zc, +size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, U64 pledgedSrcSize) { @@ -2393,17 +2393,17 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* zc, { size_t const errorCode = ZSTD_checkCParams_advanced(params.cParams, pledgedSrcSize); if (ZSTD_isError(errorCode)) return errorCode; } - return ZSTD_compressBegin_internal(zc, dict, dictSize, params, pledgedSrcSize); + return ZSTD_compressBegin_internal(cctx, dict, dictSize, params, pledgedSrcSize); } -size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* zc, const void* dict, size_t dictSize, int compressionLevel) +size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel) { ZSTD_parameters params; memset(¶ms, 0, sizeof(params)); params.cParams = ZSTD_getCParams(compressionLevel, 0, dictSize); - ZSTD_LOG_BLOCK("%p: ZSTD_compressBegin_usingDict compressionLevel=%d\n", zc->base, compressionLevel); - return ZSTD_compressBegin_internal(zc, dict, dictSize, params, 0); + ZSTD_LOG_BLOCK("%p: ZSTD_compressBegin_usingDict compressionLevel=%d\n", cctx->base, compressionLevel); + return ZSTD_compressBegin_internal(cctx, dict, dictSize, params, 0); } @@ -2532,6 +2532,81 @@ size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcS } +/* ===== Dictionary API ===== */ + +struct ZSTD_CDict_s { + void* dictContent; + size_t dictContentSize; + ZSTD_CCtx* refContext; +}; /* typedef'd tp ZSTD_CDict within zstd.h */ + +ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, ZSTD_parameters params, ZSTD_customMem customMem) +{ + if (!customMem.customAlloc && !customMem.customFree) + customMem = defaultCustomMem; + + if (!customMem.customAlloc || !customMem.customFree) + return NULL; + + { ZSTD_CDict* const cdict = (ZSTD_CDict*) customMem.customAlloc(customMem.opaque, sizeof(*cdict)); + void* const dictContent = customMem.customAlloc(customMem.opaque, dictSize); + ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(customMem); + + if (!dictContent || !cdict || !cctx) { + customMem.customFree(customMem.opaque, dictContent); + customMem.customFree(customMem.opaque, cdict); + customMem.customFree(customMem.opaque, cctx); + return NULL; + } + + memcpy(dictContent, dict, dictSize); + { size_t const errorCode = ZSTD_compressBegin_advanced(cctx, dictContent, dictSize, params, 0); + if (ZSTD_isError(errorCode)) { + customMem.customFree(customMem.opaque, dictContent); + customMem.customFree(customMem.opaque, cdict); + customMem.customFree(customMem.opaque, cctx); + return NULL; + } } + + cdict->dictContent = dictContent; + cdict->dictContentSize = dictSize; + cdict->refContext = cctx; + return cdict; + } +} + +ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel) +{ + ZSTD_customMem const allocator = { NULL, NULL, NULL }; + ZSTD_parameters params; + memset(¶ms, 0, sizeof(params)); + params.cParams = ZSTD_getCParams(compressionLevel, 0, dictSize); + params.fParams.contentSizeFlag = 1; + return ZSTD_createCDict_advanced(dict, dictSize, params, allocator); +} + +size_t ZSTD_freeCDict(ZSTD_CDict* cdict) +{ + ZSTD_freeFunction const cFree = cdict->refContext->customMem.customFree; + void* const opaque = cdict->refContext->customMem.opaque; + ZSTD_freeCCtx(cdict->refContext); + cFree(opaque, cdict->dictContent); + cFree(opaque, cdict); + return 0; +} + +ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict) +{ + return ZSTD_compress_usingPreparedCCtx(cctx, cdict->refContext, + dst, dstCapacity, + src, srcSize); +} + + + /*-===== Pre-defined compression levels =====-*/ #define ZSTD_DEFAULT_CLEVEL 1 diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 9b7b1886b..344bd0201 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -1267,3 +1267,77 @@ size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t return 0; } + + +struct ZSTD_DDict_s { + void* dictContent; + size_t dictContentSize; + ZSTD_DCtx* refContext; +}; /* typedef'd tp ZSTD_CDict within zstd.h */ + +ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, ZSTD_customMem customMem) +{ + if (!customMem.customAlloc && !customMem.customFree) + customMem = defaultCustomMem; + + if (!customMem.customAlloc || !customMem.customFree) + return NULL; + + { ZSTD_DDict* const ddict = (ZSTD_DDict*) customMem.customAlloc(customMem.opaque, sizeof(*ddict)); + void* const dictContent = customMem.customAlloc(customMem.opaque, dictSize); + ZSTD_DCtx* const dctx = ZSTD_createDCtx_advanced(customMem); + + if (!dictContent || !ddict || !dctx) { + customMem.customFree(customMem.opaque, dictContent); + customMem.customFree(customMem.opaque, ddict); + customMem.customFree(customMem.opaque, dctx); + return NULL; + } + + memcpy(dictContent, dict, dictSize); + { size_t const errorCode = ZSTD_decompressBegin_usingDict(dctx, dictContent, dictSize); + if (ZSTD_isError(errorCode)) { + customMem.customFree(customMem.opaque, dictContent); + customMem.customFree(customMem.opaque, ddict); + customMem.customFree(customMem.opaque, dctx); + return NULL; + } } + + ddict->dictContent = dictContent; + ddict->dictContentSize = dictSize; + ddict->refContext = dctx; + return ddict; + } +} + +/*! ZSTD_createDDict() : +* Create a digested dictionary, ready to start decompression operation without startup delay. +* `dict` can be released after creation */ +ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize) +{ + ZSTD_customMem const allocator = { NULL, NULL, NULL }; + return ZSTD_createDDict_advanced(dict, dictSize, allocator); +} + +size_t ZSTD_freeDDict(ZSTD_DDict* ddict) +{ + ZSTD_freeFunction const cFree = ddict->refContext->customMem.customFree; + void* const opaque = ddict->refContext->customMem.opaque; + ZSTD_freeDCtx(ddict->refContext); + cFree(opaque, ddict->dictContent); + cFree(opaque, ddict); + return 0; +} + +/*! ZSTD_decompress_usingDDict() : +* Decompression using a pre-digested Dictionary +* In contrast with older ZSTD_decompress_usingDict(), use dictionary without significant overhead. */ +ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_DDict* ddict) +{ + return ZSTD_decompress_usingPreparedDCtx(dctx, ddict->refContext, + dst, dstCapacity, + src, srcSize); +} From ee1a084852aba12b370a4214b72ccf0ca459e45f Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 7 Jun 2016 01:40:49 +0200 Subject: [PATCH 06/11] Integrated new dictionary API into bench module --- programs/bench.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index ecc609d83..51d47bde1 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -216,24 +216,21 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, UTIL_getTime(&clockStart); { U32 nbLoops = 0; + ZSTD_CDict* cdict = ZSTD_createCDict(dictBuffer, dictBufferSize, cLevel); + if (cdict==NULL) EXM_THROW(1, "ZSTD_createCDict() allocation failure"); do { U32 blockNb; - { ZSTD_parameters params; - memset(¶ms, 0, sizeof(params)); - params.cParams = ZSTD_getCParams(cLevel, blockSize, dictBufferSize); - params.fParams.contentSizeFlag = 1; - { size_t const initResult = ZSTD_compressBegin_advanced(refCtx, dictBuffer, dictBufferSize, params, blockSize); - if (ZSTD_isError(initResult)) break; - } } for (blockNb=0; blockNb Date: Tue, 7 Jun 2016 11:36:13 +0200 Subject: [PATCH 07/11] Added build commands for various versions of Visual Studio - For people who want to build the library without opening it in VS --- .gitignore | 1 + build/build.VS2010.cmd | 7 +++++++ build/build.VS2012.cmd | 6 ++++++ build/build.VS2013.cmd | 7 +++++++ build/build.VS2015.cmd | 7 +++++++ build/build.generic.cmd | 38 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 66 insertions(+) create mode 100644 build/build.VS2010.cmd create mode 100644 build/build.VS2012.cmd create mode 100644 build/build.VS2013.cmd create mode 100644 build/build.VS2015.cmd create mode 100644 build/build.generic.cmd diff --git a/.gitignore b/.gitignore index 9a71d4bcd..a06c2afa2 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,7 @@ projects/VS2010 projects/VS2012 projects/VS2013 projects/VS2015 +build/bin # IDEA solution files *.idea diff --git a/build/build.VS2010.cmd b/build/build.VS2010.cmd new file mode 100644 index 000000000..ddaeb4eee --- /dev/null +++ b/build/build.VS2010.cmd @@ -0,0 +1,7 @@ +@echo off + +rem build 32-bit +call "%~p0%build.generic.cmd" VS2010 Release "Clean,Build" Win32 v100 + +rem build 64-bit +call "%~p0%build.generic.cmd" VS2010 Release "Clean,Build" x64 v100 \ No newline at end of file diff --git a/build/build.VS2012.cmd b/build/build.VS2012.cmd new file mode 100644 index 000000000..96431e113 --- /dev/null +++ b/build/build.VS2012.cmd @@ -0,0 +1,6 @@ +@echo off + +rem build 32-bit +call "%~p0%build.generic.cmd" VS2012 Release "Clean,Build" Win32 v110 +rem build 64-bit +call "%~p0%build.generic.cmd" VS2012 Release "Clean,Build" x64 v110 \ No newline at end of file diff --git a/build/build.VS2013.cmd b/build/build.VS2013.cmd new file mode 100644 index 000000000..06aed3e4f --- /dev/null +++ b/build/build.VS2013.cmd @@ -0,0 +1,7 @@ +@echo off + +rem build 32-bit +call "%~p0%build.generic.cmd" VS2013 Release "Clean,Build" Win32 v120 + +rem build 64-bit +call "%~p0%build.generic.cmd" VS2013 Release "Clean,Build" x64 v120 \ No newline at end of file diff --git a/build/build.VS2015.cmd b/build/build.VS2015.cmd new file mode 100644 index 000000000..0e2484949 --- /dev/null +++ b/build/build.VS2015.cmd @@ -0,0 +1,7 @@ +@echo off + +rem build 32-bit +call "%~p0%build.generic.cmd" VS2015 Release "Clean,Build" Win32 v140 + +rem build 64-bit +call "%~p0%build.generic.cmd" VS2015 Release "Clean,Build" x64 v140 \ No newline at end of file diff --git a/build/build.generic.cmd b/build/build.generic.cmd new file mode 100644 index 000000000..289776bf9 --- /dev/null +++ b/build/build.generic.cmd @@ -0,0 +1,38 @@ +@echo off + +rem +rem Parameters +rem %1: vs_solution_version: VS solution version: VS2012, VS2013, VS2015 +rem %2: vs_configuration: VS configuration: Debug, Release +rem %3: vs_target: target: Build or Rebuild +rem %4: vs_platform: platform: x64 or Win32 +rem %5: vs_toolset: toolset: v100, v110, v120, v140 +rem + +set vs_solution_version=%1 +set vs_configuration=%2 +set vs_target=%3 +set vs_platform=%4 +set vs_toolset=%5 + +set msbuild="%windir%\Microsoft.NET\Framework\v4.0.30319\MSBuild.exe" +IF %vs_solution_version% == VS2013 SET msbuild="C:\Program Files (x86)\MSBuild\12.0\Bin\MSBuild.exe" +IF %vs_solution_version% == VS2015 SET msbuild="C:\Program Files (x86)\MSBuild\14.0\Bin\MSBuild.exe" +rem TODO: Visual Studio "15" (vNext) will use MSBuild 15.0 ? + +set project="%~p0\..\projects\VS2010\zstd.sln" + +set output=%~p0%bin +set output="%output%/%vs_solution_version%/%vs_platform%/" + +set msbuildparams=/verbosity:minimal /p:Platform="%vs_platform%" /p:Configuration="%vs_configuration%" /p:PlatformToolset="%vs_toolset%" /t:Clean,Build /nologo /p:OutDir=%output% + +echo ### Building %vs_solution_version% project for %vs_configuration% %vs_platform% (%vs_toolset%)... +echo ### Build Params: %msbuildparams% + +%msbuild% %project% %msbuildparams% +IF ERRORLEVEL 1 EXIT /B 1 +echo # Success +echo # OutDir: %output% +echo # + From 302fb53a762a90fb8d1fa97e066bb30c78739c09 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 7 Jun 2016 12:16:49 +0200 Subject: [PATCH 08/11] Removed `ZSTD_*_usingPrepared?Ctx()` declaration from public space --- lib/common/zstd.h | 104 ++++++++++++------------------- lib/compress/zstd_compress.c | 7 ++- lib/decompress/zstd_decompress.c | 5 ++ 3 files changed, 50 insertions(+), 66 deletions(-) diff --git a/lib/common/zstd.h b/lib/common/zstd.h index ea0840b69..3c1e16a7b 100644 --- a/lib/common/zstd.h +++ b/lib/common/zstd.h @@ -126,46 +126,10 @@ ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); /*!< @return : error ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); -/*-*********************** -* Dictionary API -*************************/ -/*! ZSTD_createCDict() : -* Create a digested dictionary, ready to start compression operation without startup delay. -* `dict` can be released after creation */ -typedef struct ZSTD_CDict_s ZSTD_CDict; -ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel); -ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict); - -/*! ZSTD_compress_usingCDict() : -* Compression using a pre-digested Dictionary -* In contrast with older ZSTD_compress_usingDict(), use dictionary without significant overhead. -* Note that compression level is decided during dictionary creation */ -ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const ZSTD_CDict* cdict); - -/*! ZSTD_createDDict() : -* Create a digested dictionary, ready to start decompression operation without startup delay. -* `dict` can be released after creation */ -typedef struct ZSTD_DDict_s ZSTD_DDict; -ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize); -ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict); - -/*! ZSTD_decompress_usingDDict() : -* Decompression using a pre-digested Dictionary -* In contrast with older ZSTD_decompress_usingDict(), use dictionary without significant overhead. */ -ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const ZSTD_DDict* ddict); - - -/*-*********************** -* Deprecated API -*************************/ +/*-************************ +* Simple dictionary API +***************************/ /*! ZSTD_compress_usingDict() : -* *** Warning : this function will soon be declared deprecated *** * Compression using a pre-defined Dictionary content (see dictBuilder). * Note 1 : This function load the dictionary, resulting in a significant startup time. * Note 2 : `dict` must remain valid and unmodified during compression operation. @@ -177,7 +141,6 @@ ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, int compressionLevel); /*! ZSTD_decompress_usingDict() : -* *** Warning : this function will soon be declared deprecated *** * Decompression using a pre-defined Dictionary content (see dictBuilder). * Dictionary must be identical to the one used during compression. * Note 1 : This function load the dictionary, resulting in a significant startup time @@ -189,6 +152,42 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, const void* dict,size_t dictSize); +/*-************************** +* Advanced Dictionary API +****************************/ +/*! ZSTD_createCDict() : +* Create a digested dictionary, ready to start compression operation without startup delay. +* `dict` can be released after creation */ +typedef struct ZSTD_CDict_s ZSTD_CDict; +ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel); +ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict); + +/*! ZSTD_compress_usingCDict() : +* Compression using a pre-digested Dictionary. +* Much faster than ZSTD_compress_usingDict() when same dictionary is used multiple times. +* Note that compression level is decided during dictionary creation */ +ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict); + +/*! ZSTD_createDDict() : +* Create a digested dictionary, ready to start decompression operation without startup delay. +* `dict` can be released after creation */ +typedef struct ZSTD_DDict_s ZSTD_DDict; +ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize); +ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict); + +/*! ZSTD_decompress_usingDDict() : +* Decompression using a pre-digested Dictionary +* Much faster than ZSTD_decompress_usingDict() when same dictionary is used multiple times. */ +ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_DDict* ddict); + + + #ifdef ZSTD_STATIC_LINKING_ONLY /* ==================================================================================== @@ -431,31 +430,6 @@ ZSTDLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult); ZSTDLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code); -/*-*********************** -* Deprecated API -*************************/ -/*! ZSTD_compress_usingPreparedCCtx() : -* *** Warning : this function will soon be deprecated ! *** -* Same as ZSTD_compress_usingDict, but using a reference context `preparedCCtx`, where dictionary has been loaded. -* It avoids reloading the dictionary each time. -* `preparedCCtx` must have been properly initialized using ZSTD_compressBegin_usingDict() or ZSTD_compressBegin_advanced(). -* Requires 2 contexts : 1 for reference (preparedCCtx) which will not be modified, and 1 to run the compression operation (cctx) */ -ZSTDLIB_API size_t ZSTD_compress_usingPreparedCCtx( - ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize); - -/*! ZSTD_decompress_usingPreparedDCtx() : -* *** Warning : this function will soon be deprecated ! *** -* Same as ZSTD_decompress_usingDict, but using a reference context `preparedDCtx`, where dictionary has been loaded. -* It avoids reloading the dictionary each time. -* `preparedDCtx` must have been properly initialized using ZSTD_decompressBegin_usingDict(). -* Requires 2 contexts : 1 for reference (preparedDCtx), which will not be modified, and 1 to run the decompression operation (dctx) */ -ZSTDLIB_API size_t ZSTD_decompress_usingPreparedDCtx( - ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize); - #endif /* ZSTD_STATIC_LINKING_ONLY */ #if defined (__cplusplus) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index e57f38d90..9a94cd97c 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2449,7 +2449,12 @@ size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity) } -size_t ZSTD_compress_usingPreparedCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, +/*! ZSTD_compress_usingPreparedCCtx() : +* Same as ZSTD_compress_usingDict, but using a reference context `preparedCCtx`, where dictionary has been loaded. +* It avoids reloading the dictionary each time. +* `preparedCCtx` must have been properly initialized using ZSTD_compressBegin_usingDict() or ZSTD_compressBegin_advanced(). +* Requires 2 contexts : 1 for reference (preparedCCtx) which will not be modified, and 1 to run the compression operation (cctx) */ +static size_t ZSTD_compress_usingPreparedCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) { diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 344bd0201..f0666e3de 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -1012,6 +1012,11 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, } +/*! ZSTD_decompress_usingPreparedDCtx() : +* Same as ZSTD_decompress_usingDict, but using a reference context `preparedDCtx`, where dictionary has been loaded. +* It avoids reloading the dictionary each time. +* `preparedDCtx` must have been properly initialized using ZSTD_decompressBegin_usingDict(). +* Requires 2 contexts : 1 for reference (preparedDCtx), which will not be modified, and 1 to run the decompression operation (dctx) */ size_t ZSTD_decompress_usingPreparedDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* refDCtx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) From 0faabe43f8d43387a9ab6d005243d8d0f0ea86c8 Mon Sep 17 00:00:00 2001 From: Christophe Chevalier Date: Tue, 7 Jun 2016 14:00:02 +0200 Subject: [PATCH 09/11] Use Rebuild target instead of Clean+Build --- build/build.VS2010.cmd | 4 ++-- build/build.VS2012.cmd | 4 ++-- build/build.VS2013.cmd | 4 ++-- build/build.VS2015.cmd | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/build/build.VS2010.cmd b/build/build.VS2010.cmd index ddaeb4eee..19c6a52ba 100644 --- a/build/build.VS2010.cmd +++ b/build/build.VS2010.cmd @@ -1,7 +1,7 @@ @echo off rem build 32-bit -call "%~p0%build.generic.cmd" VS2010 Release "Clean,Build" Win32 v100 +call "%~p0%build.generic.cmd" VS2010 Release Rebuild Win32 v100 rem build 64-bit -call "%~p0%build.generic.cmd" VS2010 Release "Clean,Build" x64 v100 \ No newline at end of file +call "%~p0%build.generic.cmd" VS2010 Release Rebuild x64 v100 \ No newline at end of file diff --git a/build/build.VS2012.cmd b/build/build.VS2012.cmd index 96431e113..66439c907 100644 --- a/build/build.VS2012.cmd +++ b/build/build.VS2012.cmd @@ -1,6 +1,6 @@ @echo off rem build 32-bit -call "%~p0%build.generic.cmd" VS2012 Release "Clean,Build" Win32 v110 +call "%~p0%build.generic.cmd" VS2012 Release Rebuild Win32 v110 rem build 64-bit -call "%~p0%build.generic.cmd" VS2012 Release "Clean,Build" x64 v110 \ No newline at end of file +call "%~p0%build.generic.cmd" VS2012 Release Rebuild x64 v110 \ No newline at end of file diff --git a/build/build.VS2013.cmd b/build/build.VS2013.cmd index 06aed3e4f..220b26b63 100644 --- a/build/build.VS2013.cmd +++ b/build/build.VS2013.cmd @@ -1,7 +1,7 @@ @echo off rem build 32-bit -call "%~p0%build.generic.cmd" VS2013 Release "Clean,Build" Win32 v120 +call "%~p0%build.generic.cmd" VS2013 Release Rebuild Win32 v120 rem build 64-bit -call "%~p0%build.generic.cmd" VS2013 Release "Clean,Build" x64 v120 \ No newline at end of file +call "%~p0%build.generic.cmd" VS2013 Release Rebuild x64 v120 \ No newline at end of file diff --git a/build/build.VS2015.cmd b/build/build.VS2015.cmd index 0e2484949..b5784340a 100644 --- a/build/build.VS2015.cmd +++ b/build/build.VS2015.cmd @@ -1,7 +1,7 @@ @echo off rem build 32-bit -call "%~p0%build.generic.cmd" VS2015 Release "Clean,Build" Win32 v140 +call "%~p0%build.generic.cmd" VS2015 Release Rebuild Win32 v140 rem build 64-bit -call "%~p0%build.generic.cmd" VS2015 Release "Clean,Build" x64 v140 \ No newline at end of file +call "%~p0%build.generic.cmd" VS2015 Release Rebuild x64 v140 \ No newline at end of file From f1510489bdbb7b0c2df5f94c806721c8adb4b65f Mon Sep 17 00:00:00 2001 From: Christophe Chevalier Date: Tue, 7 Jun 2016 14:32:05 +0200 Subject: [PATCH 10/11] Simplified and reordered the arguments to build.generic.cmd and added default values - always Clean,Rebuild - output folder is now build/bin/{CONFIGURATION}/{ARCH} --- build/build.VS2010.cmd | 4 +-- build/build.VS2012.cmd | 4 +-- build/build.VS2013.cmd | 4 +-- build/build.VS2015.cmd | 4 +-- build/build.generic.cmd | 55 +++++++++++++++++++++++++---------------- 5 files changed, 42 insertions(+), 29 deletions(-) diff --git a/build/build.VS2010.cmd b/build/build.VS2010.cmd index 19c6a52ba..c3bc1763e 100644 --- a/build/build.VS2010.cmd +++ b/build/build.VS2010.cmd @@ -1,7 +1,7 @@ @echo off rem build 32-bit -call "%~p0%build.generic.cmd" VS2010 Release Rebuild Win32 v100 +call "%~p0%build.generic.cmd" VS2010 Win32 Release v100 rem build 64-bit -call "%~p0%build.generic.cmd" VS2010 Release Rebuild x64 v100 \ No newline at end of file +call "%~p0%build.generic.cmd" VS2010 x64 Release v100 \ No newline at end of file diff --git a/build/build.VS2012.cmd b/build/build.VS2012.cmd index 66439c907..d7399a9d6 100644 --- a/build/build.VS2012.cmd +++ b/build/build.VS2012.cmd @@ -1,6 +1,6 @@ @echo off rem build 32-bit -call "%~p0%build.generic.cmd" VS2012 Release Rebuild Win32 v110 +call "%~p0%build.generic.cmd" VS2012 Win32 Release v110 rem build 64-bit -call "%~p0%build.generic.cmd" VS2012 Release Rebuild x64 v110 \ No newline at end of file +call "%~p0%build.generic.cmd" VS2012 x64 Release v110 \ No newline at end of file diff --git a/build/build.VS2013.cmd b/build/build.VS2013.cmd index 220b26b63..486ba6cf0 100644 --- a/build/build.VS2013.cmd +++ b/build/build.VS2013.cmd @@ -1,7 +1,7 @@ @echo off rem build 32-bit -call "%~p0%build.generic.cmd" VS2013 Release Rebuild Win32 v120 +call "%~p0%build.generic.cmd" VS2013 Win32 Release v120 rem build 64-bit -call "%~p0%build.generic.cmd" VS2013 Release Rebuild x64 v120 \ No newline at end of file +call "%~p0%build.generic.cmd" VS2013 x64 Release v120 \ No newline at end of file diff --git a/build/build.VS2015.cmd b/build/build.VS2015.cmd index b5784340a..abc41c9a0 100644 --- a/build/build.VS2015.cmd +++ b/build/build.VS2015.cmd @@ -1,7 +1,7 @@ @echo off rem build 32-bit -call "%~p0%build.generic.cmd" VS2015 Release Rebuild Win32 v140 +call "%~p0%build.generic.cmd" VS2015 Win32 Release v140 rem build 64-bit -call "%~p0%build.generic.cmd" VS2015 Release Rebuild x64 v140 \ No newline at end of file +call "%~p0%build.generic.cmd" VS2015 x64 Release v140 \ No newline at end of file diff --git a/build/build.generic.cmd b/build/build.generic.cmd index 289776bf9..502c8db95 100644 --- a/build/build.generic.cmd +++ b/build/build.generic.cmd @@ -1,33 +1,46 @@ @echo off -rem -rem Parameters -rem %1: vs_solution_version: VS solution version: VS2012, VS2013, VS2015 -rem %2: vs_configuration: VS configuration: Debug, Release -rem %3: vs_target: target: Build or Rebuild -rem %4: vs_platform: platform: x64 or Win32 -rem %5: vs_toolset: toolset: v100, v110, v120, v140 -rem +IF "%1%" == "" GOTO display_help -set vs_solution_version=%1 -set vs_configuration=%2 -set vs_target=%3 -set vs_platform=%4 -set vs_toolset=%5 +SET vs_version=%1 -set msbuild="%windir%\Microsoft.NET\Framework\v4.0.30319\MSBuild.exe" -IF %vs_solution_version% == VS2013 SET msbuild="C:\Program Files (x86)\MSBuild\12.0\Bin\MSBuild.exe" -IF %vs_solution_version% == VS2015 SET msbuild="C:\Program Files (x86)\MSBuild\14.0\Bin\MSBuild.exe" +SET vs_platform=%2 +IF "%vs_platform%" == "" SET vs_platform=x64 + +SET vs_configuration=%3 +IF "%vs_configuration%" == "" SET vs_configuration=Release + +SET vs_toolset=%4 + +GOTO build + +:display_help + +echo Syntax: build.generic.cmd vs_version vs_platform vs_configuration vs_toolset +echo vs_version: VS installed version (VS2012, VS2013, VS2015, ...) +echo vs_platform: Platform (x64 or Win32) +echo vs_configuration: VS configuration (Release or Debug) +echo vs_toolset: Platform Toolset (v100, v110, v120, v140) + +EXIT /B 1 + +:build + +SET msbuild="%windir%\Microsoft.NET\Framework\v4.0.30319\MSBuild.exe" +IF %vs_version% == VS2013 SET msbuild="C:\Program Files (x86)\MSBuild\12.0\Bin\MSBuild.exe" +IF %vs_version% == VS2015 SET msbuild="C:\Program Files (x86)\MSBuild\14.0\Bin\MSBuild.exe" rem TODO: Visual Studio "15" (vNext) will use MSBuild 15.0 ? -set project="%~p0\..\projects\VS2010\zstd.sln" +SET project="%~p0\..\projects\VS2010\zstd.sln" -set output=%~p0%bin -set output="%output%/%vs_solution_version%/%vs_platform%/" +SET msbuildparams=/verbosity:minimal /nologo /t:Clean,Build /p:Platform=%vs_platform% /p:Configuration=%vs_configuration% +IF NOT "%vs_toolset%" == "" SET msbuildparams=%msbuildparams% /p:PlatformToolset=%vs_toolset% -set msbuildparams=/verbosity:minimal /p:Platform="%vs_platform%" /p:Configuration="%vs_configuration%" /p:PlatformToolset="%vs_toolset%" /t:Clean,Build /nologo /p:OutDir=%output% +SET output=%~p0%bin +SET output="%output%/%vs_configuration%/%vs_platform%/" +SET msbuildparams=%msbuildparams% /p:OutDir=%output% -echo ### Building %vs_solution_version% project for %vs_configuration% %vs_platform% (%vs_toolset%)... +echo ### Building %vs_version% project for %vs_configuration% %vs_platform% (%vs_toolset%)... echo ### Build Params: %msbuildparams% %msbuild% %project% %msbuildparams% From 662a541431c19ef4997113e3820b735c45ad529c Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 8 Jun 2016 11:11:02 +0200 Subject: [PATCH 11/11] updated huff0 - now generates a common HUF_DTable type for all decoding tables --- NEWS | 1 + lib/common/huf.h | 85 ++--- lib/decompress/huf_decompress.c | 592 ++++++------------------------- lib/decompress/zstd_decompress.c | 8 +- 4 files changed, 162 insertions(+), 524 deletions(-) diff --git a/NEWS b/NEWS index e54b09967..767fe5c01 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,6 @@ v0.7.0 New : Support for directory compression, using `-r`, thanks to Przemyslaw Skibinski +New : Visual build scripts, by Christophe Chevalier New : Support for Sparse File-systems (do not use space for zero-filled sectors) New : Frame checksum support New : Support pass-through mode (when using `-df`) diff --git a/lib/common/huf.h b/lib/common/huf.h index d54e6616f..c76694576 100644 --- a/lib/common/huf.h +++ b/lib/common/huf.h @@ -31,8 +31,8 @@ You can contact the author at : - Source repository : https://github.com/Cyan4973/FiniteStateEntropy ****************************************************************** */ -#ifndef HUF_H -#define HUF_H +#ifndef HUF_H_298734234 +#define HUF_H_298734234 #if defined (__cplusplus) extern "C" { @@ -53,8 +53,9 @@ size_t HUF_decompress(void* dst, size_t dstSize, /* HUF_compress() : Compress content from buffer 'src', of size 'srcSize', into buffer 'dst'. - 'dst' buffer must be already allocated. Compression runs faster if `dstCapacity` >= HUF_compressBound(srcSize). - Note : `srcSize` must be <= `HUF_BLOCKSIZE_MAX` == 128 KB + 'dst' buffer must be already allocated. + Compression runs faster if `dstCapacity` >= HUF_compressBound(srcSize). + `srcSize` must be <= `HUF_BLOCKSIZE_MAX` == 128 KB @return : size of compressed data (<= `dstCapacity`) Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!! if return == 1, srcData is a single repeated byte symbol (RLE compression). @@ -63,7 +64,7 @@ HUF_compress() : HUF_decompress() : Decompress HUF data from buffer 'cSrc', of size 'cSrcSize', into already allocated buffer 'dst', of minimum size 'dstSize'. - `dstSize` : must be the **exact** size of original (uncompressed) data. + `dstSize` : **must** be the ***exact*** size of original (uncompressed) data. Note : in contrast with FSE, HUF_decompress can regenerate RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data, because it knows size to regenerate. @@ -121,13 +122,12 @@ size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize HUF_CElt* name = (HUF_CElt*)(name##hv) /* no final ; */ /* static allocation of HUF's DTable */ -#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1<= 64 */ /* **************************************** @@ -161,35 +160,6 @@ size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, un size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable); -/*! -HUF_decompress() does the following: -1. select the decompression algorithm (X2, X4, X6) based on pre-computed heuristics -2. build Huffman table from save, using HUF_readDTableXn() -3. decode 1 or 4 segments in parallel using HUF_decompressSXn_usingDTable -*/ -size_t HUF_readDTableX2 (unsigned short* DTable, const void* src, size_t srcSize); -size_t HUF_readDTableX4 (unsigned* DTable, const void* src, size_t srcSize); -size_t HUF_readDTableX6 (unsigned* DTable, const void* src, size_t srcSize); - -size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned short* DTable); -size_t HUF_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable); -size_t HUF_decompress4X6_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable); - - -/* single stream variants */ - -size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); -size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable); - -size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */ -size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */ -size_t HUF_decompress1X6 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* quad-symbols decoder, only works for dstSize >= 64 */ - -size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned short* DTable); -size_t HUF_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable); -size_t HUF_decompress1X6_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable); - - /*! HUF_readStats() : Read compact Huffman tree, saved by HUF_writeCTable(). `huffWeight` is destination buffer. @@ -204,6 +174,39 @@ size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, size_t HUF_readCTable (HUF_CElt* CTable, unsigned maxSymbolValue, const void* src, size_t srcSize); +/* +HUF_decompress() does the following: +1. select the decompression algorithm (X2, X4) based on pre-computed heuristics +2. build Huffman table from save, using HUF_readDTableXn() +3. decode 1 or 4 segments in parallel using HUF_decompressSXn_usingDTable +*/ + +/** HUF_selectDecoder() : +* Tells which decoder is likely to decode faster, +* based on a set of pre-determined metrics. +* @return : 0==HUF_decompress4X2, 1==HUF_decompress4X4 . +* Assumption : 0 < cSrcSize < dstSize <= 128 KB */ +U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize); + +size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize); +size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize); + +size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); +size_t HUF_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); + + +/* single stream variants */ + +size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); +size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable); + +size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */ +size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */ + +size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); +size_t HUF_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); + + #endif /* HUF_STATIC_LINKING_ONLY */ @@ -211,4 +214,4 @@ size_t HUF_readCTable (HUF_CElt* CTable, unsigned maxSymbolValue, const void* sr } #endif -#endif /* HUF_H */ +#endif /* HUF_H_298734234 */ diff --git a/lib/decompress/huf_decompress.c b/lib/decompress/huf_decompress.c index d1096e457..ed235f5e7 100644 --- a/lib/decompress/huf_decompress.c +++ b/lib/decompress/huf_decompress.c @@ -60,37 +60,26 @@ * Includes ****************************************************************/ #include /* memcpy, memset */ -#include /* printf (debug) */ #include "bitstream.h" #include "fse.h" /* header compression */ #define HUF_STATIC_LINKING_ONLY #include "huf.h" - /* ************************************************************** * Error Management ****************************************************************/ #define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ - -/* ******************************************************* -* HUF : Huffman block decompression -*********************************************************/ -typedef struct { BYTE byte; BYTE nbBits; } HUF_DEltX2; /* single-symbol decoding */ - -typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUF_DEltX4; /* double-symbols decoding */ - -typedef struct { BYTE symbol; BYTE weight; } sortedSymbol_t; - - - /*-***************************/ /* single-symbol decoding */ /*-***************************/ +typedef struct { BYTE maxTableLog; BYTE currentTableLog; } DTableDesc; -size_t HUF_readDTableX2 (U16* DTable, const void* src, size_t srcSize) +typedef struct { BYTE byte; BYTE nbBits; } HUF_DEltX2; /* single-symbol decoding */ + +size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize) { BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */ @@ -101,16 +90,19 @@ size_t HUF_readDTableX2 (U16* DTable, const void* src, size_t srcSize) U32 nextRankStart; void* const dtPtr = DTable + 1; HUF_DEltX2* const dt = (HUF_DEltX2*)dtPtr; + DTableDesc dtd; - HUF_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(U16)); /* if compilation fails here, assertion is false */ + HUF_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(HUF_DTable)); /* if compilation fails here, assertion is false */ + memcpy(&dtd, DTable, sizeof(dtd)); //memset(huffWeight, 0, sizeof(huffWeight)); /* is not necessary, even though some analyzer complain ... */ iSize = HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize); if (HUF_isError(iSize)) return iSize; /* check result */ - if (tableLog > DTable[0]) return ERROR(tableLog_tooLarge); /* DTable is too small */ - DTable[0] = (U16)tableLog; /* maybe should separate sizeof allocated DTable, from used size of DTable, in case of re-use */ + if (tableLog > dtd.maxTableLog) return ERROR(tableLog_tooLarge); /* DTable is too small */ + dtd.currentTableLog = (BYTE)tableLog; /* maybe should separate sizeof allocated DTable, from used size of DTable, in case of re-use */ + memcpy(DTable, &dtd, sizeof(dtd)); /* Prepare ranks */ nextRankStart = 0; @@ -181,14 +173,18 @@ static inline size_t HUF_decodeStreamX2(BYTE* p, BIT_DStream_t* const bitDPtr, B size_t HUF_decompress1X2_usingDTable( void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, - const U16* DTable) + const HUF_DTable* DTable) { BYTE* op = (BYTE*)dst; BYTE* const oend = op + dstSize; - const U32 dtLog = DTable[0]; const void* dtPtr = DTable; const HUF_DEltX2* const dt = ((const HUF_DEltX2*)dtPtr)+1; BIT_DStream_t bitD; + DTableDesc dtd; + U32 dtLog; + + memcpy(&dtd, DTable, sizeof(dtd)); + dtLog = dtd.currentTableLog; { size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize); if (HUF_isError(errorCode)) return errorCode; } @@ -219,7 +215,7 @@ size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cS size_t HUF_decompress4X2_usingDTable( void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, - const U16* DTable) + const HUF_DTable* DTable) { /* Check */ if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ @@ -229,18 +225,16 @@ size_t HUF_decompress4X2_usingDTable( BYTE* const oend = ostart + dstSize; const void* const dtPtr = DTable; const HUF_DEltX2* const dt = ((const HUF_DEltX2*)dtPtr) +1; - const U32 dtLog = DTable[0]; - size_t errorCode; /* Init */ BIT_DStream_t bitD1; BIT_DStream_t bitD2; BIT_DStream_t bitD3; BIT_DStream_t bitD4; - const size_t length1 = MEM_readLE16(istart); - const size_t length2 = MEM_readLE16(istart+2); - const size_t length3 = MEM_readLE16(istart+4); - size_t length4; + size_t const length1 = MEM_readLE16(istart); + size_t const length2 = MEM_readLE16(istart+2); + size_t const length3 = MEM_readLE16(istart+4); + size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); const BYTE* const istart1 = istart + 6; /* jumpTable */ const BYTE* const istart2 = istart1 + length1; const BYTE* const istart3 = istart2 + length2; @@ -254,17 +248,21 @@ size_t HUF_decompress4X2_usingDTable( BYTE* op3 = opStart3; BYTE* op4 = opStart4; U32 endSignal; + DTableDesc dtd; + U32 dtLog; + + memcpy(&dtd, DTable, sizeof(dtd)); + dtLog = dtd.currentTableLog; - length4 = cSrcSize - (length1 + length2 + length3 + 6); if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ - errorCode = BIT_initDStream(&bitD1, istart1, length1); - if (HUF_isError(errorCode)) return errorCode; - errorCode = BIT_initDStream(&bitD2, istart2, length2); - if (HUF_isError(errorCode)) return errorCode; - errorCode = BIT_initDStream(&bitD3, istart3, length3); - if (HUF_isError(errorCode)) return errorCode; - errorCode = BIT_initDStream(&bitD4, istart4, length4); - if (HUF_isError(errorCode)) return errorCode; + { size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1); + if (HUF_isError(errorCode)) return errorCode; } + { size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2); + if (HUF_isError(errorCode)) return errorCode; } + { size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3); + if (HUF_isError(errorCode)) return errorCode; } + { size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4); + if (HUF_isError(errorCode)) return errorCode; } /* 16-32 symbols per loop (4-8 symbols per stream) */ endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); @@ -315,11 +313,11 @@ size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cS HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX); const BYTE* ip = (const BYTE*) cSrc; - size_t const errorCode = HUF_readDTableX2 (DTable, cSrc, cSrcSize); - if (HUF_isError(errorCode)) return errorCode; - if (errorCode >= cSrcSize) return ERROR(srcSize_wrong); - ip += errorCode; - cSrcSize -= errorCode; + size_t const hSize = HUF_readDTableX2 (DTable, cSrc, cSrcSize); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; + cSrcSize -= hSize; return HUF_decompress4X2_usingDTable (dst, dstSize, ip, cSrcSize, DTable); } @@ -328,6 +326,9 @@ size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cS /* *************************/ /* double-symbols decoding */ /* *************************/ +typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUF_DEltX4; /* double-symbols decoding */ + +typedef struct { BYTE symbol; BYTE weight; } sortedSymbol_t; static void HUF_fillDTableX4Level2(HUF_DEltX4* DTable, U32 sizeLog, const U32 consumed, const U32* rankValOrigin, const int minWeight, @@ -413,7 +414,7 @@ static void HUF_fillDTableX4(HUF_DEltX4* DTable, const U32 targetLog, } } -size_t HUF_readDTableX4 (U32* DTable, const void* src, size_t srcSize) +size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize) { BYTE weightList[HUF_SYMBOLVALUE_MAX + 1]; sortedSymbol_t sortedSymbol[HUF_SYMBOLVALUE_MAX + 1]; @@ -422,20 +423,23 @@ size_t HUF_readDTableX4 (U32* DTable, const void* src, size_t srcSize) U32* const rankStart = rankStart0+1; rankVal_t rankVal; U32 tableLog, maxW, sizeOfSort, nbSymbols; - const U32 memLog = DTable[0]; + DTableDesc dtd; + U32 maxTableLog; size_t iSize; - void* dtPtr = DTable; - HUF_DEltX4* const dt = ((HUF_DEltX4*)dtPtr) + 1; + void* dtPtr = DTable+1; /* force compiler to avoid strict-aliasing */ + HUF_DEltX4* const dt = (HUF_DEltX4*)dtPtr; HUF_STATIC_ASSERT(sizeof(HUF_DEltX4) == sizeof(U32)); /* if compilation fails here, assertion is false */ - if (memLog > HUF_TABLELOG_ABSOLUTEMAX) return ERROR(tableLog_tooLarge); + memcpy(&dtd, DTable, sizeof(dtd)); + maxTableLog = dtd.maxTableLog-1; + if (maxTableLog > HUF_TABLELOG_ABSOLUTEMAX) return ERROR(tableLog_tooLarge); //memset(weightList, 0, sizeof(weightList)); /* is not necessary, even though some analyzer complain ... */ iSize = HUF_readStats(weightList, HUF_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize); if (HUF_isError(iSize)) return iSize; /* check result */ - if (tableLog > memLog) return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */ + if (tableLog > maxTableLog) return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */ /* find maxWeight */ for (maxW = tableLog; rankStats[maxW]==0; maxW--) {} /* necessarily finds a solution before 0 */ @@ -464,7 +468,7 @@ size_t HUF_readDTableX4 (U32* DTable, const void* src, size_t srcSize) /* Build rankVal */ { U32* const rankVal0 = rankVal[0]; - { int const rescale = (memLog-tableLog) - 1; /* tableLog <= memLog */ + { int const rescale = (maxTableLog-tableLog) - 1; /* tableLog <= maxTableLog */ U32 nextRankVal = 0; U32 w; for (w=1; w> consumed; } } } } - HUF_fillDTableX4(dt, memLog, + HUF_fillDTableX4(dt, maxTableLog, sortedSymbol, sizeOfSort, rankStart0, rankVal, maxW, tableLog+1); + dtd.currentTableLog = (BYTE)maxTableLog; + memcpy(DTable, &dtd, sizeof(dtd)); return iSize; } @@ -536,7 +542,7 @@ static inline size_t HUF_decodeStreamX4(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* c HUF_DECODE_SYMBOLX4_0(p, bitDPtr); } - /* closer to the end */ + /* closer to end : up to 2 symbols at a time */ while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p <= pEnd-2)) HUF_DECODE_SYMBOLX4_0(p, bitDPtr); @@ -553,23 +559,24 @@ static inline size_t HUF_decodeStreamX4(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* c size_t HUF_decompress1X4_usingDTable( void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, - const U32* DTable) + const HUF_DTable* DTable) { - const BYTE* const istart = (const BYTE*) cSrc; - BYTE* const ostart = (BYTE*) dst; - BYTE* const oend = ostart + dstSize; - - const U32 dtLog = DTable[0]; - const void* const dtPtr = DTable; - const HUF_DEltX4* const dt = ((const HUF_DEltX4*)dtPtr) +1; + BIT_DStream_t bitD; /* Init */ - BIT_DStream_t bitD; - { size_t const errorCode = BIT_initDStream(&bitD, istart, cSrcSize); - if (HUF_isError(errorCode)) return errorCode; } + { size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize); + if (HUF_isError(errorCode)) return errorCode; + } /* decode */ - HUF_decodeStreamX4(ostart, &bitD, oend, dt, dtLog); + { BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + const void* const dtPtr = DTable+1; /* force compiler to not use strict-aliasing */ + const HUF_DEltX4* const dt = (const HUF_DEltX4*)dtPtr; + DTableDesc dtd; + memcpy(&dtd, DTable, sizeof(dtd)); + HUF_decodeStreamX4(ostart, &bitD, oend, dt, dtd.currentTableLog); + } /* check */ if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); @@ -595,32 +602,30 @@ size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cS size_t HUF_decompress4X4_usingDTable( void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, - const U32* DTable) + const HUF_DTable* DTable) { if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ { const BYTE* const istart = (const BYTE*) cSrc; BYTE* const ostart = (BYTE*) dst; BYTE* const oend = ostart + dstSize; - const void* const dtPtr = DTable; - const HUF_DEltX4* const dt = ((const HUF_DEltX4*)dtPtr) +1; - const U32 dtLog = DTable[0]; - size_t errorCode; + const void* const dtPtr = DTable+1; + const HUF_DEltX4* const dt = (const HUF_DEltX4*)dtPtr; /* Init */ BIT_DStream_t bitD1; BIT_DStream_t bitD2; BIT_DStream_t bitD3; BIT_DStream_t bitD4; - const size_t length1 = MEM_readLE16(istart); - const size_t length2 = MEM_readLE16(istart+2); - const size_t length3 = MEM_readLE16(istart+4); - size_t length4; + size_t const length1 = MEM_readLE16(istart); + size_t const length2 = MEM_readLE16(istart+2); + size_t const length3 = MEM_readLE16(istart+4); + size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); const BYTE* const istart1 = istart + 6; /* jumpTable */ const BYTE* const istart2 = istart1 + length1; const BYTE* const istart3 = istart2 + length2; const BYTE* const istart4 = istart3 + length3; - const size_t segmentSize = (dstSize+3) / 4; + size_t const segmentSize = (dstSize+3) / 4; BYTE* const opStart2 = ostart + segmentSize; BYTE* const opStart3 = opStart2 + segmentSize; BYTE* const opStart4 = opStart3 + segmentSize; @@ -629,17 +634,21 @@ size_t HUF_decompress4X4_usingDTable( BYTE* op3 = opStart3; BYTE* op4 = opStart4; U32 endSignal; + DTableDesc dtd; + U32 dtLog; + + memcpy(&dtd, DTable, sizeof(dtd)); + dtLog = dtd.currentTableLog; - length4 = cSrcSize - (length1 + length2 + length3 + 6); if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ - errorCode = BIT_initDStream(&bitD1, istart1, length1); - if (HUF_isError(errorCode)) return errorCode; - errorCode = BIT_initDStream(&bitD2, istart2, length2); - if (HUF_isError(errorCode)) return errorCode; - errorCode = BIT_initDStream(&bitD3, istart3, length3); - if (HUF_isError(errorCode)) return errorCode; - errorCode = BIT_initDStream(&bitD4, istart4, length4); - if (HUF_isError(errorCode)) return errorCode; + { size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1); + if (HUF_isError(errorCode)) return errorCode; } + { size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2); + if (HUF_isError(errorCode)) return errorCode; } + { size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3); + if (HUF_isError(errorCode)) return errorCode; } + { size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4); + if (HUF_isError(errorCode)) return errorCode; } /* 16-32 symbols per loop (4-8 symbols per stream) */ endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); @@ -677,8 +686,8 @@ size_t HUF_decompress4X4_usingDTable( HUF_decodeStreamX4(op4, &bitD4, oend, dt, dtLog); /* check */ - endSignal = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); - if (!endSignal) return ERROR(corruption_detected); + { U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); + if (!endCheck) return ERROR(corruption_detected); } /* decoded size */ return dstSize; @@ -701,386 +710,6 @@ size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cS } -/* ********************************/ -/* quad-symbol decoding */ -/* ********************************/ -typedef struct { BYTE nbBits; BYTE nbBytes; } HUF_DDescX6; -typedef union { BYTE byte[4]; U32 sequence; } HUF_DSeqX6; - -/* recursive, up to level 3; may benefit from