diff --git a/.gitignore b/.gitignore index c4c9282bd..072fe39c1 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,21 @@ # Visual solution files *.suo *.user + +# Build results +[Dd]ebug/ +[Rr]elease/ +[Rr]eleases/ +x64/ +x86/ +[Bb]in/ +[Oo]bj/ + +# Visual C++ cache files +ipch/ +*.aps +*.ncb +*.opendb +*.opensdf +*.sdf +*.cachefile \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 846f12175..2d5a73a6e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,6 +19,7 @@ env: - ZSTD_TRAVIS_CI_ENV="-C programs test-zstd_nolegacy" - ZSTD_TRAVIS_CI_ENV=usan - ZSTD_TRAVIS_CI_ENV=asan + - ZSTD_TRAVIS_CI_ENV=asan32 - ZSTD_TRAVIS_CI_ENV="-C programs valgrindTest" matrix: diff --git a/Makefile b/Makefile index 9e7b70ee0..abec33f86 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ # ################################################################ # Version number -export VERSION := 0.4.3 +export VERSION := 0.4.4 PRGDIR = programs ZSTDDIR = lib @@ -96,6 +96,9 @@ usan: clean asan: clean $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address" +asan32: clean + $(MAKE) -C $(PRGDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address" + uasan: clean $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address -fsanitize=undefined" diff --git a/NEWS b/NEWS index ad8a737f9..079483a3c 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,8 @@ +v0.4.4 +Fixed : high compression modes for Windows 32 bits +new : external dictionary API extended to buffered mode and accessible through command line +new : windows DLL project, thanks to Christophe Chevalier + v0.4.3 : new : external dictionary API new : zstd-frugal diff --git a/lib/zstd.h b/lib/zstd.h index e4d441497..b0c841f3a 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -43,6 +43,20 @@ extern "C" { #include /* size_t */ +/* *************************************************************** +* Tuning parameters +*****************************************************************/ +/*! +* ZSTD_DLL_EXPORT : +* Enable exporting of functions when building a Windows DLL +*/ +#if defined(_WIN32) && defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZSTDLIB_API __declspec(dllexport) +#else +# define ZSTDLIB_API +#endif + + /* ************************************* * Version ***************************************/ @@ -50,18 +64,18 @@ extern "C" { #define ZSTD_VERSION_MINOR 4 /* for new (non-breaking) interface capabilities */ #define ZSTD_VERSION_RELEASE 3 /* for tweaks, bug-fixes, or development */ #define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) -unsigned ZSTD_versionNumber (void); +ZSTDLIB_API unsigned ZSTD_versionNumber (void); /* ************************************* * Simple functions ***************************************/ -size_t ZSTD_compress( void* dst, size_t maxDstSize, - const void* src, size_t srcSize, - int compressionLevel); +ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t maxDstSize, + const void* src, size_t srcSize, + int compressionLevel); -size_t ZSTD_decompress( void* dst, size_t maxOriginalSize, - const void* src, size_t compressedSize); +ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t maxOriginalSize, + const void* src, size_t compressedSize); /** ZSTD_compress() : @@ -83,25 +97,25 @@ ZSTD_decompress() : /* ************************************* * Tool functions ***************************************/ -size_t ZSTD_compressBound(size_t srcSize); /** maximum compressed size (worst case scenario) */ +ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /** maximum compressed size (worst case scenario) */ /* Error Management */ -unsigned ZSTD_isError(size_t code); /** tells if a return value is an error code */ -const char* ZSTD_getErrorName(size_t code); /** provides error code string */ +ZSTDLIB_API unsigned ZSTD_isError(size_t code); /** tells if a return value is an error code */ +ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /** provides error code string */ /* ************************************* * Advanced functions ***************************************/ typedef struct ZSTD_CCtx_s ZSTD_CCtx; /* incomplete type */ -ZSTD_CCtx* ZSTD_createCCtx(void); -size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); +ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void); +ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); /** ZSTD_compressCCtx() : Same as ZSTD_compress(), but requires a ZSTD_CCtx working space already allocated */ -size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel); +ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel); #if defined (__cplusplus) diff --git a/lib/zstd_buffered.c b/lib/zstd_buffered.c index 844c50405..99148c297 100644 --- a/lib/zstd_buffered.c +++ b/lib/zstd_buffered.c @@ -160,6 +160,12 @@ size_t ZBUFF_compressInit(ZBUFF_CCtx* zbc, int compressionLevel) } +ZSTDLIB_API size_t ZBUFF_compressWithDictionary(ZBUFF_CCtx* zbc, const void* src, size_t srcSize) +{ + ZSTD_compress_insertDictionary(zbc->zc, src, srcSize); + return 0; +} + /* *** Compression *** */ @@ -327,6 +333,8 @@ struct ZBUFF_DCtx_s { size_t outStart; size_t outEnd; size_t hPos; + const char* dict; + size_t dictSize; ZBUFF_dStage stage; unsigned char headerBuffer[ZSTD_frameHeaderSize_max]; }; /* typedef'd to ZBUFF_DCtx within "zstd_buffered.h" */ @@ -353,17 +361,23 @@ size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbc) } - /* *** Initialization *** */ size_t ZBUFF_decompressInit(ZBUFF_DCtx* zbc) { zbc->stage = ZBUFFds_readHeader; - zbc->hPos = zbc->inPos = zbc->outStart = zbc->outEnd = 0; + zbc->hPos = zbc->inPos = zbc->outStart = zbc->outEnd = zbc->dictSize = 0; return ZSTD_resetDCtx(zbc->zc); } +size_t ZBUFF_decompressWithDictionary(ZBUFF_DCtx* zbc, const void* src, size_t srcSize) +{ + zbc->dict = (const char*)src; + zbc->dictSize = srcSize; + return 0; +} + /* *** Decompression *** */ @@ -442,6 +456,8 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDstSizePt if (zbc->outBuff == NULL) return ERROR(memory_allocation); } } + if (zbc->dictSize) + ZSTD_decompress_insertDictionary(zbc->zc, zbc->dict, zbc->dictSize); if (zbc->hPos) { /* some data already loaded into headerBuffer : transfer into inBuff */ diff --git a/lib/zstd_buffered.h b/lib/zstd_buffered.h index 80ba819db..d2316a82b 100644 --- a/lib/zstd_buffered.h +++ b/lib/zstd_buffered.h @@ -47,17 +47,32 @@ extern "C" { #include /* size_t */ +/* *************************************************************** +* Tuning parameters +*****************************************************************/ +/*! +* ZSTD_DLL_EXPORT : +* Enable exporting of functions when building a Windows DLL +*/ +#if defined(_WIN32) && defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZSTDLIB_API __declspec(dllexport) +#else +# define ZSTDLIB_API +#endif + + /* ************************************* * Streaming functions ***************************************/ typedef struct ZBUFF_CCtx_s ZBUFF_CCtx; -ZBUFF_CCtx* ZBUFF_createCCtx(void); -size_t ZBUFF_freeCCtx(ZBUFF_CCtx* cctx); +ZSTDLIB_API ZBUFF_CCtx* ZBUFF_createCCtx(void); +ZSTDLIB_API size_t ZBUFF_freeCCtx(ZBUFF_CCtx* cctx); -size_t ZBUFF_compressInit(ZBUFF_CCtx* cctx, int compressionLevel); -size_t ZBUFF_compressContinue(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr, const void* src, size_t* srcSizePtr); -size_t ZBUFF_compressFlush(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr); -size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr); +ZSTDLIB_API size_t ZBUFF_compressInit(ZBUFF_CCtx* cctx, int compressionLevel); +ZSTDLIB_API size_t ZBUFF_compressWithDictionary(ZBUFF_CCtx* cctx, const void* src, size_t srcSize); +ZSTDLIB_API size_t ZBUFF_compressContinue(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr, const void* src, size_t* srcSizePtr); +ZSTDLIB_API size_t ZBUFF_compressFlush(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr); +ZSTDLIB_API size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr); /** ************************************************ * Streaming compression @@ -67,6 +82,9 @@ size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr); * Use ZBUFF_compressInit() to start a new compression operation. * ZBUFF_CCtx objects can be reused multiple times. * +* Optionally, a reference to a static dictionary can be created with ZBUFF_compressWithDictionary() +* Note that the dictionary content must remain accessible during the compression process. +* * Use ZBUFF_compressContinue() repetitively to consume input stream. * *srcSizePtr and *maxDstSizePtr can be any size. * The function will report how many bytes were read or written within *srcSizePtr and *maxDstSizePtr. @@ -97,11 +115,13 @@ size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr); typedef struct ZBUFF_DCtx_s ZBUFF_DCtx; -ZBUFF_DCtx* ZBUFF_createDCtx(void); -size_t ZBUFF_freeDCtx(ZBUFF_DCtx* dctx); +ZSTDLIB_API ZBUFF_DCtx* ZBUFF_createDCtx(void); +ZSTDLIB_API size_t ZBUFF_freeDCtx(ZBUFF_DCtx* dctx); -size_t ZBUFF_decompressInit(ZBUFF_DCtx* dctx); -size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx, void* dst, size_t* maxDstSizePtr, const void* src, size_t* srcSizePtr); +ZSTDLIB_API size_t ZBUFF_decompressInit(ZBUFF_DCtx* dctx); +ZSTDLIB_API size_t ZBUFF_decompressWithDictionary(ZBUFF_DCtx* dctx, const void* src, size_t srcSize); + +ZSTDLIB_API size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx, void* dst, size_t* maxDstSizePtr, const void* src, size_t* srcSizePtr); /** ************************************************ * Streaming decompression @@ -111,6 +131,10 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx, void* dst, size_t* maxDstSizeP * Use ZBUFF_decompressInit() to start a new decompression operation. * ZBUFF_DCtx objects can be reused multiple times. * +* Optionally, a reference to a static dictionary can be set, using ZBUFF_decompressWithDictionary() +* It must be the same content as the one set during compression phase. +* Dictionary content must remain accessible during the decompression process. +* * Use ZBUFF_decompressContinue() repetitively to consume your input. * *srcSizePtr and *maxDstSizePtr can be any size. * The function will report how many bytes were read or written by modifying *srcSizePtr and *maxDstSizePtr. @@ -129,15 +153,15 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx, void* dst, size_t* maxDstSizeP /* ************************************* * Tool functions ***************************************/ -unsigned ZBUFF_isError(size_t errorCode); -const char* ZBUFF_getErrorName(size_t errorCode); +ZSTDLIB_API unsigned ZBUFF_isError(size_t errorCode); +ZSTDLIB_API const char* ZBUFF_getErrorName(size_t errorCode); /** The below functions provide recommended buffer sizes for Compression or Decompression operations. * These sizes are not compulsory, they just tend to offer better latency */ -size_t ZBUFF_recommendedCInSize(void); -size_t ZBUFF_recommendedCOutSize(void); -size_t ZBUFF_recommendedDInSize(void); -size_t ZBUFF_recommendedDOutSize(void); +ZSTDLIB_API size_t ZBUFF_recommendedCInSize(void); +ZSTDLIB_API size_t ZBUFF_recommendedCOutSize(void); +ZSTDLIB_API size_t ZBUFF_recommendedDInSize(void); +ZSTDLIB_API size_t ZBUFF_recommendedDOutSize(void); #if defined (__cplusplus) diff --git a/lib/zstd_buffered_static.h b/lib/zstd_buffered_static.h index cba1d6fca..7d9ee27d2 100644 --- a/lib/zstd_buffered_static.h +++ b/lib/zstd_buffered_static.h @@ -52,7 +52,7 @@ extern "C" { /* ************************************* * Advanced Streaming functions ***************************************/ -size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* cctx, ZSTD_parameters params); +ZSTDLIB_API size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* cctx, ZSTD_parameters params); #if defined (__cplusplus) diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index c18d8f86c..83f85c770 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -65,6 +65,7 @@ /* ************************************* * Constants ***************************************/ +unsigned ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; } static const U32 g_searchStrength = 8; @@ -599,7 +600,7 @@ static unsigned ZSTD_NbCommonBytes (register size_t val) # if defined(_MSC_VER) && defined(_WIN64) unsigned long r = 0; _BitScanForward64( &r, (U64)val ); - return (int)(r>>3); + return (unsigned)(r>>3); # elif defined(__GNUC__) && (__GNUC__ >= 3) return (__builtin_ctzll((U64)val) >> 3); # else @@ -612,7 +613,7 @@ static unsigned ZSTD_NbCommonBytes (register size_t val) # if defined(_MSC_VER) unsigned long r=0; _BitScanForward( &r, (U32)val ); - return (int)(r>>3); + return (unsigned)(r>>3); # elif defined(__GNUC__) && (__GNUC__ >= 3) return (__builtin_ctz((U32)val) >> 3); # else @@ -1025,7 +1026,7 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co size_t commonLengthSmaller=0, commonLengthLarger=0; const BYTE* const base = zc->base; const BYTE* match = base + matchIndex; - U32 current = (U32)(ip-base); + const U32 current = (U32)(ip-base); const U32 btLow = btMask >= current ? 0 : current - btMask; U32* smallerPtr = bt + 2*(current&btMask); U32* largerPtr = bt + 2*(current&btMask) + 1; @@ -1033,13 +1034,14 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co const U32 windowLow = zc->lowLimit; if ( (current-matchIndex == 1) /* RLE */ + && (matchIndex > windowLow) && (MEM_read64(match) == MEM_read64(ip)) ) { size_t rleLength = ZSTD_count(ip+8, match+8, iend) + 8; return (U32)(rleLength - mls); } - hashTable[h] = (U32)(ip - base); /* Update Hash Table */ + hashTable[h] = current; /* Update Hash Table */ while (nbCompares-- && (matchIndex > windowLow)) { @@ -1102,7 +1104,7 @@ size_t ZSTD_insertBtAndFindBestMatch ( size_t bestLength = 0; U32 dummy32; /* to be nullified at the end */ - hashTable[h] = (U32)(ip-base); /* Update Hash Table */ + hashTable[h] = current; /* Update Hash Table */ while (nbCompares-- && (matchIndex > windowLow)) { @@ -1212,7 +1214,7 @@ static U32 ZSTD_insertBt1_extDict(ZSTD_CCtx* zc, const BYTE* const ip, const U32 const BYTE* const dictEnd = dictBase + dictLimit; const BYTE* const prefixStart = base + dictLimit; const BYTE* match = base + matchIndex; - U32 current = (U32)(ip-base); + const U32 current = (U32)(ip-base); const U32 btLow = btMask >= current ? 0 : current - btMask; U32* smallerPtr = bt + 2*(current&btMask); U32* largerPtr = bt + 2*(current&btMask) + 1; @@ -1226,7 +1228,7 @@ static U32 ZSTD_insertBt1_extDict(ZSTD_CCtx* zc, const BYTE* const ip, const U32 return (U32)(rleLength - mls); } - hashTable[h] = (U32)(ip - base); /* Update Hash Table */ + hashTable[h] = current; /* Update Hash Table */ while (nbCompares-- && (matchIndex > windowLow)) { @@ -1317,7 +1319,7 @@ size_t ZSTD_insertBtAndFindBestMatch_extDict ( size_t bestLength = 0; U32 dummy32; /* to be nullified at the end */ - hashTable[h] = (U32)(ip-base); /* Update Hash Table */ + hashTable[h] = current; /* Update Hash Table */ while (nbCompares-- && (matchIndex > windowLow)) { @@ -1725,7 +1727,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx, /* init */ ZSTD_resetSeqStore(seqStorePtr); - if (((ip-base) - dictLimit) < REPCODE_STARTVALUE) ip += REPCODE_STARTVALUE; + if ((ip - prefixStart) < REPCODE_STARTVALUE) ip += REPCODE_STARTVALUE; /* Match Loop */ while (ip < ilimit) @@ -2007,13 +2009,16 @@ size_t ZSTD_compressContinue (ZSTD_CCtx* zc, } /* preemptive overflow correction */ - if ((zc->base > ip) || (zc->lowLimit > (1<<30) )) + if (zc->lowLimit > (1<<30)) { - U32 correction = zc->lowLimit-1; + U32 btplus = (zc->params.strategy == ZSTD_btlazy2); + U32 contentMask = (1 << (zc->params.contentLog - btplus)) - 1; + U32 newLowLimit = zc->lowLimit & contentMask; /* preserve position % contentSize */ + U32 correction = zc->lowLimit - newLowLimit; ZSTD_reduceIndex(zc, correction); zc->base += correction; zc->dictBase += correction; - zc->lowLimit -= correction; + zc->lowLimit = newLowLimit; zc->dictLimit -= correction; if (zc->nextToUpdate < correction) zc->nextToUpdate = 0; else zc->nextToUpdate -= correction; @@ -2097,7 +2102,7 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* ctx, ZSTD_parameters ZSTD_getParams(int compressionLevel, U64 srcSizeHint) { ZSTD_parameters result; - int tableID = ((srcSizeHint-1) <= 128 KB) + ((srcSizeHint-1) <= 16 KB); /* intentional underflow for srcSizeHint == 0 */ + int tableID = ((srcSizeHint-1) <= 256 KB) + ((srcSizeHint-1) <= 128 KB) + ((srcSizeHint-1) <= 16 KB); /* intentional underflow for srcSizeHint == 0 */ if (compressionLevel<=0) compressionLevel = 1; if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL; result = ZSTD_defaultParameters[tableID][compressionLevel]; diff --git a/lib/zstd_decompress.c b/lib/zstd_decompress.c index d1e01c3e5..e1c30a8d6 100644 --- a/lib/zstd_decompress.c +++ b/lib/zstd_decompress.c @@ -527,14 +527,11 @@ FORCE_INLINE size_t ZSTD_execSequence(BYTE* op, *litPtr = litEnd; /* update for next sequence */ /* copy Match */ - /* check */ - //if (match > oLitEnd) return ERROR(corruption_detected); /* address space overflow test (is clang optimizer wrongly removing this test ?) */ - if (sequence.offset > (size_t)oLitEnd) return ERROR(corruption_detected); /* address space overflow test (this test seems preserved by clang optimizer) */ - - if (match < base) + if (sequence.offset > (size_t)(oLitEnd - base)) { /* offset beyond prefix */ - if (match < vBase) return ERROR(corruption_detected); + if (sequence.offset > (size_t)(oLitEnd - vBase)) + return ERROR(corruption_detected); match = dictEnd - (base-match); if (match + sequence.matchLength <= dictEnd) { @@ -773,8 +770,6 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* ctx, void* dst, size_t maxDstSize, con if (srcSize != ctx->expected) return ERROR(srcSize_wrong); if (dst != ctx->previousDstEnd) /* not contiguous */ { - if ((dst > ctx->base) && (dst < ctx->previousDstEnd)) /* rolling buffer : new segment into dictionary */ - ctx->base = (char*)dst; /* temporary affectation, for vBase calculation */ ctx->dictEnd = ctx->previousDstEnd; ctx->vBase = (const char*)dst - ((const char*)(ctx->previousDstEnd) - (const char*)(ctx->base)); ctx->base = dst; diff --git a/lib/zstd_static.h b/lib/zstd_static.h index 5c3156248..ebb8c83cf 100644 --- a/lib/zstd_static.h +++ b/lib/zstd_static.h @@ -85,29 +85,29 @@ typedef struct /** ZSTD_getParams * return ZSTD_parameters structure for a selected compression level and srcSize. * srcSizeHint value is optional, select 0 if not known */ -ZSTD_parameters ZSTD_getParams(int compressionLevel, U64 srcSizeHint); +ZSTDLIB_API ZSTD_parameters ZSTD_getParams(int compressionLevel, U64 srcSizeHint); /** ZSTD_validateParams * correct params value to remain within authorized range */ -void ZSTD_validateParams(ZSTD_parameters* params); +ZSTDLIB_API void ZSTD_validateParams(ZSTD_parameters* params); /** ZSTD_compress_advanced * Same as ZSTD_compressCCtx(), with fine-tune control of each compression parameter */ -size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx, - void* dst, size_t maxDstSize, - const void* src, size_t srcSize, - ZSTD_parameters params); +ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx, + void* dst, size_t maxDstSize, + const void* src, size_t srcSize, + ZSTD_parameters params); /* ************************************** * Streaming functions (bufferless mode) ****************************************/ -size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize, int compressionLevel); -size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, ZSTD_parameters params); -size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* ctx, const void* src, size_t srcSize); +ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize, int compressionLevel); +ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, ZSTD_parameters params); +ZSTDLIB_API size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* ctx, const void* src, size_t srcSize); -size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); -size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize); +ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); +ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize); /** Streaming compression, bufferless mode @@ -120,7 +120,7 @@ size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize); Use ZSTD_compressBegin(). You may also prefer the advanced derivative ZSTD_compressBegin_advanced(), for finer parameter control. - It's then possible to add a dictionary with ZSTD_compressDictionary() + It's then possible to add a dictionary with ZSTD_compress_insertDictionary() Note that dictionary presence is a "hidden" information, the decoder needs to be aware that it is required for proper decoding, or decoding will fail. @@ -136,15 +136,15 @@ size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize); typedef struct ZSTD_DCtx_s ZSTD_DCtx; -ZSTD_DCtx* ZSTD_createDCtx(void); -size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); +ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void); +ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); -size_t ZSTD_resetDCtx(ZSTD_DCtx* dctx); -size_t ZSTD_getFrameParams(ZSTD_parameters* params, const void* src, size_t srcSize); -void ZSTD_decompress_insertDictionary(ZSTD_DCtx* ctx, const void* src, size_t srcSize); +ZSTDLIB_API size_t ZSTD_resetDCtx(ZSTD_DCtx* dctx); +ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_parameters* params, const void* src, size_t srcSize); +ZSTDLIB_API void ZSTD_decompress_insertDictionary(ZSTD_DCtx* ctx, const void* src, size_t srcSize); -size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx); -size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); +ZSTDLIB_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx); +ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); /** Streaming decompression, bufferless mode @@ -180,7 +180,8 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, co * Pre-defined compression levels ***************************************/ #define ZSTD_MAX_CLEVEL 20 -static const ZSTD_parameters ZSTD_defaultParameters[3][ZSTD_MAX_CLEVEL+1] = { +ZSTDLIB_API unsigned ZSTD_maxCLevel (void); +static const ZSTD_parameters ZSTD_defaultParameters[4][ZSTD_MAX_CLEVEL+1] = { { /* "default" */ /* W, C, H, S, L, strat */ { 0, 18, 12, 12, 1, 4, ZSTD_fast }, /* level 0 - never used */ @@ -205,6 +206,30 @@ static const ZSTD_parameters ZSTD_defaultParameters[3][ZSTD_MAX_CLEVEL+1] = { { 0, 25, 26, 23, 5, 5, ZSTD_btlazy2 }, /* level 19 */ { 0, 26, 27, 25, 9, 5, ZSTD_btlazy2 }, /* level 20 */ }, +{ /* for srcSize <= 256 KB */ + /* W, C, H, S, L, strat */ + { 0, 0, 0, 0, 0, 0, ZSTD_fast }, /* level 0 - never used */ + { 0, 18, 16, 15, 1, 7, ZSTD_fast }, /* level 1 */ + { 0, 18, 16, 16, 1, 7, ZSTD_fast }, /* level 2 */ + { 0, 18, 18, 18, 1, 7, ZSTD_fast }, /* level 3 */ + { 0, 18, 14, 15, 4, 6, ZSTD_greedy }, /* level 4 */ + { 0, 18, 16, 16, 1, 6, ZSTD_lazy }, /* level 5 */ + { 0, 18, 15, 15, 3, 6, ZSTD_lazy }, /* level 6 */ + { 0, 18, 15, 15, 4, 6, ZSTD_lazy }, /* level 7 */ + { 0, 18, 16, 18, 4, 6, ZSTD_lazy }, /* level 8 */ + { 0, 18, 18, 18, 4, 6, ZSTD_lazy }, /* level 9 */ + { 0, 18, 18, 18, 5, 6, ZSTD_lazy }, /* level 10 */ + { 0, 18, 18, 19, 6, 6, ZSTD_lazy }, /* level 11 */ + { 0, 18, 18, 19, 7, 6, ZSTD_lazy }, /* level 12 */ + { 0, 18, 19, 15, 7, 5, ZSTD_btlazy2 }, /* level 13 */ + { 0, 18, 19, 16, 8, 5, ZSTD_btlazy2 }, /* level 14 */ + { 0, 18, 19, 17, 9, 5, ZSTD_btlazy2 }, /* level 15 */ + { 0, 18, 19, 17, 10, 5, ZSTD_btlazy2 }, /* level 16 */ + { 0, 18, 19, 17, 11, 5, ZSTD_btlazy2 }, /* level 17 */ + { 0, 18, 19, 17, 12, 5, ZSTD_btlazy2 }, /* level 18 */ + { 0, 18, 19, 17, 13, 5, ZSTD_btlazy2 }, /* level 19 */ + { 0, 18, 19, 17, 14, 5, ZSTD_btlazy2 }, /* level 20 */ +}, { /* for srcSize <= 128 KB */ /* W, C, H, S, L, strat */ { 0, 17, 12, 12, 1, 4, ZSTD_fast }, /* level 0 - never used */ diff --git a/programs/Makefile b/programs/Makefile index 6e9d6b56f..822f2d24b 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -30,7 +30,7 @@ # fullbench32: Same as fullbench, but forced to compile in 32-bits mode # ########################################################################## -VERSION?= 0.4.3 +VERSION?= 0.4.4 DESTDIR?= PREFIX ?= /usr/local @@ -94,8 +94,8 @@ zstd-noBench: $(ZSTD_FILES) $(ZSTDDIR)/zstd_buffered.c \ zstdcli.c fileio.c $(ZSTD_FILEIO_LEGACY) $(CC) $(FLAGS) -DZSTD_NOBENCH $^ -o zstd$(EXT) -zstd-frugal: clean - CFLAGS=-Os $(MAKE) zstd-noBench ZSTD_LEGACY_SUPPORT=0 +zstd-frugal: clean + $(MAKE) zstd-noBench ZSTD_LEGACY_SUPPORT=0 fullbench : $(ZSTD_FILES) \ datagen.c fullbench.c @@ -169,77 +169,7 @@ test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zbuff32 test-all: test test32 valgrindTest zstd-playTests: datagen - @echo "\n**** frame concatenation **** " - @echo "hello " > hello.tmp - @echo "world!" > world.tmp - @cat hello.tmp world.tmp > helloworld.tmp - $(ZSTD) hello.tmp > hello.zstd - $(ZSTD) world.tmp > world.zstd - @cat hello.zstd world.zstd > helloworld.zstd - $(ZSTD) -df helloworld.zstd > result.tmp - cat result.tmp - sdiff helloworld.tmp result.tmp - @rm *.tmp *.zstd - @echo frame concatenation test completed - @echo "**** flush write error test **** " - echo foo | $(ZSTD) > /dev/full; if [ $$? -eq 0 ] ; then echo "write error not detected!"; false; fi - echo foo | $(ZSTD) | $(ZSTD) -d > /dev/full; if [ $$? -eq 0 ] ; then echo "write error not detected!"; false; fi - @echo "**** zstd round-trip tests **** " - @./datagen | md5sum > tmp1 - ./datagen | $(ZSTD) -v | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen | $(ZSTD) -6 -v | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - @./datagen -g270000000 | md5sum > tmp1 - ./datagen -g270000000 | $(ZSTD) -v | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g270000000 | $(ZSTD) -v2 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g270000000 | $(ZSTD) -v3 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - @./datagen -g140000000 -P60| md5sum > tmp1 - ./datagen -g140000000 -P60 | $(ZSTD) -v4 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g140000000 -P60 | $(ZSTD) -v5 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g140000000 -P60 | $(ZSTD) -v6 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - @./datagen -g70000000 -P70 | md5sum > tmp1 - ./datagen -g70000000 -P70 | $(ZSTD) -v7 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g70000000 -P70 | $(ZSTD) -v8 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g70000000 -P70 | $(ZSTD) -v9 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - @./datagen -g35000000 -P75 | md5sum > tmp1 - ./datagen -g35000000 -P75 | $(ZSTD) -v10 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g35000000 -P75 | $(ZSTD) -v11 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g35000000 -P75 | $(ZSTD) -v12 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - @./datagen -g18000000 -P80 | md5sum > tmp1 - ./datagen -g18000000 -P80 | $(ZSTD) -v13 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g18000000 -P80 | $(ZSTD) -v14 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g18000000 -P80 | $(ZSTD) -v15 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g18000000 -P80 | $(ZSTD) -v16 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g18000000 -P80 | $(ZSTD) -v17 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - @./datagen -g50000000 -P94 | md5sum > tmp1 - ./datagen -g50000000 -P94 | $(ZSTD) -v18 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g50000000 -P94 | $(ZSTD) -v19 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - @./datagen -g99000000 -P99 | md5sum > tmp1 - ./datagen -g99000000 -P99 | $(ZSTD) -v20 | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 - ./datagen -g6000000000 -P99| md5sum > tmp1 - ./datagen -g6000000000 -P99| $(ZSTD) -vq | $(ZSTD) -d | md5sum > tmp2 - @diff tmp1 tmp2 + ZSTD=$(ZSTD) ./playTests.sh --test-large-data test-zstd: ZSTD = ./zstd test-zstd: zstd zstd-playTests diff --git a/programs/fileio.c b/programs/fileio.c index 27c72c574..81635b934 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -121,6 +121,7 @@ #define CACHELINE 64 +#define MAX_DICT_SIZE (512 KB) /* ************************************* * Macros @@ -235,12 +236,13 @@ static U64 FIO_getFileSize(const char* infilename) } -unsigned long long FIO_compressFilename(const char* output_filename, const char* input_filename, int cLevel) +unsigned long long FIO_compressFilename(const char* output_filename, const char* input_filename, + const char* dictFileName, int cLevel) { U64 filesize = 0; U64 compressedfilesize = 0; - BYTE* inBuff; - BYTE* outBuff; + U64 dictSize = 0; + BYTE* inBuff, *outBuff, *dictBuff=NULL; size_t inBuffSize = ZBUFF_recommendedCInSize(); size_t outBuffSize = ZBUFF_recommendedCOutSize(); FILE* finput; @@ -252,16 +254,43 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char* ctx = ZBUFF_createCCtx(); inBuff = (BYTE*)malloc(inBuffSize); outBuff = (BYTE*)malloc(outBuffSize); - if (!inBuff || !outBuff || !ctx) EXM_THROW(21, "Allocation error : not enough memory"); + if (!inBuff || !outBuff || !ctx) EXM_THROW(20, "Allocation error : not enough memory"); + + /* dictionary */ + if (dictFileName) + { + FILE* dictHandle; + size_t readSize; + DISPLAYLEVEL(4,"Using %s as dictionary \n", dictFileName); + dictHandle = fopen(dictFileName, "rb"); + if (dictHandle==0) EXM_THROW(21, "Error opening dictionary file %s", dictFileName); + dictSize = FIO_getFileSize(dictFileName); + if (dictSize > MAX_DICT_SIZE) + { + int seekResult; + if (dictSize > 1 GB) EXM_THROW(21, "Dictionary file %s is too large", dictFileName); /* avoid extreme cases */ + DISPLAYLEVEL(2,"Dictionary %s is too large : using last %u bytes only \n", dictFileName, MAX_DICT_SIZE); + seekResult = fseek(dictHandle, (size_t)(dictSize-MAX_DICT_SIZE), SEEK_SET); /* use end of file */ + if (seekResult != 0) EXM_THROW(21, "Error seeking into dictionary file %s", dictFileName); + dictSize = MAX_DICT_SIZE; + } + dictBuff = (BYTE*)malloc((size_t)dictSize); + if (dictBuff==NULL) EXM_THROW(20, "Allocation error : not enough memory for dictBuff"); + readSize = fread(dictBuff, 1, (size_t)dictSize, dictHandle); + if (readSize!=dictSize) EXM_THROW(21, "Error reading dictionary file %s", dictFileName); + fclose(dictHandle); + } /* init */ FIO_getFileHandles(&finput, &foutput, input_filename, output_filename); - filesize = FIO_getFileSize(input_filename); + filesize = FIO_getFileSize(input_filename) + dictSize; errorCode = ZBUFF_compressInit_advanced(ctx, ZSTD_getParams(cLevel, filesize)); if (ZBUFF_isError(errorCode)) EXM_THROW(22, "Error initializing compression"); - filesize = 0; + errorCode = ZBUFF_compressWithDictionary(ctx, dictBuff, (size_t)dictSize); + if (ZBUFF_isError(errorCode)) EXM_THROW(22, "Error initializing dictionary"); /* Main compression loop */ + filesize = 0; while (1) { size_t inSize; @@ -311,6 +340,7 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char* /* clean */ free(inBuff); free(outBuff); + free(dictBuff); ZBUFF_freeCCtx(ctx); fclose(finput); if (fclose(foutput)) EXM_THROW(28, "Write error : cannot properly close %s", output_filename); @@ -322,6 +352,7 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char* unsigned long long FIO_decompressFrame(FILE* foutput, FILE* finput, BYTE* inBuff, size_t inBuffSize, size_t alreadyLoaded, BYTE* outBuff, size_t outBuffSize, + BYTE* dictBuff, size_t dictSize, ZBUFF_DCtx* dctx) { U64 frameSize = 0; @@ -329,6 +360,7 @@ unsigned long long FIO_decompressFrame(FILE* foutput, FILE* finput, /* Main decompression Loop */ ZBUFF_decompressInit(dctx); + ZBUFF_decompressWithDictionary(dctx, dictBuff, dictSize); while (1) { /* Decode */ @@ -359,16 +391,42 @@ unsigned long long FIO_decompressFrame(FILE* foutput, FILE* finput, } -unsigned long long FIO_decompressFilename(const char* output_filename, const char* input_filename) +unsigned long long FIO_decompressFilename(const char* output_filename, const char* input_filename, const char* dictFileName) { FILE* finput, *foutput; BYTE* inBuff=NULL; size_t inBuffSize = ZBUFF_recommendedDInSize(); BYTE* outBuff=NULL; size_t outBuffSize = ZBUFF_recommendedDOutSize(); + BYTE* dictBuff=NULL; + size_t dictSize = 0; U64 filesize = 0; size_t toRead; + /* dictionary */ + if (dictFileName) + { + FILE* dictHandle; + size_t readSize; + DISPLAYLEVEL(4,"Using %s as dictionary \n", dictFileName); + dictHandle = fopen(dictFileName, "rb"); + if (dictHandle==0) EXM_THROW(21, "Error opening dictionary file %s", dictFileName); + dictSize = (size_t)FIO_getFileSize(dictFileName); + if (dictSize > MAX_DICT_SIZE) + { + int seekResult; + if (dictSize > 1 GB) EXM_THROW(21, "Dictionary file %s is too large", dictFileName); /* avoid extreme cases */ + DISPLAYLEVEL(2,"Dictionary %s is too large : using last %u bytes only \n", dictFileName, MAX_DICT_SIZE); + seekResult = fseek(dictHandle, dictSize-MAX_DICT_SIZE, SEEK_SET); /* use end of file */ + if (seekResult != 0) EXM_THROW(21, "Error seeking into dictionary file %s", dictFileName); + dictSize = MAX_DICT_SIZE; + } + dictBuff = (BYTE*)malloc(dictSize); + if (dictBuff==NULL) EXM_THROW(20, "Allocation error : not enough memory for dictBuff"); + readSize = fread(dictBuff, 1, (size_t)dictSize, dictHandle); + if (readSize!=dictSize) EXM_THROW(21, "Error reading dictionary file %s", dictFileName); + fclose(dictHandle); + } /* Init */ ZBUFF_DCtx* dctx = ZBUFF_createDCtx(); @@ -396,7 +454,11 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha } #endif /* ZSTD_LEGACY_SUPPORT */ - filesize += FIO_decompressFrame(foutput, finput, inBuff, inBuffSize, toRead, outBuff, outBuffSize, dctx); + filesize += FIO_decompressFrame(foutput, finput, + inBuff, inBuffSize, toRead, + outBuff, outBuffSize, + dictBuff, dictSize, + dctx); } DISPLAYLEVEL(2, "\r%79s\r", ""); @@ -405,6 +467,7 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha /* clean */ free(inBuff); free(outBuff); + free(dictBuff); ZBUFF_freeDCtx(dctx); fclose(finput); if (fclose(foutput)) EXM_THROW(38, "Write error : cannot properly close %s", output_filename); diff --git a/programs/fileio.h b/programs/fileio.h index 037c819ea..0f5087879 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -52,8 +52,8 @@ void FIO_setNotificationLevel(unsigned level); /* ************************************* * Stream/File functions ***************************************/ -unsigned long long FIO_compressFilename (const char* outfilename, const char* infilename, int compressionLevel); -unsigned long long FIO_decompressFilename (const char* outfilename, const char* infilename); +unsigned long long FIO_compressFilename (const char* outfilename, const char* infilename, const char* dictFileName, int compressionLevel); +unsigned long long FIO_decompressFilename (const char* outfilename, const char* infilename, const char* dictFileName); /** FIO_compressFilename : @result : size of compressed file diff --git a/programs/fullbench.c b/programs/fullbench.c index cb4822022..0eeb88e85 100644 --- a/programs/fullbench.c +++ b/programs/fullbench.c @@ -235,7 +235,7 @@ size_t local_ZSTD_decodeLiteralsBlock(void* dst, size_t dstSize, void* buff2, co size_t local_ZSTD_decodeSeqHeaders(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize) { - U32 DTableML[1<<11], DTableLL[1<<10], DTableOffb[1<<9]; + U32 DTableML[FSE_DTABLE_SIZE_U32(10)], DTableLL[FSE_DTABLE_SIZE_U32(10)], DTableOffb[FSE_DTABLE_SIZE_U32(9)]; /* MLFSELog, LLFSELog and OffFSELog are not public values */ const BYTE* dumps; size_t length; int nbSeq; @@ -245,7 +245,6 @@ size_t local_ZSTD_decodeSeqHeaders(void* dst, size_t dstSize, void* buff2, const - /********************************************************* * Bench functions *********************************************************/ @@ -429,7 +428,7 @@ int benchFiles(char** fileNamesTable, int nbFiles, U32 benchNb) return 11; } - // Memory allocation & restrictions + /* Memory allocation & restrictions */ inFileSize = BMK_GetFileSize(inFileName); benchedSize = (size_t) BMK_findMaxMem(inFileSize*3) / 3; if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize; diff --git a/programs/playTests.sh b/programs/playTests.sh new file mode 100755 index 000000000..25c7fb23a --- /dev/null +++ b/programs/playTests.sh @@ -0,0 +1,88 @@ +#!/bin/sh -e + +die() { + echo "$@" 1>&2 + exit 1 +} + +roundTripTest() { + if [ -n "$3" ]; then + local c="$3" + local p="$2" + else + local c="$2" + fi + + rm -f tmp1 tmp2 + echo "roundTripTest: ./datagen $1 $p | $ZSTD -v$c | $ZSTD -d" + ./datagen $1 $p | md5sum > tmp1 + ./datagen $1 $p | $ZSTD -v$c | $ZSTD -d | md5sum > tmp2 + diff -q tmp1 tmp2 +} + +[ -n "$ZSTD" ] || die "ZSTD variable must be defined!" + +printf "\n**** frame concatenation **** " + +echo "hello " > hello.tmp +echo "world!" > world.tmp +cat hello.tmp world.tmp > helloworld.tmp +$ZSTD hello.tmp > hello.zstd +$ZSTD world.tmp > world.zstd +cat hello.zstd world.zstd > helloworld.zstd +$ZSTD -df helloworld.zstd > result.tmp +cat result.tmp +sdiff helloworld.tmp result.tmp +rm ./*.tmp ./*.zstd + +echo frame concatenation test completed + +echo "**** flush write error test **** " + +echo foo | $ZSTD > /dev/full && die "write error not detected!" +echo foo | $ZSTD | $ZSTD -d > /dev/full && die "write error not detected!" + +echo "*** dictionary tests *** " + +./datagen > tmpDict +./datagen -g1M | md5sum > tmp1 +./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dv | md5sum > tmp2 +diff -q tmp1 tmp2 + +echo "**** zstd round-trip tests **** " + +roundTripTest +roundTripTest '' 6 + +if [ "$1" != "--test-large-data" ]; then + echo "Skipping large data tests" + exit 0 +fi + +roundTripTest -g270000000 1 +roundTripTest -g270000000 2 +roundTripTest -g270000000 3 + +roundTripTest -g140000000 -P60 4 +roundTripTest -g140000000 -P60 5 +roundTripTest -g140000000 -P60 6 + +roundTripTest -g70000000 -P70 7 +roundTripTest -g70000000 -P70 8 +roundTripTest -g70000000 -P70 9 + +roundTripTest -g35000000 -P75 10 +roundTripTest -g35000000 -P75 11 +roundTripTest -g35000000 -P75 12 + +roundTripTest -g18000000 -P80 13 +roundTripTest -g18000000 -P80 14 +roundTripTest -g18000000 -P80 15 +roundTripTest -g18000000 -P80 16 +roundTripTest -g18000000 -P80 17 + +roundTripTest -g50000000 -P94 18 +roundTripTest -g50000000 -P94 19 + +roundTripTest -g99000000 -P99 20 +roundTripTest -g6000000000 -P99 q diff --git a/programs/zbufftest.c b/programs/zbufftest.c index 590393561..ab8aa3486 100644 --- a/programs/zbufftest.c +++ b/programs/zbufftest.c @@ -161,6 +161,7 @@ static int basicUnitTests(U32 seed, double compressibility) ZBUFF_compressInit(zc, 1); readSize = CNBufferSize; genSize = compressedBufferSize; + ZBUFF_compressWithDictionary(zc, CNBuffer, 128 KB); result = ZBUFF_compressContinue(zc, compressedBuffer, &genSize, CNBuffer, &readSize); if (ZBUFF_isError(result)) goto _output_error; if (readSize != CNBufferSize) goto _output_error; /* entire input should be consumed */ @@ -174,6 +175,7 @@ static int basicUnitTests(U32 seed, double compressibility) /* Basic decompression test */ DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); ZBUFF_decompressInit(zd); + ZBUFF_decompressWithDictionary(zd, CNBuffer, 128 KB); readSize = cSize; genSize = CNBufferSize; result = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, compressedBuffer, &readSize); @@ -244,7 +246,6 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit U32 coreSeed = seed, lseed = 0; ZBUFF_CCtx* zc; ZBUFF_DCtx* zd; - XXH64_state_t crc64; U32 startTime = FUZ_GetMilliStart(); /* allocation */ @@ -280,10 +281,12 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit for ( ; (testNb <= nbTests) || (FUZ_GetMilliSpan(startTime) < g_testTime); testNb++ ) { size_t sampleSize, sampleStart; - size_t cSize; + const BYTE* dict; + size_t cSize, dictSize; size_t maxTestSize, totalTestSize, readSize, totalCSize, genSize, totalGenSize; size_t errorCode; U32 sampleSizeLog, buffNb, n, nbChunks; + XXH64_state_t crc64; U64 crcOrig, crcDest; /* init */ @@ -316,6 +319,15 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit maxTestSize = (size_t)1 << sampleSizeLog; maxTestSize += FUZ_rand(&lseed) & (maxTestSize-1); ZBUFF_compressInit(zc, (FUZ_rand(&lseed) % (20 - (sampleSizeLog/3))) + 1); + + sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog; + sampleSize = (size_t)1 << sampleSizeLog; + sampleSize += FUZ_rand(&lseed) & (sampleSize-1); + sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize); + dict = srcBuffer + sampleStart; + dictSize = sampleSize; + ZBUFF_compressWithDictionary(zc, dict, dictSize); + totalTestSize = 0; cSize = 0; for (n=0; n Application true - v110 + v120 Unicode Application true - v110 + v120 Unicode Application false - v110 + v120 true Unicode Application false - v110 + v120 true Unicode @@ -111,7 +111,7 @@ WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) true true - /analyze:stacksize19000 %(AdditionalOptions) + /analyze:stacksize25000 %(AdditionalOptions) Console @@ -129,7 +129,7 @@ WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) true true - /analyze:stacksize19000 %(AdditionalOptions) + /analyze:stacksize25000 %(AdditionalOptions) Console @@ -149,7 +149,7 @@ WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) true true - /analyze:stacksize19000 %(AdditionalOptions) + /analyze:stacksize25000 %(AdditionalOptions) Console diff --git a/visual/2012/fullbench/fullbench.vcxproj.filters b/visual/2013/fullbench/fullbench.vcxproj.filters similarity index 100% rename from visual/2012/fullbench/fullbench.vcxproj.filters rename to visual/2013/fullbench/fullbench.vcxproj.filters diff --git a/visual/2012/fuzzer/fuzzer.vcxproj b/visual/2013/fuzzer/fuzzer.vcxproj similarity index 94% rename from visual/2012/fuzzer/fuzzer.vcxproj rename to visual/2013/fuzzer/fuzzer.vcxproj index 63ab47205..0844efe74 100644 --- a/visual/2012/fuzzer/fuzzer.vcxproj +++ b/visual/2013/fuzzer/fuzzer.vcxproj @@ -27,26 +27,26 @@ Application true - v110 + v120 Unicode Application true - v110 + v120 Unicode Application false - v110 + v120 true Unicode Application false - v110 + v120 true Unicode @@ -111,7 +111,7 @@ WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) true true - /analyze:stacksize19000 %(AdditionalOptions) + /analyze:stacksize25000 %(AdditionalOptions) Console @@ -129,7 +129,7 @@ WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) true true - /analyze:stacksize19000 %(AdditionalOptions) + /analyze:stacksize25000 %(AdditionalOptions) Console @@ -149,7 +149,7 @@ WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) true true - /analyze:stacksize19000 %(AdditionalOptions) + /analyze:stacksize25000 %(AdditionalOptions) Console diff --git a/visual/2012/fuzzer/fuzzer.vcxproj.filters b/visual/2013/fuzzer/fuzzer.vcxproj.filters similarity index 100% rename from visual/2012/fuzzer/fuzzer.vcxproj.filters rename to visual/2013/fuzzer/fuzzer.vcxproj.filters diff --git a/visual/2012/zstd.sln b/visual/2013/zstd.sln similarity index 74% rename from visual/2012/zstd.sln rename to visual/2013/zstd.sln index 2b5eb9895..4f2447fbb 100644 --- a/visual/2012/zstd.sln +++ b/visual/2013/zstd.sln @@ -1,12 +1,16 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Express 2012 for Windows Desktop +# Visual Studio 14 +VisualStudioVersion = 14.0.24720.0 +MinimumVisualStudioVersion = 10.0.40219.1 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zstd", "zstd\zstd.vcxproj", "{4E52A41A-F33B-4C7A-8C36-A1A6B4F4277C}" EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fuzzer", "fuzzer\fuzzer.vcxproj", "{6FD4352B-346C-4703-96EA-D4A8B9A6976E}" EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fullbench", "fullbench\fullbench.vcxproj", "{61ABD629-1CC8-4FD7-9281-6B8DBB9D3DF8}" EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zstdlib", "zstdlib\zstdlib.vcxproj", "{8BFD8150-94D5-4BF9-8A50-7BD9929A0850}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Win32 = Debug|Win32 @@ -39,6 +43,14 @@ Global {61ABD629-1CC8-4FD7-9281-6B8DBB9D3DF8}.Release|Win32.Build.0 = Release|Win32 {61ABD629-1CC8-4FD7-9281-6B8DBB9D3DF8}.Release|x64.ActiveCfg = Release|x64 {61ABD629-1CC8-4FD7-9281-6B8DBB9D3DF8}.Release|x64.Build.0 = Release|x64 + {8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Debug|Win32.ActiveCfg = Debug|Win32 + {8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Debug|Win32.Build.0 = Debug|Win32 + {8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Debug|x64.ActiveCfg = Debug|x64 + {8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Debug|x64.Build.0 = Debug|x64 + {8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Release|Win32.ActiveCfg = Release|Win32 + {8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Release|Win32.Build.0 = Release|Win32 + {8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Release|x64.ActiveCfg = Release|x64 + {8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/visual/2012/zstd/zstd.vcxproj b/visual/2013/zstd/zstd.vcxproj similarity index 95% rename from visual/2012/zstd/zstd.vcxproj rename to visual/2013/zstd/zstd.vcxproj index 524cf6276..2ab12858b 100644 --- a/visual/2012/zstd/zstd.vcxproj +++ b/visual/2013/zstd/zstd.vcxproj @@ -63,27 +63,27 @@ Application true - v110 Unicode + v120 Application true - v110 + v120 Unicode Application false - v110 true + v120 Unicode Application false - v110 true + v120 Unicode @@ -104,7 +104,7 @@ true - $(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\legacy;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); true diff --git a/visual/2012/zstd/zstd.vcxproj.filters b/visual/2013/zstd/zstd.vcxproj.filters similarity index 100% rename from visual/2012/zstd/zstd.vcxproj.filters rename to visual/2013/zstd/zstd.vcxproj.filters diff --git a/visual/2013/zstdlib/resource.h b/visual/2013/zstdlib/resource.h new file mode 100644 index 000000000..db984ef4d Binary files /dev/null and b/visual/2013/zstdlib/resource.h differ diff --git a/visual/2013/zstdlib/zstdlib.rc b/visual/2013/zstdlib/zstdlib.rc new file mode 100644 index 000000000..d5b107e33 Binary files /dev/null and b/visual/2013/zstdlib/zstdlib.rc differ diff --git a/visual/2013/zstdlib/zstdlib.vcxproj b/visual/2013/zstdlib/zstdlib.vcxproj new file mode 100644 index 000000000..124595556 --- /dev/null +++ b/visual/2013/zstdlib/zstdlib.vcxproj @@ -0,0 +1,216 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + {8BFD8150-94D5-4BF9-8A50-7BD9929A0850} + Win32Proj + zstdlib + + + + DynamicLibrary + true + Unicode + v120 + + + DynamicLibrary + true + v120 + Unicode + + + DynamicLibrary + false + true + v120 + Unicode + + + DynamicLibrary + false + true + v120 + Unicode + + + + + + + + + + + + + + + + + + + true + zstdlib_x86 + $(Platform)\$(Configuration)\ + $(SolutionDir)$(Platform)\$(Configuration)\ + $(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + true + + + true + zstdlib_x64 + $(Platform)\$(Configuration)\ + $(SolutionDir)$(Platform)\$(Configuration)\ + $(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + true + + + false + zstdlib_x86 + $(SolutionDir)$(Platform)\$(Configuration)\ + $(Platform)\$(Configuration)\ + $(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + false + + + false + zstdlib_x64 + $(SolutionDir)$(Platform)\$(Configuration)\ + $(Platform)\$(Configuration)\ + $(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + false + + + + + + Level4 + Disabled + ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + true + EnableFastChecks + MultiThreadedDebugDLL + EditAndContinue + true + true + /analyze:stacksize25000 %(AdditionalOptions) + + + Console + true + MachineX86 + + + + + + + Level4 + Disabled + ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + false + EnableFastChecks + MultiThreadedDebugDLL + ProgramDatabase + true + /analyze:stacksize25000 %(AdditionalOptions) + + + Console + true + + + + + Level4 + + + MaxSpeed + true + true + ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + true + /analyze:stacksize25000 %(AdditionalOptions) + MultiThreadedDLL + ProgramDatabase + All + + + Console + true + true + true + MachineX86 + + + + + Level4 + + + MaxSpeed + true + true + ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + false + false + MultiThreadedDLL + ProgramDatabase + true + true + All + + + Console + true + true + true + + + + + + \ No newline at end of file diff --git a/visual/2013/zstdlib/zstdlib.vcxproj.filters b/visual/2013/zstdlib/zstdlib.vcxproj.filters new file mode 100644 index 000000000..9e930d7b1 --- /dev/null +++ b/visual/2013/zstdlib/zstdlib.vcxproj.filters @@ -0,0 +1,80 @@ + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hh;hpp;hxx;hm;inl;inc;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + + + Resource Files + + + \ No newline at end of file