diff --git a/.gitignore b/.gitignore index 01a4dd917..5f2ca97de 100644 --- a/.gitignore +++ b/.gitignore @@ -18,18 +18,15 @@ zstd *.out *.app -# IDEA solution files -*.idea +# Test artefacts +tmp* +dictionary # Other files .directory _codelite/ _zstdbench/ .clang_complete - -# Test artefacts -tmp* -dictionary - -# tmp files +*.idea *.swp +.DS_Store diff --git a/NEWS b/NEWS index 36bd15242..70429644d 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,7 @@ +v0.8.2 +API : ZDICT_getDictID() +Small decompression speed improvement + v0.8.1 New streaming API Changed : --ultra now enables levels beyond 19 @@ -5,6 +9,7 @@ Changed : -i# now selects benchmark time in second Fixed : ZSTD_compress* can now compress > 4 GB in a single pass, reported by Nick Terrell Fixed : speed regression on specific patterns (#272) Fixed : support for Z_SYNC_FLUSH, by Dmitry Krot (#291) +Fixed : ICC compilation, by Przemyslaw Skibinski v0.8.0 Improved : better speed on clang and gcc -O2, thanks to Eric Biggers diff --git a/lib/common/fse_decompress.c b/lib/common/fse_decompress.c index 918de64c5..032e65771 100644 --- a/lib/common/fse_decompress.c +++ b/lib/common/fse_decompress.c @@ -68,6 +68,9 @@ #define FSE_isError ERR_isError #define FSE_STATIC_ASSERT(c) { enum { FSE_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ +/* check and forward error code */ +#define CHECK_F(f) { size_t const e = f; if (FSE_isError(e)) return e; } + /* ************************************************************** * Complex types @@ -152,7 +155,6 @@ size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned position = (position + step) & tableMask; while (position > highThreshold) position = (position + step) & tableMask; /* lowprob area */ } } - if (position!=0) return ERROR(GENERIC); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ } @@ -169,7 +171,6 @@ size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned } - #ifndef FSE_COMMONDEFS_ONLY /*-******************************************************* @@ -234,8 +235,7 @@ FORCE_INLINE size_t FSE_decompress_usingDTable_generic( FSE_DState_t state2; /* Init */ - { size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize); /* replaced last arg by maxCompressed Size */ - if (FSE_isError(errorCode)) return errorCode; } + CHECK_F(BIT_initDStream(&bitD, cSrc, cSrcSize)); FSE_initDState(&state1, &bitD, dt); FSE_initDState(&state2, &bitD, dt); @@ -243,7 +243,7 @@ FORCE_INLINE size_t FSE_decompress_usingDTable_generic( #define FSE_GETSYMBOL(statePtr) fast ? FSE_decodeSymbolFast(statePtr, &bitD) : FSE_decodeSymbol(statePtr, &bitD) /* 4 symbols per loop */ - for ( ; (BIT_reloadDStream(&bitD)==BIT_DStream_unfinished) && (op sizeof(bitD.bitContainer)*8) /* This test must be static */ @@ -266,18 +266,14 @@ FORCE_INLINE size_t FSE_decompress_usingDTable_generic( /* note : BIT_reloadDStream(&bitD) >= FSE_DStream_partiallyFilled; Ends at exactly BIT_DStream_completed */ while (1) { if (op>(omax-2)) return ERROR(dstSize_tooSmall); - *op++ = FSE_GETSYMBOL(&state1); - if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) { *op++ = FSE_GETSYMBOL(&state2); break; } if (op>(omax-2)) return ERROR(dstSize_tooSmall); - *op++ = FSE_GETSYMBOL(&state2); - if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) { *op++ = FSE_GETSYMBOL(&state1); break; @@ -320,8 +316,7 @@ size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, size_t cSr cSrcSize -= NCountLength; } - { size_t const errorCode = FSE_buildDTable (dt, counting, maxSymbolValue, tableLog); - if (FSE_isError(errorCode)) return errorCode; } + CHECK_F( FSE_buildDTable (dt, counting, maxSymbolValue, tableLog) ); return FSE_decompress_usingDTable (dst, maxDstSize, ip, cSrcSize, dt); /* always return, even if it is an error code */ } diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 00b1408ca..a696b2bc6 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -154,7 +154,7 @@ size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx) return 0; /* reserved as a potential error code in the future */ } -size_t ZSTD_sizeofCCtx(const ZSTD_CCtx* cctx) +size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx) { return sizeof(*cctx) + cctx->workSpaceSize; } @@ -2867,6 +2867,10 @@ size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel) return ZSTD_initCStream_usingDict(zcs, NULL, 0, compressionLevel); } +size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs) +{ + return sizeof(zcs) + ZSTD_sizeof_CCtx(zcs->zc) + zcs->outBuffSize + zcs->inBuffSize; +} /*====== Compression ======*/ diff --git a/lib/compress/zstd_opt.h b/lib/compress/zstd_opt.h index f30cdbfda..612ac3b4d 100644 --- a/lib/compress/zstd_opt.h +++ b/lib/compress/zstd_opt.h @@ -529,7 +529,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, continue; mlen = opt[cur].mlen; - if (opt[cur].off > ZSTD_REP_MOVE_OPT) { + if (opt[cur].off > ZSTD_REP_MOVE_OPT) { opt[cur].rep[2] = opt[cur-mlen].rep[1]; opt[cur].rep[1] = opt[cur-mlen].rep[0]; opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE_OPT; @@ -610,12 +610,12 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off-1, mlen - MINMATCH); } - // ZSTD_LOG_PARSER("%d: Found2 mlen=%d best_mlen=%d off=%d price=%d litlen=%d\n", (int)(inr-base), mlen, best_mlen, matches[u].off, price, litlen); + // ZSTD_LOG_PARSER("%d: Found2 mlen=%d best_mlen=%d off=%d price=%d litlen=%d\n", (int)(inr-base), mlen, best_mlen, matches[u].off, price, litlen); if (cur + mlen > last_pos || (price < opt[cur + mlen].price)) SET_PRICE(cur + mlen, mlen, matches[u].off, litlen, price); mlen++; - } } } // for (cur = 1; cur <= last_pos; cur++) + } } } // for (cur = 1; cur <= last_pos; cur++) best_mlen = opt[last_pos].mlen; best_off = opt[last_pos].off; diff --git a/lib/decompress/huf_decompress.c b/lib/decompress/huf_decompress.c index 1580b3750..a5521bd36 100644 --- a/lib/decompress/huf_decompress.c +++ b/lib/decompress/huf_decompress.c @@ -37,7 +37,7 @@ ****************************************************************/ #if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) /* inline is defined */ -#elif defined(_MSC_VER) +#elif defined(_MSC_VER) || defined(__GNUC__) # define inline __inline #else # define inline /* disable inline */ @@ -57,10 +57,10 @@ /* ************************************************************** -* Includes +* Dependencies ****************************************************************/ #include /* memcpy, memset */ -#include "bitstream.h" +#include "bitstream.h" /* BIT_* */ #include "fse.h" /* header compression */ #define HUF_STATIC_LINKING_ONLY #include "huf.h" @@ -103,7 +103,7 @@ size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize) HUF_DEltX2* const dt = (HUF_DEltX2*)dtPtr; HUF_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUF_DTable)); - //memset(huffWeight, 0, sizeof(huffWeight)); /* is not necessary, even though some analyzer complain ... */ + /* memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */ iSize = HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize); if (HUF_isError(iSize)) return iSize; @@ -388,22 +388,22 @@ static void HUF_fillDTableX4Level2(HUF_DEltX4* DTable, U32 sizeLog, const U32 co } /* fill DTable */ - { U32 s; for (s=0; s= 1 */ + MEM_writeLE16(&(DElt.sequence), (U16)(baseSeq + (symbol << 8))); + DElt.nbBits = (BYTE)(nbBits + consumed); + DElt.length = 2; + do { DTable[i++] = DElt; } while (i= 1 */ - rankVal[weight] += length; - }} + rankVal[weight] += length; + } } } typedef U32 rankVal_t[HUF_TABLELOG_ABSOLUTEMAX][HUF_TABLELOG_ABSOLUTEMAX + 1]; @@ -442,8 +442,8 @@ static void HUF_fillDTableX4(HUF_DEltX4* DTable, const U32 targetLog, MEM_writeLE16(&(DElt.sequence), symbol); DElt.nbBits = (BYTE)(nbBits); DElt.length = 1; - { U32 u; - const U32 end = start + length; + { U32 const end = start + length; + U32 u; for (u = start; u < end; u++) DTable[u] = DElt; } } rankVal[weight] += length; @@ -467,7 +467,7 @@ size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize) HUF_STATIC_ASSERT(sizeof(HUF_DEltX4) == sizeof(HUF_DTable)); /* if compilation fails here, assertion is false */ if (maxTableLog > HUF_TABLELOG_ABSOLUTEMAX) return ERROR(tableLog_tooLarge); - //memset(weightList, 0, sizeof(weightList)); /* is not necessary, even though some analyzer complain ... */ + /* memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */ iSize = HUF_readStats(weightList, HUF_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize); if (HUF_isError(iSize)) return iSize; @@ -533,7 +533,7 @@ size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize) static U32 HUF_decodeSymbolX4(void* op, BIT_DStream_t* DStream, const HUF_DEltX4* dt, const U32 dtLog) { - const size_t val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ + size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ memcpy(op, dt+val, 2); BIT_skipBits(DStream, dt[val].nbBits); return dt[val].length; @@ -541,7 +541,7 @@ static U32 HUF_decodeSymbolX4(void* op, BIT_DStream_t* DStream, const HUF_DEltX4 static U32 HUF_decodeLastSymbolX4(void* op, BIT_DStream_t* DStream, const HUF_DEltX4* dt, const U32 dtLog) { - const size_t val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ + size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ memcpy(op, dt+val, 1); if (dt[val].length==1) BIT_skipBits(DStream, dt[val].nbBits); else { @@ -570,7 +570,7 @@ static inline size_t HUF_decodeStreamX4(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* c BYTE* const pStart = p; /* up to 8 symbols at a time */ - while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p < pEnd-7)) { + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-(sizeof(bitDPtr->bitContainer)-1))) { HUF_DECODE_SYMBOLX4_2(p, bitDPtr); HUF_DECODE_SYMBOLX4_1(p, bitDPtr); HUF_DECODE_SYMBOLX4_2(p, bitDPtr); @@ -578,7 +578,7 @@ static inline size_t HUF_decodeStreamX4(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* c } /* closer to end : up to 2 symbols at a time */ - while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p <= pEnd-2)) + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p <= pEnd-2)) HUF_DECODE_SYMBOLX4_0(p, bitDPtr); while (p <= pEnd-2) @@ -697,7 +697,7 @@ static size_t HUF_decompress4X4_usingDTable_internal( /* 16-32 symbols per loop (4-8 symbols per stream) */ endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); - for ( ; (endSignal==BIT_DStream_unfinished) && (op4<(oend-7)) ; ) { + for ( ; (endSignal==BIT_DStream_unfinished) & (op4<(oend-(sizeof(bitD4.bitContainer)-1))) ; ) { HUF_DECODE_SYMBOLX4_2(op1, &bitD1); HUF_DECODE_SYMBOLX4_2(op2, &bitD2); HUF_DECODE_SYMBOLX4_2(op3, &bitD3); @@ -722,7 +722,7 @@ static size_t HUF_decompress4X4_usingDTable_internal( if (op1 > opStart2) return ERROR(corruption_detected); if (op2 > opStart3) return ERROR(corruption_detected); if (op3 > opStart4) return ERROR(corruption_detected); - /* note : op4 supposed already verified within main loop */ + /* note : op4 already verified within main loop */ /* finish bitStreams one by one */ HUF_decodeStreamX4(op1, &bitD1, opStart2, dt, dtLog); @@ -848,9 +848,6 @@ size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcS { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); return decompress[algoNb](dst, dstSize, cSrc, cSrcSize); } - - //return HUF_decompress4X2(dst, dstSize, cSrc, cSrcSize); /* multi-streams single-symbol decoding */ - //return HUF_decompress4X4(dst, dstSize, cSrc, cSrcSize); /* multi-streams double-symbols decoding */ } size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 74e166add..524c2f56e 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -50,6 +50,16 @@ #endif +/*! +* STREAM_WINDOW_MAX : +* maximum window size accepted by DStream. +* frames requiring more memory will be rejected. +*/ +#ifndef ZSTD_STREAM_WINDOW_MAX +# define ZSTD_STREAM_WINDOW_MAX (257 << 20) /* 257 MB */ +#endif + + /*-******************************************************* * Dependencies *********************************************************/ @@ -138,7 +148,7 @@ struct ZSTD_DCtx_s BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; }; /* typedef'd to ZSTD_DCtx within "zstd_static.h" */ -size_t ZSTD_sizeofDCtx (const ZSTD_DCtx* dctx) { return sizeof(*dctx); } +size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx) { return sizeof(*dctx); } size_t ZSTD_estimateDCtxSize(void) { return sizeof(ZSTD_DCtx); } @@ -153,7 +163,8 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) dctx->hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); dctx->litEntropy = dctx->fseEntropy = 0; dctx->dictID = 0; - { int i; for (i=0; irep[i] = repStartValue[i]; } + MEM_STATIC_ASSERT(sizeof(dctx->rep)==sizeof(repStartValue)); + memcpy(dctx->rep, repStartValue, sizeof(repStartValue)); return 0; } @@ -161,15 +172,12 @@ ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) { ZSTD_DCtx* dctx; - if (!customMem.customAlloc && !customMem.customFree) - customMem = defaultCustomMem; - - if (!customMem.customAlloc || !customMem.customFree) - return NULL; + if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem; + if (!customMem.customAlloc || !customMem.customFree) return NULL; dctx = (ZSTD_DCtx*) customMem.customAlloc(customMem.opaque, sizeof(ZSTD_DCtx)); if (!dctx) return NULL; - memcpy(&dctx->customMem, &customMem, sizeof(ZSTD_customMem)); + memcpy(&dctx->customMem, &customMem, sizeof(customMem)); ZSTD_decompressBegin(dctx); return dctx; } @@ -188,8 +196,8 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx) void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) { - memcpy(dstDCtx, srcDCtx, - sizeof(ZSTD_DCtx) - (ZSTD_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH + ZSTD_frameHeaderSize_max)); /* no need to copy workspace */ + size_t const workSpaceSize = (ZSTD_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH) + ZSTD_frameHeaderSize_max; + memcpy(dstDCtx, srcDCtx, sizeof(ZSTD_DCtx) - workSpaceSize); /* no need to copy workspace */ } @@ -210,7 +218,7 @@ static size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize) U32 const singleSegment = (fhd >> 5) & 1; U32 const fcsId = fhd >> 6; return ZSTD_frameHeaderSize_min + !singleSegment + ZSTD_did_fieldSize[dictID] + ZSTD_fcs_fieldSize[fcsId] - + (singleSegment && !ZSTD_fcs_fieldSize[fcsId]); + + (singleSegment && !fcsId); } } @@ -592,9 +600,9 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState) { seq_t seq; - U32 const llCode = FSE_peekSymbol(&(seqState->stateLL)); - U32 const mlCode = FSE_peekSymbol(&(seqState->stateML)); - U32 const ofCode = FSE_peekSymbol(&(seqState->stateOffb)); /* <= maxOff, by table construction */ + U32 const llCode = FSE_peekSymbol(&seqState->stateLL); + U32 const mlCode = FSE_peekSymbol(&seqState->stateML); + U32 const ofCode = FSE_peekSymbol(&seqState->stateOffb); /* <= maxOff, by table construction */ U32 const llBits = LL_bits[llCode]; U32 const mlBits = ML_bits[mlCode]; @@ -623,8 +631,8 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState) if (!ofCode) offset = 0; else { - offset = OF_base[ofCode] + BIT_readBits(&(seqState->DStream), ofBits); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */ - if (MEM_32bits()) BIT_reloadDStream(&(seqState->DStream)); + offset = OF_base[ofCode] + BIT_readBits(&seqState->DStream, ofBits); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */ + if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); } if (ofCode <= 1) { @@ -645,18 +653,18 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState) seq.offset = offset; } - seq.matchLength = ML_base[mlCode] + ((mlCode>31) ? BIT_readBits(&(seqState->DStream), mlBits) : 0); /* <= 16 bits */ - if (MEM_32bits() && (mlBits+llBits>24)) BIT_reloadDStream(&(seqState->DStream)); + seq.matchLength = ML_base[mlCode] + ((mlCode>31) ? BIT_readBits(&seqState->DStream, mlBits) : 0); /* <= 16 bits */ + if (MEM_32bits() && (mlBits+llBits>24)) BIT_reloadDStream(&seqState->DStream); - seq.litLength = LL_base[llCode] + ((llCode>15) ? BIT_readBits(&(seqState->DStream), llBits) : 0); /* <= 16 bits */ + seq.litLength = LL_base[llCode] + ((llCode>15) ? BIT_readBits(&seqState->DStream, llBits) : 0); /* <= 16 bits */ if (MEM_32bits() || - (totalBits > 64 - 7 - (LLFSELog+MLFSELog+OffFSELog)) ) BIT_reloadDStream(&(seqState->DStream)); + (totalBits > 64 - 7 - (LLFSELog+MLFSELog+OffFSELog)) ) BIT_reloadDStream(&seqState->DStream); /* ANS state update */ - FSE_updateState(&(seqState->stateLL), &(seqState->DStream)); /* <= 9 bits */ - FSE_updateState(&(seqState->stateML), &(seqState->DStream)); /* <= 9 bits */ - if (MEM_32bits()) BIT_reloadDStream(&(seqState->DStream)); /* <= 18 bits */ - FSE_updateState(&(seqState->stateOffb), &(seqState->DStream)); /* <= 8 bits */ + FSE_updateState(&seqState->stateLL, &seqState->DStream); /* <= 9 bits */ + FSE_updateState(&seqState->stateML, &seqState->DStream); /* <= 9 bits */ + if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */ + FSE_updateState(&seqState->stateOffb, &seqState->DStream); /* <= 8 bits */ return seq; } @@ -680,7 +688,9 @@ size_t ZSTD_execSequence(BYTE* op, if (iLitEnd > litLimit_w) return ERROR(corruption_detected); /* over-read beyond lit buffer */ /* copy Literals */ - ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ + ZSTD_copy8(op, *litPtr); + if (sequence.litLength > 8) + ZSTD_wildcopy(op+8, (*litPtr)+8, sequence.litLength - 8); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ op = oLitEnd; *litPtr = iLitEnd; /* update for next sequence */ @@ -980,9 +990,10 @@ size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t sr } -/*-********************************** -* Streaming Decompression API -************************************/ +/*-************************************** +* Advanced Streaming Decompression API +* Bufferless and synchronous +****************************************/ size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx) { return dctx->expected; } ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx) { @@ -1382,6 +1393,11 @@ size_t ZSTD_initDStream(ZSTD_DStream* zds) return ZSTD_initDStream_usingDict(zds, NULL, 0); } +size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds) +{ + return sizeof(*zds) + ZSTD_sizeof_DCtx(zds->zd) + zds->inBuffSize + zds->outBuffSize; +} + /* *** Decompression *** */ @@ -1439,6 +1455,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB /* Frame header instruct buffer sizes */ { size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX); size_t const neededOutSize = zds->fParams.windowSize + blockSize; + if (zds->fParams.windowSize > ZSTD_STREAM_WINDOW_MAX) return ERROR(frameParameter_unsupported); zds->blockSize = blockSize; if (zds->inBuffSize < blockSize) { zds->customMem.customFree(zds->customMem.opaque, zds->inBuff); @@ -1531,9 +1548,3 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB return nextSrcSizeHint; } } - - - - - - diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index 3416a979a..b2fb7c08f 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -126,6 +126,13 @@ unsigned ZDICT_isError(size_t errorCode) { return ERR_isError(errorCode); } const char* ZDICT_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); } +unsigned ZDICT_getDictID(const void* dictBuffer, size_t dictSize) +{ + if (dictSize < 8) return 0; + if (MEM_readLE32(dictBuffer) != ZSTD_DICT_MAGIC) return 0; + return MEM_readLE32((const char*)dictBuffer + 4); +} + /*-******************************************************** * Dictionary training functions @@ -692,7 +699,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, esr.workPlace = malloc(ZSTD_BLOCKSIZE_ABSOLUTEMAX); if (!esr.ref || !esr.zc || !esr.workPlace) { eSize = ERROR(memory_allocation); - DISPLAYLEVEL(1, "Not enough memory"); + DISPLAYLEVEL(1, "Not enough memory \n"); goto _cleanup; } if (offcodeMax>OFFCODE_MAX) { eSize = ERROR(dictionary_wrong); goto _cleanup; } /* too large dictionary */ @@ -708,7 +715,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, { size_t const beginResult = ZSTD_compressBegin_advanced(esr.ref, dictBuffer, dictBufferSize, params, 0); if (ZSTD_isError(beginResult)) { eSize = ERROR(GENERIC); - DISPLAYLEVEL(1, "error : ZSTD_compressBegin_advanced failed "); + DISPLAYLEVEL(1, "error : ZSTD_compressBegin_advanced failed \n"); goto _cleanup; } } @@ -724,7 +731,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, errorCode = HUF_buildCTable (hufTable, countLit, 255, huffLog); if (HUF_isError(errorCode)) { eSize = ERROR(GENERIC); - DISPLAYLEVEL(1, "HUF_buildCTable error"); + DISPLAYLEVEL(1, "HUF_buildCTable error \n"); goto _cleanup; } huffLog = (U32)errorCode; @@ -740,7 +747,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, errorCode = FSE_normalizeCount(offcodeNCount, Offlog, offcodeCount, total, offcodeMax); if (FSE_isError(errorCode)) { eSize = ERROR(GENERIC); - DISPLAYLEVEL(1, "FSE_normalizeCount error with offcodeCount"); + DISPLAYLEVEL(1, "FSE_normalizeCount error with offcodeCount \n"); goto _cleanup; } Offlog = (U32)errorCode; @@ -749,7 +756,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, errorCode = FSE_normalizeCount(matchLengthNCount, mlLog, matchLengthCount, total, MaxML); if (FSE_isError(errorCode)) { eSize = ERROR(GENERIC); - DISPLAYLEVEL(1, "FSE_normalizeCount error with matchLengthCount"); + DISPLAYLEVEL(1, "FSE_normalizeCount error with matchLengthCount \n"); goto _cleanup; } mlLog = (U32)errorCode; @@ -758,7 +765,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, errorCode = FSE_normalizeCount(litLengthNCount, llLog, litLengthCount, total, MaxLL); if (FSE_isError(errorCode)) { eSize = ERROR(GENERIC); - DISPLAYLEVEL(1, "FSE_normalizeCount error with litLengthCount"); + DISPLAYLEVEL(1, "FSE_normalizeCount error with litLengthCount \n"); goto _cleanup; } llLog = (U32)errorCode; @@ -768,7 +775,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, { size_t const hhSize = HUF_writeCTable(dstPtr, maxDstSize, hufTable, 255, huffLog); if (HUF_isError(hhSize)) { eSize = ERROR(GENERIC); - DISPLAYLEVEL(1, "HUF_writeCTable error"); + DISPLAYLEVEL(1, "HUF_writeCTable error \n"); goto _cleanup; } dstPtr += hhSize; @@ -779,7 +786,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, { size_t const ohSize = FSE_writeNCount(dstPtr, maxDstSize, offcodeNCount, OFFCODE_MAX, Offlog); if (FSE_isError(ohSize)) { eSize = ERROR(GENERIC); - DISPLAYLEVEL(1, "FSE_writeNCount error with offcodeNCount"); + DISPLAYLEVEL(1, "FSE_writeNCount error with offcodeNCount \n"); goto _cleanup; } dstPtr += ohSize; @@ -790,7 +797,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, { size_t const mhSize = FSE_writeNCount(dstPtr, maxDstSize, matchLengthNCount, MaxML, mlLog); if (FSE_isError(mhSize)) { eSize = ERROR(GENERIC); - DISPLAYLEVEL(1, "FSE_writeNCount error with matchLengthNCount"); + DISPLAYLEVEL(1, "FSE_writeNCount error with matchLengthNCount \n"); goto _cleanup; } dstPtr += mhSize; @@ -801,7 +808,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, { size_t const lhSize = FSE_writeNCount(dstPtr, maxDstSize, litLengthNCount, MaxLL, llLog); if (FSE_isError(lhSize)) { eSize = ERROR(GENERIC); - DISPLAYLEVEL(1, "FSE_writeNCount error with litlengthNCount"); + DISPLAYLEVEL(1, "FSE_writeNCount error with litlengthNCount \n"); goto _cleanup; } dstPtr += lhSize; @@ -811,7 +818,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, if (maxDstSize<12) { eSize = ERROR(GENERIC); - DISPLAYLEVEL(1, "not enough space to write RepOffsets"); + DISPLAYLEVEL(1, "not enough space to write RepOffsets \n"); goto _cleanup; } # if 0 @@ -856,10 +863,14 @@ size_t ZDICT_addEntropyTablesFromBuffer_advanced(void* dictBuffer, size_t dictCo /* entropy tables */ DISPLAYLEVEL(2, "\r%70s\r", ""); /* clean display line */ DISPLAYLEVEL(2, "statistics ... \n"); - hSize += ZDICT_analyzeEntropy((char*)dictBuffer+hSize, dictBufferCapacity-hSize, + { size_t const eSize = ZDICT_analyzeEntropy((char*)dictBuffer+hSize, dictBufferCapacity-hSize, compressionLevel, samplesBuffer, samplesSizes, nbSamples, (char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize); + if (ZDICT_isError(eSize)) return eSize; + hSize += eSize; + } + if (hSize + dictContentSize < dictBufferCapacity) memmove((char*)dictBuffer + hSize, (char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize); @@ -902,7 +913,7 @@ size_t ZDICT_trainFromBuffer_unsafe( /* display best matches */ if (g_displayLevel>= 3) { - U32 const nb = 25; + U32 const nb = MIN(25, dictList[0].pos); U32 const dictContentSize = ZDICT_dictSize(dictList); U32 u; DISPLAYLEVEL(3, "\n %u segments found, of total size %u \n", dictList[0].pos, dictContentSize); diff --git a/lib/dictBuilder/zdict.h b/lib/dictBuilder/zdict.h index d61b5922c..68349da05 100644 --- a/lib/dictBuilder/zdict.h +++ b/lib/dictBuilder/zdict.h @@ -38,6 +38,19 @@ extern "C" { #endif + +/*====== Export for Windows ======*/ +/*! +* ZSTD_DLL_EXPORT : +* Enable exporting of functions when building a Windows DLL +*/ +#if defined(_WIN32) && defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZDICTLIB_API __declspec(dllexport) +#else +# define ZDICTLIB_API +#endif + + /*! ZDICT_trainFromBuffer() : Train a dictionary from an array of samples. Samples must be stored concatenated in a single flat buffer `samplesBuffer`, @@ -50,13 +63,14 @@ extern "C" { In general, it's recommended to provide a few thousands samples, but this can vary a lot. It's recommended that total size of all samples be about ~x100 times the target size of dictionary. */ -size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity, +ZDICTLIB_API size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity, const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples); /*====== Helper functions ======*/ -unsigned ZDICT_isError(size_t errorCode); -const char* ZDICT_getErrorName(size_t errorCode); +ZDICTLIB_API unsigned ZDICT_getDictID(const void* dictBuffer, size_t dictSize); /**< extracts dictID; @return zero if error (not a valid dictionary) */ +ZDICTLIB_API unsigned ZDICT_isError(size_t errorCode); +ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode); diff --git a/lib/zstd.h b/lib/zstd.h index 056940ff1..cb667484b 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -55,7 +55,7 @@ extern "C" { /*======= Version =======*/ #define ZSTD_VERSION_MAJOR 0 #define ZSTD_VERSION_MINOR 8 -#define ZSTD_VERSION_RELEASE 1 +#define ZSTD_VERSION_RELEASE 2 #define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE #define ZSTD_QUOTE(str) #str @@ -92,7 +92,7 @@ ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity, * Always ensure result fits within application's authorized limits ! * Each application can set its own limits. * note 4 : when `return==0`, if precise failure cause is needed, use ZSTD_getFrameParams() to know more. */ -unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); +ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); /*! ZSTD_decompress() : `compressedSize` : must be the _exact_ size of compressed input, otherwise decompression will fail. @@ -191,6 +191,107 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); +/*-************************** +* Streaming +****************************/ + +typedef struct ZSTD_inBuffer_s { + const void* src; /**< start of input buffer */ + size_t size; /**< size of input buffer */ + size_t pos; /**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */ +} ZSTD_inBuffer; + +typedef struct ZSTD_outBuffer_s { + void* dst; /**< start of output buffer */ + size_t size; /**< size of output buffer */ + size_t pos; /**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */ +} ZSTD_outBuffer; + + +/*====== compression ======*/ + +/*-*********************************************************************** +* Streaming compression - howto +* +* A ZSTD_CStream object is required to track streaming operation. +* Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources. +* ZSTD_CStream objects can be reused multiple times on consecutive compression operations. +* +* Start by initializing ZSTD_CStream. +* Use ZSTD_initCStream() to start a new compression operation. +* Use ZSTD_initCStream_usingDict() for a compression which requires a dictionary. +* +* Use ZSTD_compressStream() repetitively to consume input stream. +* The function will automatically update both `pos`. +* Note that it may not consume the entire input, in which case `pos < size`, +* and it's up to the caller to present again remaining data. +* @return : a hint to preferred nb of bytes to use as input for next function call (it's just a hint, to improve latency) +* or an error code, which can be tested using ZSTD_isError(). +* +* At any moment, it's possible to flush whatever data remains within buffer, using ZSTD_flushStream(). +* `output->pos` will be updated. +* Note some content might still be left within internal buffer if `output->size` is too small. +* @return : nb of bytes still present within internal buffer (0 if it's empty) +* or an error code, which can be tested using ZSTD_isError(). +* +* ZSTD_endStream() instructs to finish a frame. +* It will perform a flush and write frame epilogue. +* The epilogue is required for decoders to consider a frame completed. +* Similar to ZSTD_flushStream(), it may not be able to flush the full content if `output->size` is too small. +* In which case, call again ZSTD_endStream() to complete the flush. +* @return : nb of bytes still present within internal buffer (0 if it's empty) +* or an error code, which can be tested using ZSTD_isError(). +* +* *******************************************************************/ + +typedef struct ZSTD_CStream_s ZSTD_CStream; +ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void); +ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs); + +ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */ +ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer */ + +ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); +ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input); +ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); +ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); + + +/*====== decompression ======*/ + +/*-*************************************************************************** +* Streaming decompression howto +* +* A ZSTD_DStream object is required to track streaming operations. +* Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources. +* ZSTD_DStream objects can be re-init multiple times. +* +* Use ZSTD_initDStream() to start a new decompression operation, +* or ZSTD_initDStream_usingDict() if decompression requires a dictionary. +* +* Use ZSTD_decompressStream() repetitively to consume your input. +* The function will update both `pos`. +* Note that it may not consume the entire input (pos < size), +* in which case it's up to the caller to present remaining input again. +* @return : 0 when a frame is completely decoded and fully flushed, +* 1 when there is still some data left within internal buffer to flush, +* >1 when more data is expected, with value being a suggested next input size (it's just a hint, which helps latency, any size is accepted), +* or an error code, which can be tested using ZSTD_isError(). +* +* *******************************************************************************/ + +typedef struct ZSTD_DStream_s ZSTD_DStream; +ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void); +ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds); + +ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */ +ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer */ + +ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds); +ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input); + + + #ifdef ZSTD_STATIC_LINKING_ONLY /* ==================================================================================== @@ -275,12 +376,12 @@ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictS /*! ZSTD_sizeofCCtx() : * Gives the amount of memory used by a given ZSTD_CCtx */ -ZSTDLIB_API size_t ZSTD_sizeofCCtx(const ZSTD_CCtx* cctx); +ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx); /*! ZSTD_getParams() : * same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of a `ZSTD_compressionParameters`. * All fields of `ZSTD_frameParameters` are set to default (0) */ -ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSize, size_t dictSize); +ZSTDLIB_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSize, size_t dictSize); /*! ZSTD_getCParams() : * @return ZSTD_compressionParameters structure for a selected compression level and srcSize. @@ -317,118 +418,28 @@ ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); /*! ZSTD_sizeofDCtx() : * Gives the amount of memory used by a given ZSTD_DCtx */ -ZSTDLIB_API size_t ZSTD_sizeofDCtx(const ZSTD_DCtx* dctx); +ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx); /* ****************************************************************** -* Streaming +* Advanced Streaming ********************************************************************/ -typedef struct ZSTD_inBuffer_s { - const void* src; /**< start of input buffer */ - size_t size; /**< size of input buffer */ - size_t pos; /**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */ -} ZSTD_inBuffer; - -typedef struct ZSTD_outBuffer_s { - void* dst; /**< start of output buffer */ - size_t size; /**< size of output buffer */ - size_t pos; /**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */ -} ZSTD_outBuffer; - - /*====== compression ======*/ -/*-*********************************************************************** -* Streaming compression - howto -* -* A ZSTD_CStream object is required to track streaming operation. -* Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources. -* ZSTD_CStream objects can be reused multiple times on consecutive compression operations. -* -* Start by initializing ZSTD_CStream. -* Use ZSTD_initCStream() to start a new compression operation. -* Use ZSTD_initCStream_usingDict() for a compression which requires a dictionary. -* -* Use ZSTD_compressStream() repetitively to consume input stream. -* The function will automatically update both `pos`. -* Note that it may not consume the entire input, in which case `pos < size`, -* and it's up to the caller to present again remaining data. -* @return : a hint to preferred nb of bytes to use as input for next function call (it's just a hint, to improve latency) -* or an error code, which can be tested using ZSTD_isError(). -* -* At any moment, it's possible to flush whatever data remains within buffer, using ZSTD_flushStream(). -* `output->pos` will be updated. -* Note some content might still be left within internal buffer if `output->size` is too small. -* @return : nb of bytes still present within internal buffer (0 if it's empty) -* or an error code, which can be tested using ZSTD_isError(). -* -* ZSTD_endStream() instructs to finish a frame. -* It will perform a flush and write frame epilogue. -* The epilogue is required for decoders to consider a frame completed. -* Similar to ZSTD_flushStream(), it may not be able to flush the full content if `output->size` is too small. -* In which case, call again ZSTD_endStream() to complete the flush. -* @return : nb of bytes still present within internal buffer (0 if it's empty) -* or an error code, which can be tested using ZSTD_isError(). -* -* *******************************************************************/ - -typedef struct ZSTD_CStream_s ZSTD_CStream; -ZSTD_CStream* ZSTD_createCStream(void); -size_t ZSTD_freeCStream(ZSTD_CStream* zcs); - -size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */ -size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer */ - -size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); -size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input); -size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); -size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); - -/* advanced */ -ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); -size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); -size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, +ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); +ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); +ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); +ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); /*====== decompression ======*/ -/*-*************************************************************************** -* Streaming decompression howto -* -* A ZSTD_DStream object is required to track streaming operations. -* Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources. -* ZSTD_DStream objects can be re-init multiple times. -* -* Use ZSTD_initDStream() to start a new decompression operation, -* or ZSTD_initDStream_usingDict() if decompression requires a dictionary. -* -* Use ZSTD_decompressStream() repetitively to consume your input. -* The function will update both `pos`. -* Note that it may not consume the entire input (pos < size), -* in which case it's up to the caller to present remaining input again. -* @return : 0 when a frame is completely decoded and fully flushed, -* 1 when there is still some data left within internal buffer to flush, -* >1 when more data is expected, with value being a suggested next input size (it's just a hint, which helps latency, any size is accepted), -* or an error code, which can be tested using ZSTD_isError(). -* -* *******************************************************************************/ - -typedef struct ZSTD_DStream_s ZSTD_DStream; -ZSTD_DStream* ZSTD_createDStream(void); -size_t ZSTD_freeDStream(ZSTD_DStream* zds); - -size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */ -size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer */ - -size_t ZSTD_initDStream(ZSTD_DStream* zds); -size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input); - /* advanced */ -ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); -size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); - +ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); +ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); +ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds); /* ****************************************************************** diff --git a/programs/bench.c b/programs/bench.c index 12c59bff6..dc36d170f 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -257,7 +257,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, (void)fastestD; (void)crcOrig; /* unused when decompression disabled */ #if 1 /* Decompression */ - memset(resultBuffer, 0xD6, srcSize); /* warm result buffer */ + if (!dCompleted) memset(resultBuffer, 0xD6, srcSize); /* warm result buffer */ UTIL_sleepMilli(1); /* give processor time to other processes */ UTIL_waitForNextTick(ticksPerSecond); diff --git a/programs/dibio.c b/programs/dibio.c index cb864ec1d..19188d76f 100644 --- a/programs/dibio.c +++ b/programs/dibio.c @@ -204,11 +204,11 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize, if ((!fileSizes) || (!srcBuffer) || (!dictBuffer)) EXM_THROW(12, "not enough memory for DiB_trainFiles"); /* should not happen */ g_displayLevel = params.notificationLevel; if (nbFiles < 5) { - DISPLAYLEVEL(2, "! Warning : nb of samples too low for proper processing \n"); - DISPLAYLEVEL(2, "! Please provide one file per sample \n"); - DISPLAYLEVEL(2, "! Avoid concatenating multiple samples into a single file \n"); - DISPLAYLEVEL(2, "! otherwise, dictBuilder will be unable to find the beginning of each sample \n"); - DISPLAYLEVEL(2, "! resulting in distorted statistics \n"); + DISPLAYLEVEL(2, "! Warning : nb of samples too low for proper processing ! \n"); + DISPLAYLEVEL(2, "! Please provide _one file per sample_. \n"); + DISPLAYLEVEL(2, "! Do not concatenate samples together into a single file, \n"); + DISPLAYLEVEL(2, "! as dictBuilder will be unable to find the beginning of each sample, \n"); + DISPLAYLEVEL(2, "! resulting in poor dictionary quality. \n"); } /* init */ diff --git a/programs/fileio.c b/programs/fileio.c index 397147739..c10c28968 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -364,9 +364,8 @@ static int FIO_compressFilename_internal(cRess_t ress, } /* Status */ - if (strlen(srcFileName) > 20) srcFileName += strlen(srcFileName)-20; /* display last 20 characters */ DISPLAYLEVEL(2, "\r%79s\r", ""); - DISPLAYLEVEL(2,"%-20.20s :%6.2f%% (%6llu => %6llu bytes, %s) \n", srcFileName, + DISPLAYLEVEL(2,"%-20s :%6.2f%% (%6llu => %6llu bytes, %s) \n", srcFileName, (double)compressedfilesize/(readsize+(!readsize) /* avoid div by zero */ )*100, (unsigned long long)readsize, (unsigned long long) compressedfilesize, dstFileName); diff --git a/tests/fuzzer.c b/tests/fuzzer.c index be36d74bb..bed86f592 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -100,7 +100,6 @@ static unsigned FUZ_rand(unsigned* src) return rand32 >> 5; } - static unsigned FUZ_highbit32(U32 v32) { unsigned nbBits = 0; @@ -110,6 +109,10 @@ static unsigned FUZ_highbit32(U32 v32) } +/*============================================= +* Basic Unit tests +=============================================*/ + #define CHECK_V(var, fn) size_t const var = fn; if (ZSTD_isError(var)) goto _output_error #define CHECK(fn) { CHECK_V(err, fn); } #define CHECKPLUS(var, fn, more) { CHECK_V(var, fn); more; } @@ -272,6 +275,12 @@ static int basicUnitTests(U32 seed, double compressibility) if (ZDICT_isError(dictSize)) goto _output_error; DISPLAYLEVEL(4, "OK, created dictionary of size %u \n", (U32)dictSize); + DISPLAYLEVEL(4, "test%3i : check dictID : ", testNb++); + { U32 const dictID = ZDICT_getDictID(dictBuffer, dictSize); + if (dictID==0) goto _output_error; + DISPLAYLEVEL(4, "OK : %u \n", dictID); + } + DISPLAYLEVEL(4, "test%3i : compress with dictionary : ", testNb++); cSize = ZSTD_compress_usingDict(cctx, compressedBuffer, ZSTD_compressBound(CNBuffSize), CNBuffer, CNBuffSize, diff --git a/tests/playTests.sh b/tests/playTests.sh index 013ba6f64..75b334ff4 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -32,7 +32,7 @@ case "$OS" in esac MD5SUM="md5sum" -if [ "$TRAVIS_OS_NAME" = "osx" ]; then +if [[ "$OSTYPE" == "darwin"* ]]; then MD5SUM="md5 -r" fi diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index 7f136b1b5..4c81e8df6 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -61,7 +61,6 @@ static const U32 prime1 = 2654435761U; static const U32 prime2 = 2246822519U; - /*-************************************ * Display Macros **************************************/ @@ -93,7 +92,6 @@ static U32 FUZ_GetMilliStart(void) return nCount; } - static U32 FUZ_GetMilliSpan(U32 nTimeStart) { U32 const nCurrent = FUZ_GetMilliStart(); @@ -117,7 +115,6 @@ unsigned int FUZ_rand(unsigned int* seedPtr) return rand32 >> 5; } - /* static unsigned FUZ_highbit32(U32 v32) { @@ -141,6 +138,11 @@ static void freeFunction(void* opaque, void* address) free(address); } + +/*====================================================== +* Basic Unit tests +======================================================*/ + static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem customMem) { int testResult = 0; @@ -187,6 +189,12 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo cSize += outBuff.pos; DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100); + DISPLAYLEVEL(4, "test%3i : check CStream size : ", testNb++); + { size_t const s = ZSTD_sizeof_CStream(zc); + if (ZSTD_isError(s)) goto _output_error; + DISPLAYLEVEL(4, "OK (%u bytes) \n", (U32)s); + } + /* skippable frame test */ DISPLAYLEVEL(4, "test%3i : decompress skippable frame : ", testNb++); ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB); @@ -218,6 +226,12 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo } } DISPLAYLEVEL(4, "OK \n"); + DISPLAYLEVEL(4, "test%3i : check DStream size : ", testNb++); + { size_t const s = ZSTD_sizeof_DStream(zd); + if (ZSTD_isError(s)) goto _output_error; + DISPLAYLEVEL(4, "OK (%u bytes) \n", (U32)s); + } + /* Byte-by-byte decompression test */ DISPLAYLEVEL(4, "test%3i : decompress byte-by-byte : ", testNb++); { size_t r = 1; diff --git a/zstd.rb b/zstd.rb index 1817c2ebe..62b1e0b8f 100644 --- a/zstd.rb +++ b/zstd.rb @@ -1,8 +1,8 @@ class Zstd < Formula desc "Zstandard - Fast real-time compression algorithm" homepage "http://www.zstd.net/" - url "https://github.com/Cyan4973/zstd/archive/v0.7.5.tar.gz" - sha256 "6800defac9a93ddb1d9673d62c78526800df71cf9f353456f8e488ba4de51061" + url "https://github.com/Cyan4973/zstd/archive/v0.8.1.tar.gz" + sha256 "4632bee45988dd0fe3edf1e67bdf0a833895cbb1a7d1eb23ef0b7d753f8bffdd" def install system "make", "install", "PREFIX=#{prefix}" diff --git a/zstd_compression_format.md b/zstd_compression_format.md index 0989912dd..61f6a57bf 100644 --- a/zstd_compression_format.md +++ b/zstd_compression_format.md @@ -443,7 +443,7 @@ using little-endian convention. In this representation, bits on the left are smallest bits. -__`Literals_Block_Type`__ +__`Literals_Block_Type`__ This field uses 2 lowest bits of first byte, describing 4 different block types : @@ -460,7 +460,7 @@ This field uses 2 lowest bits of first byte, describing 4 different block types using Huffman tree _from previous Huffman-compressed literals block_. Huffman tree description will be skipped. -__`Size_Format`__ +__`Size_Format`__ `Size_Format` is divided into 2 families : @@ -697,8 +697,13 @@ A match copy command specifies an offset and a length. The offset gives the position to copy from, which can be within a previous block. -There are 3 symbol types, literals lengths, offsets and match lengths, -which are encoded together, interleaved in a single _bitstream_. +When all _sequences_ are decoded, +if there is any literal left in the _literal section_, +these bytes are added at the end of the block. + +The _Sequences_Section_ regroup all symbols required to decode commands. +There are 3 symbol types : literals lengths, offsets and match lengths. +They are encoded together, interleaved, in a single _bitstream_. Each symbol is a _code_ in its own context, which specifies a baseline and a number of bits to add. @@ -905,8 +910,8 @@ since it will be discovered and reported by the decoding process. The bitstream starts by reporting on which scale it operates. `Accuracy_Log = low4bits + 5`. -Note that maximum `Accuracy_Log` for literal and match length is `9`, -and for offsets it is `8`. Higher values are considered errors. +Note that maximum `Accuracy_Log` for literal and match lengths is `9`, +and for offsets is `8`. Higher values are considered errors. Then follow each symbol value, from `0` to last present one. The number of bits used by each field is variable. @@ -1128,8 +1133,8 @@ _Reserved ranges :_ However, for public distribution of compressed frames, the following ranges are reserved for future use and should not be used : - - low range : 1 - 32767 - - high range : >= (2^31) + - low range : 1 - 32767 + - high range : >= (2^31) __`Entropy_Tables`__ : following the same format as a [compressed blocks]. They are stored in following order :