diff --git a/.gitignore b/.gitignore index 7e142fcd1..6ca7f8a7e 100644 --- a/.gitignore +++ b/.gitignore @@ -47,7 +47,3 @@ ipch/ .directory _codelite _zstdbench - -lib/zstd_opt_LZ5.c -lib/zstd_opt_llen.c -lib/zstd_opt_nollen.c diff --git a/lib/.debug/zstd_stats.h b/lib/.debug/zstd_stats.h new file mode 100644 index 000000000..d0189f872 --- /dev/null +++ b/lib/.debug/zstd_stats.h @@ -0,0 +1,164 @@ +/* + zstd - standard compression library + Header File for static linking only + Copyright (C) 2014-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - zstd homepage : http://www.zstd.net +*/ +#ifndef ZSTD_STATS_H +#define ZSTD_STATS_H + + +#if defined (__cplusplus) +extern "C" { +#endif + +/*-************************************* +* Dependencies +***************************************/ +//#include "zstd.h" +//#include "mem.h" + + +/*-************************************* +* Constants +***************************************/ +//#define ZSTD_MAGICNUMBER 0xFD2FB526 /* v0.6 */ + + +/*-************************************* +* Types +***************************************/ +typedef struct { + U32 priceOffset, priceOffCode, priceMatchLength, priceLiteral, priceLitLength, priceDumpsLength; + U32 totalMatchSum, totalLitSum, totalSeqSum, totalRepSum; + U32 litSum, matchLengthSum, litLengthSum, offCodeSum; + U32 matchLengthFreq[1<totalMatchSum += stats->totalSeqSum * ((searchLength == 3) ? 3 : 4); + printf("avgMatchL=%.2f avgLitL=%.2f match=%.1f%% lit=%.1f%% reps=%d seq=%d\n", (float)stats->totalMatchSum/stats->totalSeqSum, (float)stats->totalLitSum/stats->totalSeqSum, 100.0*stats->totalMatchSum/(stats->totalMatchSum+stats->totalLitSum), 100.0*stats->totalLitSum/(stats->totalMatchSum+stats->totalLitSum), stats->totalRepSum, stats->totalSeqSum); + printf("SumBytes=%d Offset=%d OffCode=%d Match=%d Literal=%d LitLength=%d DumpsLength=%d\n", (stats->priceOffset+stats->priceOffCode+stats->priceMatchLength+stats->priceLiteral+stats->priceLitLength+stats->priceDumpsLength)/8, stats->priceOffset/8, stats->priceOffCode/8, stats->priceMatchLength/8, stats->priceLiteral/8, stats->priceLitLength/8, stats->priceDumpsLength/8); +} + +MEM_STATIC void ZSTD_statsInit(ZSTD_stats_t* stats) +{ + stats->totalLitSum = stats->totalMatchSum = stats->totalSeqSum = stats->totalRepSum = 1; + stats->priceOffset = stats->priceOffCode = stats->priceMatchLength = stats->priceLiteral = stats->priceLitLength = stats->priceDumpsLength = 0; +} + +MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats) +{ + unsigned u; + + stats->litSum = (1<litLengthSum = (1<matchLengthSum = (1<offCodeSum = (1<litFreq[u] = 1; + for (u=0; u<=MaxLL; u++) + stats->litLengthFreq[u] = 1; + for (u=0; u<=MaxML; u++) + stats->matchLengthFreq[u] = 1; + for (u=0; u<=MaxOff; u++) + stats->offCodeFreq[u] = 1; +} + +MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) +{ + /* offset */ + BYTE offCode = offset ? (BYTE)ZSTD_highbit(offset+1) + 1 : 0; + stats->priceOffCode += ZSTD_highbit(stats->offCodeSum+1) - ZSTD_highbit(stats->offCodeFreq[offCode]+1); + stats->priceOffset += (offCode-1) + (!offCode); + + /* match Length */ + stats->priceDumpsLength += ((matchLength >= MaxML)<<3) + ((matchLength >= 255+MaxML)<<4) + ((matchLength>=(1<<15))<<3); + stats->priceMatchLength += ZSTD_highbit(stats->matchLengthSum+1) - ZSTD_highbit(stats->matchLengthFreq[(matchLength >= MaxML) ? MaxML : matchLength]+1); + + if (litLength) { + /* literals */ + U32 u; + stats->priceLiteral += litLength * ZSTD_highbit(stats->litSum+1); + for (u=0; u < litLength; u++) + stats->priceLiteral -= ZSTD_highbit(stats->litFreq[literals[u]]+1); + + /* literal Length */ + stats->priceDumpsLength += ((litLength >= MaxLL)<<3) + ((litLength >= 255+MaxLL)<<4) + ((litLength>=(1<<15))<<3); + stats->priceLitLength += ZSTD_highbit(stats->litLengthSum+1) - ZSTD_highbit(stats->litLengthFreq[(litLength >= MaxLL) ? MaxLL : litLength]+1); + } else { + stats->priceLitLength += ZSTD_highbit(stats->litLengthSum+1) - ZSTD_highbit(stats->litLengthFreq[0]+1); + } + + + if (offset == 0) stats->totalRepSum++; + stats->totalSeqSum++; + stats->totalMatchSum += matchLength; + stats->totalLitSum += litLength; + + U32 u; + /* literals */ + stats->litSum += litLength; + for (u=0; u < litLength; u++) + stats->litFreq[literals[u]]++; + + /* literal Length */ + stats->litLengthSum++; + if (litLength >= MaxLL) + stats->litLengthFreq[MaxLL]++; + else + stats->litLengthFreq[litLength]++; + + /* match offset */ + stats->offCodeSum++; + stats->offCodeFreq[offCode]++; + + /* match Length */ + stats->matchLengthSum++; + if (matchLength >= MaxML) + stats->matchLengthFreq[MaxML]++; + else + stats->matchLengthFreq[matchLength]++; +} + + + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_STATIC_H */ diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index fcb6e36bc..3d16d087e 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -95,6 +95,7 @@ struct ZSTD_CCtx_s U32 lowLimit; /* below that point, no more data */ U32 nextToUpdate; /* index from which to continue dictionary update */ U32 nextToUpdate3; /* index from which to continue dictionary update */ + U32 hashLog3; /* dispatch table : larger == faster, more memory */ U32 loadedDictEnd; U32 stage; ZSTD_parameters params; @@ -219,7 +220,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, const size_t tokenSpace = blockSize + 11*maxNbSeq; const size_t chainSize = (params.cParams.strategy == ZSTD_fast) ? 0 : (1 << params.cParams.chainLog); const size_t hSize = 1 << params.cParams.hashLog; - const size_t h3Size = (params.cParams.searchLength==3) ? (1 << HASHLOG3) : 0; + const size_t h3Size = (zc->hashLog3) ? 1 << zc->hashLog3 : 0; const size_t tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); /* Check if workSpace is large enough, alloc a new one if needed */ @@ -286,13 +287,14 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx) { if (srcCCtx->stage!=0) return ERROR(stage_wrong); + dstCCtx->hashLog3 = srcCCtx->hashLog3; /* must be before ZSTD_resetCCtx_advanced */ ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params); /* copy tables */ - { const size_t chaineSize = (srcCCtx->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << srcCCtx->params.cParams.chainLog); + { const size_t chainSize = (srcCCtx->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << srcCCtx->params.cParams.chainLog); const size_t hSize = 1 << srcCCtx->params.cParams.hashLog; - const size_t h3Size = (srcCCtx->params.cParams.searchLength == 3) ? (1 << HASHLOG3) : 0; - const size_t tableSpace = (chaineSize + hSize + h3Size) * sizeof(U32); + const size_t h3Size = (srcCCtx->hashLog3) ? 1 << srcCCtx->hashLog3 : 0; + const size_t tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); memcpy(dstCCtx->workSpace, srcCCtx->workSpace, tableSpace); } @@ -344,7 +346,7 @@ static void ZSTD_reduceIndex (ZSTD_CCtx* zc, const U32 reducerValue) { const U32 chainSize = (zc->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << zc->params.cParams.chainLog); ZSTD_reduceTable(zc->chainTable, chainSize, reducerValue); } - { const U32 h3Size = (zc->params.cParams.searchLength == 3) ? (1 << HASHLOG3) : 0; + { const U32 h3Size = (zc->hashLog3) ? 1 << zc->hashLog3 : 0; ZSTD_reduceTable(zc->hashTable3, h3Size, reducerValue); } } @@ -855,12 +857,7 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const B printf("Cpos %6u :%5u literals & match %3u bytes at distance %6u \n", pos, (U32)litLength, (U32)matchCode+MINMATCH, (U32)offsetCode); #endif -#if ZSTD_OPT_DEBUG == 3 - if (offsetCode == 0) seqStorePtr->realRepSum++; - seqStorePtr->realSeqSum++; - seqStorePtr->realMatchSum += matchCode; - seqStorePtr->realLitSum += litLength; -#endif + ZSTD_statsUpdatePrices(&seqStorePtr->stats, litLength, literals, offsetCode, matchCode); /* copy Literals */ ZSTD_wildcopy(seqStorePtr->lit, literals, litLength); @@ -1746,6 +1743,7 @@ _storeSequence: { size_t const lastLLSize = iend - anchor; memcpy(seqStorePtr->lit, anchor, lastLLSize); seqStorePtr->lit += lastLLSize; + ZSTD_statsUpdatePrices(&seqStorePtr->stats, lastLLSize, anchor, 0, 0); } } @@ -1992,6 +1990,8 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCa } + + static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, void* dst, size_t dstCapacity, const void* src, size_t srcSize) @@ -2002,15 +2002,13 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, BYTE* const ostart = (BYTE*)dst; BYTE* op = ostart; const U32 maxDist = 1 << zc->params.cParams.windowLog; -#if ZSTD_OPT_DEBUG == 3 - seqStore_t* ssPtr = &zc->seqStore; - static U32 priceFunc = 0; - ssPtr->realMatchSum = ssPtr->realLitSum = ssPtr->realSeqSum = ssPtr->realRepSum = 1; - ssPtr->priceFunc = priceFunc; -#endif + ZSTD_stats_t* stats = &zc->seqStore.stats; + + ZSTD_statsInit(stats); while (remaining) { size_t cSize; + ZSTD_statsResetFreqs(stats); if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */ if (remaining < blockSize) blockSize = remaining; @@ -2042,12 +2040,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc, op += cSize; } -#if ZSTD_OPT_DEBUG == 3 - ssPtr->realMatchSum += ssPtr->realSeqSum * ((zc->params.searchLength == 3) ? 3 : 4); - printf("avgMatchL=%.2f avgLitL=%.2f match=%.1f%% lit=%.1f%% reps=%d seq=%d priceFunc=%d\n", (float)ssPtr->realMatchSum/ssPtr->realSeqSum, (float)ssPtr->realLitSum/ssPtr->realSeqSum, 100.0*ssPtr->realMatchSum/(ssPtr->realMatchSum+ssPtr->realLitSum), 100.0*ssPtr->realLitSum/(ssPtr->realMatchSum+ssPtr->realLitSum), ssPtr->realRepSum, ssPtr->realSeqSum, ssPtr->priceFunc); - priceFunc++; -#endif - + ZSTD_statsPrint(stats, zc->params.cParams.searchLength); return op-ostart; } @@ -2233,13 +2226,16 @@ static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* dict, si } } - /*! ZSTD_compressBegin_internal() : * @return : 0, or an error code */ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* zc, const void* dict, size_t dictSize, ZSTD_parameters params, U64 pledgedSrcSize) { + U32 hashLog3 = (pledgedSrcSize || pledgedSrcSize >= 8192) ? ZSTD_HASHLOG3_MAX : ((pledgedSrcSize >= 2048) ? ZSTD_HASHLOG3_MIN + 1 : ZSTD_HASHLOG3_MIN); + zc->hashLog3 = (params.cParams.searchLength==3) ? hashLog3 : 0; +// printf("windowLog=%d hashLog=%d hashLog3=%d \n", params.windowLog, params.hashLog, zc->hashLog3); + { size_t const errorCode = ZSTD_resetCCtx_advanced(zc, params); if (ZSTD_isError(errorCode)) return errorCode; } @@ -2291,6 +2287,18 @@ size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* zc, const void* dict, size_t dict return ZSTD_compressBegin_internal(zc, dict, dictSize, params, 0); } + +size_t ZSTD_compressBegin_targetSrcSize(ZSTD_CCtx* zc, const void* dict, size_t dictSize, size_t targetSrcSize, int compressionLevel) +{ + ZSTD_parameters params; + params.cParams = ZSTD_getCParams(compressionLevel, targetSrcSize, dictSize); + params.fParams.contentSizeFlag = 1; + ZSTD_adjustCParams(¶ms.cParams, targetSrcSize, dictSize); + ZSTD_LOG_BLOCK("%p: ZSTD_compressBegin_targetSrcSize compressionLevel=%d\n", zc->base, compressionLevel); + return ZSTD_compressBegin_internal(zc, dict, dictSize, params, targetSrcSize); +} + + size_t ZSTD_compressBegin(ZSTD_CCtx* zc, int compressionLevel) { ZSTD_LOG_BLOCK("%p: ZSTD_compressBegin compressionLevel=%d\n", zc->base, compressionLevel); @@ -2530,9 +2538,6 @@ ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, U64 srcSize, si U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB); /* intentional underflow for srcSizeHint == 0 */ if (compressionLevel<=0) compressionLevel = 1; if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL; -#if ZSTD_OPT_DEBUG >= 1 - tableID=0; -#endif cp = ZSTD_defaultCParameters[tableID][compressionLevel]; if (MEM_32bits()) { /* auto-correction, for 32-bits mode */ if (cp.windowLog > ZSTD_WINDOWLOG_MAX) cp.windowLog = ZSTD_WINDOWLOG_MAX; diff --git a/lib/zstd_decompress.c b/lib/zstd_decompress.c index fd9b40233..436058b52 100644 --- a/lib/zstd_decompress.c +++ b/lib/zstd_decompress.c @@ -149,7 +149,7 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) dctx->hufTableX4[0] = HufLog; dctx->flagStaticTables = 0; dctx->fParams.mml = MINMATCH; /* overwritten by frame but forces ZSTD_btopt to MINMATCH in block mode */ - ZSTD_LOG_BLOCK("%p: ZSTD_decompressBegin searchLength=%d\n", dctx->base, dctx->params.searchLength); + ZSTD_LOG_BLOCK("%p: ZSTD_decompressBegin searchLength=%d\n", dctx->base, dctx->fParams.mml); return 0; } @@ -843,7 +843,7 @@ static size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, if (srcSize >= ZSTD_BLOCKSIZE_MAX) return ERROR(srcSize_wrong); - ZSTD_LOG_BLOCK("%p: ZSTD_decompressBlock_internal searchLength=%d\n", dctx->base, dctx->params.searchLength); + ZSTD_LOG_BLOCK("%p: ZSTD_decompressBlock_internal searchLength=%d\n", dctx->base, dctx->fParams.mml); /* Decode literals sub-block */ { size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize); @@ -951,7 +951,7 @@ size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) { ZSTD_decompressBegin_usingDict(dctx, dict, dictSize); - ZSTD_LOG_BLOCK("%p: ZSTD_decompressBegin_usingDict searchLength=%d\n", dctx->base, dctx->params.searchLength); + ZSTD_LOG_BLOCK("%p: ZSTD_decompressBegin_usingDict searchLength=%d\n", dctx->base, dctx->fParams.mml); ZSTD_checkContinuity(dctx, dst); return ZSTD_decompressFrame(dctx, dst, dstCapacity, src, srcSize); } diff --git a/lib/zstd_internal.h b/lib/zstd_internal.h index ff2713408..ce36aad3b 100644 --- a/lib/zstd_internal.h +++ b/lib/zstd_internal.h @@ -50,10 +50,8 @@ /*-************************************* * Common constants ***************************************/ -#define ZSTD_OPT_DEBUG 0 // 1 = tableID=0; 3 = price func tests; 5 = check encoded sequences; 9 = full logs -#if defined(ZSTD_OPT_DEBUG) && ZSTD_OPT_DEBUG>0 - #include -#endif +#define ZSTD_OPT_DEBUG 0 // 3 = compression stats; 5 = check encoded sequences; 9 = full logs +#include #if defined(ZSTD_OPT_DEBUG) && ZSTD_OPT_DEBUG>=9 #define ZSTD_LOG_PARSER(...) printf(__VA_ARGS__) #define ZSTD_LOG_ENCODE(...) printf(__VA_ARGS__) @@ -99,7 +97,6 @@ typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t; #define MINMATCH 4 #define REPCODE_STARTVALUE 1 -#define HASHLOG3 17 #define Litbits 8 #define Offbits 5 @@ -195,6 +192,16 @@ typedef struct { U32 rep2; } ZSTD_optimal_t; +#if ZSTD_OPT_DEBUG == 3 + #include ".debug/zstd_stats.h" +#else + typedef struct { U32 unused; } ZSTD_stats_t; + MEM_STATIC void ZSTD_statsPrint(ZSTD_stats_t* stats, U32 searchLength) { (void)stats; (void)searchLength; }; + MEM_STATIC void ZSTD_statsInit(ZSTD_stats_t* stats) { (void)stats; }; + MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats) { (void)stats; }; + MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) { (void)stats; (void)litLength; (void)literals; (void)offset; (void)matchLength; }; +#endif + typedef struct { void* buffer; U32* offsetStart; @@ -227,17 +234,12 @@ typedef struct { U32 log2litSum; U32 log2offCodeSum; U32 factor; -#if ZSTD_OPT_DEBUG == 3 - U32 realMatchSum; - U32 realLitSum; - U32 realSeqSum; - U32 realRepSum; - U32 priceFunc; -#endif + ZSTD_stats_t stats; } seqStore_t; const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); void ZSTD_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq); +size_t ZSTD_compressBegin_targetSrcSize(ZSTD_CCtx* zc, const void* dict, size_t dictSize, size_t targetSrcSize, int compressionLevel); #endif /* ZSTD_CCOMMON_H_MODULE */ diff --git a/lib/zstd_opt.h b/lib/zstd_opt.h index 7618fcadf..145570031 100644 --- a/lib/zstd_opt.h +++ b/lib/zstd_opt.h @@ -111,9 +111,18 @@ FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* seqStorePtr, U32 litLength, co price -= ZSTD_highbit(seqStorePtr->litFreq[literals[u]]+1); /* literal Length */ - price += ((litLength >= MaxLL)<<3) + ((litLength >= 255+MaxLL)<<4) + ((litLength>=(1<<15))<<3); - if (litLength >= MaxLL) litLength = MaxLL; - price += seqStorePtr->log2litLengthSum - ZSTD_highbit(seqStorePtr->litLengthFreq[litLength]+1); + { static const BYTE LL_Code[64] = { 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 16, 17, 17, 18, 18, 19, 19, + 20, 20, 20, 20, 21, 21, 21, 21, + 22, 22, 22, 22, 22, 22, 22, 22, + 23, 23, 23, 23, 23, 23, 23, 23, + 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24 }; + const BYTE LL_deltaCode = 19; + const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit(litLength) + LL_deltaCode : LL_Code[litLength]; + price += LL_bits[llCode] + seqStorePtr->log2litLengthSum - ZSTD_highbit(seqStorePtr->litLengthFreq[llCode]+1); + } return price; } @@ -122,25 +131,24 @@ FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* seqStorePtr, U32 litLength, co FORCE_INLINE U32 ZSTD_getPrice(seqStore_t* seqStorePtr, U32 litLength, const BYTE* literals, U32 offset, U32 matchLength) { /* offset */ - BYTE offCode = offset ? (BYTE)ZSTD_highbit(offset+1) + 1 : 0; - U32 price = (offCode-1) + (!offCode) + seqStorePtr->log2offCodeSum - ZSTD_highbit(seqStorePtr->offCodeFreq[offCode]+1); + BYTE offCode = (BYTE)ZSTD_highbit(offset+1); + U32 price = offCode + seqStorePtr->log2offCodeSum - ZSTD_highbit(seqStorePtr->offCodeFreq[offCode]+1); /* match Length */ - price += ((matchLength >= MaxML)<<3) + ((matchLength >= 255+MaxML)<<4) + ((matchLength>=(1<<15))<<3); - if (matchLength >= MaxML) matchLength = MaxML; - price += ZSTD_getLiteralPrice(seqStorePtr, litLength, literals) + seqStorePtr->log2matchLengthSum - ZSTD_highbit(seqStorePtr->matchLengthFreq[matchLength]+1); - -#if ZSTD_OPT_DEBUG == 3 - switch (seqStorePtr->priceFunc) { - default: - case 0: - return 1 + price + ((seqStorePtr->litSum>>5) / seqStorePtr->litLengthSum) + ((seqStorePtr->litSum<<1) / (seqStorePtr->litSum + seqStorePtr->matchSum)); - case 1: - return 1 + price; + { static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, + 38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, + 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, + 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 }; + const BYTE ML_deltaCode = 36; + const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit(matchLength) + ML_deltaCode : ML_Code[matchLength]; + price += ML_bits[mlCode] + seqStorePtr->log2matchLengthSum - ZSTD_highbit(seqStorePtr->matchLengthFreq[mlCode]+1); } -#else - return price + seqStorePtr->factor; -#endif + + return price + ZSTD_getLiteralPrice(seqStorePtr, litLength, literals) + seqStorePtr->factor; } @@ -154,23 +162,39 @@ MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const B seqStorePtr->litFreq[literals[u]]++; /* literal Length */ - seqStorePtr->litLengthSum++; - if (litLength >= MaxLL) - seqStorePtr->litLengthFreq[MaxLL]++; - else - seqStorePtr->litLengthFreq[litLength]++; + { static const BYTE LL_Code[64] = { 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 16, 17, 17, 18, 18, 19, 19, + 20, 20, 20, 20, 21, 21, 21, 21, + 22, 22, 22, 22, 22, 22, 22, 22, + 23, 23, 23, 23, 23, 23, 23, 23, + 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24 }; + const BYTE LL_deltaCode = 19; + const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit(litLength) + LL_deltaCode : LL_Code[litLength]; + seqStorePtr->litLengthFreq[llCode]++; + seqStorePtr->litLengthSum++; + } /* match offset */ seqStorePtr->offCodeSum++; - BYTE offCode = offset ? (BYTE)ZSTD_highbit(offset+1) + 1 : 0; + BYTE offCode = (BYTE)ZSTD_highbit(offset+1); seqStorePtr->offCodeFreq[offCode]++; /* match Length */ - seqStorePtr->matchLengthSum++; - if (matchLength >= MaxML) - seqStorePtr->matchLengthFreq[MaxML]++; - else - seqStorePtr->matchLengthFreq[matchLength]++; + { static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, + 38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, + 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, + 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 }; + const BYTE ML_deltaCode = 36; + const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit(matchLength) + ML_deltaCode : ML_Code[matchLength]; + seqStorePtr->matchLengthFreq[mlCode]++; + seqStorePtr->matchLengthSum++; + } ZSTD_setLog2Prices(seqStorePtr); } @@ -196,17 +220,18 @@ MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const B static U32 ZSTD_insertAndFindFirstIndexHash3 (ZSTD_CCtx* zc, const BYTE* ip) { U32* const hashTable3 = zc->hashTable3; + U32 const hashLog3 = zc->hashLog3; const BYTE* const base = zc->base; const U32 target = (U32)(ip - base); U32 idx = zc->nextToUpdate3; while(idx < target) { - hashTable3[ZSTD_hash3Ptr(base+idx, HASHLOG3)] = idx; + hashTable3[ZSTD_hash3Ptr(base+idx, hashLog3)] = idx; idx++; } zc->nextToUpdate3 = target; - return hashTable3[ZSTD_hash3Ptr(ip, HASHLOG3)]; + return hashTable3[ZSTD_hash3Ptr(ip, hashLog3)]; } diff --git a/lib/zstd_static.h b/lib/zstd_static.h index 333861255..bbc20240d 100644 --- a/lib/zstd_static.h +++ b/lib/zstd_static.h @@ -63,6 +63,8 @@ extern "C" { #define ZSTD_CHAINLOG_MIN 4 #define ZSTD_HASHLOG_MAX ZSTD_WINDOWLOG_MAX #define ZSTD_HASHLOG_MIN 12 +#define ZSTD_HASHLOG3_MAX 17 +#define ZSTD_HASHLOG3_MIN 15 #define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1) #define ZSTD_SEARCHLOG_MIN 1 #define ZSTD_SEARCHLENGTH_MAX 7 diff --git a/programs/.gitignore b/programs/.gitignore index a1d72aad1..fe074a767 100644 --- a/programs/.gitignore +++ b/programs/.gitignore @@ -36,3 +36,8 @@ paramgrill # Default result files dictionary grillResults.txt +_* + +# Misc files +*.bat +fileTests.sh diff --git a/programs/bench.c b/programs/bench.c index 0d18ea646..cea9634c2 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -44,32 +44,58 @@ /* ************************************* * Includes ***************************************/ +#define _POSIX_C_SOURCE 199309L /* before - needed for nanosleep() */ #include /* malloc, free */ #include /* memset */ #include /* fprintf, fopen, ftello64 */ #include /* stat64 */ #include /* stat64 */ -#include /* clock_t, clock, CLOCKS_PER_SEC */ +#include /* clock_t, nanosleep, clock, CLOCKS_PER_SEC */ /* sleep : posix - windows - others */ #if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__))) -# include /* sleep */ -# include /* setpriority */ +# include +# include /* setpriority */ # define BMK_sleep(s) sleep(s) -# define HIGH_PRIORITY setpriority(PRIO_PROCESS, 0, -20) +# define mili_sleep(mili) { struct timespec t; t.tv_sec=0; t.tv_nsec=mili*1000000ULL; nanosleep(&t, NULL); } +# define SET_HIGH_PRIORITY setpriority(PRIO_PROCESS, 0, -20) #elif defined(_WIN32) # include # define BMK_sleep(s) Sleep(1000*s) -# define HIGH_PRIORITY SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS); +# define mili_sleep(mili) Sleep(mili) +# define SET_HIGH_PRIORITY SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS) #else # define BMK_sleep(s) /* disabled */ -# define HIGH_PRIORITY +# define mili_sleep(mili) /* disabled */ +# define SET_HIGH_PRIORITY /* disabled */ +#endif + +#if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__))) + typedef clock_t BMK_time_t; +# define BMK_initTimer(ticksPerSecond) ticksPerSecond=0 +# define BMK_getTime(x) x = clock() +# define BMK_getSpanTimeMicro(ticksPerSecond, clockStart, clockEnd) (1000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC) +# define BMK_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) (1000000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC) +#elif defined(_WIN32) + typedef LARGE_INTEGER BMK_time_t; +# define BMK_initTimer(x) if (!QueryPerformanceFrequency(&x)) { fprintf(stderr, "ERROR: QueryPerformance not present\n"); } +# define BMK_getTime(x) QueryPerformanceCounter(&x) +# define BMK_getSpanTimeMicro(ticksPerSecond, clockStart, clockEnd) (1000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart) +# define BMK_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) (1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart) +#else + typedef int BMK_time_t; +# define BMK_initTimer(ticksPerSecond) ticksPerSecond=0 +# define BMK_getTimeMicro(clockStart) clockStart=1 +# define BMK_getSpanTimeMicro(ticksPerSecond, clockStart, clockEnd) (TIMELOOP_S*1000000ULL+clockEnd-clockStart) +# define BMK_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) (TIMELOOP_S*1000000000ULL+clockEnd-clockStart) #endif #include "mem.h" #include "zstd_static.h" +#include "zstd_internal.h" /* ZSTD_compressBegin_targetSrcSize */ +#include "datagen.h" /* RDG_genBuffer */ #include "xxhash.h" -#include "datagen.h" /* RDG_genBuffer */ + /* ************************************* @@ -87,6 +113,10 @@ /* ************************************* * Constants ***************************************/ +#ifndef ZSTD_VERSION +# define ZSTD_VERSION "" +#endif + #define NBLOOPS 3 #define TIMELOOP_S 1 #define ACTIVEPERIOD_S 70 @@ -97,7 +127,6 @@ #define GB *(1U<<30) static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31)); -#define DEFAULT_CHUNKSIZE (4 MB) static U32 g_compressibilityDefault = 50; @@ -132,29 +161,38 @@ static U32 g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result ***************************************/ static U32 g_nbIterations = NBLOOPS; static size_t g_blockSize = 0; +int g_additionalParam = 0; + +void BMK_setNotificationLevel(unsigned level) { g_displayLevel=level; } + +void BMK_setAdditionalParam(int additionalParam) { g_additionalParam=additionalParam; } void BMK_SetNbIterations(unsigned nbLoops) { g_nbIterations = nbLoops; - DISPLAY("- %i iterations -\n", g_nbIterations); + DISPLAYLEVEL(2, "- %i iterations -\n", g_nbIterations); } void BMK_SetBlockSize(size_t blockSize) { g_blockSize = blockSize; - DISPLAY("using blocks of size %u KB \n", (U32)(blockSize>>10)); + DISPLAYLEVEL(2, "using blocks of size %u KB \n", (U32)(blockSize>>10)); } /* ******************************************************** * Private functions **********************************************************/ -static clock_t BMK_clockSpan( clock_t clockStart ) +/* returns time span in microseconds */ +static U64 BMK_clockSpan( BMK_time_t clockStart, BMK_time_t ticksPerSecond ) { - return clock() - clockStart; /* works even if overflow, span limited to <= ~30mn */ + BMK_time_t clockEnd; + + (void)ticksPerSecond; + BMK_getTime(clockEnd); + return BMK_getSpanTimeMicro(ticksPerSecond, clockStart, clockEnd); } - static U64 BMK_getFileSize(const char* infilename) { int r; @@ -184,13 +222,22 @@ typedef struct size_t resSize; } blockParam_t; +typedef struct +{ + double ratio; + size_t cSize; + double cSpeed; + double dSpeed; +} benchResult_t; + + #define MIN(a,b) ((a)<(b) ? (a) : (b)) #define MAX(a,b) ((a)>(b) ? (a) : (b)) static int BMK_benchMem(const void* srcBuffer, size_t srcSize, const char* displayName, int cLevel, const size_t* fileSizes, U32 nbFiles, - const void* dictBuffer, size_t dictBufferSize) + const void* dictBuffer, size_t dictBufferSize, benchResult_t *result) { size_t const blockSize = (g_blockSize ? g_blockSize : srcSize) + (!srcSize); /* avoid div by 0 */ U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles; @@ -203,6 +250,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, ZSTD_DCtx* refDCtx = ZSTD_createDCtx(); ZSTD_DCtx* dctx = ZSTD_createDCtx(); U32 nbBlocks; + BMK_time_t ticksPerSecond; /* checks */ if (!compressedBuffer || !resultBuffer || !blockTable || !refCtx || !ctx || !refDCtx || !dctx) @@ -210,7 +258,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, /* init */ if (strlen(displayName)>17) displayName += strlen(displayName)-17; /* can only display 17 characters */ - HIGH_PRIORITY; + BMK_initTimer(ticksPerSecond); /* Init blockTable data */ { const char* srcPtr = (const char*)srcBuffer; @@ -240,34 +288,40 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, /* Bench */ { double fastestC = 100000000., fastestD = 100000000.; U64 const crcOrig = XXH64(srcBuffer, srcSize, 0); - clock_t coolTime = clock(); + U64 crcCheck = 0; + BMK_time_t coolTime; U32 testNb; + size_t cSize = 0; + double ratio = 0.; - DISPLAY("\r%79s\r", ""); + BMK_getTime(coolTime); + DISPLAYLEVEL(2, "\r%79s\r", ""); for (testNb = 1; testNb <= (g_nbIterations + !g_nbIterations); testNb++) { - size_t cSize; - double ratio = 0.; - clock_t clockStart; - clock_t const clockLoop = g_nbIterations ? TIMELOOP_S * CLOCKS_PER_SEC : 10; + BMK_time_t clockStart, clockEnd; + U64 clockLoop = g_nbIterations ? TIMELOOP_S*1000000ULL : 10; /* overheat protection */ - if (BMK_clockSpan(coolTime) > ACTIVEPERIOD_S * CLOCKS_PER_SEC) { + if (BMK_clockSpan(coolTime, ticksPerSecond) > ACTIVEPERIOD_S*1000000ULL) { DISPLAY("\rcooling down ... \r"); BMK_sleep(COOLPERIOD_S); - coolTime = clock(); + BMK_getTime(coolTime); } /* Compression */ - DISPLAY("%2i-%-17.17s :%10u ->\r", testNb, displayName, (U32)srcSize); + DISPLAYLEVEL(2, "%2i-%-17.17s :%10u ->\r", testNb, displayName, (U32)srcSize); memset(compressedBuffer, 0xE5, maxCompressedSize); /* warm up and erase result buffer */ - clockStart = clock(); - while (clock() == clockStart); - clockStart = clock(); + mili_sleep(1); /* give processor time to other processes */ + BMK_getTime(clockStart); + do { BMK_getTime(clockEnd); } + while (BMK_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) == 0); + BMK_getTime(clockStart); + { U32 nbLoops; - for (nbLoops = 0 ; BMK_clockSpan(clockStart) < clockLoop ; nbLoops++) { + for (nbLoops = 0 ; BMK_clockSpan(clockStart, ticksPerSecond) < clockLoop ; nbLoops++) { U32 blockNb; - ZSTD_compressBegin_usingDict(refCtx, dictBuffer, dictBufferSize, cLevel); + ZSTD_compressBegin_targetSrcSize(refCtx, dictBuffer, dictBufferSize, blockSize, cLevel); + // ZSTD_compressBegin_usingDict(refCtx, dictBuffer, dictBufferSize, cLevel); for (blockNb=0; blockNb%10u (%5.3f),%6.1f MB/s\r", + DISPLAYLEVEL(2, "%2i-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s\r", testNb, displayName, (U32)srcSize, (U32)cSize, ratio, - (double)srcSize / 1000000. / (fastestC / CLOCKS_PER_SEC) ); + (double)srcSize / fastestC ); (void)fastestD; (void)crcOrig; /* unused when decompression disabled */ #if 1 /* Decompression */ memset(resultBuffer, 0xD6, srcSize); /* warm result buffer */ - clockStart = clock(); - while (clock() == clockStart); - clockStart = clock(); + mili_sleep(1); /* give processor time to other processes */ + BMK_getTime(clockStart); + do { BMK_getTime(clockEnd); } + while (BMK_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) == 0); + BMK_getTime(clockStart); { U32 nbLoops; - for (nbLoops = 0 ; BMK_clockSpan(clockStart) < clockLoop ; nbLoops++) { + for (nbLoops = 0 ; BMK_clockSpan(clockStart, ticksPerSecond) < clockLoop ; nbLoops++) { U32 blockNb; ZSTD_decompressBegin_usingDict(refDCtx, dictBuffer, dictBufferSize); for (blockNb=0; blockNb%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r", + DISPLAYLEVEL(2, "%2i-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r", testNb, displayName, (U32)srcSize, (U32)cSize, ratio, - (double)srcSize / 1000000. / (fastestC / CLOCKS_PER_SEC), - (double)srcSize / 1000000. / (fastestD / CLOCKS_PER_SEC) ); + (double)srcSize / fastestC, + (double)srcSize / fastestD ); /* CRC Checking */ - { U64 const crcCheck = XXH64(resultBuffer, srcSize, 0); + { crcCheck = XXH64(resultBuffer, srcSize, 0); if (crcOrig!=crcCheck) { size_t u; DISPLAY("!!! WARNING !!! %14s : Invalid Checksum : %x != %x \n", displayName, (unsigned)crcOrig, (unsigned)crcCheck); @@ -346,7 +402,14 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, } } /* CRC Checking */ #endif } /* for (testNb = 1; testNb <= (g_nbIterations + !g_nbIterations); testNb++) */ - DISPLAY("%2i#\n", cLevel); + + if (crcOrig == crcCheck) { + result->ratio = ratio; + result->cSize = cSize; + result->cSpeed = (double)srcSize / fastestC; + result->dSpeed = (double)srcSize / fastestD; + } + DISPLAYLEVEL(2, "%2i#\n", cLevel); } /* Bench */ /* clean up */ @@ -379,24 +442,51 @@ static size_t BMK_findMaxMem(U64 requiredMem) } static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, - const char* displayName, int cLevel, + const char* displayName, int cLevel, int cLevelLast, const size_t* fileSizes, unsigned nbFiles, const void* dictBuffer, size_t dictBufferSize) { - if (cLevel < 0) { /* range mode : test all levels from 1 to l */ - int l; - for (l=1; l <= -cLevel; l++) { - BMK_benchMem(srcBuffer, benchedSize, - displayName, l, - fileSizes, nbFiles, - dictBuffer, dictBufferSize); + benchResult_t result, total; + int l; + + SET_HIGH_PRIORITY; + + const char* pch = strrchr(displayName, '\\'); /* Windows */ + if (!pch) pch = strrchr(displayName, '/'); /* Linux */ + if (pch) displayName = pch+1; + + memset(&result, 0, sizeof(result)); + memset(&total, 0, sizeof(total)); + + if (g_displayLevel == 1 && !g_additionalParam) + DISPLAY("bench %s: input %u bytes, %i iterations, %u KB blocks\n", ZSTD_VERSION, (U32)benchedSize, g_nbIterations, (U32)(g_blockSize>>10)); + + if (cLevelLast < cLevel) cLevelLast = cLevel; + + for (l=cLevel; l <= cLevelLast; l++) { + BMK_benchMem(srcBuffer, benchedSize, + displayName, l, + fileSizes, nbFiles, + dictBuffer, dictBufferSize, &result); + if (g_displayLevel == 1) { + if (g_additionalParam) + DISPLAY("%-3i%11i (%5.3f) %6.1f MB/s %6.1f MB/s %s (param=%d)\n", -l, (int)result.cSize, result.ratio, result.cSpeed, result.dSpeed, displayName, g_additionalParam); + else + DISPLAY("%-3i%11i (%5.3f) %6.1f MB/s %6.1f MB/s %s\n", -l, (int)result.cSize, result.ratio, result.cSpeed, result.dSpeed, displayName); + total.cSize += result.cSize; + total.cSpeed += result.cSpeed; + total.dSpeed += result.dSpeed; + total.ratio += result.ratio; } - return; } - BMK_benchMem(srcBuffer, benchedSize, - displayName, cLevel, - fileSizes, nbFiles, - dictBuffer, dictBufferSize); + if (g_displayLevel == 1 && cLevelLast > cLevel) + { + total.cSize /= 1+cLevelLast-cLevel; + total.cSpeed /= 1+cLevelLast-cLevel; + total.dSpeed /= 1+cLevelLast-cLevel; + total.ratio /= 1+cLevelLast-cLevel; + DISPLAY("avg%11i (%5.3f) %6.1f MB/s %6.1f MB/s %s\n", (int)total.cSize, total.ratio, total.cSpeed, total.dSpeed, displayName); + } } static U64 BMK_getTotalFileSize(const char** fileNamesTable, unsigned nbFiles) @@ -433,7 +523,7 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize, } static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, - const char* dictFileName, int cLevel) + const char* dictFileName, int cLevel, int cLevelLast) { void* srcBuffer; size_t benchedSize; @@ -473,7 +563,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, else displayName = fileNamesTable[0]; BMK_benchCLevel(srcBuffer, benchedSize, - displayName, cLevel, + displayName, cLevel, cLevelLast, fileSizes, nbFiles, dictBuffer, dictBufferSize); @@ -484,7 +574,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, } -static void BMK_syntheticTest(int cLevel, double compressibility) +static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility) { char name[20] = {0}; size_t benchedSize = 10000000; @@ -498,7 +588,7 @@ static void BMK_syntheticTest(int cLevel, double compressibility) /* Bench */ snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100)); - BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, &benchedSize, 1, NULL, 0); + BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, NULL, 0); /* clean up */ free(srcBuffer); @@ -506,14 +596,14 @@ static void BMK_syntheticTest(int cLevel, double compressibility) int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, - const char* dictFileName, int cLevel) + const char* dictFileName, int cLevel, int cLevelLast) { double const compressibility = (double)g_compressibilityDefault / 100; if (nbFiles == 0) - BMK_syntheticTest(cLevel, compressibility); + BMK_syntheticTest(cLevel, cLevelLast, compressibility); else - BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel); + BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast); return 0; } diff --git a/programs/bench.h b/programs/bench.h index 6d62451bd..3a1ca3a20 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -27,10 +27,11 @@ /* Main function */ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, - const char* dictFileName, int cLevel); + const char* dictFileName, int cLevel, int cLevelLast); /* Set Parameters */ void BMK_SetNbIterations(unsigned nbLoops); void BMK_SetBlockSize(size_t blockSize); - +void BMK_setAdditionalParam(int additionalParam); +void BMK_setNotificationLevel(unsigned level); diff --git a/programs/fullbench.c b/programs/fullbench.c index c5c564961..751966562 100644 --- a/programs/fullbench.c +++ b/programs/fullbench.c @@ -83,7 +83,6 @@ #define KNUTH 2654435761U #define MAX_MEM (1984 MB) -#define DEFAULT_CHUNKSIZE (4<<20) #define COMPRESSIBILITY_DEFAULT 0.50 static const size_t g_sampleSize = 10000000; diff --git a/programs/paramgrill.c b/programs/paramgrill.c index 1e875b875..8749e6a38 100644 --- a/programs/paramgrill.c +++ b/programs/paramgrill.c @@ -100,7 +100,6 @@ #define NB_LEVELS_TRACKED 30 static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31)); -#define DEFAULT_CHUNKSIZE (4<<20) #define COMPRESSIBILITY_DEFAULT 0.50 static const size_t sampleSize = 10000000; diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 86d6bff78..022d75131 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -143,9 +143,9 @@ static int usage_advanced(const char* programName) #ifndef ZSTD_NOBENCH DISPLAY( "Benchmark arguments :\n"); DISPLAY( " -b# : benchmark file(s), using # compression level (default : 1) \n"); + DISPLAY( " -r# : test all compression levels from -bX to # (default: 1)\n"); DISPLAY( " -i# : iteration loops [1-9](default : 3)\n"); DISPLAY( " -B# : cut file into independent blocks of size # (default: no block)\n"); - DISPLAY( " -r# : test all compression levels from 1 to # (default: disabled)\n"); #endif return 0; } @@ -182,19 +182,19 @@ int main(int argCount, const char** argv) nextArgumentIsOutFileName=0, nextArgumentIsMaxDict=0; unsigned cLevel = 1; + unsigned cLevelLast = 1; const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); /* argCount >= 1 */ unsigned filenameIdx = 0; const char* programName = argv[0]; const char* outFileName = NULL; const char* dictFileName = NULL; char* dynNameSpace = NULL; - int rangeBench = 1; unsigned maxDictSize = g_defaultMaxDictSize; unsigned dictCLevel = g_defaultDictCLevel; unsigned dictSelect = g_defaultSelectivityLevel; /* init */ - (void)rangeBench; (void)dictCLevel; /* not used when ZSTD_NOBENCH / ZSTD_NODICT set */ + (void)cLevelLast; (void)dictCLevel; /* not used when ZSTD_NOBENCH / ZSTD_NODICT set */ if (filenameTable==NULL) { DISPLAY("not enough memory\n"); exit(1); } displayOut = stderr; /* Pick out program name from path. Don't rely on stdlib because of conflicting behavior */ @@ -235,7 +235,6 @@ int main(int argCount, const char** argv) argument++; while (argument[0]!=0) { - /* compression Level */ if ((*argument>='0') && (*argument<='9')) { cLevel = 0; @@ -294,6 +293,7 @@ int main(int argCount, const char** argv) argument++; while ((*argument >='0') && (*argument <='9')) iters *= 10, iters += *argument++ - '0'; + BMK_setNotificationLevel(displayLevel); BMK_SetNbIterations(iters); } break; @@ -307,14 +307,20 @@ int main(int argCount, const char** argv) if (*argument=='K') bSize<<=10, argument++; /* allows using KB notation */ if (*argument=='M') bSize<<=20, argument++; if (*argument=='B') argument++; + BMK_setNotificationLevel(displayLevel); BMK_SetBlockSize(bSize); } break; /* range bench (benchmark only) */ case 'r': - rangeBench = -1; + /* compression Level */ argument++; + if ((*argument>='0') && (*argument<='9')) { + cLevelLast = 0; + while ((*argument >= '0') && (*argument <= '9')) + cLevelLast *= 10, cLevelLast += *argument++ - '0'; + } break; #endif /* ZSTD_NOBENCH */ @@ -325,9 +331,18 @@ int main(int argCount, const char** argv) dictSelect *= 10, dictSelect += *argument++ - '0'; break; - /* Pause at the end (hidden option) */ - case 'p': main_pause=1; argument++; break; - + /* Pause at the end (-p) or set an additional param (-p#) (hidden option) */ + case 'p': argument++; +#ifndef ZSTD_NOBENCH + if ((*argument>='0') && (*argument<='9')) { + int additionalParam = 0; + while ((*argument >= '0') && (*argument <= '9')) + additionalParam *= 10, additionalParam += *argument++ - '0'; + BMK_setAdditionalParam(additionalParam); + } else +#endif + main_pause=1; + break; /* unknown command */ default : CLEAN_RETURN(badusage(programName)); } @@ -368,7 +383,8 @@ int main(int argCount, const char** argv) /* Check if benchmark is selected */ if (bench) { #ifndef ZSTD_NOBENCH - BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel*rangeBench); + BMK_setNotificationLevel(displayLevel); + BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast); #endif goto _end; }