Yann Collet
@@ -47,7 +47,3 @@ ipch/
|
||||
.directory
|
||||
_codelite
|
||||
_zstdbench
|
||||
|
||||
lib/zstd_opt_LZ5.c
|
||||
lib/zstd_opt_llen.c
|
||||
lib/zstd_opt_nollen.c
|
||||
|
||||
@@ -0,0 +1,164 @@
|
||||
/*
|
||||
zstd - standard compression library
|
||||
Header File for static linking only
|
||||
Copyright (C) 2014-2016, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- zstd homepage : http://www.zstd.net
|
||||
*/
|
||||
#ifndef ZSTD_STATS_H
|
||||
#define ZSTD_STATS_H
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/*-*************************************
|
||||
* Dependencies
|
||||
***************************************/
|
||||
//#include "zstd.h"
|
||||
//#include "mem.h"
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Constants
|
||||
***************************************/
|
||||
//#define ZSTD_MAGICNUMBER 0xFD2FB526 /* v0.6 */
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Types
|
||||
***************************************/
|
||||
typedef struct {
|
||||
U32 priceOffset, priceOffCode, priceMatchLength, priceLiteral, priceLitLength, priceDumpsLength;
|
||||
U32 totalMatchSum, totalLitSum, totalSeqSum, totalRepSum;
|
||||
U32 litSum, matchLengthSum, litLengthSum, offCodeSum;
|
||||
U32 matchLengthFreq[1<<MLbits];
|
||||
U32 litLengthFreq[1<<LLbits];
|
||||
U32 litFreq[1<<Litbits];
|
||||
U32 offCodeFreq[1<<Offbits];
|
||||
} ZSTD_stats_t;
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Advanced functions
|
||||
***************************************/
|
||||
MEM_STATIC void ZSTD_statsPrint(ZSTD_stats_t* stats, U32 searchLength)
|
||||
{
|
||||
stats->totalMatchSum += stats->totalSeqSum * ((searchLength == 3) ? 3 : 4);
|
||||
printf("avgMatchL=%.2f avgLitL=%.2f match=%.1f%% lit=%.1f%% reps=%d seq=%d\n", (float)stats->totalMatchSum/stats->totalSeqSum, (float)stats->totalLitSum/stats->totalSeqSum, 100.0*stats->totalMatchSum/(stats->totalMatchSum+stats->totalLitSum), 100.0*stats->totalLitSum/(stats->totalMatchSum+stats->totalLitSum), stats->totalRepSum, stats->totalSeqSum);
|
||||
printf("SumBytes=%d Offset=%d OffCode=%d Match=%d Literal=%d LitLength=%d DumpsLength=%d\n", (stats->priceOffset+stats->priceOffCode+stats->priceMatchLength+stats->priceLiteral+stats->priceLitLength+stats->priceDumpsLength)/8, stats->priceOffset/8, stats->priceOffCode/8, stats->priceMatchLength/8, stats->priceLiteral/8, stats->priceLitLength/8, stats->priceDumpsLength/8);
|
||||
}
|
||||
|
||||
MEM_STATIC void ZSTD_statsInit(ZSTD_stats_t* stats)
|
||||
{
|
||||
stats->totalLitSum = stats->totalMatchSum = stats->totalSeqSum = stats->totalRepSum = 1;
|
||||
stats->priceOffset = stats->priceOffCode = stats->priceMatchLength = stats->priceLiteral = stats->priceLitLength = stats->priceDumpsLength = 0;
|
||||
}
|
||||
|
||||
MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats)
|
||||
{
|
||||
unsigned u;
|
||||
|
||||
stats->litSum = (1<<Litbits);
|
||||
stats->litLengthSum = (1<<LLbits);
|
||||
stats->matchLengthSum = (1<<MLbits);
|
||||
stats->offCodeSum = (1<<Offbits);
|
||||
|
||||
for (u=0; u<=MaxLit; u++)
|
||||
stats->litFreq[u] = 1;
|
||||
for (u=0; u<=MaxLL; u++)
|
||||
stats->litLengthFreq[u] = 1;
|
||||
for (u=0; u<=MaxML; u++)
|
||||
stats->matchLengthFreq[u] = 1;
|
||||
for (u=0; u<=MaxOff; u++)
|
||||
stats->offCodeFreq[u] = 1;
|
||||
}
|
||||
|
||||
MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength)
|
||||
{
|
||||
/* offset */
|
||||
BYTE offCode = offset ? (BYTE)ZSTD_highbit(offset+1) + 1 : 0;
|
||||
stats->priceOffCode += ZSTD_highbit(stats->offCodeSum+1) - ZSTD_highbit(stats->offCodeFreq[offCode]+1);
|
||||
stats->priceOffset += (offCode-1) + (!offCode);
|
||||
|
||||
/* match Length */
|
||||
stats->priceDumpsLength += ((matchLength >= MaxML)<<3) + ((matchLength >= 255+MaxML)<<4) + ((matchLength>=(1<<15))<<3);
|
||||
stats->priceMatchLength += ZSTD_highbit(stats->matchLengthSum+1) - ZSTD_highbit(stats->matchLengthFreq[(matchLength >= MaxML) ? MaxML : matchLength]+1);
|
||||
|
||||
if (litLength) {
|
||||
/* literals */
|
||||
U32 u;
|
||||
stats->priceLiteral += litLength * ZSTD_highbit(stats->litSum+1);
|
||||
for (u=0; u < litLength; u++)
|
||||
stats->priceLiteral -= ZSTD_highbit(stats->litFreq[literals[u]]+1);
|
||||
|
||||
/* literal Length */
|
||||
stats->priceDumpsLength += ((litLength >= MaxLL)<<3) + ((litLength >= 255+MaxLL)<<4) + ((litLength>=(1<<15))<<3);
|
||||
stats->priceLitLength += ZSTD_highbit(stats->litLengthSum+1) - ZSTD_highbit(stats->litLengthFreq[(litLength >= MaxLL) ? MaxLL : litLength]+1);
|
||||
} else {
|
||||
stats->priceLitLength += ZSTD_highbit(stats->litLengthSum+1) - ZSTD_highbit(stats->litLengthFreq[0]+1);
|
||||
}
|
||||
|
||||
|
||||
if (offset == 0) stats->totalRepSum++;
|
||||
stats->totalSeqSum++;
|
||||
stats->totalMatchSum += matchLength;
|
||||
stats->totalLitSum += litLength;
|
||||
|
||||
U32 u;
|
||||
/* literals */
|
||||
stats->litSum += litLength;
|
||||
for (u=0; u < litLength; u++)
|
||||
stats->litFreq[literals[u]]++;
|
||||
|
||||
/* literal Length */
|
||||
stats->litLengthSum++;
|
||||
if (litLength >= MaxLL)
|
||||
stats->litLengthFreq[MaxLL]++;
|
||||
else
|
||||
stats->litLengthFreq[litLength]++;
|
||||
|
||||
/* match offset */
|
||||
stats->offCodeSum++;
|
||||
stats->offCodeFreq[offCode]++;
|
||||
|
||||
/* match Length */
|
||||
stats->matchLengthSum++;
|
||||
if (matchLength >= MaxML)
|
||||
stats->matchLengthFreq[MaxML]++;
|
||||
else
|
||||
stats->matchLengthFreq[matchLength]++;
|
||||
}
|
||||
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ZSTD_STATIC_H */
|
||||
+32
-27
@@ -95,6 +95,7 @@ struct ZSTD_CCtx_s
|
||||
U32 lowLimit; /* below that point, no more data */
|
||||
U32 nextToUpdate; /* index from which to continue dictionary update */
|
||||
U32 nextToUpdate3; /* index from which to continue dictionary update */
|
||||
U32 hashLog3; /* dispatch table : larger == faster, more memory */
|
||||
U32 loadedDictEnd;
|
||||
U32 stage;
|
||||
ZSTD_parameters params;
|
||||
@@ -219,7 +220,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
|
||||
const size_t tokenSpace = blockSize + 11*maxNbSeq;
|
||||
const size_t chainSize = (params.cParams.strategy == ZSTD_fast) ? 0 : (1 << params.cParams.chainLog);
|
||||
const size_t hSize = 1 << params.cParams.hashLog;
|
||||
const size_t h3Size = (params.cParams.searchLength==3) ? (1 << HASHLOG3) : 0;
|
||||
const size_t h3Size = (zc->hashLog3) ? 1 << zc->hashLog3 : 0;
|
||||
const size_t tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
|
||||
|
||||
/* Check if workSpace is large enough, alloc a new one if needed */
|
||||
@@ -286,13 +287,14 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx)
|
||||
{
|
||||
if (srcCCtx->stage!=0) return ERROR(stage_wrong);
|
||||
|
||||
dstCCtx->hashLog3 = srcCCtx->hashLog3; /* must be before ZSTD_resetCCtx_advanced */
|
||||
ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params);
|
||||
|
||||
/* copy tables */
|
||||
{ const size_t chaineSize = (srcCCtx->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << srcCCtx->params.cParams.chainLog);
|
||||
{ const size_t chainSize = (srcCCtx->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << srcCCtx->params.cParams.chainLog);
|
||||
const size_t hSize = 1 << srcCCtx->params.cParams.hashLog;
|
||||
const size_t h3Size = (srcCCtx->params.cParams.searchLength == 3) ? (1 << HASHLOG3) : 0;
|
||||
const size_t tableSpace = (chaineSize + hSize + h3Size) * sizeof(U32);
|
||||
const size_t h3Size = (srcCCtx->hashLog3) ? 1 << srcCCtx->hashLog3 : 0;
|
||||
const size_t tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
|
||||
memcpy(dstCCtx->workSpace, srcCCtx->workSpace, tableSpace);
|
||||
}
|
||||
|
||||
@@ -344,7 +346,7 @@ static void ZSTD_reduceIndex (ZSTD_CCtx* zc, const U32 reducerValue)
|
||||
{ const U32 chainSize = (zc->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << zc->params.cParams.chainLog);
|
||||
ZSTD_reduceTable(zc->chainTable, chainSize, reducerValue); }
|
||||
|
||||
{ const U32 h3Size = (zc->params.cParams.searchLength == 3) ? (1 << HASHLOG3) : 0;
|
||||
{ const U32 h3Size = (zc->hashLog3) ? 1 << zc->hashLog3 : 0;
|
||||
ZSTD_reduceTable(zc->hashTable3, h3Size, reducerValue); }
|
||||
}
|
||||
|
||||
@@ -855,12 +857,7 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const B
|
||||
printf("Cpos %6u :%5u literals & match %3u bytes at distance %6u \n",
|
||||
pos, (U32)litLength, (U32)matchCode+MINMATCH, (U32)offsetCode);
|
||||
#endif
|
||||
#if ZSTD_OPT_DEBUG == 3
|
||||
if (offsetCode == 0) seqStorePtr->realRepSum++;
|
||||
seqStorePtr->realSeqSum++;
|
||||
seqStorePtr->realMatchSum += matchCode;
|
||||
seqStorePtr->realLitSum += litLength;
|
||||
#endif
|
||||
ZSTD_statsUpdatePrices(&seqStorePtr->stats, litLength, literals, offsetCode, matchCode);
|
||||
|
||||
/* copy Literals */
|
||||
ZSTD_wildcopy(seqStorePtr->lit, literals, litLength);
|
||||
@@ -1746,6 +1743,7 @@ _storeSequence:
|
||||
{ size_t const lastLLSize = iend - anchor;
|
||||
memcpy(seqStorePtr->lit, anchor, lastLLSize);
|
||||
seqStorePtr->lit += lastLLSize;
|
||||
ZSTD_statsUpdatePrices(&seqStorePtr->stats, lastLLSize, anchor, 0, 0);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1992,6 +1990,8 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCa
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
static size_t ZSTD_compress_generic (ZSTD_CCtx* zc,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize)
|
||||
@@ -2002,15 +2002,13 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc,
|
||||
BYTE* const ostart = (BYTE*)dst;
|
||||
BYTE* op = ostart;
|
||||
const U32 maxDist = 1 << zc->params.cParams.windowLog;
|
||||
#if ZSTD_OPT_DEBUG == 3
|
||||
seqStore_t* ssPtr = &zc->seqStore;
|
||||
static U32 priceFunc = 0;
|
||||
ssPtr->realMatchSum = ssPtr->realLitSum = ssPtr->realSeqSum = ssPtr->realRepSum = 1;
|
||||
ssPtr->priceFunc = priceFunc;
|
||||
#endif
|
||||
ZSTD_stats_t* stats = &zc->seqStore.stats;
|
||||
|
||||
ZSTD_statsInit(stats);
|
||||
|
||||
while (remaining) {
|
||||
size_t cSize;
|
||||
ZSTD_statsResetFreqs(stats);
|
||||
|
||||
if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */
|
||||
if (remaining < blockSize) blockSize = remaining;
|
||||
@@ -2042,12 +2040,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc,
|
||||
op += cSize;
|
||||
}
|
||||
|
||||
#if ZSTD_OPT_DEBUG == 3
|
||||
ssPtr->realMatchSum += ssPtr->realSeqSum * ((zc->params.searchLength == 3) ? 3 : 4);
|
||||
printf("avgMatchL=%.2f avgLitL=%.2f match=%.1f%% lit=%.1f%% reps=%d seq=%d priceFunc=%d\n", (float)ssPtr->realMatchSum/ssPtr->realSeqSum, (float)ssPtr->realLitSum/ssPtr->realSeqSum, 100.0*ssPtr->realMatchSum/(ssPtr->realMatchSum+ssPtr->realLitSum), 100.0*ssPtr->realLitSum/(ssPtr->realMatchSum+ssPtr->realLitSum), ssPtr->realRepSum, ssPtr->realSeqSum, ssPtr->priceFunc);
|
||||
priceFunc++;
|
||||
#endif
|
||||
|
||||
ZSTD_statsPrint(stats, zc->params.cParams.searchLength);
|
||||
return op-ostart;
|
||||
}
|
||||
|
||||
@@ -2233,13 +2226,16 @@ static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* dict, si
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*! ZSTD_compressBegin_internal() :
|
||||
* @return : 0, or an error code */
|
||||
static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* zc,
|
||||
const void* dict, size_t dictSize,
|
||||
ZSTD_parameters params, U64 pledgedSrcSize)
|
||||
{
|
||||
U32 hashLog3 = (pledgedSrcSize || pledgedSrcSize >= 8192) ? ZSTD_HASHLOG3_MAX : ((pledgedSrcSize >= 2048) ? ZSTD_HASHLOG3_MIN + 1 : ZSTD_HASHLOG3_MIN);
|
||||
zc->hashLog3 = (params.cParams.searchLength==3) ? hashLog3 : 0;
|
||||
// printf("windowLog=%d hashLog=%d hashLog3=%d \n", params.windowLog, params.hashLog, zc->hashLog3);
|
||||
|
||||
{ size_t const errorCode = ZSTD_resetCCtx_advanced(zc, params);
|
||||
if (ZSTD_isError(errorCode)) return errorCode; }
|
||||
|
||||
@@ -2291,6 +2287,18 @@ size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* zc, const void* dict, size_t dict
|
||||
return ZSTD_compressBegin_internal(zc, dict, dictSize, params, 0);
|
||||
}
|
||||
|
||||
|
||||
size_t ZSTD_compressBegin_targetSrcSize(ZSTD_CCtx* zc, const void* dict, size_t dictSize, size_t targetSrcSize, int compressionLevel)
|
||||
{
|
||||
ZSTD_parameters params;
|
||||
params.cParams = ZSTD_getCParams(compressionLevel, targetSrcSize, dictSize);
|
||||
params.fParams.contentSizeFlag = 1;
|
||||
ZSTD_adjustCParams(¶ms.cParams, targetSrcSize, dictSize);
|
||||
ZSTD_LOG_BLOCK("%p: ZSTD_compressBegin_targetSrcSize compressionLevel=%d\n", zc->base, compressionLevel);
|
||||
return ZSTD_compressBegin_internal(zc, dict, dictSize, params, targetSrcSize);
|
||||
}
|
||||
|
||||
|
||||
size_t ZSTD_compressBegin(ZSTD_CCtx* zc, int compressionLevel)
|
||||
{
|
||||
ZSTD_LOG_BLOCK("%p: ZSTD_compressBegin compressionLevel=%d\n", zc->base, compressionLevel);
|
||||
@@ -2530,9 +2538,6 @@ ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, U64 srcSize, si
|
||||
U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB); /* intentional underflow for srcSizeHint == 0 */
|
||||
if (compressionLevel<=0) compressionLevel = 1;
|
||||
if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL;
|
||||
#if ZSTD_OPT_DEBUG >= 1
|
||||
tableID=0;
|
||||
#endif
|
||||
cp = ZSTD_defaultCParameters[tableID][compressionLevel];
|
||||
if (MEM_32bits()) { /* auto-correction, for 32-bits mode */
|
||||
if (cp.windowLog > ZSTD_WINDOWLOG_MAX) cp.windowLog = ZSTD_WINDOWLOG_MAX;
|
||||
|
||||
@@ -149,7 +149,7 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
|
||||
dctx->hufTableX4[0] = HufLog;
|
||||
dctx->flagStaticTables = 0;
|
||||
dctx->fParams.mml = MINMATCH; /* overwritten by frame but forces ZSTD_btopt to MINMATCH in block mode */
|
||||
ZSTD_LOG_BLOCK("%p: ZSTD_decompressBegin searchLength=%d\n", dctx->base, dctx->params.searchLength);
|
||||
ZSTD_LOG_BLOCK("%p: ZSTD_decompressBegin searchLength=%d\n", dctx->base, dctx->fParams.mml);
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -843,7 +843,7 @@ static size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
|
||||
|
||||
if (srcSize >= ZSTD_BLOCKSIZE_MAX) return ERROR(srcSize_wrong);
|
||||
|
||||
ZSTD_LOG_BLOCK("%p: ZSTD_decompressBlock_internal searchLength=%d\n", dctx->base, dctx->params.searchLength);
|
||||
ZSTD_LOG_BLOCK("%p: ZSTD_decompressBlock_internal searchLength=%d\n", dctx->base, dctx->fParams.mml);
|
||||
|
||||
/* Decode literals sub-block */
|
||||
{ size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize);
|
||||
@@ -951,7 +951,7 @@ size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
|
||||
const void* dict, size_t dictSize)
|
||||
{
|
||||
ZSTD_decompressBegin_usingDict(dctx, dict, dictSize);
|
||||
ZSTD_LOG_BLOCK("%p: ZSTD_decompressBegin_usingDict searchLength=%d\n", dctx->base, dctx->params.searchLength);
|
||||
ZSTD_LOG_BLOCK("%p: ZSTD_decompressBegin_usingDict searchLength=%d\n", dctx->base, dctx->fParams.mml);
|
||||
ZSTD_checkContinuity(dctx, dst);
|
||||
return ZSTD_decompressFrame(dctx, dst, dstCapacity, src, srcSize);
|
||||
}
|
||||
|
||||
+14
-12
@@ -50,10 +50,8 @@
|
||||
/*-*************************************
|
||||
* Common constants
|
||||
***************************************/
|
||||
#define ZSTD_OPT_DEBUG 0 // 1 = tableID=0; 3 = price func tests; 5 = check encoded sequences; 9 = full logs
|
||||
#if defined(ZSTD_OPT_DEBUG) && ZSTD_OPT_DEBUG>0
|
||||
#include <stdio.h>
|
||||
#endif
|
||||
#define ZSTD_OPT_DEBUG 0 // 3 = compression stats; 5 = check encoded sequences; 9 = full logs
|
||||
#include <stdio.h>
|
||||
#if defined(ZSTD_OPT_DEBUG) && ZSTD_OPT_DEBUG>=9
|
||||
#define ZSTD_LOG_PARSER(...) printf(__VA_ARGS__)
|
||||
#define ZSTD_LOG_ENCODE(...) printf(__VA_ARGS__)
|
||||
@@ -99,7 +97,6 @@ typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t;
|
||||
|
||||
#define MINMATCH 4
|
||||
#define REPCODE_STARTVALUE 1
|
||||
#define HASHLOG3 17
|
||||
|
||||
#define Litbits 8
|
||||
#define Offbits 5
|
||||
@@ -195,6 +192,16 @@ typedef struct {
|
||||
U32 rep2;
|
||||
} ZSTD_optimal_t;
|
||||
|
||||
#if ZSTD_OPT_DEBUG == 3
|
||||
#include ".debug/zstd_stats.h"
|
||||
#else
|
||||
typedef struct { U32 unused; } ZSTD_stats_t;
|
||||
MEM_STATIC void ZSTD_statsPrint(ZSTD_stats_t* stats, U32 searchLength) { (void)stats; (void)searchLength; };
|
||||
MEM_STATIC void ZSTD_statsInit(ZSTD_stats_t* stats) { (void)stats; };
|
||||
MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats) { (void)stats; };
|
||||
MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) { (void)stats; (void)litLength; (void)literals; (void)offset; (void)matchLength; };
|
||||
#endif
|
||||
|
||||
typedef struct {
|
||||
void* buffer;
|
||||
U32* offsetStart;
|
||||
@@ -227,17 +234,12 @@ typedef struct {
|
||||
U32 log2litSum;
|
||||
U32 log2offCodeSum;
|
||||
U32 factor;
|
||||
#if ZSTD_OPT_DEBUG == 3
|
||||
U32 realMatchSum;
|
||||
U32 realLitSum;
|
||||
U32 realSeqSum;
|
||||
U32 realRepSum;
|
||||
U32 priceFunc;
|
||||
#endif
|
||||
ZSTD_stats_t stats;
|
||||
} seqStore_t;
|
||||
|
||||
const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx);
|
||||
void ZSTD_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq);
|
||||
size_t ZSTD_compressBegin_targetSrcSize(ZSTD_CCtx* zc, const void* dict, size_t dictSize, size_t targetSrcSize, int compressionLevel);
|
||||
|
||||
|
||||
#endif /* ZSTD_CCOMMON_H_MODULE */
|
||||
|
||||
+57
-32
@@ -111,9 +111,18 @@ FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* seqStorePtr, U32 litLength, co
|
||||
price -= ZSTD_highbit(seqStorePtr->litFreq[literals[u]]+1);
|
||||
|
||||
/* literal Length */
|
||||
price += ((litLength >= MaxLL)<<3) + ((litLength >= 255+MaxLL)<<4) + ((litLength>=(1<<15))<<3);
|
||||
if (litLength >= MaxLL) litLength = MaxLL;
|
||||
price += seqStorePtr->log2litLengthSum - ZSTD_highbit(seqStorePtr->litLengthFreq[litLength]+1);
|
||||
{ static const BYTE LL_Code[64] = { 0, 1, 2, 3, 4, 5, 6, 7,
|
||||
8, 9, 10, 11, 12, 13, 14, 15,
|
||||
16, 16, 17, 17, 18, 18, 19, 19,
|
||||
20, 20, 20, 20, 21, 21, 21, 21,
|
||||
22, 22, 22, 22, 22, 22, 22, 22,
|
||||
23, 23, 23, 23, 23, 23, 23, 23,
|
||||
24, 24, 24, 24, 24, 24, 24, 24,
|
||||
24, 24, 24, 24, 24, 24, 24, 24 };
|
||||
const BYTE LL_deltaCode = 19;
|
||||
const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit(litLength) + LL_deltaCode : LL_Code[litLength];
|
||||
price += LL_bits[llCode] + seqStorePtr->log2litLengthSum - ZSTD_highbit(seqStorePtr->litLengthFreq[llCode]+1);
|
||||
}
|
||||
|
||||
return price;
|
||||
}
|
||||
@@ -122,25 +131,24 @@ FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* seqStorePtr, U32 litLength, co
|
||||
FORCE_INLINE U32 ZSTD_getPrice(seqStore_t* seqStorePtr, U32 litLength, const BYTE* literals, U32 offset, U32 matchLength)
|
||||
{
|
||||
/* offset */
|
||||
BYTE offCode = offset ? (BYTE)ZSTD_highbit(offset+1) + 1 : 0;
|
||||
U32 price = (offCode-1) + (!offCode) + seqStorePtr->log2offCodeSum - ZSTD_highbit(seqStorePtr->offCodeFreq[offCode]+1);
|
||||
BYTE offCode = (BYTE)ZSTD_highbit(offset+1);
|
||||
U32 price = offCode + seqStorePtr->log2offCodeSum - ZSTD_highbit(seqStorePtr->offCodeFreq[offCode]+1);
|
||||
|
||||
/* match Length */
|
||||
price += ((matchLength >= MaxML)<<3) + ((matchLength >= 255+MaxML)<<4) + ((matchLength>=(1<<15))<<3);
|
||||
if (matchLength >= MaxML) matchLength = MaxML;
|
||||
price += ZSTD_getLiteralPrice(seqStorePtr, litLength, literals) + seqStorePtr->log2matchLengthSum - ZSTD_highbit(seqStorePtr->matchLengthFreq[matchLength]+1);
|
||||
|
||||
#if ZSTD_OPT_DEBUG == 3
|
||||
switch (seqStorePtr->priceFunc) {
|
||||
default:
|
||||
case 0:
|
||||
return 1 + price + ((seqStorePtr->litSum>>5) / seqStorePtr->litLengthSum) + ((seqStorePtr->litSum<<1) / (seqStorePtr->litSum + seqStorePtr->matchSum));
|
||||
case 1:
|
||||
return 1 + price;
|
||||
{ static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
|
||||
16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
|
||||
32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37,
|
||||
38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39,
|
||||
40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
|
||||
41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41,
|
||||
42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
|
||||
42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 };
|
||||
const BYTE ML_deltaCode = 36;
|
||||
const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit(matchLength) + ML_deltaCode : ML_Code[matchLength];
|
||||
price += ML_bits[mlCode] + seqStorePtr->log2matchLengthSum - ZSTD_highbit(seqStorePtr->matchLengthFreq[mlCode]+1);
|
||||
}
|
||||
#else
|
||||
return price + seqStorePtr->factor;
|
||||
#endif
|
||||
|
||||
return price + ZSTD_getLiteralPrice(seqStorePtr, litLength, literals) + seqStorePtr->factor;
|
||||
}
|
||||
|
||||
|
||||
@@ -154,23 +162,39 @@ MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const B
|
||||
seqStorePtr->litFreq[literals[u]]++;
|
||||
|
||||
/* literal Length */
|
||||
seqStorePtr->litLengthSum++;
|
||||
if (litLength >= MaxLL)
|
||||
seqStorePtr->litLengthFreq[MaxLL]++;
|
||||
else
|
||||
seqStorePtr->litLengthFreq[litLength]++;
|
||||
{ static const BYTE LL_Code[64] = { 0, 1, 2, 3, 4, 5, 6, 7,
|
||||
8, 9, 10, 11, 12, 13, 14, 15,
|
||||
16, 16, 17, 17, 18, 18, 19, 19,
|
||||
20, 20, 20, 20, 21, 21, 21, 21,
|
||||
22, 22, 22, 22, 22, 22, 22, 22,
|
||||
23, 23, 23, 23, 23, 23, 23, 23,
|
||||
24, 24, 24, 24, 24, 24, 24, 24,
|
||||
24, 24, 24, 24, 24, 24, 24, 24 };
|
||||
const BYTE LL_deltaCode = 19;
|
||||
const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit(litLength) + LL_deltaCode : LL_Code[litLength];
|
||||
seqStorePtr->litLengthFreq[llCode]++;
|
||||
seqStorePtr->litLengthSum++;
|
||||
}
|
||||
|
||||
/* match offset */
|
||||
seqStorePtr->offCodeSum++;
|
||||
BYTE offCode = offset ? (BYTE)ZSTD_highbit(offset+1) + 1 : 0;
|
||||
BYTE offCode = (BYTE)ZSTD_highbit(offset+1);
|
||||
seqStorePtr->offCodeFreq[offCode]++;
|
||||
|
||||
/* match Length */
|
||||
seqStorePtr->matchLengthSum++;
|
||||
if (matchLength >= MaxML)
|
||||
seqStorePtr->matchLengthFreq[MaxML]++;
|
||||
else
|
||||
seqStorePtr->matchLengthFreq[matchLength]++;
|
||||
{ static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
|
||||
16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
|
||||
32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37,
|
||||
38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39,
|
||||
40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
|
||||
41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41,
|
||||
42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
|
||||
42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 };
|
||||
const BYTE ML_deltaCode = 36;
|
||||
const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit(matchLength) + ML_deltaCode : ML_Code[matchLength];
|
||||
seqStorePtr->matchLengthFreq[mlCode]++;
|
||||
seqStorePtr->matchLengthSum++;
|
||||
}
|
||||
|
||||
ZSTD_setLog2Prices(seqStorePtr);
|
||||
}
|
||||
@@ -196,17 +220,18 @@ MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const B
|
||||
static U32 ZSTD_insertAndFindFirstIndexHash3 (ZSTD_CCtx* zc, const BYTE* ip)
|
||||
{
|
||||
U32* const hashTable3 = zc->hashTable3;
|
||||
U32 const hashLog3 = zc->hashLog3;
|
||||
const BYTE* const base = zc->base;
|
||||
const U32 target = (U32)(ip - base);
|
||||
U32 idx = zc->nextToUpdate3;
|
||||
|
||||
while(idx < target) {
|
||||
hashTable3[ZSTD_hash3Ptr(base+idx, HASHLOG3)] = idx;
|
||||
hashTable3[ZSTD_hash3Ptr(base+idx, hashLog3)] = idx;
|
||||
idx++;
|
||||
}
|
||||
|
||||
zc->nextToUpdate3 = target;
|
||||
return hashTable3[ZSTD_hash3Ptr(ip, HASHLOG3)];
|
||||
return hashTable3[ZSTD_hash3Ptr(ip, hashLog3)];
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -63,6 +63,8 @@ extern "C" {
|
||||
#define ZSTD_CHAINLOG_MIN 4
|
||||
#define ZSTD_HASHLOG_MAX ZSTD_WINDOWLOG_MAX
|
||||
#define ZSTD_HASHLOG_MIN 12
|
||||
#define ZSTD_HASHLOG3_MAX 17
|
||||
#define ZSTD_HASHLOG3_MIN 15
|
||||
#define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1)
|
||||
#define ZSTD_SEARCHLOG_MIN 1
|
||||
#define ZSTD_SEARCHLENGTH_MAX 7
|
||||
|
||||
@@ -36,3 +36,8 @@ paramgrill
|
||||
# Default result files
|
||||
dictionary
|
||||
grillResults.txt
|
||||
_*
|
||||
|
||||
# Misc files
|
||||
*.bat
|
||||
fileTests.sh
|
||||
|
||||
+153
-63
@@ -44,32 +44,58 @@
|
||||
/* *************************************
|
||||
* Includes
|
||||
***************************************/
|
||||
#define _POSIX_C_SOURCE 199309L /* before <time.h> - needed for nanosleep() */
|
||||
#include <stdlib.h> /* malloc, free */
|
||||
#include <string.h> /* memset */
|
||||
#include <stdio.h> /* fprintf, fopen, ftello64 */
|
||||
#include <sys/types.h> /* stat64 */
|
||||
#include <sys/stat.h> /* stat64 */
|
||||
#include <time.h> /* clock_t, clock, CLOCKS_PER_SEC */
|
||||
#include <time.h> /* clock_t, nanosleep, clock, CLOCKS_PER_SEC */
|
||||
|
||||
/* sleep : posix - windows - others */
|
||||
#if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__)))
|
||||
# include <unistd.h> /* sleep */
|
||||
# include <sys/resource.h> /* setpriority */
|
||||
# include <unistd.h>
|
||||
# include <sys/resource.h> /* setpriority */
|
||||
# define BMK_sleep(s) sleep(s)
|
||||
# define HIGH_PRIORITY setpriority(PRIO_PROCESS, 0, -20)
|
||||
# define mili_sleep(mili) { struct timespec t; t.tv_sec=0; t.tv_nsec=mili*1000000ULL; nanosleep(&t, NULL); }
|
||||
# define SET_HIGH_PRIORITY setpriority(PRIO_PROCESS, 0, -20)
|
||||
#elif defined(_WIN32)
|
||||
# include <windows.h>
|
||||
# define BMK_sleep(s) Sleep(1000*s)
|
||||
# define HIGH_PRIORITY SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS);
|
||||
# define mili_sleep(mili) Sleep(mili)
|
||||
# define SET_HIGH_PRIORITY SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS)
|
||||
#else
|
||||
# define BMK_sleep(s) /* disabled */
|
||||
# define HIGH_PRIORITY
|
||||
# define mili_sleep(mili) /* disabled */
|
||||
# define SET_HIGH_PRIORITY /* disabled */
|
||||
#endif
|
||||
|
||||
#if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__)))
|
||||
typedef clock_t BMK_time_t;
|
||||
# define BMK_initTimer(ticksPerSecond) ticksPerSecond=0
|
||||
# define BMK_getTime(x) x = clock()
|
||||
# define BMK_getSpanTimeMicro(ticksPerSecond, clockStart, clockEnd) (1000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC)
|
||||
# define BMK_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) (1000000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC)
|
||||
#elif defined(_WIN32)
|
||||
typedef LARGE_INTEGER BMK_time_t;
|
||||
# define BMK_initTimer(x) if (!QueryPerformanceFrequency(&x)) { fprintf(stderr, "ERROR: QueryPerformance not present\n"); }
|
||||
# define BMK_getTime(x) QueryPerformanceCounter(&x)
|
||||
# define BMK_getSpanTimeMicro(ticksPerSecond, clockStart, clockEnd) (1000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart)
|
||||
# define BMK_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) (1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart)
|
||||
#else
|
||||
typedef int BMK_time_t;
|
||||
# define BMK_initTimer(ticksPerSecond) ticksPerSecond=0
|
||||
# define BMK_getTimeMicro(clockStart) clockStart=1
|
||||
# define BMK_getSpanTimeMicro(ticksPerSecond, clockStart, clockEnd) (TIMELOOP_S*1000000ULL+clockEnd-clockStart)
|
||||
# define BMK_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) (TIMELOOP_S*1000000000ULL+clockEnd-clockStart)
|
||||
#endif
|
||||
|
||||
#include "mem.h"
|
||||
#include "zstd_static.h"
|
||||
#include "zstd_internal.h" /* ZSTD_compressBegin_targetSrcSize */
|
||||
#include "datagen.h" /* RDG_genBuffer */
|
||||
#include "xxhash.h"
|
||||
#include "datagen.h" /* RDG_genBuffer */
|
||||
|
||||
|
||||
|
||||
/* *************************************
|
||||
@@ -87,6 +113,10 @@
|
||||
/* *************************************
|
||||
* Constants
|
||||
***************************************/
|
||||
#ifndef ZSTD_VERSION
|
||||
# define ZSTD_VERSION ""
|
||||
#endif
|
||||
|
||||
#define NBLOOPS 3
|
||||
#define TIMELOOP_S 1
|
||||
#define ACTIVEPERIOD_S 70
|
||||
@@ -97,7 +127,6 @@
|
||||
#define GB *(1U<<30)
|
||||
|
||||
static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31));
|
||||
#define DEFAULT_CHUNKSIZE (4 MB)
|
||||
|
||||
static U32 g_compressibilityDefault = 50;
|
||||
|
||||
@@ -132,29 +161,38 @@ static U32 g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result
|
||||
***************************************/
|
||||
static U32 g_nbIterations = NBLOOPS;
|
||||
static size_t g_blockSize = 0;
|
||||
int g_additionalParam = 0;
|
||||
|
||||
void BMK_setNotificationLevel(unsigned level) { g_displayLevel=level; }
|
||||
|
||||
void BMK_setAdditionalParam(int additionalParam) { g_additionalParam=additionalParam; }
|
||||
|
||||
void BMK_SetNbIterations(unsigned nbLoops)
|
||||
{
|
||||
g_nbIterations = nbLoops;
|
||||
DISPLAY("- %i iterations -\n", g_nbIterations);
|
||||
DISPLAYLEVEL(2, "- %i iterations -\n", g_nbIterations);
|
||||
}
|
||||
|
||||
void BMK_SetBlockSize(size_t blockSize)
|
||||
{
|
||||
g_blockSize = blockSize;
|
||||
DISPLAY("using blocks of size %u KB \n", (U32)(blockSize>>10));
|
||||
DISPLAYLEVEL(2, "using blocks of size %u KB \n", (U32)(blockSize>>10));
|
||||
}
|
||||
|
||||
|
||||
/* ********************************************************
|
||||
* Private functions
|
||||
**********************************************************/
|
||||
static clock_t BMK_clockSpan( clock_t clockStart )
|
||||
/* returns time span in microseconds */
|
||||
static U64 BMK_clockSpan( BMK_time_t clockStart, BMK_time_t ticksPerSecond )
|
||||
{
|
||||
return clock() - clockStart; /* works even if overflow, span limited to <= ~30mn */
|
||||
BMK_time_t clockEnd;
|
||||
|
||||
(void)ticksPerSecond;
|
||||
BMK_getTime(clockEnd);
|
||||
return BMK_getSpanTimeMicro(ticksPerSecond, clockStart, clockEnd);
|
||||
}
|
||||
|
||||
|
||||
static U64 BMK_getFileSize(const char* infilename)
|
||||
{
|
||||
int r;
|
||||
@@ -184,13 +222,22 @@ typedef struct
|
||||
size_t resSize;
|
||||
} blockParam_t;
|
||||
|
||||
typedef struct
|
||||
{
|
||||
double ratio;
|
||||
size_t cSize;
|
||||
double cSpeed;
|
||||
double dSpeed;
|
||||
} benchResult_t;
|
||||
|
||||
|
||||
#define MIN(a,b) ((a)<(b) ? (a) : (b))
|
||||
#define MAX(a,b) ((a)>(b) ? (a) : (b))
|
||||
|
||||
static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
||||
const char* displayName, int cLevel,
|
||||
const size_t* fileSizes, U32 nbFiles,
|
||||
const void* dictBuffer, size_t dictBufferSize)
|
||||
const void* dictBuffer, size_t dictBufferSize, benchResult_t *result)
|
||||
{
|
||||
size_t const blockSize = (g_blockSize ? g_blockSize : srcSize) + (!srcSize); /* avoid div by 0 */
|
||||
U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles;
|
||||
@@ -203,6 +250,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
||||
ZSTD_DCtx* refDCtx = ZSTD_createDCtx();
|
||||
ZSTD_DCtx* dctx = ZSTD_createDCtx();
|
||||
U32 nbBlocks;
|
||||
BMK_time_t ticksPerSecond;
|
||||
|
||||
/* checks */
|
||||
if (!compressedBuffer || !resultBuffer || !blockTable || !refCtx || !ctx || !refDCtx || !dctx)
|
||||
@@ -210,7 +258,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
||||
|
||||
/* init */
|
||||
if (strlen(displayName)>17) displayName += strlen(displayName)-17; /* can only display 17 characters */
|
||||
HIGH_PRIORITY;
|
||||
BMK_initTimer(ticksPerSecond);
|
||||
|
||||
/* Init blockTable data */
|
||||
{ const char* srcPtr = (const char*)srcBuffer;
|
||||
@@ -240,34 +288,40 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
||||
/* Bench */
|
||||
{ double fastestC = 100000000., fastestD = 100000000.;
|
||||
U64 const crcOrig = XXH64(srcBuffer, srcSize, 0);
|
||||
clock_t coolTime = clock();
|
||||
U64 crcCheck = 0;
|
||||
BMK_time_t coolTime;
|
||||
U32 testNb;
|
||||
size_t cSize = 0;
|
||||
double ratio = 0.;
|
||||
|
||||
DISPLAY("\r%79s\r", "");
|
||||
BMK_getTime(coolTime);
|
||||
DISPLAYLEVEL(2, "\r%79s\r", "");
|
||||
for (testNb = 1; testNb <= (g_nbIterations + !g_nbIterations); testNb++) {
|
||||
size_t cSize;
|
||||
double ratio = 0.;
|
||||
clock_t clockStart;
|
||||
clock_t const clockLoop = g_nbIterations ? TIMELOOP_S * CLOCKS_PER_SEC : 10;
|
||||
BMK_time_t clockStart, clockEnd;
|
||||
U64 clockLoop = g_nbIterations ? TIMELOOP_S*1000000ULL : 10;
|
||||
|
||||
/* overheat protection */
|
||||
if (BMK_clockSpan(coolTime) > ACTIVEPERIOD_S * CLOCKS_PER_SEC) {
|
||||
if (BMK_clockSpan(coolTime, ticksPerSecond) > ACTIVEPERIOD_S*1000000ULL) {
|
||||
DISPLAY("\rcooling down ... \r");
|
||||
BMK_sleep(COOLPERIOD_S);
|
||||
coolTime = clock();
|
||||
BMK_getTime(coolTime);
|
||||
}
|
||||
|
||||
/* Compression */
|
||||
DISPLAY("%2i-%-17.17s :%10u ->\r", testNb, displayName, (U32)srcSize);
|
||||
DISPLAYLEVEL(2, "%2i-%-17.17s :%10u ->\r", testNb, displayName, (U32)srcSize);
|
||||
memset(compressedBuffer, 0xE5, maxCompressedSize); /* warm up and erase result buffer */
|
||||
|
||||
clockStart = clock();
|
||||
while (clock() == clockStart);
|
||||
clockStart = clock();
|
||||
mili_sleep(1); /* give processor time to other processes */
|
||||
BMK_getTime(clockStart);
|
||||
do { BMK_getTime(clockEnd); }
|
||||
while (BMK_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) == 0);
|
||||
BMK_getTime(clockStart);
|
||||
|
||||
{ U32 nbLoops;
|
||||
for (nbLoops = 0 ; BMK_clockSpan(clockStart) < clockLoop ; nbLoops++) {
|
||||
for (nbLoops = 0 ; BMK_clockSpan(clockStart, ticksPerSecond) < clockLoop ; nbLoops++) {
|
||||
U32 blockNb;
|
||||
ZSTD_compressBegin_usingDict(refCtx, dictBuffer, dictBufferSize, cLevel);
|
||||
ZSTD_compressBegin_targetSrcSize(refCtx, dictBuffer, dictBufferSize, blockSize, cLevel);
|
||||
// ZSTD_compressBegin_usingDict(refCtx, dictBuffer, dictBufferSize, cLevel);
|
||||
for (blockNb=0; blockNb<nbBlocks; blockNb++) {
|
||||
size_t const rSize = ZSTD_compress_usingPreparedCCtx(ctx, refCtx,
|
||||
blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
|
||||
@@ -275,28 +329,30 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
||||
if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_compress_usingPreparedCCtx() failed : %s", ZSTD_getErrorName(rSize));
|
||||
blockTable[blockNb].cSize = rSize;
|
||||
} }
|
||||
{ clock_t const clockSpan = BMK_clockSpan(clockStart);
|
||||
{ U64 const clockSpan = BMK_clockSpan(clockStart, ticksPerSecond);
|
||||
if ((double)clockSpan < fastestC*nbLoops) fastestC = (double)clockSpan / nbLoops;
|
||||
} }
|
||||
|
||||
cSize = 0;
|
||||
{ U32 blockNb; for (blockNb=0; blockNb<nbBlocks; blockNb++) cSize += blockTable[blockNb].cSize; }
|
||||
ratio = (double)srcSize / (double)cSize;
|
||||
DISPLAY("%2i-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s\r",
|
||||
DISPLAYLEVEL(2, "%2i-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s\r",
|
||||
testNb, displayName, (U32)srcSize, (U32)cSize, ratio,
|
||||
(double)srcSize / 1000000. / (fastestC / CLOCKS_PER_SEC) );
|
||||
(double)srcSize / fastestC );
|
||||
|
||||
(void)fastestD; (void)crcOrig; /* unused when decompression disabled */
|
||||
#if 1
|
||||
/* Decompression */
|
||||
memset(resultBuffer, 0xD6, srcSize); /* warm result buffer */
|
||||
|
||||
clockStart = clock();
|
||||
while (clock() == clockStart);
|
||||
clockStart = clock();
|
||||
mili_sleep(1); /* give processor time to other processes */
|
||||
BMK_getTime(clockStart);
|
||||
do { BMK_getTime(clockEnd); }
|
||||
while (BMK_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) == 0);
|
||||
BMK_getTime(clockStart);
|
||||
|
||||
{ U32 nbLoops;
|
||||
for (nbLoops = 0 ; BMK_clockSpan(clockStart) < clockLoop ; nbLoops++) {
|
||||
for (nbLoops = 0 ; BMK_clockSpan(clockStart, ticksPerSecond) < clockLoop ; nbLoops++) {
|
||||
U32 blockNb;
|
||||
ZSTD_decompressBegin_usingDict(refDCtx, dictBuffer, dictBufferSize);
|
||||
for (blockNb=0; blockNb<nbBlocks; blockNb++) {
|
||||
@@ -306,22 +362,22 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
||||
if (ZSTD_isError(regenSize)) {
|
||||
DISPLAY("ZSTD_decompress_usingPreparedDCtx() failed on block %u : %s \n",
|
||||
blockNb, ZSTD_getErrorName(regenSize));
|
||||
clockStart -= clockLoop+1; /* force immediate test end */
|
||||
clockLoop = 0; /* force immediate test end */
|
||||
break;
|
||||
}
|
||||
blockTable[blockNb].resSize = regenSize;
|
||||
} }
|
||||
{ clock_t const clockSpan = BMK_clockSpan(clockStart);
|
||||
{ U64 const clockSpan = BMK_clockSpan(clockStart, ticksPerSecond);
|
||||
if ((double)clockSpan < fastestD*nbLoops) fastestD = (double)clockSpan / nbLoops;
|
||||
} }
|
||||
|
||||
DISPLAY("%2i-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r",
|
||||
DISPLAYLEVEL(2, "%2i-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r",
|
||||
testNb, displayName, (U32)srcSize, (U32)cSize, ratio,
|
||||
(double)srcSize / 1000000. / (fastestC / CLOCKS_PER_SEC),
|
||||
(double)srcSize / 1000000. / (fastestD / CLOCKS_PER_SEC) );
|
||||
(double)srcSize / fastestC,
|
||||
(double)srcSize / fastestD );
|
||||
|
||||
/* CRC Checking */
|
||||
{ U64 const crcCheck = XXH64(resultBuffer, srcSize, 0);
|
||||
{ crcCheck = XXH64(resultBuffer, srcSize, 0);
|
||||
if (crcOrig!=crcCheck) {
|
||||
size_t u;
|
||||
DISPLAY("!!! WARNING !!! %14s : Invalid Checksum : %x != %x \n", displayName, (unsigned)crcOrig, (unsigned)crcCheck);
|
||||
@@ -346,7 +402,14 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
||||
} } /* CRC Checking */
|
||||
#endif
|
||||
} /* for (testNb = 1; testNb <= (g_nbIterations + !g_nbIterations); testNb++) */
|
||||
DISPLAY("%2i#\n", cLevel);
|
||||
|
||||
if (crcOrig == crcCheck) {
|
||||
result->ratio = ratio;
|
||||
result->cSize = cSize;
|
||||
result->cSpeed = (double)srcSize / fastestC;
|
||||
result->dSpeed = (double)srcSize / fastestD;
|
||||
}
|
||||
DISPLAYLEVEL(2, "%2i#\n", cLevel);
|
||||
} /* Bench */
|
||||
|
||||
/* clean up */
|
||||
@@ -379,24 +442,51 @@ static size_t BMK_findMaxMem(U64 requiredMem)
|
||||
}
|
||||
|
||||
static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
|
||||
const char* displayName, int cLevel,
|
||||
const char* displayName, int cLevel, int cLevelLast,
|
||||
const size_t* fileSizes, unsigned nbFiles,
|
||||
const void* dictBuffer, size_t dictBufferSize)
|
||||
{
|
||||
if (cLevel < 0) { /* range mode : test all levels from 1 to l */
|
||||
int l;
|
||||
for (l=1; l <= -cLevel; l++) {
|
||||
BMK_benchMem(srcBuffer, benchedSize,
|
||||
displayName, l,
|
||||
fileSizes, nbFiles,
|
||||
dictBuffer, dictBufferSize);
|
||||
benchResult_t result, total;
|
||||
int l;
|
||||
|
||||
SET_HIGH_PRIORITY;
|
||||
|
||||
const char* pch = strrchr(displayName, '\\'); /* Windows */
|
||||
if (!pch) pch = strrchr(displayName, '/'); /* Linux */
|
||||
if (pch) displayName = pch+1;
|
||||
|
||||
memset(&result, 0, sizeof(result));
|
||||
memset(&total, 0, sizeof(total));
|
||||
|
||||
if (g_displayLevel == 1 && !g_additionalParam)
|
||||
DISPLAY("bench %s: input %u bytes, %i iterations, %u KB blocks\n", ZSTD_VERSION, (U32)benchedSize, g_nbIterations, (U32)(g_blockSize>>10));
|
||||
|
||||
if (cLevelLast < cLevel) cLevelLast = cLevel;
|
||||
|
||||
for (l=cLevel; l <= cLevelLast; l++) {
|
||||
BMK_benchMem(srcBuffer, benchedSize,
|
||||
displayName, l,
|
||||
fileSizes, nbFiles,
|
||||
dictBuffer, dictBufferSize, &result);
|
||||
if (g_displayLevel == 1) {
|
||||
if (g_additionalParam)
|
||||
DISPLAY("%-3i%11i (%5.3f) %6.1f MB/s %6.1f MB/s %s (param=%d)\n", -l, (int)result.cSize, result.ratio, result.cSpeed, result.dSpeed, displayName, g_additionalParam);
|
||||
else
|
||||
DISPLAY("%-3i%11i (%5.3f) %6.1f MB/s %6.1f MB/s %s\n", -l, (int)result.cSize, result.ratio, result.cSpeed, result.dSpeed, displayName);
|
||||
total.cSize += result.cSize;
|
||||
total.cSpeed += result.cSpeed;
|
||||
total.dSpeed += result.dSpeed;
|
||||
total.ratio += result.ratio;
|
||||
}
|
||||
return;
|
||||
}
|
||||
BMK_benchMem(srcBuffer, benchedSize,
|
||||
displayName, cLevel,
|
||||
fileSizes, nbFiles,
|
||||
dictBuffer, dictBufferSize);
|
||||
if (g_displayLevel == 1 && cLevelLast > cLevel)
|
||||
{
|
||||
total.cSize /= 1+cLevelLast-cLevel;
|
||||
total.cSpeed /= 1+cLevelLast-cLevel;
|
||||
total.dSpeed /= 1+cLevelLast-cLevel;
|
||||
total.ratio /= 1+cLevelLast-cLevel;
|
||||
DISPLAY("avg%11i (%5.3f) %6.1f MB/s %6.1f MB/s %s\n", (int)total.cSize, total.ratio, total.cSpeed, total.dSpeed, displayName);
|
||||
}
|
||||
}
|
||||
|
||||
static U64 BMK_getTotalFileSize(const char** fileNamesTable, unsigned nbFiles)
|
||||
@@ -433,7 +523,7 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize,
|
||||
}
|
||||
|
||||
static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles,
|
||||
const char* dictFileName, int cLevel)
|
||||
const char* dictFileName, int cLevel, int cLevelLast)
|
||||
{
|
||||
void* srcBuffer;
|
||||
size_t benchedSize;
|
||||
@@ -473,7 +563,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles,
|
||||
else displayName = fileNamesTable[0];
|
||||
|
||||
BMK_benchCLevel(srcBuffer, benchedSize,
|
||||
displayName, cLevel,
|
||||
displayName, cLevel, cLevelLast,
|
||||
fileSizes, nbFiles,
|
||||
dictBuffer, dictBufferSize);
|
||||
|
||||
@@ -484,7 +574,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles,
|
||||
}
|
||||
|
||||
|
||||
static void BMK_syntheticTest(int cLevel, double compressibility)
|
||||
static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility)
|
||||
{
|
||||
char name[20] = {0};
|
||||
size_t benchedSize = 10000000;
|
||||
@@ -498,7 +588,7 @@ static void BMK_syntheticTest(int cLevel, double compressibility)
|
||||
|
||||
/* Bench */
|
||||
snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100));
|
||||
BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, &benchedSize, 1, NULL, 0);
|
||||
BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, NULL, 0);
|
||||
|
||||
/* clean up */
|
||||
free(srcBuffer);
|
||||
@@ -506,14 +596,14 @@ static void BMK_syntheticTest(int cLevel, double compressibility)
|
||||
|
||||
|
||||
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
|
||||
const char* dictFileName, int cLevel)
|
||||
const char* dictFileName, int cLevel, int cLevelLast)
|
||||
{
|
||||
double const compressibility = (double)g_compressibilityDefault / 100;
|
||||
|
||||
if (nbFiles == 0)
|
||||
BMK_syntheticTest(cLevel, compressibility);
|
||||
BMK_syntheticTest(cLevel, cLevelLast, compressibility);
|
||||
else
|
||||
BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel);
|
||||
BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
+3
-2
@@ -27,10 +27,11 @@
|
||||
|
||||
/* Main function */
|
||||
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
|
||||
const char* dictFileName, int cLevel);
|
||||
const char* dictFileName, int cLevel, int cLevelLast);
|
||||
|
||||
/* Set Parameters */
|
||||
void BMK_SetNbIterations(unsigned nbLoops);
|
||||
void BMK_SetBlockSize(size_t blockSize);
|
||||
|
||||
void BMK_setAdditionalParam(int additionalParam);
|
||||
void BMK_setNotificationLevel(unsigned level);
|
||||
|
||||
|
||||
@@ -83,7 +83,6 @@
|
||||
|
||||
#define KNUTH 2654435761U
|
||||
#define MAX_MEM (1984 MB)
|
||||
#define DEFAULT_CHUNKSIZE (4<<20)
|
||||
|
||||
#define COMPRESSIBILITY_DEFAULT 0.50
|
||||
static const size_t g_sampleSize = 10000000;
|
||||
|
||||
@@ -100,7 +100,6 @@
|
||||
#define NB_LEVELS_TRACKED 30
|
||||
|
||||
static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31));
|
||||
#define DEFAULT_CHUNKSIZE (4<<20)
|
||||
|
||||
#define COMPRESSIBILITY_DEFAULT 0.50
|
||||
static const size_t sampleSize = 10000000;
|
||||
|
||||
+25
-9
@@ -143,9 +143,9 @@ static int usage_advanced(const char* programName)
|
||||
#ifndef ZSTD_NOBENCH
|
||||
DISPLAY( "Benchmark arguments :\n");
|
||||
DISPLAY( " -b# : benchmark file(s), using # compression level (default : 1) \n");
|
||||
DISPLAY( " -r# : test all compression levels from -bX to # (default: 1)\n");
|
||||
DISPLAY( " -i# : iteration loops [1-9](default : 3)\n");
|
||||
DISPLAY( " -B# : cut file into independent blocks of size # (default: no block)\n");
|
||||
DISPLAY( " -r# : test all compression levels from 1 to # (default: disabled)\n");
|
||||
#endif
|
||||
return 0;
|
||||
}
|
||||
@@ -182,19 +182,19 @@ int main(int argCount, const char** argv)
|
||||
nextArgumentIsOutFileName=0,
|
||||
nextArgumentIsMaxDict=0;
|
||||
unsigned cLevel = 1;
|
||||
unsigned cLevelLast = 1;
|
||||
const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); /* argCount >= 1 */
|
||||
unsigned filenameIdx = 0;
|
||||
const char* programName = argv[0];
|
||||
const char* outFileName = NULL;
|
||||
const char* dictFileName = NULL;
|
||||
char* dynNameSpace = NULL;
|
||||
int rangeBench = 1;
|
||||
unsigned maxDictSize = g_defaultMaxDictSize;
|
||||
unsigned dictCLevel = g_defaultDictCLevel;
|
||||
unsigned dictSelect = g_defaultSelectivityLevel;
|
||||
|
||||
/* init */
|
||||
(void)rangeBench; (void)dictCLevel; /* not used when ZSTD_NOBENCH / ZSTD_NODICT set */
|
||||
(void)cLevelLast; (void)dictCLevel; /* not used when ZSTD_NOBENCH / ZSTD_NODICT set */
|
||||
if (filenameTable==NULL) { DISPLAY("not enough memory\n"); exit(1); }
|
||||
displayOut = stderr;
|
||||
/* Pick out program name from path. Don't rely on stdlib because of conflicting behavior */
|
||||
@@ -235,7 +235,6 @@ int main(int argCount, const char** argv)
|
||||
argument++;
|
||||
|
||||
while (argument[0]!=0) {
|
||||
|
||||
/* compression Level */
|
||||
if ((*argument>='0') && (*argument<='9')) {
|
||||
cLevel = 0;
|
||||
@@ -294,6 +293,7 @@ int main(int argCount, const char** argv)
|
||||
argument++;
|
||||
while ((*argument >='0') && (*argument <='9'))
|
||||
iters *= 10, iters += *argument++ - '0';
|
||||
BMK_setNotificationLevel(displayLevel);
|
||||
BMK_SetNbIterations(iters);
|
||||
}
|
||||
break;
|
||||
@@ -307,14 +307,20 @@ int main(int argCount, const char** argv)
|
||||
if (*argument=='K') bSize<<=10, argument++; /* allows using KB notation */
|
||||
if (*argument=='M') bSize<<=20, argument++;
|
||||
if (*argument=='B') argument++;
|
||||
BMK_setNotificationLevel(displayLevel);
|
||||
BMK_SetBlockSize(bSize);
|
||||
}
|
||||
break;
|
||||
|
||||
/* range bench (benchmark only) */
|
||||
case 'r':
|
||||
rangeBench = -1;
|
||||
/* compression Level */
|
||||
argument++;
|
||||
if ((*argument>='0') && (*argument<='9')) {
|
||||
cLevelLast = 0;
|
||||
while ((*argument >= '0') && (*argument <= '9'))
|
||||
cLevelLast *= 10, cLevelLast += *argument++ - '0';
|
||||
}
|
||||
break;
|
||||
#endif /* ZSTD_NOBENCH */
|
||||
|
||||
@@ -325,9 +331,18 @@ int main(int argCount, const char** argv)
|
||||
dictSelect *= 10, dictSelect += *argument++ - '0';
|
||||
break;
|
||||
|
||||
/* Pause at the end (hidden option) */
|
||||
case 'p': main_pause=1; argument++; break;
|
||||
|
||||
/* Pause at the end (-p) or set an additional param (-p#) (hidden option) */
|
||||
case 'p': argument++;
|
||||
#ifndef ZSTD_NOBENCH
|
||||
if ((*argument>='0') && (*argument<='9')) {
|
||||
int additionalParam = 0;
|
||||
while ((*argument >= '0') && (*argument <= '9'))
|
||||
additionalParam *= 10, additionalParam += *argument++ - '0';
|
||||
BMK_setAdditionalParam(additionalParam);
|
||||
} else
|
||||
#endif
|
||||
main_pause=1;
|
||||
break;
|
||||
/* unknown command */
|
||||
default : CLEAN_RETURN(badusage(programName));
|
||||
}
|
||||
@@ -368,7 +383,8 @@ int main(int argCount, const char** argv)
|
||||
/* Check if benchmark is selected */
|
||||
if (bench) {
|
||||
#ifndef ZSTD_NOBENCH
|
||||
BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel*rangeBench);
|
||||
BMK_setNotificationLevel(displayLevel);
|
||||
BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast);
|
||||
#endif
|
||||
goto _end;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user