ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Changed Library source tree

Browse Source 
- no more zstdhc ; zstd.h is enough
- ZSTD_compress() now needs a compressionLevel
- separated zstd_compress.c and zstd_decompress.c
- updated zstdcli, fullbench, fuzzer with new API
This commit is contained in:
Yann Collet
2015-11-11 13:43:58 +01:00
parent 530918b409
commit 5be2dd25f2
12 changed files with 1425 additions and 523 deletions
Download Patch File Download Diff File Expand all files Collapse all files
programs/Makefile
+7 -7
View File
@@ -58,37 +58,37 @@ default: zstd
all: zstd zstd32 fullbench fullbench32 fuzzer fuzzer32 paramgrill datagen
zstd: $(ZSTDDIR)/zstd.c $(ZSTDDIR)/zstdhc.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
zstd: $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
$(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
xxhash.c bench.c fileio.c zstdcli.c legacy/fileio_legacy.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
zstd32: $(ZSTDDIR)/zstd.c $(ZSTDDIR)/zstdhc.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
zstd32: $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
$(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
xxhash.c bench.c fileio.c zstdcli.c legacy/fileio_legacy.c
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
fullbench : $(ZSTDDIR)/zstd.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
fullbench : $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
$(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
datagen.c fullbench.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
fullbench32: $(ZSTDDIR)/zstd.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
fullbench32: $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
$(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
datagen.c fullbench.c
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
fuzzer : $(ZSTDDIR)/zstd.c $(ZSTDDIR)/zstdhc.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
fuzzer : $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
$(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
datagen.c xxhash.c fuzzer.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
fuzzer32: $(ZSTDDIR)/zstd.c $(ZSTDDIR)/zstdhc.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
fuzzer32: $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
$(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
datagen.c xxhash.c fuzzer.c
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
paramgrill : $(ZSTDDIR)/zstdhc.c $(ZSTDDIR)/zstd.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
paramgrill : $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
$(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
datagen.c xxhash.c paramgrill.c
$(CC) $(FLAGS) $^ -lm -o $@$(EXT)
programs/bench.c
+3 -10
View File
@@ -61,7 +61,6 @@
#include "mem.h"
#include "zstd.h"
#include "zstdhc.h"
#include "xxhash.h"
@@ -231,12 +230,6 @@ typedef struct
typedef size_t (*compressor_t) (void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel);
static size_t local_compress_fast (void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel)
{
(void)compressionLevel;
return ZSTD_compress(dst, maxDstSize, src, srcSize);
}
#define MIN(a,b) ((a)<(b) ? (a) : (b))
static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, int cLevel)
@@ -247,7 +240,7 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, i
const size_t maxCompressedSize = (size_t)nbBlocks * ZSTD_compressBound(blockSize);
void* const compressedBuffer = malloc(maxCompressedSize);
void* const resultBuffer = malloc(srcSize);
const compressor_t compressor = (cLevel <= 1) ? local_compress_fast : ZSTD_HC_compress;
const compressor_t compressor = ZSTD_compress;
U64 crcOrig;
/* init */
@@ -413,7 +406,7 @@ static size_t BMK_findMaxMem(U64 requiredMem)
return (size_t)(requiredMem - step);
}
static int BMK_benchOneFile(char* inFileName, int cLevel)
static int BMK_benchOneFile(const char* inFileName, int cLevel)
{
FILE* inFile;
U64 inFileSize;
@@ -513,7 +506,7 @@ static int BMK_syntheticTest(int cLevel, double compressibility)
}
int BMK_benchFiles(char** fileNamesTable, unsigned nbFiles, unsigned cLevel)
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, unsigned cLevel)
{
double compressibility = (double)g_compressibilityDefault / 100;
programs/fileio.c
+7 -69
View File
@@ -68,7 +68,6 @@
#include "mem.h"
#include "fileio.h"
#include "zstd_static.h"
#include "zstdhc_static.h"
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
# include "zstd_legacy.h" /* legacy */
@@ -237,46 +236,6 @@ static U64 FIO_getFileSize(const char* infilename)
}
typedef void* (*FIO_createC) (void);
static void* local_ZSTD_createCCtx(void) { return (void*) ZSTD_createCCtx(); }
static void* local_ZSTD_HC_createCCtx(void) { return (void*) ZSTD_HC_createCCtx(); }
typedef size_t (*FIO_initC) (void* ctx, void* dst, size_t maxDstSize, int cLevel, U64 srcSizeHint);
static size_t local_ZSTD_compressBegin (void* ctx, void* dst, size_t maxDstSize, int cLevel, U64 srcSizeHint)
{
(void)cLevel; (void)srcSizeHint;
return ZSTD_compressBegin((ZSTD_CCtx*)ctx, dst, maxDstSize);
}
static size_t local_ZSTD_HC_compressBegin (void* ctx, void* dst, size_t maxDstSize, int cLevel, U64 srcSizeHint)
{
return ZSTD_HC_compressBegin((ZSTD_HC_CCtx*)ctx, dst, maxDstSize, cLevel, srcSizeHint);
}
typedef size_t (*FIO_continueC) (void* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
static size_t local_ZSTD_compressContinue (void* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
{
return ZSTD_compressContinue((ZSTD_CCtx*)ctx, dst, maxDstSize, src, srcSize);
}
static size_t local_ZSTD_HC_compressContinue (void* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
{
return ZSTD_HC_compressContinue((ZSTD_HC_CCtx*)ctx, dst, maxDstSize, src, srcSize);
}
typedef size_t (*FIO_endC) (void* ctx, void* dst, size_t maxDstSize);
static size_t local_ZSTD_compressEnd (void* ctx, void* dst, size_t maxDstSize)
{
return ZSTD_compressEnd((ZSTD_CCtx*)ctx, dst, maxDstSize);
}
static size_t local_ZSTD_HC_compressEnd (void* ctx, void* dst, size_t maxDstSize)
{
return ZSTD_HC_compressEnd((ZSTD_HC_CCtx*)ctx, dst, maxDstSize);
}
typedef void (*FIO_freeC) (void* ctx);
static void local_ZSTD_freeCCtx(void* ctx) { ZSTD_freeCCtx((ZSTD_CCtx*)ctx); }
static void local_ZSTD_HC_freeCCtx(void* ctx) { ZSTD_HC_freeCCtx((ZSTD_HC_CCtx*)ctx); }
unsigned long long FIO_compressFilename(const char* output_filename, const char* input_filename, int cLevel)
{
U64 filesize = 0;
@@ -291,35 +250,14 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
FILE* finput;
FILE* foutput;
size_t sizeCheck, cSize;
void* ctx;
FIO_createC createC=NULL;
FIO_initC initC=NULL;
FIO_continueC continueC = NULL;
FIO_endC endC = NULL;
FIO_freeC freeC = NULL;
ZSTD_CCtx* ctx;
/* Init */
if (cLevel <= 1)
{
createC = local_ZSTD_createCCtx;
initC = local_ZSTD_compressBegin;
continueC = local_ZSTD_compressContinue;
endC = local_ZSTD_compressEnd;
freeC = local_ZSTD_freeCCtx;
}
else
{
createC = local_ZSTD_HC_createCCtx;
initC = local_ZSTD_HC_compressBegin;
continueC = local_ZSTD_HC_compressContinue;
endC = local_ZSTD_HC_compressEnd;
freeC = local_ZSTD_HC_freeCCtx;
}
/* init */
FIO_getFileHandles(&finput, &foutput, input_filename, output_filename);
filesize = FIO_getFileSize(input_filename);
/* Allocate Memory */
ctx = createC();
ctx = ZSTD_createCCtx();
inBuff = (BYTE*)malloc(inBuffSize);
outBuff = (BYTE*)malloc(outBuffSize);
if (!inBuff || !outBuff || !ctx) EXM_THROW(21, "Allocation error : not enough memory");
@@ -327,7 +265,7 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
inEnd = inBuff + inBuffSize;
/* Write Frame Header */
cSize = initC(ctx, outBuff, outBuffSize, cLevel, filesize);
cSize = ZSTD_compressBegin(ctx, outBuff, outBuffSize, cLevel, filesize);
if (ZSTD_isError(cSize)) EXM_THROW(22, "Compression error : cannot create frame header");
sizeCheck = fwrite(outBuff, 1, cSize, foutput);
@@ -348,7 +286,7 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
DISPLAYUPDATE(2, "\rRead : %u MB ", (U32)(filesize>>20));
/* Compress Block */
cSize = continueC(ctx, outBuff, outBuffSize, inSlot, inSize);
cSize = ZSTD_compressContinue(ctx, outBuff, outBuffSize, inSlot, inSize);
if (ZSTD_isError(cSize))
EXM_THROW(24, "Compression error : %s ", ZSTD_getErrorName(cSize));
@@ -362,7 +300,7 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
}
/* End of Frame */
cSize = endC(ctx, outBuff, outBuffSize);
cSize = ZSTD_compressEnd(ctx, outBuff, outBuffSize);
if (ZSTD_isError(cSize)) EXM_THROW(26, "Compression error : cannot create frame end");
sizeCheck = fwrite(outBuff, 1, cSize, foutput);
@@ -377,7 +315,7 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
/* clean */
free(inBuff);
free(outBuff);
freeC(ctx);
ZSTD_freeCCtx(ctx);
fclose(finput);
if (fclose(foutput)) EXM_THROW(28, "Write error : cannot properly close %s", output_filename);
programs/fullbench.c
+5 -5
View File
@@ -217,7 +217,7 @@ extern size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d
size_t local_ZSTD_compress(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
{
(void)buff2;
return ZSTD_compress(dst, dstSize, src, srcSize);
return ZSTD_compress(dst, dstSize, src, srcSize, 1);
}
size_t local_ZSTD_decompress(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
@@ -325,12 +325,12 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
switch(benchNb)
{
case 11:
g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize);
g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, 1);
break;
case 31: /* ZSTD_decodeLiteralsBlock */
{
blockProperties_t bp;
g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize);
g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1);
ZSTD_getcBlockSize(dstBuff+4, dstBuffSize, &bp); // Get first block type
if (bp.blockType != bt_compressed)
{
@@ -349,7 +349,7 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
const BYTE* ip = dstBuff;
const BYTE* iend;
size_t blockSize;
ZSTD_compress(dstBuff, dstBuffSize, src, srcSize);
ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1);
ip += 4; // Jump magic Number
blockSize = ZSTD_getcBlockSize(ip, dstBuffSize, &bp); // Get first block type
if (bp.blockType != bt_compressed)
@@ -380,7 +380,7 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
case 102: /* local_decodeLiteralsForward */
{
blockProperties_t bp;
ZSTD_compress(dstBuff, dstBuffSize, src, srcSize);
ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1);
g_cSize = ZSTD_getcBlockSize(dstBuff+7, dstBuffSize, &bp);
memcpy(buff2, dstBuff+10, g_cSize);
//srcSize = benchFunction(dstBuff, dstBuffSize, buff2, src, srcSize); // real speed
programs/fuzzer.c
+10 -11
View File
@@ -47,7 +47,6 @@
#include <sys/timeb.h> /* timeb */
#include <string.h> /* strcmp */
#include "zstd_static.h"
#include "zstdhc_static.h"
#include "datagen.h" /* RDG_genBuffer */
#include "xxhash.h" /* XXH64 */
#include "mem.h"
@@ -159,7 +158,7 @@ static int basicUnitTests(U32 seed, double compressibility)
/* Basic tests */
DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
result = ZSTD_compress(compressedBuffer, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH), CNBuffer, COMPRESSIBLE_NOISE_LENGTH);
result = ZSTD_compress(compressedBuffer, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH), CNBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
if (ZSTD_isError(result)) goto _output_error;
cSize = result;
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
@@ -213,7 +212,7 @@ static int basicUnitTests(U32 seed, double compressibility)
sampleSize += 256 KB - 1;
RDG_genBuffer((char*)CNBuffer+sampleSize, 96 KB, compressibility, 0., randState);
sampleSize += 96 KB;
cSize = ZSTD_compress(compressedBuffer, ZSTD_compressBound(sampleSize), CNBuffer, sampleSize);
cSize = ZSTD_compress(compressedBuffer, ZSTD_compressBound(sampleSize), CNBuffer, sampleSize, 1);
if (ZSTD_isError(cSize)) goto _output_error;
result = ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize);
if (ZSTD_isError(result)) goto _output_error;
@@ -265,11 +264,11 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
U32 testNb = 0;
U32 coreSeed = seed, lseed = 0;
ZSTD_CCtx* ctx;
ZSTD_HC_CCtx* hcctx;
ZSTD_CCtx* hcctx;
/* allocation */
ctx = ZSTD_createCCtx();
hcctx = ZSTD_HC_createCCtx();
hcctx = ZSTD_createCCtx();
cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
@@ -332,8 +331,8 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
#define MAX(a,b) ((a)>(b)?(a):(b))
cLevelMod = MAX(1, 38 - (int)(MAX(9, sampleSizeLog) * 2)); /* use high compression levels with small samples, for speed */
cLevel = (FUZ_rand(&lseed) % cLevelMod) +1;
cSize = ZSTD_HC_compressCCtx(hcctx, cBuffer, cBufferSize, srcBuffer + sampleStart, sampleSize, cLevel);
CHECK(ZSTD_isError(cSize), "ZSTD_HC_compressCCtx failed");
cSize = ZSTD_compressCCtx(hcctx, cBuffer, cBufferSize, srcBuffer + sampleStart, sampleSize, cLevel);
CHECK(ZSTD_isError(cSize), "ZSTD_compressCCtx failed");
/* compression failure test : too small dest buffer */
if (cSize > 3)
@@ -344,10 +343,10 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
static const U32 endMark = 0x4DC2B1A9;
U32 endCheck;
memcpy(dstBuffer+tooSmallSize, &endMark, 4);
errorCode = ZSTD_HC_compressCCtx(hcctx, dstBuffer, tooSmallSize, srcBuffer + sampleStart, sampleSize, cLevel);
CHECK(!ZSTD_isError(errorCode), "ZSTD_HC_compressCCtx should have failed ! (buffer too small : %u < %u)", (U32)tooSmallSize, (U32)cSize);
errorCode = ZSTD_compressCCtx(hcctx, dstBuffer, tooSmallSize, srcBuffer + sampleStart, sampleSize, cLevel);
CHECK(!ZSTD_isError(errorCode), "ZSTD_compressCCtx should have failed ! (buffer too small : %u < %u)", (U32)tooSmallSize, (U32)cSize);
memcpy(&endCheck, dstBuffer+tooSmallSize, 4);
CHECK(endCheck != endMark, "ZSTD_HC_compressCCtx : dst buffer overflow");
CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow");
}
/* successfull decompression tests*/
@@ -434,7 +433,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
_cleanup:
ZSTD_freeCCtx(ctx);
ZSTD_HC_freeCCtx(hcctx);
ZSTD_freeCCtx(hcctx);
free(cNoiseBuffer[0]);
free(cNoiseBuffer[1]);
free(cNoiseBuffer[2]);
programs/paramgrill.c
+56 -57
View File
@@ -62,8 +62,7 @@
#endif
#include "mem.h"
#include "zstdhc_static.h"
#include "zstd.h"
#include "zstd_static.h"
#include "datagen.h"
#include "xxhash.h"
@@ -122,8 +121,8 @@ static U32 g_rand = 1;
static U32 g_singleRun = 0;
static U32 g_target = 0;
static U32 g_noSeed = 0;
static const ZSTD_HC_parameters* g_seedParams = ZSTD_HC_defaultParameters[0];
static ZSTD_HC_parameters g_params = { 0, 0, 0, 0, 0, ZSTD_HC_greedy };
static const ZSTD_parameters* g_seedParams = ZSTD_defaultParameters[0];
static ZSTD_parameters g_params = { 0, 0, 0, 0, 0, ZSTD_greedy };
void BMK_SetNbIterations(int nbLoops)
{
@@ -270,8 +269,8 @@ typedef struct
static size_t BMK_benchParam(BMK_result_t* resultPtr,
const void* srcBuffer, size_t srcSize,
ZSTD_HC_CCtx* ctx,
const ZSTD_HC_parameters params)
ZSTD_CCtx* ctx,
const ZSTD_parameters params)
{
const size_t blockSize = g_blockSize ? g_blockSize : srcSize;
const U32 nbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize);
@@ -284,7 +283,7 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
U32 Hlog = params.hashLog;
U32 Slog = params.searchLog;
U32 Slength = params.searchLength;
ZSTD_HC_strategy strat = params.strategy;
ZSTD_strategy strat = params.strategy;
char name[30] = { 0 };
U64 crcOrig;
@@ -358,7 +357,7 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
while (BMK_GetMilliSpan(milliTime) < TIMELOOP)
{
for (blockNb=0; blockNb<nbBlocks; blockNb++)
blockTable[blockNb].cSize = ZSTD_HC_compress_advanced(ctx,
blockTable[blockNb].cSize = ZSTD_compress_advanced(ctx,
blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
blockTable[blockNb].srcPtr, blockTable[blockNb].srcSize,
params);
@@ -429,13 +428,13 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
}
const char* g_stratName[] = { "ZSTD_HC_fast ",
"ZSTD_HC_greedy ",
"ZSTD_HC_lazy ",
"ZSTD_HC_lazy2 ",
"ZSTD_HC_btlazy2" };
const char* g_stratName[] = { "ZSTD_fast ",
"ZSTD_greedy ",
"ZSTD_lazy ",
"ZSTD_lazy2 ",
"ZSTD_btlazy2" };
static void BMK_printWinner(FILE* f, U32 cLevel, BMK_result_t result, ZSTD_HC_parameters params, size_t srcSize)
static void BMK_printWinner(FILE* f, U32 cLevel, BMK_result_t result, ZSTD_parameters params, size_t srcSize)
{
DISPLAY("\r%79s\r", "");
fprintf(f," {%3u,%3u,%3u,%3u,%3u, %s }, ",
@@ -447,11 +446,11 @@ static void BMK_printWinner(FILE* f, U32 cLevel, BMK_result_t result, ZSTD_HC_pa
}
static U32 g_cSpeedTarget[ZSTD_HC_MAX_CLEVEL+1] = { 0 };
static U32 g_cSpeedTarget[ZSTD_MAX_CLEVEL+1] = { 0 };
typedef struct {
BMK_result_t result;
ZSTD_HC_parameters params;
ZSTD_parameters params;
} winnerInfo_t;
static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSize)
@@ -459,11 +458,11 @@ static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSi
int cLevel;
fprintf(f, "\n /* Selected configurations : */ \n");
fprintf(f, "#define ZSTD_HC_MAX_CLEVEL %2u \n", ZSTD_HC_MAX_CLEVEL);
fprintf(f, "static const ZSTD_HC_parameters ZSTD_HC_defaultParameters[ZSTD_HC_MAX_CLEVEL+1] = {\n");
fprintf(f, "#define ZSTD_MAX_CLEVEL %2u \n", ZSTD_MAX_CLEVEL);
fprintf(f, "static const ZSTD_parameters ZSTD_defaultParameters[ZSTD_MAX_CLEVEL+1] = {\n");
fprintf(f, " /* W, C, H, S, L, strat */ \n");
for (cLevel=0; cLevel <= ZSTD_HC_MAX_CLEVEL; cLevel++)
for (cLevel=0; cLevel <= ZSTD_MAX_CLEVEL; cLevel++)
BMK_printWinner(f, cLevel, winners[cLevel].result, winners[cLevel].params, srcSize);
}
@@ -477,9 +476,9 @@ static void BMK_printWinners(FILE* f, const winnerInfo_t* winners, size_t srcSiz
}
static int BMK_seed(winnerInfo_t* winners, const ZSTD_HC_parameters params,
static int BMK_seed(winnerInfo_t* winners, const ZSTD_parameters params,
const void* srcBuffer, size_t srcSize,
ZSTD_HC_CCtx* ctx)
ZSTD_CCtx* ctx)
{
BMK_result_t testResult;
int better = 0;
@@ -487,7 +486,7 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_HC_parameters params,
BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, params);
for (cLevel = 1; cLevel <= ZSTD_HC_MAX_CLEVEL; cLevel++)
for (cLevel = 1; cLevel <= ZSTD_MAX_CLEVEL; cLevel++)
{
if (testResult.cSpeed < g_cSpeedTarget[cLevel])
continue; /* not fast enough for this level */
@@ -514,9 +513,9 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_HC_parameters params,
double O_DMemUsed_note = O_ratioNote * ( 40 + 9*cLevel) - log((double)O_DMemUsed);
size_t W_CMemUsed = (1 << params.windowLog) + 4 * (1 << params.hashLog) +
((params.strategy==ZSTD_HC_fast) ? 0 : 4 * (1 << params.contentLog));
((params.strategy==ZSTD_fast) ? 0 : 4 * (1 << params.contentLog));
size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + 4 * (1 << winners[cLevel].params.hashLog) +
((winners[cLevel].params.strategy==ZSTD_HC_fast) ? 0 : 4 * (1 << winners[cLevel].params.contentLog));
((winners[cLevel].params.strategy==ZSTD_fast) ? 0 : 4 * (1 << winners[cLevel].params.contentLog));
double W_CMemUsed_note = W_ratioNote * ( 50 + 13*cLevel) - log((double)W_CMemUsed);
double O_CMemUsed_note = O_ratioNote * ( 50 + 13*cLevel) - log((double)O_CMemUsed);
@@ -581,10 +580,10 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_HC_parameters params,
/* nullified useless params, to ensure count stats */
static ZSTD_HC_parameters* sanitizeParams(ZSTD_HC_parameters params)
static ZSTD_parameters* sanitizeParams(ZSTD_parameters params)
{
g_params = params;
if (params.strategy == ZSTD_HC_fast)
if (params.strategy == ZSTD_fast)
{
g_params.contentLog = 0;
g_params.searchLog = 0;
@@ -604,16 +603,16 @@ static BYTE g_alreadyTested[PARAMTABLESIZE] = {0}; /* init to zero */
#define MAX(a,b) ( (a) > (b) ? (a) : (b) )
static void playAround(FILE* f, winnerInfo_t* winners,
ZSTD_HC_parameters params,
ZSTD_parameters params,
const void* srcBuffer, size_t srcSize,
ZSTD_HC_CCtx* ctx)
ZSTD_CCtx* ctx)
{
int nbVariations = 0;
const int startTime = BMK_GetMilliStart();
while (BMK_GetMilliSpan(startTime) < g_maxVariationTime)
{
ZSTD_HC_parameters p = params;
ZSTD_parameters p = params;
U32 nbChanges = (FUZ_rand(&g_rand) & 3) + 1;
if (nbVariations++ > g_maxNbVariations) break;
@@ -643,16 +642,16 @@ static void playAround(FILE* f, winnerInfo_t* winners,
case 9:
p.searchLength--; break;
case 10:
p.strategy = (ZSTD_HC_strategy)(((U32)p.strategy)+1); break;
p.strategy = (ZSTD_strategy)(((U32)p.strategy)+1); break;
case 11:
p.strategy = (ZSTD_HC_strategy)(((U32)p.strategy)-1); break;
p.strategy = (ZSTD_strategy)(((U32)p.strategy)-1); break;
}
}
/* validate new conf */
{
ZSTD_HC_parameters saved = p;
ZSTD_HC_validateParams(&p, g_blockSize ? g_blockSize : srcSize);
ZSTD_parameters saved = p;
ZSTD_validateParams(&p, g_blockSize ? g_blockSize : srcSize);
if (memcmp(&p, &saved, sizeof(p))) continue; /* p was invalid */
}
@@ -675,19 +674,19 @@ static void playAround(FILE* f, winnerInfo_t* winners,
static void BMK_selectRandomStart(
FILE* f, winnerInfo_t* winners,
const void* srcBuffer, size_t srcSize,
ZSTD_HC_CCtx* ctx)
ZSTD_CCtx* ctx)
{
U32 id = (FUZ_rand(&g_rand) % (ZSTD_HC_MAX_CLEVEL+1));
U32 id = (FUZ_rand(&g_rand) % (ZSTD_MAX_CLEVEL+1));
if ((id==0) || (winners[id].params.windowLog==0))
{
/* totally random entry */
ZSTD_HC_parameters p;
p.contentLog = FUZ_rand(&g_rand) % (ZSTD_HC_CONTENTLOG_MAX+1 - ZSTD_HC_CONTENTLOG_MIN) + ZSTD_HC_CONTENTLOG_MIN;
p.hashLog = FUZ_rand(&g_rand) % (ZSTD_HC_HASHLOG_MAX+1 - ZSTD_HC_HASHLOG_MIN) + ZSTD_HC_HASHLOG_MIN;
p.searchLog = FUZ_rand(&g_rand) % (ZSTD_HC_SEARCHLOG_MAX+1 - ZSTD_HC_SEARCHLOG_MIN) + ZSTD_HC_SEARCHLOG_MIN;
p.windowLog = FUZ_rand(&g_rand) % (ZSTD_HC_WINDOWLOG_MAX+1 - ZSTD_HC_WINDOWLOG_MIN) + ZSTD_HC_WINDOWLOG_MIN;
p.searchLength=FUZ_rand(&g_rand) % (ZSTD_HC_SEARCHLENGTH_MAX+1 - ZSTD_HC_SEARCHLENGTH_MIN) + ZSTD_HC_SEARCHLENGTH_MIN;
p.strategy = (ZSTD_HC_strategy) (FUZ_rand(&g_rand) % (ZSTD_HC_btlazy2+1));
ZSTD_parameters p;
p.contentLog = FUZ_rand(&g_rand) % (ZSTD_CONTENTLOG_MAX+1 - ZSTD_CONTENTLOG_MIN) + ZSTD_CONTENTLOG_MIN;
p.hashLog = FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN) + ZSTD_HASHLOG_MIN;
p.searchLog = FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN) + ZSTD_SEARCHLOG_MIN;
p.windowLog = FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN) + ZSTD_WINDOWLOG_MIN;
p.searchLength=FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN) + ZSTD_SEARCHLENGTH_MIN;
p.strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btlazy2+1));
playAround(f, winners, p, srcBuffer, srcSize, ctx);
}
else
@@ -697,9 +696,9 @@ static void BMK_selectRandomStart(
static void BMK_benchMem(void* srcBuffer, size_t srcSize)
{
ZSTD_HC_CCtx* ctx = ZSTD_HC_createCCtx();
ZSTD_HC_parameters params;
winnerInfo_t winners[ZSTD_HC_MAX_CLEVEL+1];
ZSTD_CCtx* ctx = ZSTD_createCCtx();
ZSTD_parameters params;
winnerInfo_t winners[ZSTD_MAX_CLEVEL+1];
int i;
const char* rfName = "grillResults.txt";
FILE* f;
@@ -709,7 +708,7 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize)
if (g_singleRun)
{
BMK_result_t testResult;
ZSTD_HC_validateParams(&g_params, blockSize);
ZSTD_validateParams(&g_params, blockSize);
BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, g_params);
DISPLAY("\n");
return;
@@ -731,24 +730,24 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize)
params.contentLog = 1;
params.searchLog = 1;
params.searchLength = 7;
params.strategy = ZSTD_HC_fast;
ZSTD_HC_validateParams(&params, blockSize);
params.strategy = ZSTD_fast;
ZSTD_validateParams(&params, blockSize);
BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, params);
g_cSpeedTarget[1] = (testResult.cSpeed * 15) >> 4;
}
/* establish speed objectives (relative to level 1) */
for (i=2; i<=ZSTD_HC_MAX_CLEVEL; i++)
for (i=2; i<=ZSTD_MAX_CLEVEL; i++)
g_cSpeedTarget[i] = (g_cSpeedTarget[i-1] * 25) >> 5;
/* populate initial solution */
{
const int tableID = (blockSize > 128 KB);
const int maxSeeds = g_noSeed ? 1 : ZSTD_HC_MAX_CLEVEL;
g_seedParams = ZSTD_HC_defaultParameters[tableID];
const int maxSeeds = g_noSeed ? 1 : ZSTD_MAX_CLEVEL;
g_seedParams = ZSTD_defaultParameters[tableID];
for (i=1; i<=maxSeeds; i++)
{
const U32 btPlus = (params.strategy == ZSTD_HC_btlazy2);
const U32 btPlus = (params.strategy == ZSTD_btlazy2);
params = g_seedParams[i];
params.windowLog = MIN(srcLog, params.windowLog);
params.contentLog = MIN(params.windowLog+btPlus, params.contentLog);
@@ -775,7 +774,7 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize)
/* clean up*/
fclose(f);
ZSTD_HC_freeCCtx(ctx);
ZSTD_freeCCtx(ctx);
}
@@ -994,12 +993,12 @@ int main(int argc, char** argv)
g_params.searchLength *= 10, g_params.searchLength += *argument++ - '0';
continue;
case 't': /* strategy */
g_params.strategy = (ZSTD_HC_strategy)0;
g_params.strategy = (ZSTD_strategy)0;
argument++;
while ((*argument>= '0') && (*argument<='9'))
{
g_params.strategy = (ZSTD_HC_strategy)((U32)g_params.strategy *10);
g_params.strategy = (ZSTD_HC_strategy)((U32)g_params.strategy + *argument++ - '0');
g_params.strategy = (ZSTD_strategy)((U32)g_params.strategy *10);
g_params.strategy = (ZSTD_strategy)((U32)g_params.strategy + *argument++ - '0');
}
continue;
case 'L':
@@ -1009,7 +1008,7 @@ int main(int argc, char** argv)
while ((*argument>= '0') && (*argument<='9'))
cLevel *= 10, cLevel += *argument++ - '0';
if (cLevel < 1) cLevel = 1;
if (cLevel > ZSTD_HC_MAX_CLEVEL) cLevel = ZSTD_HC_MAX_CLEVEL;
if (cLevel > ZSTD_MAX_CLEVEL) cLevel = ZSTD_MAX_CLEVEL;
g_params = g_seedParams[cLevel];
continue;
}