ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1275 from GeorgeLu97/multilvl

Browse Source 
MultiLvl
This commit is contained in:
Yann Collet
2018-08-14 14:17:28 -07:00
committed by GitHub
parent a25d8000ec 76acba025d
commit e19c79900c
2 changed files with 766 additions and 443 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/README.md
+15 -5
View File
@@ -107,15 +107,25 @@ Full list of arguments
L# - level L# - level
--zstd= : Single run, parameter selection syntax same as zstdcli --zstd= : Single run, parameter selection syntax same as zstdcli
--optimize= : find parameters to maximize compression ratio given parameters --optimize= : find parameters to maximize compression ratio given parameters
Can use all --zstd= commands to constrain the type of solution found in addition to the following constraints Can use all --zstd= commands to constrain the type of solution found in addition to the following constraints
cSpeed= - Minimum compression speed cSpeed= : Minimum compression speed
dSpeed= - Minimum decompression speed dSpeed= : Minimum decompression speed
cMem= - compression memory cMem= : Maximum compression memory
lvl= - Automatically sets compression speed constraint to the speed of that level lvl= : Searches for solutions which are strictly better than that compression lvl in ratio and cSpeed,
stc= : When invoked with lvl=, represents percentage slack in ratio/cSpeed allowed for a solution to be considered (Default 99%)
: In normal operation, represents percentage slack in choosing viable starting strategy selection in choosing the default parameters
(Lower value will begin with stronger strategies) (Default 90%)
preferSpeed= / preferRatio=
: Only affects lvl = invocations. Defines value placed on compression speed or ratio
when determining overall winner (default 1 for both, higher = more valued).
tries= : Maximum number of random restarts on a single strategy before switching (Default 5)
Higher values will make optimizer run longer, more chances to find better solution.
--optimize= : same as -O with more verbose syntax
-P# : generated sample compressibility -P# : generated sample compressibility
-t# : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours )) -t# : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours ))
-v : Prints Benchmarking output -v : Prints Benchmarking output
-D : Next argument dictionary file -D : Next argument dictionary file
-s : Benchmark all files separately
``` ```
Any inputs afterwards are treated as files to benchmark. Any inputs afterwards are treated as files to benchmark.
tests/paramgrill.c
+751 -438
View File
@@ -74,10 +74,11 @@ typedef enum {
hlog_ind = 2, hlog_ind = 2,
slog_ind = 3, slog_ind = 3,
slen_ind = 4, slen_ind = 4,
tlen_ind = 5 tlen_ind = 5,
strt_ind = 6
} varInds_t; } varInds_t;
#define NUM_PARAMS 6 #define NUM_PARAMS 7
/* just don't use strategy as a param. */ /* just don't use strategy as a param. */
#undef ZSTD_WINDOWLOG_MAX #undef ZSTD_WINDOWLOG_MAX
@@ -92,12 +93,13 @@ typedef enum {
#define SLOG_RANGE (ZSTD_SEARCHLOG_MAX - ZSTD_SEARCHLOG_MIN + 1) #define SLOG_RANGE (ZSTD_SEARCHLOG_MAX - ZSTD_SEARCHLOG_MIN + 1)
#define SLEN_RANGE (ZSTD_SEARCHLENGTH_MAX - ZSTD_SEARCHLENGTH_MIN + 1) #define SLEN_RANGE (ZSTD_SEARCHLENGTH_MAX - ZSTD_SEARCHLENGTH_MIN + 1)
#define TLEN_RANGE 17 #define TLEN_RANGE 17
#define STRT_RANGE (ZSTD_btultra - ZSTD_fast + 1)
/* TLEN_RANGE picked manually */ /* TLEN_RANGE picked manually */
static const int rangetable[NUM_PARAMS] = { WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE }; static const int rangetable[NUM_PARAMS] = { WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE, STRT_RANGE };
static const U32 tlen_table[TLEN_RANGE] = { 0, 1, 2, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 256, 512, 999 }; static const U32 tlen_table[TLEN_RANGE] = { 0, 1, 2, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 256, 512, 999 };
/*-************************************ /*-************************************
* Benchmark Parameters * Benchmark Parameters/Global Variables
**************************************/ **************************************/
typedef BYTE U8; typedef BYTE U8;
@@ -108,17 +110,57 @@ static double g_compressibility = COMPRESSIBILITY_DEFAULT;
static U32 g_blockSize = 0; static U32 g_blockSize = 0;
static U32 g_rand = 1; static U32 g_rand = 1;
static U32 g_singleRun = 0; static U32 g_singleRun = 0;
static U32 g_optimizer = 0;
static U32 g_target = 0; static U32 g_target = 0;
static U32 g_noSeed = 0; static U32 g_noSeed = 0;
static ZSTD_compressionParameters g_params = { 0, 0, 0, 0, 0, 0, ZSTD_greedy }; static ZSTD_compressionParameters g_params; /* Initialized at the beginning of main w/ emptyParams() function */
static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */ static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */
typedef struct {
BMK_result_t result;
ZSTD_compressionParameters params;
} winnerInfo_t;
typedef struct {
U32 cSpeed; /* bytes / sec */
U32 dSpeed;
U32 cMem; /* bytes */
} constraint_t;
typedef struct winner_ll_node winner_ll_node;
struct winner_ll_node {
winnerInfo_t res;
winner_ll_node* next;
};
static winner_ll_node* g_winners; /* linked list sorted ascending by cSize & cSpeed */
static BMK_result_t g_lvltarget;
static int g_optmode = 0;
static U32 g_speedMultiplier = 1;
static U32 g_ratioMultiplier = 1;
/* g_mode? */
/* range 0 - 99, measure of how strict */
#define DEFAULT_STRICTNESS 99999
static U32 g_strictness = DEFAULT_STRICTNESS;
void BMK_SetNbIterations(int nbLoops) void BMK_SetNbIterations(int nbLoops)
{ {
g_nbIterations = nbLoops; g_nbIterations = nbLoops;
DISPLAY("- %u iterations -\n", g_nbIterations); DISPLAY("- %u iterations -\n", g_nbIterations);
} }
/*
* Additional Global Variables (Defined Above Use)
* g_stratName
* g_level_constraint
* g_alreadyTested
* g_maxTries
*/
/*-******************************************************* /*-*******************************************************
* Private functions * Private functions
*********************************************************/ *********************************************************/
@@ -134,14 +176,14 @@ static size_t BMK_findMaxMem(U64 requiredMem)
requiredMem = (((requiredMem >> 26) + 1) << 26); requiredMem = (((requiredMem >> 26) + 1) << 26);
if (requiredMem > maxMemory) requiredMem = maxMemory; if (requiredMem > maxMemory) requiredMem = maxMemory;
requiredMem += 2*step; requiredMem += 2 * step;
while (!testmem) { while (!testmem && requiredMem > 0) {
requiredMem -= step;
testmem = malloc ((size_t)requiredMem); testmem = malloc ((size_t)requiredMem);
requiredMem -= step;
} }
free (testmem); free (testmem);
return (size_t) (requiredMem - step); return (size_t) requiredMem;
} }
@@ -198,12 +240,6 @@ static void findClockGranularity(void) {
DEBUGOUTPUT("Granularity: %llu\n", (unsigned long long)g_clockGranularity); DEBUGOUTPUT("Granularity: %llu\n", (unsigned long long)g_clockGranularity);
} }
typedef struct {
U32 cSpeed; /* bytes / sec */
U32 dSpeed;
U32 cMem; /* bytes */
} constraint_t;
#define CLAMPCHECK(val,min,max) { \ #define CLAMPCHECK(val,min,max) { \
if (val && (((val)<(min)) | ((val)>(max)))) { \ if (val && (((val)<(min)) | ((val)>(max)))) { \
DISPLAY("INVALID PARAMETER CONSTRAINTS\n"); \ DISPLAY("INVALID PARAMETER CONSTRAINTS\n"); \
@@ -247,7 +283,7 @@ static void BMK_translateAdvancedParams(const ZSTD_compressionParameters params)
/* checks results are feasible */ /* checks results are feasible */
static int feasible(const BMK_result_t results, const constraint_t target) { static int feasible(const BMK_result_t results, const constraint_t target) {
return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.cMem); return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.cMem) && (!g_optmode || results.cSize <= g_lvltarget.cSize);
} }
/* hill climbing value for part 1 */ /* hill climbing value for part 1 */
@@ -267,24 +303,37 @@ static double resultScore(const BMK_result_t res, const size_t srcSize, const co
ret = (MIN(1, cs) + MIN(1, ds) + MIN(1, cm))*r1 + rt * rtr + ret = (MIN(1, cs) + MIN(1, ds) + MIN(1, cm))*r1 + rt * rtr +
(MAX(0, log(cs))+ MAX(0, log(ds))+ MAX(0, log(cm))) * r2; (MAX(0, log(cs))+ MAX(0, log(ds))+ MAX(0, log(cm))) * r2;
return ret; return ret;
} }
/* calculates normalized squared euclidean distance of result1 if it is in the first quadrant relative to lvlRes */
static double resultDistLvl(const BMK_result_t result1, const BMK_result_t lvlRes) {
double normalizedCSpeedGain1 = (result1.cSpeed / lvlRes.cSpeed) - 1;
double normalizedRatioGain1 = ((double)lvlRes.cSize / result1.cSize) - 1;
if(normalizedRatioGain1 < 0 || normalizedCSpeedGain1 < 0) {
return 0.0;
}
return normalizedRatioGain1 * g_ratioMultiplier + normalizedCSpeedGain1 * g_speedMultiplier;
}
/* return true if r2 strictly better than r1 */ /* return true if r2 strictly better than r1 */
static int compareResultLT(const BMK_result_t result1, const BMK_result_t result2, const constraint_t target, size_t srcSize) { static int compareResultLT(const BMK_result_t result1, const BMK_result_t result2, const constraint_t target, size_t srcSize) {
if(feasible(result1, target) && feasible(result2, target)) { if(feasible(result1, target) && feasible(result2, target)) {
return (result1.cSize > result2.cSize) || (result1.cSize == result2.cSize && result2.cSpeed > result1.cSpeed) if(g_optmode) {
|| (result1.cSize == result2.cSize && result2.cSpeed == result1.cSpeed && result2.dSpeed > result1.dSpeed); return resultDistLvl(result1, g_lvltarget) < resultDistLvl(result2, g_lvltarget);
} else {
return (result1.cSize > result2.cSize) || (result1.cSize == result2.cSize && result2.cSpeed > result1.cSpeed)
|| (result1.cSize == result2.cSize && result2.cSpeed == result1.cSpeed && result2.dSpeed > result1.dSpeed);
}
} }
return feasible(result2, target) || (!feasible(result1, target) && (resultScore(result1, srcSize, target) < resultScore(result2, srcSize, target))); return feasible(result2, target) || (!feasible(result1, target) && (resultScore(result1, srcSize, target) < resultScore(result2, srcSize, target)));
} }
/* factor sort of arbitrary */
static constraint_t relaxTarget(constraint_t target) { static constraint_t relaxTarget(constraint_t target) {
target.cMem = (U32)-1; target.cMem = (U32)-1;
target.cSpeed *= 0.9; target.cSpeed *= ((double)g_strictness) / 100;
target.dSpeed *= 0.9; target.dSpeed *= ((double)g_strictness) / 100;
return target; return target;
} }
@@ -297,22 +346,6 @@ const char* g_stratName[ZSTD_btultra+1] = {
"ZSTD_greedy ", "ZSTD_lazy ", "ZSTD_lazy2 ", "ZSTD_greedy ", "ZSTD_lazy ", "ZSTD_lazy2 ",
"ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra "}; "ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra "};
/* benchParam but only takes in one input buffer. */
static int
BMK_benchParam1(BMK_result_t* resultPtr,
const void* srcBuffer, size_t srcSize,
const ZSTD_compressionParameters cParams) {
BMK_return_t res = BMK_benchMem(srcBuffer,srcSize, &srcSize, 1, BASE_CLEVEL, &cParams, NULL, 0, 0, "File");
*resultPtr = res.result;
return res.error;
}
typedef struct {
BMK_result_t result;
ZSTD_compressionParameters params;
} winnerInfo_t;
static ZSTD_compressionParameters emptyParams(void) { static ZSTD_compressionParameters emptyParams(void) {
ZSTD_compressionParameters p = { 0, 0, 0, 0, 0, 0, (ZSTD_strategy)0 }; ZSTD_compressionParameters p = { 0, 0, 0, 0, 0, 0, (ZSTD_strategy)0 };
return p; return p;
@@ -348,15 +381,6 @@ typedef struct {
ZSTD_DCtx* dctx; ZSTD_DCtx* dctx;
} contexts_t; } contexts_t;
static int
BMK_benchParam(BMK_result_t* resultPtr,
const buffers_t buf, const contexts_t ctx,
const ZSTD_compressionParameters cParams) {
BMK_return_t res = BMK_benchMem(buf.srcPtrs[0], buf.srcSize, buf.srcSizes, (unsigned)buf.nbBlocks, 0, &cParams, ctx.dictBuffer, ctx.dictSize, 0, "Files");
*resultPtr = res.result;
return res.error;
}
/*-******************************************************* /*-*******************************************************
* From Paramgrill * From Paramgrill
*********************************************************/ *********************************************************/
@@ -482,6 +506,189 @@ static size_t local_defaultDecompress(
* From Paramgrill End * From Paramgrill End
*********************************************************/ *********************************************************/
static void freeBuffers(const buffers_t b) {
if(b.srcPtrs != NULL) {
free(b.srcBuffer);
}
free(b.srcPtrs);
free(b.srcSizes);
if(b.dstPtrs != NULL) {
free(b.dstPtrs[0]);
}
free(b.dstPtrs);
free(b.dstCapacities);
free(b.dstSizes);
if(b.resPtrs != NULL) {
free(b.resPtrs[0]);
}
free(b.resPtrs);
free(b.resSizes);
}
/* allocates buffer's arguments. returns success / failuere */
static int createBuffers(buffers_t* buff, const char* const * const fileNamesTable,
const size_t nbFiles)
{
size_t pos = 0;
size_t n;
U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, (U32)nbFiles);
const size_t benchedSize = MIN(BMK_findMaxMem(totalSizeToLoad * 3) / 3, totalSizeToLoad);
const size_t blockSize = g_blockSize ? g_blockSize : totalSizeToLoad;
U32 const maxNbBlocks = (U32) ((totalSizeToLoad + (blockSize-1)) / MAX(blockSize, 1)) + (U32)nbFiles;
U32 blockNb = 0;
if(!totalSizeToLoad || !benchedSize) {
DISPLAY("Nothing to Bench\n");
return 1;
}
buff->srcPtrs = (const void**)calloc(maxNbBlocks, sizeof(void*));
buff->srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
buff->dstPtrs = (void**)calloc(maxNbBlocks, sizeof(void*));
buff->dstCapacities = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
buff->dstSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
buff->resPtrs = (void**)calloc(maxNbBlocks, sizeof(void*));
buff->resSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
if(!buff->srcPtrs || !buff->srcSizes || !buff->dstPtrs || !buff->dstCapacities || !buff->dstSizes || !buff->resPtrs || !buff->resSizes) {
DISPLAY("alloc error\n");
freeBuffers(*buff);
return 1;
}
buff->srcBuffer = malloc(benchedSize);
buff->srcPtrs[0] = (const void*)buff->srcBuffer;
buff->dstPtrs[0] = malloc(ZSTD_compressBound(benchedSize) + (maxNbBlocks * 1024));
buff->resPtrs[0] = malloc(benchedSize);
if(!buff->srcPtrs[0] || !buff->dstPtrs[0] || !buff->resPtrs[0]) {
DISPLAY("alloc error\n");
freeBuffers(*buff);
return 1;
}
for(n = 0; n < nbFiles; n++) {
FILE* f;
U64 fileSize = UTIL_getFileSize(fileNamesTable[n]);
if (UTIL_isDirectory(fileNamesTable[n])) {
DISPLAY("Ignoring %s directory... \n", fileNamesTable[n]);
continue;
}
if (fileSize == UTIL_FILESIZE_UNKNOWN) {
DISPLAY("Cannot evaluate size of %s, ignoring ... \n", fileNamesTable[n]);
continue;
}
f = fopen(fileNamesTable[n], "rb");
if (f==NULL) {
DISPLAY("impossible to open file %s\n", fileNamesTable[n]);
freeBuffers(*buff);
fclose(f);
return 10;
}
DISPLAY("Loading %s... \r", fileNamesTable[n]);
if (fileSize + pos > benchedSize) fileSize = benchedSize - pos, n=nbFiles; /* buffer too small - stop after this file */
{
char* buffer = (char*)(buff->srcBuffer);
size_t const readSize = fread((buffer)+pos, 1, (size_t)fileSize, f);
size_t blocked = 0;
while(blocked < readSize) {
buff->srcPtrs[blockNb] = (const void*)((buffer) + (pos + blocked));
buff->srcSizes[blockNb] = blockSize;
blocked += blockSize;
blockNb++;
}
if(readSize > 0) { buff->srcSizes[blockNb - 1] = ((readSize - 1) % blockSize) + 1; }
if (readSize != (size_t)fileSize) {
DISPLAY("could not read %s", fileNamesTable[n]);
freeBuffers(*buff);
fclose(f);
return 1;
}
pos += readSize;
}
fclose(f);
}
if(!blockNb) {
DISPLAY("Failed to load any files\n");
freeBuffers(*buff);
return 1;
}
buff->dstCapacities[0] = ZSTD_compressBound(buff->srcSizes[0]);
buff->dstSizes[0] = buff->dstCapacities[0];
buff->resSizes[0] = buff->srcSizes[0];
for(n = 1; n < blockNb; n++) {
buff->dstPtrs[n] = ((char*)buff->dstPtrs[n-1]) + buff->dstCapacities[n-1];
buff->resPtrs[n] = ((char*)buff->resPtrs[n-1]) + buff->resSizes[n-1];
buff->dstCapacities[n] = ZSTD_compressBound(buff->srcSizes[n]);
buff->dstSizes[n] = buff->dstCapacities[n];
buff->resSizes[n] = buff->srcSizes[n];
}
buff->srcSize = pos;
buff->nbBlocks = blockNb;
if (pos == 0) { DISPLAY("\nno data to bench\n"); return 1; }
return 0;
}
static void freeContexts(const contexts_t ctx) {
free(ctx.dictBuffer);
ZSTD_freeCCtx(ctx.cctx);
ZSTD_freeDCtx(ctx.dctx);
}
static int createContexts(contexts_t* ctx, const char* dictFileName) {
FILE* f;
size_t readSize;
ctx->cctx = ZSTD_createCCtx();
ctx->dctx = ZSTD_createDCtx();
if(dictFileName == NULL) {
ctx->dictSize = 0;
ctx->dictBuffer = NULL;
return 0;
}
ctx->dictSize = UTIL_getFileSize(dictFileName);
ctx->dictBuffer = malloc(ctx->dictSize);
f = fopen(dictFileName, "rb");
if(!f) {
DISPLAY("unable to open file\n");
fclose(f);
freeContexts(*ctx);
return 1;
}
if(ctx->dictSize > 64 MB || !(ctx->dictBuffer)) {
DISPLAY("dictionary too large\n");
fclose(f);
freeContexts(*ctx);
return 1;
}
readSize = fread(ctx->dictBuffer, 1, ctx->dictSize, f);
if(readSize != ctx->dictSize) {
DISPLAY("unable to read file\n");
fclose(f);
freeContexts(*ctx);
return 1;
}
fclose(f);
return 0;
}
/* Replicate functionality of benchMemAdvanced, but with pre-split src / dst buffers */ /* Replicate functionality of benchMemAdvanced, but with pre-split src / dst buffers */
/* The purpose is so that sufficient information is returned so that a decompression call to benchMemInvertible is possible */ /* The purpose is so that sufficient information is returned so that a decompression call to benchMemInvertible is possible */
/* BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); */ /* BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); */
@@ -629,14 +836,141 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t
return results; return results;
} }
static int BMK_benchParam(BMK_result_t* resultPtr,
buffers_t buf, contexts_t ctx,
const ZSTD_compressionParameters cParams) {
BMK_return_t res = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_both, BMK_timeMode, 3);
*resultPtr = res.result;
return res.error;
}
/* comparison function: */
/* strictly better, strictly worse, equal, speed-side adv, size-side adv */
//Maybe use compress_only for benchmark first run?
#define WORSE_RESULT 0
#define BETTER_RESULT 1
#define ERROR_RESULT 2
#define SPEED_RESULT 4
#define SIZE_RESULT 5
/* maybe have epsilon-eq to limit table size? */
static int speedSizeCompare(BMK_result_t r1, BMK_result_t r2) {
if(r1.cSpeed < r2.cSpeed) {
if(r1.cSize >= r2.cSize) {
return BETTER_RESULT;
}
return SPEED_RESULT; /* r2 is smaller but not faster. */
} else {
if(r1.cSize <= r2.cSize) {
return WORSE_RESULT;
}
return SIZE_RESULT; /* r2 is faster but not smaller */
}
}
/* 0 for insertion, 1 for no insert */
/* maintain invariant speedSizeCompare(n, n->next) = SPEED_RESULT */
static int insertWinner(winnerInfo_t w, constraint_t targetConstraints) {
BMK_result_t r = w.result;
winner_ll_node* cur_node = g_winners;
/* first node to insert */
if(!feasible(r, targetConstraints)) {
return 1;
}
if(g_winners == NULL) {
winner_ll_node* first_node = malloc(sizeof(winner_ll_node));
if(first_node == NULL) {
return 1;
}
first_node->next = NULL;
first_node->res = w;
g_winners = first_node;
return 0;
}
while(cur_node->next != NULL) {
switch(speedSizeCompare(cur_node->res.result, r)) {
case WORSE_RESULT:
{
return 1; /* never insert if better */
}
case BETTER_RESULT:
{
winner_ll_node* tmp;
cur_node->res = cur_node->next->res;
tmp = cur_node->next;
cur_node->next = cur_node->next->next;
free(tmp);
break;
}
case SIZE_RESULT:
{
cur_node = cur_node->next;
break;
}
case SPEED_RESULT: /* insert after first size result, then return */
{
winner_ll_node* newnode = malloc(sizeof(winner_ll_node));
if(newnode == NULL) {
return 1;
}
newnode->res = cur_node->res;
cur_node->res = w;
newnode->next = cur_node->next;
cur_node->next = newnode;
return 0;
}
}
}
assert(cur_node->next == NULL);
switch(speedSizeCompare(cur_node->res.result, r)) {
case WORSE_RESULT:
{
return 1; /* never insert if better */
}
case BETTER_RESULT:
{
cur_node->res = w;
return 0;
}
case SIZE_RESULT:
{
winner_ll_node* newnode = malloc(sizeof(winner_ll_node));
if(newnode == NULL) {
return 1;
}
newnode->res = w;
newnode->next = NULL;
cur_node->next = newnode;
return 0;
}
case SPEED_RESULT: /* insert before first size result, then return */
{
winner_ll_node* newnode = malloc(sizeof(winner_ll_node));
if(newnode == NULL) {
return 1;
}
newnode->res = cur_node->res;
cur_node->res = w;
newnode->next = cur_node->next;
cur_node->next = newnode;
return 0;
}
default:
return 1;
}
}
/* Writes to f the results of a parameter benchmark */
/* when used with --optimize, will only print results better than previously discovered */
static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result, const ZSTD_compressionParameters params, const size_t srcSize) static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result, const ZSTD_compressionParameters params, const size_t srcSize)
{ {
char lvlstr[15] = "Custom Level"; char lvlstr[15] = "Custom Level";
const U64 time = UTIL_clockSpanNano(g_time);
const U64 minutes = time / (60ULL * TIMELOOP_NANOSEC);
DISPLAY("\r%79s\r", ""); fprintf(f, "\r%79s\r", "");
fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ",
params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength,
@@ -648,9 +982,13 @@ static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result
fprintf(f, fprintf(f,
"/* %s */ /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */", "/* %s */ /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */",
lvlstr, (double)srcSize / result.cSize, (double)result.cSpeed / (1 << 20), (double)result.dSpeed / (1 << 20)); lvlstr, (double)srcSize / result.cSize, (double)result.cSpeed / (1 MB), (double)result.dSpeed / (1 MB));
if(TIMED) { fprintf(f, " - %1lu:%2lu:%05.2f", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); } if(TIMED) {
const U64 time = UTIL_clockSpanNano(g_time);
const U64 minutes = time / (60ULL * TIMELOOP_NANOSEC);
fprintf(f, " - %1lu:%2lu:%05.2f", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC);
}
fprintf(f, "\n"); fprintf(f, "\n");
} }
@@ -658,7 +996,7 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res
{ {
/* global winner used for constraints */ /* global winner used for constraints */
static winnerInfo_t g_winner = { { 0, 0, (size_t)-1, (size_t)-1 } , { 0, 0, 0, 0, 0, 0, ZSTD_fast } }; static winnerInfo_t g_winner = { { 0, 0, (size_t)-1, (size_t)-1 } , { 0, 0, 0, 0, 0, 0, ZSTD_fast } };
if(DEBUG || compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { if(DEBUG || compareResultLT(g_winner.result, result, targetConstraints, srcSize)) {
if(DEBUG && compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { if(DEBUG && compareResultLT(g_winner.result, result, targetConstraints, srcSize)) {
DISPLAY("New Winner: \n"); DISPLAY("New Winner: \n");
@@ -672,6 +1010,54 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res
g_winner.params = params; g_winner.params = params;
} }
} }
//prints out tradeoff table if using lvloptimize
if(g_optmode && g_optimizer) {
winnerInfo_t w;
winner_ll_node* n;
w.result = result;
w.params = params;
insertWinner(w, targetConstraints);
if(!DEBUG) { fprintf(f, "\033c"); }
fprintf(f, "\n");
/* the table */
fprintf(f, "================================\n");
for(n = g_winners; n != NULL; n = n->next) {
fprintf(f, "\r%79s\r", "");
fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ",
n->res.params.windowLog, n->res.params.chainLog, n->res.params.hashLog, n->res.params.searchLog, n->res.params.searchLength,
n->res.params.targetLength, g_stratName[(U32)(n->res.params.strategy)]);
fprintf(f,
" /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n",
(double)srcSize / n->res.result.cSize, (double)n->res.result.cSpeed / (1 MB), (double)n->res.result.dSpeed / (1 MB));
}
fprintf(f, "================================\n");
fprintf(f, "Level Bounds: R: > %.3f AND C: < %.1f MB/s \n\n",
(double)srcSize / g_lvltarget.cSize, (double)g_lvltarget.cSpeed / (1 MB));
fprintf(f, "Overall Winner: \n");
fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ",
g_winner.params.windowLog, g_winner.params.chainLog, g_winner.params.hashLog, g_winner.params.searchLog, g_winner.params.searchLength,
g_winner.params.targetLength, g_stratName[(U32)(g_winner.params.strategy)]);
fprintf(f,
" /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n",
(double)srcSize / g_winner.result.cSize, (double)g_winner.result.cSpeed / (1 MB), (double)g_winner.result.dSpeed / (1 MB));
BMK_translateAdvancedParams(g_winner.params);
fprintf(f, "Latest BMK: \n");
fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ",
params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength,
params.targetLength, g_stratName[(U32)(params.strategy)]);
fprintf(f,
" /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n",
(double)srcSize / result.cSize, (double)result.cSpeed / (1 MB), (double)result.dSpeed / (1 MB));
}
} }
static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSize) static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSize)
@@ -723,14 +1109,14 @@ static void BMK_init_level_constraints(int bytePerSec_level1)
} } } }
} }
static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters params, static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters params,
const void* srcBuffer, size_t srcSize) buffers_t buf, contexts_t ctx)
{ {
BMK_result_t testResult; BMK_result_t testResult;
int better = 0; int better = 0;
int cLevel; int cLevel;
BMK_benchParam1(&testResult, srcBuffer, srcSize, params); BMK_benchParam(&testResult, buf, ctx, params);
for (cLevel = 1; cLevel <= NB_LEVELS_TRACKED; cLevel++) { for (cLevel = 1; cLevel <= NB_LEVELS_TRACKED; cLevel++) {
@@ -746,15 +1132,15 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para
/* first solution for this cLevel */ /* first solution for this cLevel */
winners[cLevel].result = testResult; winners[cLevel].result = testResult;
winners[cLevel].params = params; winners[cLevel].params = params;
BMK_printWinner(stdout, cLevel, testResult, params, srcSize); BMK_printWinner(stdout, cLevel, testResult, params, buf.srcSize);
better = 1; better = 1;
continue; continue;
} }
if ((double)testResult.cSize <= ((double)winners[cLevel].result.cSize * (1. + (0.02 / cLevel))) ) { if ((double)testResult.cSize <= ((double)winners[cLevel].result.cSize * (1. + (0.02 / cLevel))) ) {
/* Validate solution is "good enough" */ /* Validate solution is "good enough" */
double W_ratio = (double)srcSize / testResult.cSize; double W_ratio = (double)buf.srcSize / testResult.cSize;
double O_ratio = (double)srcSize / winners[cLevel].result.cSize; double O_ratio = (double)buf.srcSize / winners[cLevel].result.cSize;
double W_ratioNote = log (W_ratio); double W_ratioNote = log (W_ratio);
double O_ratioNote = log (O_ratio); double O_ratioNote = log (O_ratio);
size_t W_DMemUsed = (1 << params.windowLog) + (16 KB); size_t W_DMemUsed = (1 << params.windowLog) + (16 KB);
@@ -811,7 +1197,7 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para
winners[cLevel].result = testResult; winners[cLevel].result = testResult;
winners[cLevel].params = params; winners[cLevel].params = params;
BMK_printWinner(stdout, cLevel, testResult, params, srcSize); BMK_printWinner(stdout, cLevel, testResult, params, buf.srcSize);
better = 1; better = 1;
} } } }
@@ -846,8 +1232,10 @@ static int sanitizeVarArray(varInds_t* varNew, const int varLength, const varInd
int i, j = 0; int i, j = 0;
for(i = 0; i < varLength; i++) { for(i = 0; i < varLength; i++) {
if( !((varArray[i] == clog_ind && strat == ZSTD_fast) if( !((varArray[i] == clog_ind && strat == ZSTD_fast)
|| (varArray[i] == slog_ind && strat == ZSTD_fast)
|| (varArray[i] == slog_ind && strat == ZSTD_dfast) || (varArray[i] == slog_ind && strat == ZSTD_dfast)
|| (varArray[i] == tlen_ind && strat != ZSTD_btopt && strat != ZSTD_btultra && strat != ZSTD_fast))) { || (varArray[i] == tlen_ind && strat != ZSTD_btopt && strat != ZSTD_btultra && strat != ZSTD_fast)
/* || varArray[i] == strt_ind */ )) {
varNew[j] = varArray[i]; varNew[j] = varArray[i];
j++; j++;
} }
@@ -884,6 +1272,10 @@ static int variableParams(const ZSTD_compressionParameters paramConstraints, var
res[j] = tlen_ind; res[j] = tlen_ind;
j++; j++;
} }
if(!paramConstraints.strategy) {
res[j] = strt_ind;
j++;
}
return j; return j;
} }
@@ -918,6 +1310,7 @@ static void paramVaryOnce(const varInds_t paramIndex, const int amt, ZSTD_compre
case tlen_ind: case tlen_ind:
ptr->targetLength = tlen_table[MAX(0, MIN(TLEN_RANGE - 1, tlen_inv(ptr->targetLength) + amt))]; ptr->targetLength = tlen_table[MAX(0, MIN(TLEN_RANGE - 1, tlen_inv(ptr->targetLength) + amt))];
break; break;
case strt_ind: ptr->strategy += amt; break;
default: break; default: break;
} }
} }
@@ -934,7 +1327,7 @@ static void paramVariation(ZSTD_compressionParameters* ptr, const varInds_t* var
const U32 changeID = FUZ_rand(&g_rand) % (varyLen << 1); const U32 changeID = FUZ_rand(&g_rand) % (varyLen << 1);
paramVaryOnce(varyParams[changeID >> 1], ((changeID & 1) << 1) - 1, &p); paramVaryOnce(varyParams[changeID >> 1], ((changeID & 1) << 1) - 1, &p);
} }
validated = !ZSTD_isError(ZSTD_checkCParams(p)); validated = !ZSTD_isError(ZSTD_checkCParams(p)) && p.strategy > 0;
} }
*ptr = p; *ptr = p;
} }
@@ -944,7 +1337,9 @@ static size_t memoTableLen(const varInds_t* varyParams, const int varyLen) {
size_t arrayLen = 1; size_t arrayLen = 1;
int i; int i;
for(i = 0; i < varyLen; i++) { for(i = 0; i < varyLen; i++) {
arrayLen *= rangetable[varyParams[i]]; if(varyParams[i] != strt_ind) {
arrayLen *= rangetable[varyParams[i]];
}
} }
return arrayLen; return arrayLen;
} }
@@ -967,6 +1362,7 @@ static unsigned memoTableInd(const ZSTD_compressionParameters* ptr, const varInd
- ZSTD_SEARCHLENGTH_MIN; break; - ZSTD_SEARCHLENGTH_MIN; break;
case tlen_ind: ind *= TLEN_RANGE; ind += tlen_inv(ptr->targetLength) case tlen_ind: ind *= TLEN_RANGE; ind += tlen_inv(ptr->targetLength)
- ZSTD_TARGETLENGTH_MIN; break; - ZSTD_TARGETLENGTH_MIN; break;
case strt_ind: break;
} }
} }
return ind; return ind;
@@ -989,6 +1385,7 @@ static void memoTableIndInv(ZSTD_compressionParameters* ptr, const varInds_t* va
ind /= SLEN_RANGE; break; ind /= SLEN_RANGE; break;
case tlen_ind: ptr->targetLength = tlen_table[(ind % TLEN_RANGE)]; case tlen_ind: ptr->targetLength = tlen_table[(ind % TLEN_RANGE)];
ind /= TLEN_RANGE; break; ind /= TLEN_RANGE; break;
case strt_ind: break;
} }
} }
} }
@@ -1068,7 +1465,7 @@ static void freeMemoTableArray(U8** mtAll) {
/* takes unsanitized varyParams */ /* takes unsanitized varyParams */
static U8** createMemoTableArray(ZSTD_compressionParameters paramConstraints, constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { static U8** createMemoTableArray(ZSTD_compressionParameters paramConstraints, constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) {
varInds_t varNew[NUM_PARAMS]; varInds_t varNew[NUM_PARAMS];
U8** mtAll = calloc(sizeof(U8*),(ZSTD_btultra + 1)); U8** mtAll = (U8**)calloc(sizeof(U8*),(ZSTD_btultra + 1));
int i; int i;
if(mtAll == NULL) { if(mtAll == NULL) {
return NULL; return NULL;
@@ -1087,6 +1484,17 @@ static U8** createMemoTableArray(ZSTD_compressionParameters paramConstraints, co
return mtAll; return mtAll;
} }
static ZSTD_compressionParameters overwriteParams(ZSTD_compressionParameters base, ZSTD_compressionParameters mask) {
base.windowLog = mask.windowLog ? mask.windowLog : base.windowLog;
base.chainLog = mask.chainLog ? mask.chainLog : base.chainLog;
base.hashLog = mask.hashLog ? mask.hashLog : base.hashLog;
base.searchLog = mask.searchLog ? mask.searchLog : base.searchLog;
base.searchLength = mask.searchLength ? mask.searchLength : base.searchLength;
base.targetLength = mask.targetLength ? mask.targetLength : base.targetLength;
base.strategy = mask.strategy ? mask.strategy : base.strategy;
return base;
}
#define PARAMTABLELOG 25 #define PARAMTABLELOG 25
#define PARAMTABLESIZE (1<<PARAMTABLELOG) #define PARAMTABLESIZE (1<<PARAMTABLELOG)
#define PARAMTABLEMASK (PARAMTABLESIZE-1) #define PARAMTABLEMASK (PARAMTABLESIZE-1)
@@ -1103,11 +1511,11 @@ static BYTE* NB_TESTS_PLAYED(ZSTD_compressionParameters p) {
static void playAround(FILE* f, winnerInfo_t* winners, static void playAround(FILE* f, winnerInfo_t* winners,
ZSTD_compressionParameters params, ZSTD_compressionParameters params,
const void* srcBuffer, size_t srcSize) buffers_t buf, contexts_t ctx)
{ {
int nbVariations = 0; int nbVariations = 0;
UTIL_time_t const clockStart = UTIL_getTime(); UTIL_time_t const clockStart = UTIL_getTime();
const U32 unconstrained[NUM_PARAMS] = { 0, 1, 2, 3, 4, 5 }; const U32 unconstrained[NUM_PARAMS] = { 0, 1, 2, 3, 4, 5, 6 };
while (UTIL_clockSpanMicro(clockStart) < g_maxVariationTime) { while (UTIL_clockSpanMicro(clockStart) < g_maxVariationTime) {
@@ -1115,7 +1523,7 @@ static void playAround(FILE* f, winnerInfo_t* winners,
BYTE* b; BYTE* b;
if (nbVariations++ > g_maxNbVariations) break; if (nbVariations++ > g_maxNbVariations) break;
paramVariation(&p, unconstrained, 7, 4); paramVariation(&p, unconstrained, NUM_PARAMS, 4);
/* exclude faster if already played params */ /* exclude faster if already played params */
if (FUZ_rand(&g_rand) & ((1 << *NB_TESTS_PLAYED(p))-1)) if (FUZ_rand(&g_rand) & ((1 << *NB_TESTS_PLAYED(p))-1))
@@ -1124,11 +1532,11 @@ static void playAround(FILE* f, winnerInfo_t* winners,
/* test */ /* test */
b = NB_TESTS_PLAYED(p); b = NB_TESTS_PLAYED(p);
(*b)++; (*b)++;
if (!BMK_seed(winners, p, srcBuffer, srcSize)) continue; if (!BMK_seed(winners, p, buf, ctx)) continue;
/* improvement found => search more */ /* improvement found => search more */
BMK_printWinners(f, winners, srcSize); BMK_printWinners(f, winners, buf.srcSize);
playAround(f, winners, p, srcBuffer, srcSize); playAround(f, winners, p, buf, ctx);
} }
} }
@@ -1175,36 +1583,24 @@ static void randomConstrainedParams(ZSTD_compressionParameters* pc, varInds_t* v
static void BMK_selectRandomStart( static void BMK_selectRandomStart(
FILE* f, winnerInfo_t* winners, FILE* f, winnerInfo_t* winners,
const void* srcBuffer, size_t srcSize) buffers_t buf, contexts_t ctx)
{ {
U32 const id = FUZ_rand(&g_rand) % (NB_LEVELS_TRACKED+1); U32 const id = FUZ_rand(&g_rand) % (NB_LEVELS_TRACKED+1);
if ((id==0) || (winners[id].params.windowLog==0)) { if ((id==0) || (winners[id].params.windowLog==0)) {
/* use some random entry */ /* use some random entry */
ZSTD_compressionParameters const p = ZSTD_adjustCParams(randomParams(), srcSize, 0); ZSTD_compressionParameters const p = ZSTD_adjustCParams(randomParams(), buf.srcSize, 0);
playAround(f, winners, p, srcBuffer, srcSize); playAround(f, winners, p, buf, ctx);
} else { } else {
playAround(f, winners, winners[id].params, srcBuffer, srcSize); playAround(f, winners, winners[id].params, buf, ctx);
} }
} }
static void BMK_benchFullTable(buffers_t buf, contexts_t ctx, const size_t maxBlockSize)
static void BMK_benchOnce(const void* srcBuffer, size_t srcSize)
{
BMK_result_t testResult;
g_params = ZSTD_adjustCParams(g_params, srcSize, 0);
BMK_benchParam1(&testResult, srcBuffer, srcSize, g_params);
DISPLAY("Compression Ratio: %.3f Compress Speed: %.1f MB/s Decompress Speed: %.1f MB/s\n", (double)srcSize / testResult.cSize,
(double)testResult.cSpeed / (1 MB), (double)testResult.dSpeed / (1 MB));
return;
}
static void BMK_benchFullTable(const void* srcBuffer, size_t srcSize)
{ {
ZSTD_compressionParameters params; ZSTD_compressionParameters params;
winnerInfo_t winners[NB_LEVELS_TRACKED+1]; winnerInfo_t winners[NB_LEVELS_TRACKED+1];
const char* const rfName = "grillResults.txt"; const char* const rfName = "grillResults.txt";
FILE* const f = fopen(rfName, "w"); FILE* const f = fopen(rfName, "w");
const size_t blockSize = g_blockSize ? g_blockSize : srcSize; /* cut by block or not ? */
/* init */ /* init */
assert(g_singleRun==0); assert(g_singleRun==0);
@@ -1212,12 +1608,12 @@ static void BMK_benchFullTable(const void* srcBuffer, size_t srcSize)
if (f==NULL) { DISPLAY("error opening %s \n", rfName); exit(1); } if (f==NULL) { DISPLAY("error opening %s \n", rfName); exit(1); }
if (g_target) { if (g_target) {
BMK_init_level_constraints(g_target*(1 MB)); BMK_init_level_constraints(g_target * (1 MB));
} else { } else {
/* baseline config for level 1 */ /* baseline config for level 1 */
ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, blockSize, 0); ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, maxBlockSize, ctx.dictSize); //is dictionary ever even useful here?
BMK_result_t testResult; BMK_result_t testResult;
BMK_benchParam1(&testResult, srcBuffer, srcSize, l1params); BMK_benchParam(&testResult, buf, ctx, l1params);
BMK_init_level_constraints((int)((testResult.cSpeed * 31) / 32)); BMK_init_level_constraints((int)((testResult.cSpeed * 31) / 32));
} }
@@ -1225,124 +1621,161 @@ static void BMK_benchFullTable(const void* srcBuffer, size_t srcSize)
{ const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); { const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel();
int i; int i;
for (i=0; i<=maxSeeds; i++) { for (i=0; i<=maxSeeds; i++) {
params = ZSTD_getCParams(i, blockSize, 0); params = ZSTD_getCParams(i, maxBlockSize, 0);
BMK_seed(winners, params, srcBuffer, srcSize); BMK_seed(winners, params, buf, ctx);
} } } }
BMK_printWinners(f, winners, srcSize); BMK_printWinners(f, winners, buf.srcSize);
/* start tests */ /* start tests */
{ const time_t grillStart = time(NULL); { const time_t grillStart = time(NULL);
do { do {
BMK_selectRandomStart(f, winners, srcBuffer, srcSize); BMK_selectRandomStart(f, winners, buf, ctx);
} while (BMK_timeSpan(grillStart) < g_grillDuration_s); } while (BMK_timeSpan(grillStart) < g_grillDuration_s);
} }
/* end summary */ /* end summary */
BMK_printWinners(f, winners, srcSize); BMK_printWinners(f, winners, buf.srcSize);
DISPLAY("grillParams operations completed \n"); DISPLAY("grillParams operations completed \n");
/* clean up*/ /* clean up*/
fclose(f); fclose(f);
} }
static void BMK_benchMemInit(const void* srcBuffer, size_t srcSize)
{
if (g_singleRun)
return BMK_benchOnce(srcBuffer, srcSize);
else
return BMK_benchFullTable(srcBuffer, srcSize);
}
static int benchSample(void) static int benchSample(void)
{ {
const char* const name = "Sample 10MB"; const char* const name = "Sample 10MB";
size_t const benchedSize = (10 MB); size_t const benchedSize = 10 MB;
U32 blockSize = g_blockSize ? g_blockSize : benchedSize;
U32 const maxNbBlocks = (U32) ((benchedSize + (blockSize-1)) / blockSize) + 1;
size_t splitSize = 0;
void* origBuff = malloc(benchedSize); buffers_t buf;
if (!origBuff) { perror("not enough memory"); return 12; } contexts_t ctx;
/* Fill buffer */ buf.srcPtrs = (const void**)calloc(maxNbBlocks, sizeof(void*));
RDG_genBuffer(origBuff, benchedSize, g_compressibility, 0.0, 0); buf.dstPtrs = (void**)calloc(maxNbBlocks, sizeof(void*));
buf.resPtrs = (void**)calloc(maxNbBlocks, sizeof(void*));
buf.srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
buf.dstSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
buf.dstCapacities = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
buf.resSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
buf.srcSize = benchedSize;
if(!buf.srcPtrs || !buf.dstPtrs || !buf.resPtrs || !buf.srcSizes || !buf.dstSizes || !buf.dstCapacities || !buf.resSizes) {
DISPLAY("Allocation Error\n");
freeBuffers(buf);
return 1;
}
buf.srcBuffer = malloc(benchedSize);
buf.srcPtrs[0] = (const void*)buf.srcBuffer;
buf.dstPtrs[0] = malloc(ZSTD_compressBound(benchedSize) + 1024 * maxNbBlocks);
buf.resPtrs[0] = malloc(benchedSize);
if(!buf.srcPtrs[0] || !buf.dstPtrs[0] || !buf.resPtrs[0]) {
DISPLAY("Allocation Error\n");
freeBuffers(buf);
return 1;
}
splitSize = MIN(benchedSize, blockSize);
buf.srcSizes[0] = splitSize;
buf.dstCapacities[0] = ZSTD_compressBound(splitSize);
buf.resSizes[0] = splitSize;
for(buf.nbBlocks = 1; splitSize < benchedSize; buf.nbBlocks++) {
const size_t i = buf.nbBlocks;
const size_t nextBlockSize = MIN(benchedSize - splitSize, blockSize);
buf.srcSizes[i] = nextBlockSize;
buf.dstCapacities[i] = ZSTD_compressBound(nextBlockSize);
buf.resSizes[i] = nextBlockSize;
buf.srcPtrs[i] = (const void*)(((const char*)buf.srcPtrs[i-1]) + buf.srcSizes[i-1]);
buf.dstPtrs[i] = (void*)(((char*)buf.dstPtrs[i-1]) + buf.dstSizes[i-1]);
buf.resPtrs[i] = (void*)(((char*)buf.resPtrs[i-1]) + buf.resSizes[i-1]);
splitSize += nextBlockSize;
}
if(createContexts(&ctx, NULL)) {
DISPLAY("Context Creation Error\n");
freeBuffers(buf);
return 1;
}
RDG_genBuffer(buf.srcBuffer, benchedSize, g_compressibility, 0.0, 0);
/* bench */ /* bench */
DISPLAY("\r%79s\r", ""); DISPLAY("\r%79s\r", "");
DISPLAY("using %s %i%%: \n", name, (int)(g_compressibility*100)); DISPLAY("using %s %i%%: \n", name, (int)(g_compressibility*100));
BMK_benchMemInit(origBuff, benchedSize);
free(origBuff); BMK_benchFullTable(buf, ctx, MIN(blockSize, benchedSize));
freeBuffers(buf);
freeContexts(ctx);
return 0; return 0;
} }
static int benchOnce(buffers_t buf, contexts_t ctx) {
BMK_result_t testResult;
if(BMK_benchParam(&testResult, buf, ctx, g_params)) {
DISPLAY("Error during benchmarking\n");
return 1;
}
BMK_printWinner(stdout, CUSTOM_LEVEL, testResult, g_params, buf.srcSize);
return 0;
}
/* benchFiles() : /* benchFiles() :
* note: while this function takes a table of filenames, * note: while this function takes a table of filenames,
* in practice, only the first filename will be used */ * in practice, only the first filename will be used */
int benchFiles(const char** fileNamesTable, int nbFiles) int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileName, int cLevel)
{ {
int fileIdx=0; buffers_t buf;
contexts_t ctx;
size_t maxBlockSize = 0, i;
int ret = 0;
/* Loop for each file */ if(createBuffers(&buf, fileNamesTable, nbFiles)) {
while (fileIdx<nbFiles) { DISPLAY("unable to load files\n");
const char* const inFileName = fileNamesTable[fileIdx++]; return 1;
FILE* const inFile = fopen( inFileName, "rb" );
U64 const inFileSize = UTIL_getFileSize(inFileName);
size_t benchedSize;
void* origBuff;
/* Check file existence */
if (inFile==NULL) {
DISPLAY( "Pb opening %s\n", inFileName);
return 11;
}
if (inFileSize == UTIL_FILESIZE_UNKNOWN) {
DISPLAY("Pb evaluating size of %s \n", inFileName);
fclose(inFile);
return 11;
}
/* Memory allocation */
benchedSize = BMK_findMaxMem(inFileSize*3) / 3;
if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
if (benchedSize < inFileSize)
DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20));
origBuff = malloc(benchedSize);
if (origBuff==NULL) {
DISPLAY("\nError: not enough memory!\n");
fclose(inFile);
return 12;
}
/* Fill input buffer */
DISPLAY("Loading %s... \r", inFileName);
{ size_t const readSize = fread(origBuff, 1, benchedSize, inFile);
fclose(inFile);
if(readSize != benchedSize) {
DISPLAY("\nError: problem reading file '%s' !! \n", inFileName);
free(origBuff);
return 13;
} }
/* bench */
DISPLAY("\r%79s\r", "");
DISPLAY("using %s : \n", inFileName);
BMK_benchMemInit(origBuff, benchedSize);
/* clean */
free(origBuff);
} }
return 0; if(createContexts(&ctx, dictFileName)) {
DISPLAY("unable to load dictionary\n");
freeBuffers(buf);
return 2;
}
for(i = 0; i < buf.nbBlocks; i++) {
maxBlockSize = MAX(maxBlockSize, buf.srcSizes[i]);
}
DISPLAY("\r%79s\r", "");
if(nbFiles == 1) {
DISPLAY("using %s : \n", fileNamesTable[0]);
} else {
DISPLAY("using %d Files : \n", nbFiles);
}
g_params = ZSTD_adjustCParams(overwriteParams(ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize), g_params), maxBlockSize, ctx.dictSize);
if(g_singleRun) {
ret = benchOnce(buf, ctx);
} else {
BMK_benchFullTable(buf, ctx, maxBlockSize);
}
freeBuffers(buf);
freeContexts(ctx);
return ret;
} }
#define WORSE_RESULT 0
#define BETTER_RESULT 1
#define ERROR_RESULT 2
/* Benchmarking which stops when we are sufficiently sure the solution is infeasible / worse than the winner */ /* Benchmarking which stops when we are sufficiently sure the solution is infeasible / worse than the winner */
#define VARIANCE 1.1 #define VARIANCE 1.2
static int allBench(BMK_result_t* resultPtr, static int allBench(BMK_result_t* resultPtr,
const buffers_t buf, const contexts_t ctx, const buffers_t buf, const contexts_t ctx,
const ZSTD_compressionParameters cParams, const ZSTD_compressionParameters cParams,
@@ -1384,7 +1817,7 @@ static int allBench(BMK_result_t* resultPtr,
} }
/* anything with worse ratio in feas is definitely worse, discard */ /* anything with worse ratio in feas is definitely worse, discard */
if(feas && benchres.result.cSize < winnerResult->cSize) { if(feas && benchres.result.cSize < winnerResult->cSize && !g_optmode) {
return WORSE_RESULT; return WORSE_RESULT;
} }
@@ -1413,7 +1846,7 @@ static int allBench(BMK_result_t* resultPtr,
/* disregard infeasible results in feas mode */ /* disregard infeasible results in feas mode */
/* disregard if resultMax < winner in infeas mode */ /* disregard if resultMax < winner in infeas mode */
if((feas && !feasible(resultMax, target)) || if((feas && !feasible(resultMax, target)) ||
(!feas && (winnerRS > resultScore(resultMax, buf.srcSize, target)))) { (!feas && (winnerRS > resultScore(resultMax, buf.srcSize, target)))) {
return WORSE_RESULT; return WORSE_RESULT;
} }
@@ -1445,7 +1878,6 @@ static int allBench(BMK_result_t* resultPtr,
} else { } else {
return WORSE_RESULT; return WORSE_RESULT;
} }
} }
#define INFEASIBLE_THRESHOLD 200 #define INFEASIBLE_THRESHOLD 200
@@ -1477,6 +1909,7 @@ static int benchMemo(BMK_result_t* resultPtr,
return res; return res;
} }
/* One iteration of hill climbing. Specifically, it first tries all /* One iteration of hill climbing. Specifically, it first tries all
* valid parameter configurations w/ manhattan distance 1 and picks the best one * valid parameter configurations w/ manhattan distance 1 and picks the best one
* failing that, it progressively tries candidates further and further away (up to #dim + 2) * failing that, it progressively tries candidates further and further away (up to #dim + 2)
@@ -1489,10 +1922,14 @@ static int benchMemo(BMK_result_t* resultPtr,
* Phase 2 optimizes in accordance with what the original function sets out to maximize, with * Phase 2 optimizes in accordance with what the original function sets out to maximize, with
* all feasible solutions valued over all infeasible solutions. * all feasible solutions valued over all infeasible solutions.
*/ */
/* sanitize all params here.
* all generation after random should be sanitized. (maybe sanitize random)
*/
static winnerInfo_t climbOnce(const constraint_t target, static winnerInfo_t climbOnce(const constraint_t target,
const varInds_t* varArray, const int varLen, const varInds_t* varArray, const int varLen, ZSTD_strategy strat,
U8* const memoTable, U8** memoTableArray,
const buffers_t buf, const contexts_t ctx, buffers_t buf, contexts_t ctx,
const ZSTD_compressionParameters init) { const ZSTD_compressionParameters init) {
/* /*
* cparam - currently considered 'center' * cparam - currently considered 'center'
@@ -1503,13 +1940,15 @@ static winnerInfo_t climbOnce(const constraint_t target,
winnerInfo_t candidateInfo, winnerInfo; winnerInfo_t candidateInfo, winnerInfo;
int better = 1; int better = 1;
int feas = 0; int feas = 0;
varInds_t varNew[NUM_PARAMS];
int varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat);
winnerInfo = initWinnerInfo(init); winnerInfo = initWinnerInfo(init);
candidateInfo = winnerInfo; candidateInfo = winnerInfo;
{ {
winnerInfo_t bestFeasible1 = initWinnerInfo(cparam); winnerInfo_t bestFeasible1 = initWinnerInfo(cparam);
DISPLAY("Climb Part 1\n"); DEBUGOUTPUT("Climb Part 1\n");
while(better) { while(better) {
int i, dist, offset; int i, dist, offset;
@@ -1523,15 +1962,20 @@ static winnerInfo_t climbOnce(const constraint_t target,
for(offset = -1; offset <= 1; offset += 2) { for(offset = -1; offset <= 1; offset += 2) {
candidateInfo.params = cparam; candidateInfo.params = cparam;
paramVaryOnce(varArray[i], offset, &candidateInfo.params); paramVaryOnce(varArray[i], offset, &candidateInfo.params);
candidateInfo.params = sanitizeParams(candidateInfo.params);
if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params)) && candidateInfo.params.strategy > 0) {
int res = benchMemo(&candidateInfo.result, int res;
if(strat != candidateInfo.params.strategy) { /* maybe only try strategy switching after exhausting non-switching solutions? */
strat = candidateInfo.params.strategy;
varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat);
}
res = benchMemo(&candidateInfo.result,
buf, ctx, buf, ctx,
candidateInfo.params, target, &winnerInfo.result, memoTable, sanitizeParams(candidateInfo.params), target, &winnerInfo.result, memoTableArray[strat],
varArray, varLen, feas); varNew, varLenNew, feas);
if(res == BETTER_RESULT) { /* synonymous with better when called w/ infeasibleBM */ if(res == BETTER_RESULT) { /* synonymous with better when called w/ infeasibleBM */
winnerInfo = candidateInfo; winnerInfo = candidateInfo;
BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize); BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, sanitizeParams(winnerInfo.params), target, buf.srcSize);
better = 1; better = 1;
if(compareResultLT(bestFeasible1.result, winnerInfo.result, target, buf.srcSize)) { if(compareResultLT(bestFeasible1.result, winnerInfo.result, target, buf.srcSize)) {
bestFeasible1 = winnerInfo; bestFeasible1 = winnerInfo;
@@ -1546,22 +1990,29 @@ static winnerInfo_t climbOnce(const constraint_t target,
} }
for(dist = 2; dist < varLen + 2; dist++) { /* varLen is # dimensions */ for(dist = 2; dist < varLen + 2; dist++) { /* varLen is # dimensions */
for(i = 0; i < 2 * varLen + 2; i++) { for(i = 0; i < (1 << varLen) / varLen + 2; i++) {
int res; int res;
candidateInfo.params = cparam; candidateInfo.params = cparam;
/* param error checking already done here */ /* param error checking already done here */
paramVariation(&candidateInfo.params, varArray, varLen, dist); paramVariation(&candidateInfo.params, varArray, varLen, dist);
if(strat != candidateInfo.params.strategy) {
strat = candidateInfo.params.strategy;
varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat);
}
res = benchMemo(&candidateInfo.result, res = benchMemo(&candidateInfo.result,
buf, ctx, buf, ctx,
candidateInfo.params, target, &winnerInfo.result, memoTable, sanitizeParams(candidateInfo.params), target, &winnerInfo.result, memoTableArray[strat],
varArray, varLen, feas); varNew, varLenNew, feas);
if(res == BETTER_RESULT) { /* synonymous with better in this case*/ if(res == BETTER_RESULT) { /* synonymous with better in this case*/
winnerInfo = candidateInfo; winnerInfo = candidateInfo;
BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize); BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, sanitizeParams(winnerInfo.params), target, buf.srcSize);
better = 1; better = 1;
if(compareResultLT(bestFeasible1.result, winnerInfo.result, target, buf.srcSize)) { if(compareResultLT(bestFeasible1.result, winnerInfo.result, target, buf.srcSize)) {
bestFeasible1 = winnerInfo; bestFeasible1 = winnerInfo;
} }
break;
} }
} }
@@ -1576,7 +2027,7 @@ static winnerInfo_t climbOnce(const constraint_t target,
feas = 1; feas = 1;
better = 1; better = 1;
winnerInfo = bestFeasible1; /* note with change, bestFeasible may not necessarily be feasible, but if one has been benchmarked, it will be. */ winnerInfo = bestFeasible1; /* note with change, bestFeasible may not necessarily be feasible, but if one has been benchmarked, it will be. */
DISPLAY("Climb Part 2\n"); DEBUGOUTPUT("Climb Part 2\n");
} }
} }
winnerInfo = bestFeasible1; winnerInfo = bestFeasible1;
@@ -1598,10 +2049,11 @@ static winnerInfo_t optimizeFixedStrategy(
const constraint_t target, ZSTD_compressionParameters paramTarget, const constraint_t target, ZSTD_compressionParameters paramTarget,
const ZSTD_strategy strat, const ZSTD_strategy strat,
const varInds_t* varArray, const int varLen, const varInds_t* varArray, const int varLen,
U8* const memoTable, const int tries) { U8** memoTableArray, const int tries) {
int i = 0; int i = 0;
varInds_t varNew[NUM_PARAMS]; varInds_t varNew[NUM_PARAMS];
int varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat); int varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat);
ZSTD_compressionParameters init; ZSTD_compressionParameters init;
winnerInfo_t winnerInfo, candidateInfo; winnerInfo_t winnerInfo, candidateInfo;
winnerInfo = initWinnerInfo(emptyParams()); winnerInfo = initWinnerInfo(emptyParams());
@@ -1615,208 +2067,19 @@ static winnerInfo_t optimizeFixedStrategy(
while(i < tries) { while(i < tries) {
DEBUGOUTPUT("Restart\n"); DEBUGOUTPUT("Restart\n");
randomConstrainedParams(&init, varNew, varLenNew, memoTable); randomConstrainedParams(&init, varNew, varLenNew, memoTableArray[strat]);
candidateInfo = climbOnce(target, varNew, varLenNew, memoTable, buf, ctx, init); candidateInfo = climbOnce(target, varArray, varLen, strat, memoTableArray, buf, ctx, init);
if(compareResultLT(winnerInfo.result, candidateInfo.result, target, buf.srcSize)) { if(compareResultLT(winnerInfo.result, candidateInfo.result, target, buf.srcSize)) {
winnerInfo = candidateInfo; winnerInfo = candidateInfo;
BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize); BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize);
i = 0; i = 0;
} }
i++; i++;
} }
return winnerInfo; return winnerInfo;
} }
static void freeBuffers(const buffers_t b) {
if(b.srcPtrs != NULL) {
free(b.srcBuffer);
}
free(b.srcPtrs);
free(b.srcSizes);
if(b.dstPtrs != NULL) {
free(b.dstPtrs[0]);
}
free(b.dstPtrs);
free(b.dstCapacities);
free(b.dstSizes);
if(b.resPtrs != NULL) {
free(b.resPtrs[0]);
}
free(b.resPtrs);
}
/* allocates buffer's arguments. returns 0 = success / 1 = failuere */
static int createBuffers(buffers_t* const buff, const char* const * const fileNamesTable,
const size_t nbFiles)
{
size_t pos = 0;
size_t n;
U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, (U32)nbFiles);
size_t benchedSize = MIN(BMK_findMaxMem(totalSizeToLoad * 3) / 3, totalSizeToLoad);
const size_t blockSize = g_blockSize ? g_blockSize : totalSizeToLoad; //(largest fileSize or total fileSize)
U32 const maxNbBlocks = (U32) ((totalSizeToLoad + (blockSize-1)) / blockSize) + (U32)nbFiles;
U32 blockNb = 0;
memset(buff, 0, sizeof(buffers_t));
buff->srcPtrs = (const void**)calloc(maxNbBlocks, sizeof(void*));
buff->srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
buff->dstPtrs = (void**)calloc(maxNbBlocks, sizeof(void*));
buff->dstCapacities = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
buff->dstSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
buff->resPtrs = (void**)calloc(maxNbBlocks, sizeof(void*));
buff->resSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
if(!buff->srcPtrs || !buff->srcSizes || !buff->dstPtrs || !buff->dstCapacities || !buff->dstSizes || !buff->resPtrs || !buff->resSizes) {
DISPLAY("alloc error\n");
freeBuffers(*buff);
return 1;
}
buff->srcBuffer = malloc(benchedSize);
buff->srcPtrs[0] = (const void*)buff->srcBuffer;
buff->dstPtrs[0] = malloc(ZSTD_compressBound(benchedSize) + (maxNbBlocks * 1024));
buff->resPtrs[0] = malloc(benchedSize);
if(!buff->srcPtrs[0] || !buff->dstPtrs[0] || !buff->resPtrs[0]) {
DISPLAY("alloc error\n");
freeBuffers(*buff);
return 1;
}
for(n = 0; n < nbFiles; n++) {
FILE* f;
U64 fileSize = UTIL_getFileSize(fileNamesTable[n]);
if (UTIL_isDirectory(fileNamesTable[n])) {
DISPLAY("Ignoring %s directory... \n", fileNamesTable[n]);
continue;
}
if (fileSize == UTIL_FILESIZE_UNKNOWN) {
DISPLAY("Cannot evaluate size of %s, ignoring ... \n", fileNamesTable[n]);
continue;
}
f = fopen(fileNamesTable[n], "rb");
if (f==NULL) {
DISPLAY("impossible to open file %s\n", fileNamesTable[n]);
freeBuffers(*buff);
fclose(f);
return 10;
}
DISPLAY("Loading %s... \r", fileNamesTable[n]);
if (fileSize + pos > benchedSize) fileSize = benchedSize - pos, n = nbFiles; /* buffer too small - stop after this file */
{
char* buffer = (char*)(buff->srcBuffer);
size_t const readSize = fread(((buffer)+pos), 1, (size_t)fileSize, f);
size_t blocked = 0;
while(blocked < readSize) {
buff->srcPtrs[blockNb] = (const void*)((buffer) + (pos + blocked));
buff->srcSizes[blockNb] = blockSize;
blocked += blockSize;
blockNb++;
}
if(readSize > 0) { buff->srcSizes[blockNb - 1] = ((readSize - 1) % blockSize) + 1; }
if (readSize != (size_t)fileSize) {
DISPLAY("could not read %s", fileNamesTable[n]);
freeBuffers(*buff);
fclose(f);
return 1;
}
pos += readSize;
}
fclose(f);
}
buff->dstCapacities[0] = ZSTD_compressBound(buff->srcSizes[0]);
buff->dstSizes[0] = buff->dstCapacities[0];
buff->resSizes[0] = buff->srcSizes[0];
for(n = 1; n < blockNb; n++) {
buff->dstPtrs[n] = ((char*)buff->dstPtrs[n-1]) + buff->dstCapacities[n-1];
buff->resPtrs[n] = ((char*)buff->resPtrs[n-1]) + buff->resSizes[n-1];
buff->dstCapacities[n] = ZSTD_compressBound(buff->srcSizes[n]);
buff->dstSizes[n] = buff->dstCapacities[n];
buff->resSizes[n] = buff->srcSizes[n];
}
buff->srcSize = pos;
buff->nbBlocks = blockNb;
if (pos == 0) { DISPLAY("\nno data to bench\n"); return 1; }
return 0;
}
static void freeContexts(const contexts_t ctx) {
free(ctx.dictBuffer);
ZSTD_freeCCtx(ctx.cctx);
ZSTD_freeDCtx(ctx.dctx);
}
/* Creates struct holding contexts and dictionary buffers. returns 0 on success, 1 on failure. */
static int createContexts(contexts_t* const ctx, const char* dictFileName) {
FILE* f;
size_t readSize;
U64 dictSize;
ctx->cctx = ZSTD_createCCtx();
ctx->dctx = ZSTD_createDCtx();
ctx->dictSize = 0;
ctx->dictBuffer = NULL;
if(!ctx->cctx || !ctx->dctx) {
DISPLAY("context allocation error\n");
freeContexts(*ctx);
return 1;
}
if(dictFileName == NULL) {
return 0;
}
dictSize = UTIL_getFileSize(dictFileName);
if(dictSize == UTIL_FILESIZE_UNKNOWN) {
DISPLAY("Unable to get dictionary size\n");
freeContexts(*ctx);
return 1;
} else {
ctx->dictSize = (size_t)dictSize;
}
ctx->dictBuffer = malloc(ctx->dictSize);
f = fopen(dictFileName, "rb");
if(!f) {
DISPLAY("unable to open file\n");
fclose(f);
freeContexts(*ctx);
return 1;
}
if(ctx->dictSize > 64 MB || !(ctx->dictBuffer)) {
DISPLAY("dictionary too large\n");
fclose(f);
freeContexts(*ctx);
return 1;
}
readSize = fread(ctx->dictBuffer, 1, ctx->dictSize, f);
if(readSize != ctx->dictSize) {
DISPLAY("unable to read file\n");
fclose(f);
freeContexts(*ctx);
return 1;
}
return 0;
}
/* goes best, best-1, best+1, best-2, ... */ /* goes best, best-1, best+1, best-2, ... */
/* return 0 if nothing remaining */ /* return 0 if nothing remaining */
static int nextStrategy(const int currentStrategy, const int bestStrategy) { static int nextStrategy(const int currentStrategy, const int bestStrategy) {
@@ -1847,18 +2110,8 @@ static int nextStrategy(const int currentStrategy, const int bestStrategy) {
} }
} }
static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZSTD_compressionParameters mask) { /* experiment with playing with this and decay value */
base.windowLog = mask.windowLog ? mask.windowLog : base.windowLog;
base.chainLog = mask.chainLog ? mask.chainLog : base.chainLog;
base.hashLog = mask.hashLog ? mask.hashLog : base.hashLog;
base.searchLog = mask.searchLog ? mask.searchLog : base.searchLog;
base.searchLength = mask.searchLength ? mask.searchLength : base.searchLength;
base.targetLength = mask.targetLength ? mask.targetLength : base.targetLength;
base.strategy = mask.strategy ? mask.strategy : base.strategy;
return base;
}
#define MAX_TRIES 8
/* main fn called when using --optimize */ /* main fn called when using --optimize */
/* Does strategy selection by benchmarking default compression levels /* Does strategy selection by benchmarking default compression levels
* then optimizes by strategy, starting with the best one and moving * then optimizes by strategy, starting with the best one and moving
@@ -1871,6 +2124,10 @@ static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZS
* paramTarget - parameter constraints (i.e. restriction search space to where strategy = ZSTD_fast) * paramTarget - parameter constraints (i.e. restriction search space to where strategy = ZSTD_fast)
* cLevel - compression level to exceed (all solutions must be > lvl in cSpeed + ratio) * cLevel - compression level to exceed (all solutions must be > lvl in cSpeed + ratio)
*/ */
static int g_maxTries = 5;
#define TRY_DECAY 1
static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget, int cLevel) static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget, int cLevel)
{ {
varInds_t varArray [NUM_PARAMS]; varInds_t varArray [NUM_PARAMS];
@@ -1883,6 +2140,8 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
contexts_t ctx; contexts_t ctx;
buffers_t buf; buffers_t buf;
g_time = UTIL_getTime();
/* Init */ /* Init */
if(!cParamValid(paramTarget)) { if(!cParamValid(paramTarget)) {
return 1; return 1;
@@ -1915,7 +2174,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
if(paramTarget.strategy) { if(paramTarget.strategy) {
varInds_t varNew[NUM_PARAMS]; varInds_t varNew[NUM_PARAMS];
int varLenNew = sanitizeVarArray(varNew, varLen, varArray, paramTarget.strategy); int varLenNew = sanitizeVarArray(varNew, varLen, varArray, paramTarget.strategy);
allMT = calloc(sizeof(U8), (ZSTD_btultra + 1)); allMT = (U8**)calloc(sizeof(U8*), (ZSTD_btultra + 1));
if(allMT == NULL) { if(allMT == NULL) {
ret = 57; ret = 57;
goto _cleanUp; goto _cleanUp;
@@ -1940,14 +2199,38 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
goto _cleanUp; goto _cleanUp;
} }
if(cLevel) { /* default strictness = Maximum for */
if(g_strictness == DEFAULT_STRICTNESS) {
if(g_optmode) {
g_strictness = 99;
} else {
g_strictness = 90;
}
} else {
if(0 >= g_strictness || g_strictness > 100) {
DISPLAY("Strictness Outside of Bounds\n");
ret = 4;
goto _cleanUp;
}
}
/* use level'ing mode instead of normal target mode */
/* Should lvl be parameter-masked here? */
if(g_optmode) {
winner.params = ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize); winner.params = ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize);
if(BMK_benchParam(&winner.result, buf, ctx, winner.params)) { if(BMK_benchParam(&winner.result, buf, ctx, winner.params)) {
ret = 3; ret = 3;
goto _cleanUp; goto _cleanUp;
} }
g_lvltarget = winner.result;
g_lvltarget.cSpeed *= ((double)g_strictness) / 100;
g_lvltarget.dSpeed *= ((double)g_strictness) / 100;
g_lvltarget.cSize /= ((double)g_strictness) / 100;
target.cSpeed = (U32)g_lvltarget.cSpeed;
target.dSpeed = (U32)g_lvltarget.dSpeed; //See if this is reasonable.
target.cSpeed = (U32)winner.result.cSpeed;
BMK_printWinnerOpt(stdout, cLevel, winner.result, winner.params, target, buf.srcSize); BMK_printWinnerOpt(stdout, cLevel, winner.result, winner.params, target, buf.srcSize);
} }
@@ -1958,6 +2241,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
} else { } else {
DISPLAY("optimizing for %lu Files", (unsigned long)nbFiles); DISPLAY("optimizing for %lu Files", (unsigned long)nbFiles);
} }
if(target.cSpeed != 0) { DISPLAY(" - limit compression speed %u MB/s", target.cSpeed >> 20); } if(target.cSpeed != 0) { DISPLAY(" - limit compression speed %u MB/s", target.cSpeed >> 20); }
if(target.dSpeed != 0) { DISPLAY(" - limit decompression speed %u MB/s", target.dSpeed >> 20); } if(target.dSpeed != 0) { DISPLAY(" - limit decompression speed %u MB/s", target.dSpeed >> 20); }
if(target.cMem != (U32)-1) { DISPLAY(" - limit memory %u MB", target.cMem >> 20); } if(target.cMem != (U32)-1) { DISPLAY(" - limit memory %u MB", target.cMem >> 20); }
@@ -1966,20 +2250,19 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
findClockGranularity(); findClockGranularity();
{ {
varInds_t varNew[NUM_PARAMS]; ZSTD_compressionParameters CParams;
/* find best solution from default params */ /* find best solution from default params */
{ {
/* strategy selection */ /* strategy selection */
const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel();
DEBUGOUTPUT("Strategy Selection\n"); DEBUGOUTPUT("Strategy Selection\n");
if(paramTarget.strategy == 0) { /* no variable based constraints */ if(paramTarget.strategy == 0) {
BMK_result_t candidate; BMK_result_t candidate;
int i; int i;
for (i=1; i<=maxSeeds; i++) { for (i=1; i<=maxSeeds; i++) {
int ec; int ec;
ZSTD_compressionParameters CParams = ZSTD_getCParams(i, maxBlockSize, ctx.dictSize); CParams = overwriteParams(ZSTD_getCParams(i, maxBlockSize, ctx.dictSize), paramTarget);
CParams = maskParams(CParams, paramTarget);
ec = BMK_benchParam(&candidate, buf, ctx, CParams); ec = BMK_benchParam(&candidate, buf, ctx, CParams);
BMK_printWinnerOpt(stdout, i, candidate, CParams, target, buf.srcSize); BMK_printWinnerOpt(stdout, i, candidate, CParams, target, buf.srcSize);
@@ -1987,6 +2270,9 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
winner.result = candidate; winner.result = candidate;
winner.params = CParams; winner.params = CParams;
} }
/* if the current params are too slow, just stop. */
if(target.cSpeed > candidate.cSpeed * 3 / 2) { break; }
} }
} }
} }
@@ -1999,31 +2285,36 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
int bestStrategy = (int)winner.params.strategy; int bestStrategy = (int)winner.params.strategy;
if(paramTarget.strategy == 0) { if(paramTarget.strategy == 0) {
int st = (int)winner.params.strategy; int st = (int)winner.params.strategy;
int tries = MAX_TRIES; int tries = g_maxTries;
{ {
int varLenNew = sanitizeVarArray(varNew, varLen, varArray, st); /* one iterations of hill climbing with the level-defined parameters. */
winnerInfo_t w1 = climbOnce(target, varNew, varLenNew, allMT[st], winnerInfo_t w1 = climbOnce(target, varArray, varLen, st, allMT,
buf, ctx, winner.params); buf, ctx, winner.params);
if(compareResultLT(winner.result, w1.result, target, buf.srcSize)) { if(compareResultLT(winner.result, w1.result, target, buf.srcSize)) {
winner = w1; winner = w1;
} }
} }
while(st && tries) { while(st && tries > 0) {
winnerInfo_t wc = optimizeFixedStrategy(buf, ctx, target, paramTarget, winnerInfo_t wc;
st, varArray, varLen, allMT[st], tries); DEBUGOUTPUT("StrategySwitch: %s\n", g_stratName[st]);
DEBUGOUTPUT("StratNum %d\n", st);
wc = optimizeFixedStrategy(buf, ctx, target, paramTarget,
st, varArray, varLen, allMT, tries);
if(compareResultLT(winner.result, wc.result, target, buf.srcSize)) { if(compareResultLT(winner.result, wc.result, target, buf.srcSize)) {
winner = wc; winner = wc;
tries = g_maxTries;
bestStrategy = st;
} else {
st = nextStrategy(st, bestStrategy);
tries -= TRY_DECAY;
} }
st = nextStrategy(st, bestStrategy);
tries--;
} }
} else { } else {
winner = optimizeFixedStrategy(buf, ctx, target, paramTarget, paramTarget.strategy, winner = optimizeFixedStrategy(buf, ctx, target, paramTarget, paramTarget.strategy,
varArray, varLen, allMT[paramTarget.strategy], 10); varArray, varLen, allMT, g_maxTries);
} }
} }
@@ -2104,6 +2395,7 @@ static int usage_advanced(void)
DISPLAY( " -t# : Caps runtime of operation in seconds (default : %u seconds (%.1f hours)) \n", (U32)g_grillDuration_s, g_grillDuration_s / 3600); DISPLAY( " -t# : Caps runtime of operation in seconds (default : %u seconds (%.1f hours)) \n", (U32)g_grillDuration_s, g_grillDuration_s / 3600);
DISPLAY( " -v : Prints Benchmarking output\n"); DISPLAY( " -v : Prints Benchmarking output\n");
DISPLAY( " -D : Next argument dictionary file\n"); DISPLAY( " -D : Next argument dictionary file\n");
DISPLAY( " -s : Seperate Files\n");
return 0; return 0;
} }
@@ -2115,15 +2407,15 @@ static int badusage(const char* exename)
} }
#define PARSE_SUB_ARGS(stringLong, stringShort, variable) { if (longCommandWArg(&argument, stringLong) || longCommandWArg(&argument, stringShort)) { variable = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } } #define PARSE_SUB_ARGS(stringLong, stringShort, variable) { if (longCommandWArg(&argument, stringLong) || longCommandWArg(&argument, stringShort)) { variable = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } }
#define PARSE_CPARAMS(variable) \ #define PARSE_CPARAMS(variable) \
{ \ { \
PARSE_SUB_ARGS("windowLog=", "wlog=", variable.windowLog); \ PARSE_SUB_ARGS("windowLog=", "wlog=", variable.windowLog); \
PARSE_SUB_ARGS("chainLog=" , "clog=", variable.chainLog); \ PARSE_SUB_ARGS("chainLog=" , "clog=", variable.chainLog); \
PARSE_SUB_ARGS("hashLog=", "hlog=", variable.hashLog); \ PARSE_SUB_ARGS("hashLog=", "hlog=", variable.hashLog); \
PARSE_SUB_ARGS("searchLog=" , "slog=", variable.searchLog); \ PARSE_SUB_ARGS("searchLog=" , "slog=", variable.searchLog); \
PARSE_SUB_ARGS("searchLength=", "slen=", variable.searchLength); \ PARSE_SUB_ARGS("searchLength=", "slen=", variable.searchLength); \
PARSE_SUB_ARGS("targetLength=" , "tlen=", variable.targetLength); \ PARSE_SUB_ARGS("targetLength=" , "tlen=", variable.targetLength); \
PARSE_SUB_ARGS("strategy=", "strat=", variable.strategy); \ PARSE_SUB_ARGS("strategy=", "strat=", variable.strategy); \
} }
int main(int argc, const char** argv) int main(int argc, const char** argv)
@@ -2134,38 +2426,42 @@ int main(int argc, const char** argv)
const char* exename=argv[0]; const char* exename=argv[0];
const char* input_filename = NULL; const char* input_filename = NULL;
const char* dictFileName = NULL; const char* dictFileName = NULL;
U32 optimizer = 0;
U32 main_pause = 0; U32 main_pause = 0;
int optimizerCLevel = 0; int cLevel = 0;
int seperateFiles = 0;
constraint_t target = { 0, 0, (U32)-1 }; constraint_t target = { 0, 0, (U32)-1 };
ZSTD_compressionParameters paramTarget = { 0, 0, 0, 0, 0, 0, 0 };
ZSTD_compressionParameters paramTarget = emptyParams();
g_params = emptyParams();
assert(argc>=1); /* for exename */ assert(argc>=1); /* for exename */
g_time = UTIL_getTime();
/* Welcome message */ /* Welcome message */
DISPLAY(WELCOME_MESSAGE); DISPLAY(WELCOME_MESSAGE);
for(i=1; i<argc; i++) { for(i=1; i<argc; i++) {
const char* argument = argv[i]; const char* argument = argv[i];
DISPLAY("%d: ", i); DEBUGOUTPUT("%d: ", i);
DISPLAY("%s\n", argument); DEBUGOUTPUT("%s\n", argument);
assert(argument != NULL); assert(argument != NULL);
if(!strcmp(argument,"--no-seed")) { g_noSeed = 1; continue; } if(!strcmp(argument,"--no-seed")) { g_noSeed = 1; continue; }
if (longCommandWArg(&argument, "--optimize=")) { if (longCommandWArg(&argument, "--optimize=")) {
optimizer = 1; g_optimizer = 1;
for ( ; ;) { for ( ; ;) {
PARSE_CPARAMS(paramTarget); PARSE_CPARAMS(paramTarget);
PARSE_SUB_ARGS("compressionSpeed=" , "cSpeed=", target.cSpeed); PARSE_SUB_ARGS("compressionSpeed=" , "cSpeed=", target.cSpeed);
PARSE_SUB_ARGS("decompressionSpeed=", "dSpeed=", target.dSpeed); PARSE_SUB_ARGS("decompressionSpeed=", "dSpeed=", target.dSpeed);
PARSE_SUB_ARGS("compressionMemory=" , "cMem=", target.cMem); PARSE_SUB_ARGS("compressionMemory=" , "cMem=", target.cMem);
PARSE_SUB_ARGS("level=", "lvl=", optimizerCLevel); PARSE_SUB_ARGS("strict=", "stc=", g_strictness);
PARSE_SUB_ARGS("preferSpeed=", "prfSpd=", g_speedMultiplier);
PARSE_SUB_ARGS("preferRatio=", "prfRto=", g_ratioMultiplier);
PARSE_SUB_ARGS("maxTries=", "tries=", g_maxTries);
if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevel = readU32FromChar(&argument); g_optmode = 1; if (argument[0]==',') { argument++; continue; } else break; }
DISPLAY("invalid optimization parameter \n"); DISPLAY("invalid optimization parameter \n");
return 1; return 1;
} }
@@ -2178,10 +2474,11 @@ int main(int argc, const char** argv)
} else if (longCommandWArg(&argument, "--zstd=")) { } else if (longCommandWArg(&argument, "--zstd=")) {
/* Decode command (note : aggregated commands are allowed) */ /* Decode command (note : aggregated commands are allowed) */
g_singleRun = 1; g_singleRun = 1;
g_params = ZSTD_getCParams(2, g_blockSize, 0); cLevel = 2;
for ( ; ;) { for ( ; ;) {
PARSE_CPARAMS(g_params) PARSE_CPARAMS(g_params)
if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { g_params = ZSTD_getCParams(readU32FromChar(&argument), g_blockSize, 0); if (argument[0]==',') { argument++; continue; } else break; } if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevel = readU32FromChar(&argument); g_params = emptyParams(); if (argument[0]==',') { argument++; continue; } else break; }
DISPLAY("invalid compression parameter \n"); DISPLAY("invalid compression parameter \n");
return 1; return 1;
} }
@@ -2257,10 +2554,9 @@ int main(int argc, const char** argv)
g_params.strategy = (ZSTD_strategy)readU32FromChar(&argument); g_params.strategy = (ZSTD_strategy)readU32FromChar(&argument);
continue; continue;
case 'L': case 'L':
{ int cLevel; { argument++;
argument++;
cLevel = readU32FromChar(&argument); cLevel = readU32FromChar(&argument);
g_params = ZSTD_getCParams(cLevel, g_blockSize, 0); g_params = emptyParams();
continue; continue;
} }
default : ; default : ;
@@ -2289,6 +2585,10 @@ int main(int argc, const char** argv)
g_grillDuration_s = (double)readU32FromChar(&argument); g_grillDuration_s = (double)readU32FromChar(&argument);
break; break;
case 's':
seperateFiles = 1;
break;
/* load dictionary file (only applicable for optimizer rn) */ /* load dictionary file (only applicable for optimizer rn) */
case 'D': case 'D':
if(i == argc - 1) { /* last argument, return error. */ if(i == argc - 1) { /* last argument, return error. */
@@ -2312,18 +2612,31 @@ int main(int argc, const char** argv)
if (!input_filename) { input_filename=argument; filenamesStart=i; continue; } if (!input_filename) { input_filename=argument; filenamesStart=i; continue; }
} }
if (filenamesStart==0) { if (filenamesStart==0) {
if (optimizer) { if (g_optimizer) {
DISPLAY("Optimizer Expects File\n"); DISPLAY("Optimizer Expects File\n");
return 1; return 1;
} else { } else {
result = benchSample(); result = benchSample();
} }
} else { } else {
if (optimizer) { if(seperateFiles) {
result = optimizeForSize(argv+filenamesStart, argc-filenamesStart, dictFileName, target, paramTarget, optimizerCLevel); for(i = 0; i < argc - filenamesStart; i++) {
if (g_optimizer) {
result = optimizeForSize(argv+filenamesStart + i, 1, dictFileName, target, paramTarget, cLevel);
if(result) { DISPLAY("Error on File %d", i); return result; }
} else {
result = benchFiles(argv+filenamesStart + i, 1, dictFileName, cLevel);
if(result) { DISPLAY("Error on File %d", i); return result; }
}
}
} else { } else {
result = benchFiles(argv+filenamesStart, argc-filenamesStart); if (g_optimizer) {
} } result = optimizeForSize(argv+filenamesStart, argc-filenamesStart, dictFileName, target, paramTarget, cLevel);
} else {
result = benchFiles(argv+filenamesStart, argc-filenamesStart, dictFileName, cLevel);
}
}
}
if (main_pause) { int unused; printf("press enter...\n"); unused = getchar(); (void)unused; } if (main_pause) { int unused; printf("press enter...\n"); unused = getchar(); (void)unused; }