ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[zstd] Remove global variables in dictBuilder

Browse Source 
D50949782 fixed a race condition updating `g_displayLevel` by disabling display.
Instead of disabling display, delete the global variable and always "capture" a local `displayLevel` variable.
This also fixes `DISPLAYUPDATE()` by requiring the user to pass in the last update time as the first parameter.
This commit is contained in:
Nick Terrell
2025-03-05 10:35:01 -05:00
committed by Nick Terrell
parent d5b84f5a27
commit 190a620974
2 changed files with 82 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/dictBuilder/cover.c
+43 -45
View File
@@ -62,38 +62,30 @@
/*-************************************* /*-*************************************
* Console display * Console display
*
* Captures the `displayLevel` variable in the local scope.
***************************************/ ***************************************/
#ifndef LOCALDISPLAYLEVEL
static int g_displayLevel = 0;
#endif
#undef DISPLAY #undef DISPLAY
#define DISPLAY(...) \ #define DISPLAY(...) \
{ \ { \
fprintf(stderr, __VA_ARGS__); \ fprintf(stderr, __VA_ARGS__); \
fflush(stderr); \ fflush(stderr); \
} }
#undef LOCALDISPLAYLEVEL #undef DISPLAYLEVEL
#define LOCALDISPLAYLEVEL(displayLevel, l, ...) \ #define DISPLAYLEVEL(l, ...) \
if (displayLevel >= l) { \ if (displayLevel >= l) { \
DISPLAY(__VA_ARGS__); \ DISPLAY(__VA_ARGS__); \
} /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */ } /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */
#undef DISPLAYLEVEL
#define DISPLAYLEVEL(l, ...) LOCALDISPLAYLEVEL(g_displayLevel, l, __VA_ARGS__)
#ifndef LOCALDISPLAYUPDATE #undef DISPLAYUPDATE
static const clock_t g_refreshRate = CLOCKS_PER_SEC * 15 / 100; #define DISPLAYUPDATE(lastUpdateTime, l, ...) \
static clock_t g_time = 0;
#endif
#undef LOCALDISPLAYUPDATE
#define LOCALDISPLAYUPDATE(displayLevel, l, ...) \
if (displayLevel >= l) { \ if (displayLevel >= l) { \
if ((clock() - g_time > g_refreshRate) || (displayLevel >= 4)) { \ const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100; \
g_time = clock(); \ if ((clock() - lastUpdateTime > refreshRate) || (displayLevel >= 4)) { \
lastUpdateTime = clock(); \
DISPLAY(__VA_ARGS__); \ DISPLAY(__VA_ARGS__); \
} \ } \
} }
#undef DISPLAYUPDATE
#define DISPLAYUPDATE(l, ...) LOCALDISPLAYUPDATE(g_displayLevel, l, __VA_ARGS__)
/*-************************************* /*-*************************************
* Hash table * Hash table
@@ -239,6 +231,7 @@ typedef struct {
U32 *freqs; U32 *freqs;
U32 *dmerAt; U32 *dmerAt;
unsigned d; unsigned d;
int displayLevel;
} COVER_ctx_t; } COVER_ctx_t;
#if defined(ZSTD_USE_C90_QSORT) \ #if defined(ZSTD_USE_C90_QSORT) \
@@ -602,7 +595,7 @@ static void COVER_ctx_destroy(COVER_ctx_t *ctx) {
*/ */
static size_t COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer, static size_t COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer,
const size_t *samplesSizes, unsigned nbSamples, const size_t *samplesSizes, unsigned nbSamples,
unsigned d, double splitPoint) unsigned d, double splitPoint, int displayLevel)
{ {
const BYTE *const samples = (const BYTE *)samplesBuffer; const BYTE *const samples = (const BYTE *)samplesBuffer;
const size_t totalSamplesSize = COVER_sum(samplesSizes, nbSamples); const size_t totalSamplesSize = COVER_sum(samplesSizes, nbSamples);
@@ -611,6 +604,7 @@ static size_t COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer,
const unsigned nbTestSamples = splitPoint < 1.0 ? nbSamples - nbTrainSamples : nbSamples; const unsigned nbTestSamples = splitPoint < 1.0 ? nbSamples - nbTrainSamples : nbSamples;
const size_t trainingSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes, nbTrainSamples) : totalSamplesSize; const size_t trainingSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes, nbTrainSamples) : totalSamplesSize;
const size_t testSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes + nbTrainSamples, nbTestSamples) : totalSamplesSize; const size_t testSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes + nbTrainSamples, nbTestSamples) : totalSamplesSize;
ctx->displayLevel = displayLevel;
/* Checks */ /* Checks */
if (totalSamplesSize < MAX(d, sizeof(U64)) || if (totalSamplesSize < MAX(d, sizeof(U64)) ||
totalSamplesSize >= (size_t)COVER_MAX_SAMPLES_SIZE) { totalSamplesSize >= (size_t)COVER_MAX_SAMPLES_SIZE) {
@@ -695,14 +689,14 @@ void COVER_warnOnSmallCorpus(size_t maxDictSize, size_t nbDmers, int displayLeve
if (ratio >= 10) { if (ratio >= 10) {
return; return;
} }
LOCALDISPLAYLEVEL(displayLevel, 1, DISPLAYLEVEL(1,
"WARNING: The maximum dictionary size %u is too large " "WARNING: The maximum dictionary size %u is too large "
"compared to the source size %u! " "compared to the source size %u! "
"size(source)/size(dictionary) = %f, but it should be >= " "size(source)/size(dictionary) = %f, but it should be >= "
"10! This may lead to a subpar dictionary! We recommend " "10! This may lead to a subpar dictionary! We recommend "
"training on sources at least 10x, and preferably 100x " "training on sources at least 10x, and preferably 100x "
"the size of the dictionary! \n", (U32)maxDictSize, "the size of the dictionary! \n", (U32)maxDictSize,
(U32)nbDmers, ratio); (U32)nbDmers, ratio);
} }
COVER_epoch_info_t COVER_computeEpochs(U32 maxDictSize, COVER_epoch_info_t COVER_computeEpochs(U32 maxDictSize,
@@ -737,6 +731,8 @@ static size_t COVER_buildDictionary(const COVER_ctx_t *ctx, U32 *freqs,
const size_t maxZeroScoreRun = MAX(10, MIN(100, epochs.num >> 3)); const size_t maxZeroScoreRun = MAX(10, MIN(100, epochs.num >> 3));
size_t zeroScoreRun = 0; size_t zeroScoreRun = 0;
size_t epoch; size_t epoch;
clock_t lastUpdateTime = 0;
const int displayLevel = ctx->displayLevel;
DISPLAYLEVEL(2, "Breaking content into %u epochs of size %u\n", DISPLAYLEVEL(2, "Breaking content into %u epochs of size %u\n",
(U32)epochs.num, (U32)epochs.size); (U32)epochs.num, (U32)epochs.size);
/* Loop through the epochs until there are no more segments or the dictionary /* Loop through the epochs until there are no more segments or the dictionary
@@ -770,6 +766,7 @@ static size_t COVER_buildDictionary(const COVER_ctx_t *ctx, U32 *freqs,
tail -= segmentSize; tail -= segmentSize;
memcpy(dict + tail, ctx->samples + segment.begin, segmentSize); memcpy(dict + tail, ctx->samples + segment.begin, segmentSize);
DISPLAYUPDATE( DISPLAYUPDATE(
lastUpdateTime,
2, "\r%u%% ", 2, "\r%u%% ",
(unsigned)(((dictBufferCapacity - tail) * 100) / dictBufferCapacity)); (unsigned)(((dictBufferCapacity - tail) * 100) / dictBufferCapacity));
} }
@@ -785,9 +782,8 @@ ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_cover(
BYTE* const dict = (BYTE*)dictBuffer; BYTE* const dict = (BYTE*)dictBuffer;
COVER_ctx_t ctx; COVER_ctx_t ctx;
COVER_map_t activeDmers; COVER_map_t activeDmers;
const int displayLevel = parameters.zParams.notificationLevel;
parameters.splitPoint = 1.0; parameters.splitPoint = 1.0;
/* Initialize global data */
g_displayLevel = (int)parameters.zParams.notificationLevel;
/* Checks */ /* Checks */
if (!COVER_checkParameters(parameters, dictBufferCapacity)) { if (!COVER_checkParameters(parameters, dictBufferCapacity)) {
DISPLAYLEVEL(1, "Cover parameters incorrect\n"); DISPLAYLEVEL(1, "Cover parameters incorrect\n");
@@ -805,12 +801,12 @@ ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_cover(
/* Initialize context and activeDmers */ /* Initialize context and activeDmers */
{ {
size_t const initVal = COVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, size_t const initVal = COVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples,
parameters.d, parameters.splitPoint); parameters.d, parameters.splitPoint, displayLevel);
if (ZSTD_isError(initVal)) { if (ZSTD_isError(initVal)) {
return initVal; return initVal;
} }
} }
COVER_warnOnSmallCorpus(dictBufferCapacity, ctx.suffixSize, g_displayLevel); COVER_warnOnSmallCorpus(dictBufferCapacity, ctx.suffixSize, displayLevel);
if (!COVER_map_init(&activeDmers, parameters.k - parameters.d + 1)) { if (!COVER_map_init(&activeDmers, parameters.k - parameters.d + 1)) {
DISPLAYLEVEL(1, "Failed to allocate dmer map: out of memory\n"); DISPLAYLEVEL(1, "Failed to allocate dmer map: out of memory\n");
COVER_ctx_destroy(&ctx); COVER_ctx_destroy(&ctx);
@@ -1133,6 +1129,7 @@ static void COVER_tryParameters(void *opaque)
BYTE* const dict = (BYTE*)malloc(dictBufferCapacity); BYTE* const dict = (BYTE*)malloc(dictBufferCapacity);
COVER_dictSelection_t selection = COVER_dictSelectionError(ERROR(GENERIC)); COVER_dictSelection_t selection = COVER_dictSelectionError(ERROR(GENERIC));
U32* const freqs = (U32*)malloc(ctx->suffixSize * sizeof(U32)); U32* const freqs = (U32*)malloc(ctx->suffixSize * sizeof(U32));
const int displayLevel = ctx->displayLevel;
if (!COVER_map_init(&activeDmers, parameters.k - parameters.d + 1)) { if (!COVER_map_init(&activeDmers, parameters.k - parameters.d + 1)) {
DISPLAYLEVEL(1, "Failed to allocate dmer map: out of memory\n"); DISPLAYLEVEL(1, "Failed to allocate dmer map: out of memory\n");
goto _cleanup; goto _cleanup;
@@ -1184,21 +1181,22 @@ ZDICTLIB_STATIC_API size_t ZDICT_optimizeTrainFromBuffer_cover(
(1 + (kMaxD - kMinD) / 2) * (1 + (kMaxK - kMinK) / kStepSize); (1 + (kMaxD - kMinD) / 2) * (1 + (kMaxK - kMinK) / kStepSize);
const unsigned shrinkDict = 0; const unsigned shrinkDict = 0;
/* Local variables */ /* Local variables */
const int displayLevel = (int)parameters->zParams.notificationLevel; int displayLevel = (int)parameters->zParams.notificationLevel;
unsigned iteration = 1; unsigned iteration = 1;
unsigned d; unsigned d;
unsigned k; unsigned k;
COVER_best_t best; COVER_best_t best;
POOL_ctx *pool = NULL; POOL_ctx *pool = NULL;
int warned = 0; int warned = 0;
clock_t lastUpdateTime = 0;
/* Checks */ /* Checks */
if (splitPoint <= 0 || splitPoint > 1) { if (splitPoint <= 0 || splitPoint > 1) {
LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect parameters\n"); DISPLAYLEVEL(1, "Incorrect parameters\n");
return ERROR(parameter_outOfBound); return ERROR(parameter_outOfBound);
} }
if (kMinK < kMaxD || kMaxK < kMinK) { if (kMinK < kMaxD || kMaxK < kMinK) {
LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect parameters\n"); DISPLAYLEVEL(1, "Incorrect parameters\n");
return ERROR(parameter_outOfBound); return ERROR(parameter_outOfBound);
} }
if (nbSamples == 0) { if (nbSamples == 0) {
@@ -1218,19 +1216,19 @@ ZDICTLIB_STATIC_API size_t ZDICT_optimizeTrainFromBuffer_cover(
} }
/* Initialization */ /* Initialization */
COVER_best_init(&best); COVER_best_init(&best);
/* Turn down global display level to clean up display at level 2 and below */
g_displayLevel = displayLevel == 0 ? 0 : displayLevel - 1;
/* Loop through d first because each new value needs a new context */ /* Loop through d first because each new value needs a new context */
LOCALDISPLAYLEVEL(displayLevel, 2, "Trying %u different sets of parameters\n", DISPLAYLEVEL(2, "Trying %u different sets of parameters\n",
kIterations); kIterations);
for (d = kMinD; d <= kMaxD; d += 2) { for (d = kMinD; d <= kMaxD; d += 2) {
/* Initialize the context for this value of d */ /* Initialize the context for this value of d */
COVER_ctx_t ctx; COVER_ctx_t ctx;
LOCALDISPLAYLEVEL(displayLevel, 3, "d=%u\n", d); DISPLAYLEVEL(3, "d=%u\n", d);
{ {
const size_t initVal = COVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, d, splitPoint); /* Turn down global display level to clean up display at level 2 and below */
const int childDisplayLevel = (displayLevel == 0) ? 0 : displayLevel - 1;
const size_t initVal = COVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, d, splitPoint, childDisplayLevel);
if (ZSTD_isError(initVal)) { if (ZSTD_isError(initVal)) {
LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to initialize context\n"); DISPLAYLEVEL(1, "Failed to initialize context\n");
COVER_best_destroy(&best); COVER_best_destroy(&best);
POOL_free(pool); POOL_free(pool);
return initVal; return initVal;
@@ -1245,9 +1243,9 @@ ZDICTLIB_STATIC_API size_t ZDICT_optimizeTrainFromBuffer_cover(
/* Prepare the arguments */ /* Prepare the arguments */
COVER_tryParameters_data_t *data = (COVER_tryParameters_data_t *)malloc( COVER_tryParameters_data_t *data = (COVER_tryParameters_data_t *)malloc(
sizeof(COVER_tryParameters_data_t)); sizeof(COVER_tryParameters_data_t));
LOCALDISPLAYLEVEL(displayLevel, 3, "k=%u\n", k); DISPLAYLEVEL(3, "k=%u\n", k);
if (!data) { if (!data) {
LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to allocate parameters\n"); DISPLAYLEVEL(1, "Failed to allocate parameters\n");
COVER_best_destroy(&best); COVER_best_destroy(&best);
COVER_ctx_destroy(&ctx); COVER_ctx_destroy(&ctx);
POOL_free(pool); POOL_free(pool);
@@ -1262,7 +1260,7 @@ ZDICTLIB_STATIC_API size_t ZDICT_optimizeTrainFromBuffer_cover(
data->parameters.splitPoint = splitPoint; data->parameters.splitPoint = splitPoint;
data->parameters.steps = kSteps; data->parameters.steps = kSteps;
data->parameters.shrinkDict = shrinkDict; data->parameters.shrinkDict = shrinkDict;
data->parameters.zParams.notificationLevel = (unsigned)g_displayLevel; data->parameters.zParams.notificationLevel = (unsigned)ctx.displayLevel;
/* Check the parameters */ /* Check the parameters */
if (!COVER_checkParameters(data->parameters, dictBufferCapacity)) { if (!COVER_checkParameters(data->parameters, dictBufferCapacity)) {
DISPLAYLEVEL(1, "Cover parameters incorrect\n"); DISPLAYLEVEL(1, "Cover parameters incorrect\n");
@@ -1277,14 +1275,14 @@ ZDICTLIB_STATIC_API size_t ZDICT_optimizeTrainFromBuffer_cover(
COVER_tryParameters(data); COVER_tryParameters(data);
} }
/* Print status */ /* Print status */
LOCALDISPLAYUPDATE(displayLevel, 2, "\r%u%% ", DISPLAYUPDATE(lastUpdateTime, 2, "\r%u%% ",
(unsigned)((iteration * 100) / kIterations)); (unsigned)((iteration * 100) / kIterations));
++iteration; ++iteration;
} }
COVER_best_wait(&best); COVER_best_wait(&best);
COVER_ctx_destroy(&ctx); COVER_ctx_destroy(&ctx);
} }
LOCALDISPLAYLEVEL(displayLevel, 2, "\r%79s\r", ""); DISPLAYLEVEL(2, "\r%79s\r", "");
/* Fill the output buffer and parameters with output of the best parameters */ /* Fill the output buffer and parameters with output of the best parameters */
{ {
const size_t dictSize = best.dictSize; const size_t dictSize = best.dictSize;
lib/dictBuilder/fastcover.c
+39 -40
View File
@@ -49,38 +49,30 @@
/*-************************************* /*-*************************************
* Console display * Console display
*
* Captures the `displayLevel` variable in the local scope.
***************************************/ ***************************************/
#ifndef LOCALDISPLAYLEVEL
static int g_displayLevel = 0;
#endif
#undef DISPLAY #undef DISPLAY
#define DISPLAY(...) \ #define DISPLAY(...) \
{ \ { \
fprintf(stderr, __VA_ARGS__); \ fprintf(stderr, __VA_ARGS__); \
fflush(stderr); \ fflush(stderr); \
} }
#undef LOCALDISPLAYLEVEL #undef DISPLAYLEVEL
#define LOCALDISPLAYLEVEL(displayLevel, l, ...) \ #define DISPLAYLEVEL(l, ...) \
if (displayLevel >= l) { \ if (displayLevel >= l) { \
DISPLAY(__VA_ARGS__); \ DISPLAY(__VA_ARGS__); \
} /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */ } /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */
#undef DISPLAYLEVEL
#define DISPLAYLEVEL(l, ...) LOCALDISPLAYLEVEL(g_displayLevel, l, __VA_ARGS__)
#ifndef LOCALDISPLAYUPDATE #undef DISPLAYUPDATE
static const clock_t g_refreshRate = CLOCKS_PER_SEC * 15 / 100; #define DISPLAYUPDATE(lastUpdateTime, l, ...) \
static clock_t g_time = 0;
#endif
#undef LOCALDISPLAYUPDATE
#define LOCALDISPLAYUPDATE(displayLevel, l, ...) \
if (displayLevel >= l) { \ if (displayLevel >= l) { \
if ((clock() - g_time > g_refreshRate) || (displayLevel >= 4)) { \ const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100; \
g_time = clock(); \ if ((clock() - lastUpdateTime > refreshRate) || (displayLevel >= 4)) { \
lastUpdateTime = clock(); \
DISPLAY(__VA_ARGS__); \ DISPLAY(__VA_ARGS__); \
} \ } \
} }
#undef DISPLAYUPDATE
#define DISPLAYUPDATE(l, ...) LOCALDISPLAYUPDATE(g_displayLevel, l, __VA_ARGS__)
/*-************************************* /*-*************************************
@@ -136,6 +128,7 @@ typedef struct {
unsigned d; unsigned d;
unsigned f; unsigned f;
FASTCOVER_accel_t accelParams; FASTCOVER_accel_t accelParams;
int displayLevel;
} FASTCOVER_ctx_t; } FASTCOVER_ctx_t;
@@ -314,7 +307,8 @@ FASTCOVER_ctx_init(FASTCOVER_ctx_t* ctx,
const void* samplesBuffer, const void* samplesBuffer,
const size_t* samplesSizes, unsigned nbSamples, const size_t* samplesSizes, unsigned nbSamples,
unsigned d, double splitPoint, unsigned f, unsigned d, double splitPoint, unsigned f,
FASTCOVER_accel_t accelParams) FASTCOVER_accel_t accelParams,
int displayLevel)
{ {
const BYTE* const samples = (const BYTE*)samplesBuffer; const BYTE* const samples = (const BYTE*)samplesBuffer;
const size_t totalSamplesSize = COVER_sum(samplesSizes, nbSamples); const size_t totalSamplesSize = COVER_sum(samplesSizes, nbSamples);
@@ -323,6 +317,7 @@ FASTCOVER_ctx_init(FASTCOVER_ctx_t* ctx,
const unsigned nbTestSamples = splitPoint < 1.0 ? nbSamples - nbTrainSamples : nbSamples; const unsigned nbTestSamples = splitPoint < 1.0 ? nbSamples - nbTrainSamples : nbSamples;
const size_t trainingSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes, nbTrainSamples) : totalSamplesSize; const size_t trainingSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes, nbTrainSamples) : totalSamplesSize;
const size_t testSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes + nbTrainSamples, nbTestSamples) : totalSamplesSize; const size_t testSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes + nbTrainSamples, nbTestSamples) : totalSamplesSize;
ctx->displayLevel = displayLevel;
/* Checks */ /* Checks */
if (totalSamplesSize < MAX(d, sizeof(U64)) || if (totalSamplesSize < MAX(d, sizeof(U64)) ||
@@ -409,7 +404,9 @@ FASTCOVER_buildDictionary(const FASTCOVER_ctx_t* ctx,
const COVER_epoch_info_t epochs = COVER_computeEpochs( const COVER_epoch_info_t epochs = COVER_computeEpochs(
(U32)dictBufferCapacity, (U32)ctx->nbDmers, parameters.k, 1); (U32)dictBufferCapacity, (U32)ctx->nbDmers, parameters.k, 1);
const size_t maxZeroScoreRun = 10; const size_t maxZeroScoreRun = 10;
const int displayLevel = ctx->displayLevel;
size_t zeroScoreRun = 0; size_t zeroScoreRun = 0;
clock_t lastUpdateTime = 0;
size_t epoch; size_t epoch;
DISPLAYLEVEL(2, "Breaking content into %u epochs of size %u\n", DISPLAYLEVEL(2, "Breaking content into %u epochs of size %u\n",
(U32)epochs.num, (U32)epochs.size); (U32)epochs.num, (U32)epochs.size);
@@ -447,6 +444,7 @@ FASTCOVER_buildDictionary(const FASTCOVER_ctx_t* ctx,
tail -= segmentSize; tail -= segmentSize;
memcpy(dict + tail, ctx->samples + segment.begin, segmentSize); memcpy(dict + tail, ctx->samples + segment.begin, segmentSize);
DISPLAYUPDATE( DISPLAYUPDATE(
lastUpdateTime,
2, "\r%u%% ", 2, "\r%u%% ",
(unsigned)(((dictBufferCapacity - tail) * 100) / dictBufferCapacity)); (unsigned)(((dictBufferCapacity - tail) * 100) / dictBufferCapacity));
} }
@@ -484,6 +482,7 @@ static void FASTCOVER_tryParameters(void* opaque)
BYTE *const dict = (BYTE*)malloc(dictBufferCapacity); BYTE *const dict = (BYTE*)malloc(dictBufferCapacity);
COVER_dictSelection_t selection = COVER_dictSelectionError(ERROR(GENERIC)); COVER_dictSelection_t selection = COVER_dictSelectionError(ERROR(GENERIC));
U32* freqs = (U32*) malloc(((U64)1 << ctx->f) * sizeof(U32)); U32* freqs = (U32*) malloc(((U64)1 << ctx->f) * sizeof(U32));
const int displayLevel = ctx->displayLevel;
if (!segmentFreqs || !dict || !freqs) { if (!segmentFreqs || !dict || !freqs) {
DISPLAYLEVEL(1, "Failed to allocate buffers: out of memory\n"); DISPLAYLEVEL(1, "Failed to allocate buffers: out of memory\n");
goto _cleanup; goto _cleanup;
@@ -555,8 +554,7 @@ ZDICT_trainFromBuffer_fastCover(void* dictBuffer, size_t dictBufferCapacity,
FASTCOVER_ctx_t ctx; FASTCOVER_ctx_t ctx;
ZDICT_cover_params_t coverParams; ZDICT_cover_params_t coverParams;
FASTCOVER_accel_t accelParams; FASTCOVER_accel_t accelParams;
/* Initialize global data */ const int displayLevel = (int)parameters.zParams.notificationLevel;
g_displayLevel = (int)parameters.zParams.notificationLevel;
/* Assign splitPoint and f if not provided */ /* Assign splitPoint and f if not provided */
parameters.splitPoint = 1.0; parameters.splitPoint = 1.0;
parameters.f = parameters.f == 0 ? DEFAULT_F : parameters.f; parameters.f = parameters.f == 0 ? DEFAULT_F : parameters.f;
@@ -585,13 +583,13 @@ ZDICT_trainFromBuffer_fastCover(void* dictBuffer, size_t dictBufferCapacity,
{ {
size_t const initVal = FASTCOVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, size_t const initVal = FASTCOVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples,
coverParams.d, parameters.splitPoint, parameters.f, coverParams.d, parameters.splitPoint, parameters.f,
accelParams); accelParams, displayLevel);
if (ZSTD_isError(initVal)) { if (ZSTD_isError(initVal)) {
DISPLAYLEVEL(1, "Failed to initialize context\n"); DISPLAYLEVEL(1, "Failed to initialize context\n");
return initVal; return initVal;
} }
} }
COVER_warnOnSmallCorpus(dictBufferCapacity, ctx.nbDmers, g_displayLevel); COVER_warnOnSmallCorpus(dictBufferCapacity, ctx.nbDmers, displayLevel);
/* Build the dictionary */ /* Build the dictionary */
DISPLAYLEVEL(2, "Building dictionary\n"); DISPLAYLEVEL(2, "Building dictionary\n");
{ {
@@ -646,25 +644,26 @@ ZDICT_optimizeTrainFromBuffer_fastCover(
COVER_best_t best; COVER_best_t best;
POOL_ctx *pool = NULL; POOL_ctx *pool = NULL;
int warned = 0; int warned = 0;
clock_t lastUpdateTime = 0;
/* Checks */ /* Checks */
if (splitPoint <= 0 || splitPoint > 1) { if (splitPoint <= 0 || splitPoint > 1) {
LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect splitPoint\n"); DISPLAYLEVEL(1, "Incorrect splitPoint\n");
return ERROR(parameter_outOfBound); return ERROR(parameter_outOfBound);
} }
if (accel == 0 || accel > FASTCOVER_MAX_ACCEL) { if (accel == 0 || accel > FASTCOVER_MAX_ACCEL) {
LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect accel\n"); DISPLAYLEVEL(1, "Incorrect accel\n");
return ERROR(parameter_outOfBound); return ERROR(parameter_outOfBound);
} }
if (kMinK < kMaxD || kMaxK < kMinK) { if (kMinK < kMaxD || kMaxK < kMinK) {
LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect k\n"); DISPLAYLEVEL(1, "Incorrect k\n");
return ERROR(parameter_outOfBound); return ERROR(parameter_outOfBound);
} }
if (nbSamples == 0) { if (nbSamples == 0) {
LOCALDISPLAYLEVEL(displayLevel, 1, "FASTCOVER must have at least one input file\n"); DISPLAYLEVEL(1, "FASTCOVER must have at least one input file\n");
return ERROR(srcSize_wrong); return ERROR(srcSize_wrong);
} }
if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) { if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) {
LOCALDISPLAYLEVEL(displayLevel, 1, "dictBufferCapacity must be at least %u\n", DISPLAYLEVEL(1, "dictBufferCapacity must be at least %u\n",
ZDICT_DICTSIZE_MIN); ZDICT_DICTSIZE_MIN);
return ERROR(dstSize_tooSmall); return ERROR(dstSize_tooSmall);
} }
@@ -679,19 +678,18 @@ ZDICT_optimizeTrainFromBuffer_fastCover(
memset(&coverParams, 0 , sizeof(coverParams)); memset(&coverParams, 0 , sizeof(coverParams));
FASTCOVER_convertToCoverParams(*parameters, &coverParams); FASTCOVER_convertToCoverParams(*parameters, &coverParams);
accelParams = FASTCOVER_defaultAccelParameters[accel]; accelParams = FASTCOVER_defaultAccelParameters[accel];
/* Turn down global display level to clean up display at level 2 and below */
g_displayLevel = displayLevel == 0 ? 0 : displayLevel - 1;
/* Loop through d first because each new value needs a new context */ /* Loop through d first because each new value needs a new context */
LOCALDISPLAYLEVEL(displayLevel, 2, "Trying %u different sets of parameters\n", DISPLAYLEVEL(2, "Trying %u different sets of parameters\n", kIterations);
kIterations);
for (d = kMinD; d <= kMaxD; d += 2) { for (d = kMinD; d <= kMaxD; d += 2) {
/* Initialize the context for this value of d */ /* Initialize the context for this value of d */
FASTCOVER_ctx_t ctx; FASTCOVER_ctx_t ctx;
LOCALDISPLAYLEVEL(displayLevel, 3, "d=%u\n", d); DISPLAYLEVEL(3, "d=%u\n", d);
{ {
size_t const initVal = FASTCOVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, d, splitPoint, f, accelParams); /* Turn down global display level to clean up display at level 2 and below */
const int childDisplayLevel = displayLevel == 0 ? 0 : displayLevel - 1;
size_t const initVal = FASTCOVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, d, splitPoint, f, accelParams, childDisplayLevel);
if (ZSTD_isError(initVal)) { if (ZSTD_isError(initVal)) {
LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to initialize context\n"); DISPLAYLEVEL(1, "Failed to initialize context\n");
COVER_best_destroy(&best); COVER_best_destroy(&best);
POOL_free(pool); POOL_free(pool);
return initVal; return initVal;
@@ -706,9 +704,9 @@ ZDICT_optimizeTrainFromBuffer_fastCover(
/* Prepare the arguments */ /* Prepare the arguments */
FASTCOVER_tryParameters_data_t *data = (FASTCOVER_tryParameters_data_t *)malloc( FASTCOVER_tryParameters_data_t *data = (FASTCOVER_tryParameters_data_t *)malloc(
sizeof(FASTCOVER_tryParameters_data_t)); sizeof(FASTCOVER_tryParameters_data_t));
LOCALDISPLAYLEVEL(displayLevel, 3, "k=%u\n", k); DISPLAYLEVEL(3, "k=%u\n", k);
if (!data) { if (!data) {
LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to allocate parameters\n"); DISPLAYLEVEL(1, "Failed to allocate parameters\n");
COVER_best_destroy(&best); COVER_best_destroy(&best);
FASTCOVER_ctx_destroy(&ctx); FASTCOVER_ctx_destroy(&ctx);
POOL_free(pool); POOL_free(pool);
@@ -723,7 +721,7 @@ ZDICT_optimizeTrainFromBuffer_fastCover(
data->parameters.splitPoint = splitPoint; data->parameters.splitPoint = splitPoint;
data->parameters.steps = kSteps; data->parameters.steps = kSteps;
data->parameters.shrinkDict = shrinkDict; data->parameters.shrinkDict = shrinkDict;
data->parameters.zParams.notificationLevel = (unsigned)g_displayLevel; data->parameters.zParams.notificationLevel = (unsigned)ctx.displayLevel;
/* Check the parameters */ /* Check the parameters */
if (!FASTCOVER_checkParameters(data->parameters, dictBufferCapacity, if (!FASTCOVER_checkParameters(data->parameters, dictBufferCapacity,
data->ctx->f, accel)) { data->ctx->f, accel)) {
@@ -739,14 +737,15 @@ ZDICT_optimizeTrainFromBuffer_fastCover(
FASTCOVER_tryParameters(data); FASTCOVER_tryParameters(data);
} }
/* Print status */ /* Print status */
LOCALDISPLAYUPDATE(displayLevel, 2, "\r%u%% ", DISPLAYUPDATE(lastUpdateTime,
(unsigned)((iteration * 100) / kIterations)); 2, "\r%u%% ",
(unsigned)((iteration * 100) / kIterations));
++iteration; ++iteration;
} }
COVER_best_wait(&best); COVER_best_wait(&best);
FASTCOVER_ctx_destroy(&ctx); FASTCOVER_ctx_destroy(&ctx);
} }
LOCALDISPLAYLEVEL(displayLevel, 2, "\r%79s\r", ""); DISPLAYLEVEL(2, "\r%79s\r", "");
/* Fill the output buffer and parameters with output of the best parameters */ /* Fill the output buffer and parameters with output of the best parameters */
{ {
const size_t dictSize = best.dictSize; const size_t dictSize = best.dictSize;