From 96b39f65fa0e11664c64b5b82bbfdd433125d203 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Sat, 31 Dec 2016 21:07:44 -0800 Subject: [PATCH 1/6] Add COVER dictionary builder --- lib/dictBuilder/cover.c | 1041 +++++++++++++++++++++++++++++++++++++++ lib/dictBuilder/zdict.h | 52 ++ 2 files changed, 1093 insertions(+) create mode 100644 lib/dictBuilder/cover.c diff --git a/lib/dictBuilder/cover.c b/lib/dictBuilder/cover.c new file mode 100644 index 000000000..1bad01869 --- /dev/null +++ b/lib/dictBuilder/cover.c @@ -0,0 +1,1041 @@ +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +/*-************************************* +* Dependencies +***************************************/ +#include /* fprintf */ +#include /* malloc, free, qsort */ +#include /* memset */ +#include /* clock */ +#ifdef ZSTD_PTHREAD +#include "threading.h" +#endif + +#include "mem.h" /* read */ +#include "zstd_internal.h" /* includes zstd.h */ +#ifndef ZDICT_STATIC_LINKING_ONLY +#define ZDICT_STATIC_LINKING_ONLY +#endif +#include "zdict.h" + +/*-************************************* +* Constants +***************************************/ +#define COVER_MAX_SAMPLES_SIZE ((U32)-1) + +/*-************************************* +* Console display +***************************************/ +static int g_displayLevel = 2; +#define DISPLAY(...) \ + { \ + fprintf(stderr, __VA_ARGS__); \ + fflush(stderr); \ + } +#define LOCALDISPLAYLEVEL(displayLevel, l, ...) \ + if (displayLevel >= l) { \ + DISPLAY(__VA_ARGS__); \ + } /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */ +#define DISPLAYLEVEL(l, ...) LOCALDISPLAYLEVEL(g_displayLevel, l, __VA_ARGS__) + +#define LOCALDISPLAYUPDATE(displayLevel, l, ...) \ + if (displayLevel >= l) { \ + if ((clock() - g_time > refreshRate) || (displayLevel >= 4)) { \ + g_time = clock(); \ + DISPLAY(__VA_ARGS__); \ + if (displayLevel >= 4) \ + fflush(stdout); \ + } \ + } +#define DISPLAYUPDATE(l, ...) LOCALDISPLAYUPDATE(g_displayLevel, l, __VA_ARGS__) +static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100; +static clock_t g_time = 0; + +/*-************************************* +* Hash table +*************************************** +* A small specialized hash map for storing activeDmers. +* The map does not resize, so if it becomes full it will loop forever. +* Thus, the map must be large enough to store every value. +* The map implements linear probing and keeps its load less than 0.5. +*/ + +#define MAP_EMPTY_VALUE ((U32)-1) +typedef struct COVER_map_pair_t_s { + U32 key; + U32 value; +} COVER_map_pair_t; + +typedef struct COVER_map_s { + COVER_map_pair_t *data; + U32 sizeLog; + U32 size; + U32 sizeMask; +} COVER_map_t; + +/** + * Clear the map. + */ +static void COVER_map_clear(COVER_map_t *map) { + memset(map->data, MAP_EMPTY_VALUE, map->size * sizeof(COVER_map_pair_t)); +} + +/** + * Initializes a map of the given size. + * Returns 1 on success and 0 on failure. + * The map must be destroyed with COVER_map_destroy(). + * The map is only guaranteed to be large enough to hold size elements. + */ +static int COVER_map_init(COVER_map_t *map, U32 size) { + map->sizeLog = ZSTD_highbit32(size) + 2; + map->size = (U32)1 << map->sizeLog; + map->sizeMask = map->size - 1; + map->data = (COVER_map_pair_t *)malloc(map->size * sizeof(COVER_map_pair_t)); + if (!map->data) { + map->sizeLog = 0; + map->size = 0; + return 0; + } + COVER_map_clear(map); + return 1; +} + +/** + * Internal hash function + */ +static const U32 prime4bytes = 2654435761U; +static U32 COVER_map_hash(COVER_map_t *map, U32 key) { + return (key * prime4bytes) >> (32 - map->sizeLog); +} + +/** + * Helper function that returns the index that a key should be placed into. + */ +static U32 COVER_map_index(COVER_map_t *map, U32 key) { + const U32 hash = COVER_map_hash(map, key); + U32 i; + for (i = hash;; i = (i + 1) & map->sizeMask) { + COVER_map_pair_t *pos = &map->data[i]; + if (pos->value == MAP_EMPTY_VALUE) { + return i; + } + if (pos->key == key) { + return i; + } + } +} + +/** + * Returns the pointer to the value for key. + * If key is not in the map, it is inserted and the value is set to 0. + * The map must not be full. + */ +static U32 *COVER_map_at(COVER_map_t *map, U32 key) { + COVER_map_pair_t *pos = &map->data[COVER_map_index(map, key)]; + if (pos->value == MAP_EMPTY_VALUE) { + pos->key = key; + pos->value = 0; + } + return &pos->value; +} + +/** + * Deletes key from the map if present. + */ +static void COVER_map_remove(COVER_map_t *map, U32 key) { + U32 i = COVER_map_index(map, key); + COVER_map_pair_t *del = &map->data[i]; + U32 shift = 1; + if (del->value == MAP_EMPTY_VALUE) { + return; + } + for (i = (i + 1) & map->sizeMask;; i = (i + 1) & map->sizeMask) { + COVER_map_pair_t *const pos = &map->data[i]; + /* If the position is empty we are done */ + if (pos->value == MAP_EMPTY_VALUE) { + del->value = MAP_EMPTY_VALUE; + return; + } + /* If pos can be moved to del do so */ + if (((i - COVER_map_hash(map, pos->key)) & map->sizeMask) >= shift) { + del->key = pos->key; + del->value = pos->value; + del = pos; + shift = 1; + } else { + ++shift; + } + } +} + +/** + * Destroyes a map that is inited with COVER_map_init(). + */ +static void COVER_map_destroy(COVER_map_t *map) { + if (map->data) { + free(map->data); + } + map->data = NULL; + map->size = 0; +} + +/*-************************************* +* Context +***************************************/ + +typedef struct { + const BYTE *samples; + size_t *offsets; + const size_t *samplesSizes; + size_t nbSamples; + U32 *suffix; + size_t suffixSize; + U32 *freqs; + U32 *dmerAt; + unsigned d; +} COVER_ctx_t; + +/* We need a global context for qsort... */ +static COVER_ctx_t *g_ctx = NULL; + +/*-************************************* +* Helper functions +***************************************/ + +/** + * Returns the sum of the sample sizes. + */ +static size_t COVER_sum(const size_t *samplesSizes, unsigned nbSamples) { + size_t sum = 0; + size_t i; + for (i = 0; i < nbSamples; ++i) { + sum += samplesSizes[i]; + } + return sum; +} + +/** + * Returns -1 if the dmer at lp is less than the dmer at rp. + * Return 0 if the dmers at lp and rp are equal. + * Returns 1 if the dmer at lp is greater than the dmer at rp. + */ +static int COVER_cmp(COVER_ctx_t *ctx, const void *lp, const void *rp) { + const U32 lhs = *(const U32 *)lp; + const U32 rhs = *(const U32 *)rp; + return memcmp(ctx->samples + lhs, ctx->samples + rhs, ctx->d); +} + +/** + * Same as COVER_cmp() except ties are broken by pointer value + * NOTE: g_ctx must be set to call this function. A global is required because + * qsort doesn't take an opaque pointer. + */ +static int COVER_strict_cmp(const void *lp, const void *rp) { + int result = COVER_cmp(g_ctx, lp, rp); + if (result == 0) { + result = lp < rp ? -1 : 1; + } + return result; +} + +/** + * Returns the first pointer in [first, last) whose element does not compare + * less than value. If no such element exists it returns last. + */ +static const size_t *COVER_lower_bound(const size_t *first, const size_t *last, + size_t value) { + size_t count = last - first; + while (count != 0) { + size_t step = count / 2; + const size_t *ptr = first; + ptr += step; + if (*ptr < value) { + first = ++ptr; + count -= step + 1; + } else { + count = step; + } + } + return first; +} + +/** + * Generic groupBy function. + * Groups an array sorted by cmp into groups with equivalent values. + * Calls grp for each group. + */ +static void +COVER_groupBy(const void *data, size_t count, size_t size, COVER_ctx_t *ctx, + int (*cmp)(COVER_ctx_t *, const void *, const void *), + void (*grp)(COVER_ctx_t *, const void *, const void *)) { + const BYTE *ptr = (const BYTE *)data; + size_t num = 0; + while (num < count) { + const BYTE *grpEnd = ptr + size; + ++num; + while (num < count && cmp(ctx, ptr, grpEnd) == 0) { + grpEnd += size; + ++num; + } + grp(ctx, ptr, grpEnd); + ptr = grpEnd; + } +} + +/*-************************************* +* Cover functions +***************************************/ + +/** + * Called on each group of positions with the same dmer. + * Counts the frequency of each dmer and saves it in the suffix array. + * Fills `ctx->dmerAt`. + */ +static void COVER_group(COVER_ctx_t *ctx, const void *group, + const void *groupEnd) { + /* The group consists of all the positions with the same first d bytes. */ + const U32 *grpPtr = (const U32 *)group; + const U32 *grpEnd = (const U32 *)groupEnd; + /* The dmerId is how we will reference this dmer. + * This allows us to map the whole dmer space to a much smaller space, the + * size of the suffix array. + */ + const U32 dmerId = (U32)(grpPtr - ctx->suffix); + /* Count the number of samples this dmer shows up in */ + U32 freq = 0; + /* Details */ + const size_t *curOffsetPtr = ctx->offsets; + const size_t *offsetsEnd = ctx->offsets + ctx->nbSamples; + /* Once *grpPtr >= curSampleEnd this occurrence of the dmer is in a + * different sample than the last. + */ + size_t curSampleEnd = ctx->offsets[0]; + for (; grpPtr != grpEnd; ++grpPtr) { + /* Save the dmerId for this position so we can get back to it. */ + ctx->dmerAt[*grpPtr] = dmerId; + /* Dictionaries only help for the first reference to the dmer. + * After that zstd can reference the match from the previous reference. + * So only count each dmer once for each sample it is in. + */ + if (*grpPtr < curSampleEnd) { + continue; + } + freq += 1; + /* Binary search to find the end of the sample *grpPtr is in. + * In the common case that grpPtr + 1 == grpEnd we can skip the binary + * search because the loop is over. + */ + if (grpPtr + 1 != grpEnd) { + const size_t *sampleEndPtr = + COVER_lower_bound(curOffsetPtr, offsetsEnd, *grpPtr); + curSampleEnd = *sampleEndPtr; + curOffsetPtr = sampleEndPtr + 1; + } + } + /* At this point we are never going to look at this segment of the suffix + * array again. We take advantage of this fact to save memory. + * We store the frequency of the dmer in the first position of the group, + * which is dmerId. + */ + ctx->suffix[dmerId] = freq; +} + +/** + * A segment is a range in the source as well as the score of the segment. + */ +typedef struct { + U32 begin; + U32 end; + double score; +} COVER_segment_t; + +/** + * Selects the best segment in an epoch. + * Segments of are scored according to the function: + * + * Let F(d) be the frequency of dmer d. + * Let L(S) be the length of segment S. + * Let S_i be the dmer at position i of segment S. + * + * F(S_1) + F(S_2) + ... + F(S_{L(S)-d+1}) + * Score(S) = -------------------------------------- + * smoothing + L(S) + * + * We try kStep segment lengths in the range [kMin, kMax]. + * For each segment length we find the best segment according to Score. + * We then take the best segment overall according to Score and return it. + * + * The difference from the paper is that we try multiple segment lengths. + * We want to fit the segment length closer to the length of the useful part. + * Longer segments allow longer matches, so they are worth more than shorter + * ones. However, if the extra length isn't high frequency it hurts us. + * We add the smoothing in to give an advantage to longer segments. + * The larger smoothing is, the more longer matches are favored. + */ +static COVER_segment_t COVER_selectSegment(const COVER_ctx_t *ctx, U32 *freqs, + COVER_map_t *activeDmers, U32 begin, + U32 end, COVER_params_t parameters) { + /* Saves the best segment of any length tried */ + COVER_segment_t globalBestSegment = {0, 0, 0}; + /* For each segment length */ + U32 k; + U32 step = MAX((parameters.kMax - parameters.kMin) / parameters.kStep, 1); + for (k = parameters.kMin; k <= parameters.kMax; k += step) { + /* Save the best segment of this length */ + COVER_segment_t bestSegment = {0, 0, 0}; + COVER_segment_t activeSegment; + const size_t dmersInK = k - ctx->d + 1; + /* Reset the activeDmers in the segment */ + COVER_map_clear(activeDmers); + activeSegment.begin = begin; + activeSegment.end = begin; + activeSegment.score = 0; + /* Slide the active segment through the whole epoch. + * Save the best segment in bestSegment. + */ + while (activeSegment.end < end) { + /* The dmerId for the dmer at the next position */ + U32 newDmer = ctx->dmerAt[activeSegment.end]; + /* The entry in activeDmers for this dmerId */ + U32 *newDmerOcc = COVER_map_at(activeDmers, newDmer); + /* If the dmer isn't already present in the segment add its score. */ + if (*newDmerOcc == 0) { + /* The paper suggest using the L-0.5 norm, but experiments show that it + * doesn't help. + */ + activeSegment.score += freqs[newDmer]; + } + /* Add the dmer to the segment */ + activeSegment.end += 1; + *newDmerOcc += 1; + + /* If the window is now too large, drop the first position */ + if (activeSegment.end - activeSegment.begin == dmersInK + 1) { + U32 delDmer = ctx->dmerAt[activeSegment.begin]; + U32 *delDmerOcc = COVER_map_at(activeDmers, delDmer); + activeSegment.begin += 1; + *delDmerOcc -= 1; + /* If this is the last occurence of the dmer, subtract its score */ + if (*delDmerOcc == 0) { + COVER_map_remove(activeDmers, delDmer); + activeSegment.score -= freqs[delDmer]; + } + } + + /* If this segment is the best so far save it */ + if (activeSegment.score > bestSegment.score) { + bestSegment = activeSegment; + } + } + { + /* Trim off the zero frequency head and tail from the segment. */ + U32 newBegin = bestSegment.end; + U32 newEnd = bestSegment.begin; + U32 pos; + for (pos = bestSegment.begin; pos != bestSegment.end; ++pos) { + U32 freq = freqs[ctx->dmerAt[pos]]; + if (freq != 0) { + newBegin = MIN(newBegin, pos); + newEnd = pos + 1; + } + } + bestSegment.begin = newBegin; + bestSegment.end = newEnd; + /* Calculate the final score normalizing for segment length */ + bestSegment.score /= + (parameters.smoothing + (bestSegment.end - bestSegment.begin)); + } + /* If this segment is the best so far for any length save it */ + if (bestSegment.score > globalBestSegment.score) { + globalBestSegment = bestSegment; + } + } + { + /* Zero out the frequency of each dmer covered by the chosen segment. */ + size_t pos; + for (pos = globalBestSegment.begin; pos != globalBestSegment.end; ++pos) { + freqs[ctx->dmerAt[pos]] = 0; + } + } + return globalBestSegment; +} + +/** + * Check the validity of the parameters. + * If the parameters are valid and any are default, set them to the correct + * values. + * Returns 1 on success, 0 on failure. + */ +static int COVER_defaultParameters(COVER_params_t *parameters) { + /* kMin and d are required parameters */ + if (parameters->d == 0 || parameters->kMin == 0) { + return 0; + } + /* d <= kMin */ + if (parameters->d > parameters->kMin) { + return 0; + } + /* If kMax is set (non-zero) then kMin <= kMax */ + if (parameters->kMax != 0 && parameters->kMax < parameters->kMin) { + return 0; + } + /* If kMax is set, then kStep must be as well */ + if (parameters->kMax != 0 && parameters->kStep == 0) { + return 0; + } + parameters->kMax = MAX(parameters->kMin, parameters->kMax); + parameters->kStep = MAX(1, parameters->kStep); + return 1; +} + +/** + * Clean up a context initialized with `COVER_ctx_init()`. + */ +static void COVER_ctx_destroy(COVER_ctx_t *ctx) { + if (!ctx) { + return; + } + if (ctx->suffix) { + free(ctx->suffix); + ctx->suffix = NULL; + } + if (ctx->freqs) { + free(ctx->freqs); + ctx->freqs = NULL; + } + if (ctx->dmerAt) { + free(ctx->dmerAt); + ctx->dmerAt = NULL; + } + if (ctx->offsets) { + free(ctx->offsets); + ctx->offsets = NULL; + } +} + +/** + * Prepare a context for dictionary building. + * The context is only dependent on the parameter `d` and can used multiple + * times. + * Returns 1 on success or zero on error. + * The context must be destroyed with `COVER_ctx_destroy()`. + */ +static int COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer, + const size_t *samplesSizes, unsigned nbSamples, + unsigned d) { + const BYTE *const samples = (const BYTE *)samplesBuffer; + const size_t totalSamplesSize = COVER_sum(samplesSizes, nbSamples); + /* Checks */ + if (totalSamplesSize < d || + totalSamplesSize > (size_t)COVER_MAX_SAMPLES_SIZE) { + return 0; + } + /* Zero the context */ + memset(ctx, 0, sizeof(*ctx)); + DISPLAYLEVEL(2, "Training on %u samples of total size %u\n", nbSamples, + (U32)totalSamplesSize); + ctx->samples = samples; + ctx->samplesSizes = samplesSizes; + ctx->nbSamples = nbSamples; + /* Partial suffix array */ + ctx->suffixSize = totalSamplesSize - d + 1; + ctx->suffix = (U32 *)malloc(ctx->suffixSize * sizeof(U32)); + /* Maps index to the dmerID */ + ctx->dmerAt = (U32 *)malloc(ctx->suffixSize * sizeof(U32)); + /* The offsets of each file */ + ctx->offsets = (size_t *)malloc((nbSamples + 1) * sizeof(size_t)); + if (!ctx->suffix || !ctx->dmerAt || !ctx->offsets) { + COVER_ctx_destroy(ctx); + return 0; + } + ctx->freqs = NULL; + ctx->d = d; + + /* Fill offsets from the samlesSizes */ + { + U32 i; + ctx->offsets[0] = 0; + for (i = 1; i <= nbSamples; ++i) { + ctx->offsets[i] = ctx->offsets[i - 1] + samplesSizes[i - 1]; + } + } + DISPLAYLEVEL(2, "Constructing partial suffix array\n"); + { + /* suffix is a partial suffix array. + * It only sorts suffixes by their first parameters.d bytes. + * The sort is stable, so each dmer group is sorted by position in input. + */ + U32 i; + for (i = 0; i < ctx->suffixSize; ++i) { + ctx->suffix[i] = i; + } + /* qsort doesn't take an opaque pointer, so pass as a global */ + g_ctx = ctx; + qsort(ctx->suffix, ctx->suffixSize, sizeof(U32), &COVER_strict_cmp); + } + DISPLAYLEVEL(2, "Computing frequencies\n"); + /* For each dmer group (group of positions with the same first d bytes): + * 1. For each position we set dmerAt[position] = dmerID. The dmerID is + * (groupBeginPtr - suffix). This allows us to go from position to + * dmerID so we can look up values in freq. + * 2. We calculate how many samples the dmer occurs in and save it in + * freqs[dmerId]. + */ + COVER_groupBy(ctx->suffix, ctx->suffixSize, sizeof(U32), ctx, &COVER_cmp, + &COVER_group); + ctx->freqs = ctx->suffix; + ctx->suffix = NULL; + return 1; +} + +/** + * Given the prepared context build the dictionary. + */ +static size_t COVER_buildDictionary(const COVER_ctx_t *ctx, U32 *freqs, + COVER_map_t *activeDmers, void *dictBuffer, + size_t dictBufferCapacity, + COVER_params_t parameters) { + BYTE *const dict = (BYTE *)dictBuffer; + size_t tail = dictBufferCapacity; + /* Divide the data up into epochs of equal size. + * We will select at least one segment from each epoch. + */ + const U32 epochs = (U32)(dictBufferCapacity / parameters.kMax); + const U32 epochSize = (U32)(ctx->suffixSize / epochs); + size_t epoch; + DISPLAYLEVEL(2, "Breaking content into %u epochs of size %u\n", epochs, + epochSize); + for (epoch = 0; tail > 0; epoch = (epoch + 1) % epochs) { + const U32 epochBegin = (U32)(epoch * epochSize); + const U32 epochEnd = epochBegin + epochSize; + size_t segmentSize; + COVER_segment_t segment = COVER_selectSegment( + ctx, freqs, activeDmers, epochBegin, epochEnd, parameters); + segmentSize = MIN(segment.end - segment.begin + parameters.d - 1, tail); + if (segmentSize == 0) { + break; + } + /* We fill the dictionary from the back to allow the best segments to be + * referenced with the smallest offsets. + */ + tail -= segmentSize; + memcpy(dict + tail, ctx->samples + segment.begin, segmentSize); + DISPLAYUPDATE( + 2, "\r%u%% ", + (U32)(((dictBufferCapacity - tail) * 100) / dictBufferCapacity)); + } + DISPLAYLEVEL(2, "\r%79s\r", ""); + return tail; +} + +/** + * Translate from COVER_params_t to ZDICT_params_t required for finalizing the + * dictionary. + */ +static ZDICT_params_t COVER_translateParams(COVER_params_t parameters) { + ZDICT_params_t zdictParams; + memset(&zdictParams, 0, sizeof(zdictParams)); + zdictParams.notificationLevel = 1; + zdictParams.dictID = parameters.dictID; + zdictParams.compressionLevel = parameters.compressionLevel; + return zdictParams; +} + +/** + * Constructs a dictionary using a heuristic based on the following paper: + * + * Liao, Petri, Moffat, Wirth + * Effective Construction of Relative Lempel-Ziv Dictionaries + * Published in WWW 2016. + */ +ZDICTLIB_API size_t COVER_trainFromBuffer( + void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer, + const size_t *samplesSizes, unsigned nbSamples, COVER_params_t parameters) { + BYTE *const dict = (BYTE *)dictBuffer; + COVER_ctx_t ctx; + COVER_map_t activeDmers; + size_t rc; + /* Checks */ + if (!COVER_defaultParameters(¶meters)) { + DISPLAYLEVEL(1, "Cover parameters incorrect\n"); + return ERROR(GENERIC); + } + if (nbSamples == 0) { + DISPLAYLEVEL(1, "Cover must have at least one input file\n"); + return ERROR(GENERIC); + } + if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) { + return ERROR(dstSize_tooSmall); + } + /* Initialize global data */ + g_displayLevel = parameters.notificationLevel; + /* Initialize context and activeDmers */ + if (!COVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, + parameters.d)) { + DISPLAYLEVEL(1, "Failed to initialize context\n"); + return ERROR(GENERIC); + } + if (!COVER_map_init(&activeDmers, parameters.kMax - parameters.d + 1)) { + DISPLAYLEVEL(1, "Failed to allocate dmer map: out of memory\n"); + COVER_ctx_destroy(&ctx); + return ERROR(GENERIC); + } + + DISPLAYLEVEL(2, "Building dictionary\n"); + { + const size_t tail = + COVER_buildDictionary(&ctx, ctx.freqs, &activeDmers, dictBuffer, + dictBufferCapacity, parameters); + ZDICT_params_t zdictParams = COVER_translateParams(parameters); + DISPLAYLEVEL(2, "Dictionary content size: %u", + (U32)(dictBufferCapacity - tail)); + rc = ZDICT_finalizeDictionary(dict, dictBufferCapacity, dict + tail, + dictBufferCapacity - tail, samplesBuffer, + samplesSizes, nbSamples, zdictParams); + } + if (!ZSTD_isError(rc)) { + DISPLAYLEVEL(2, "Constructed dictionary of size %u\n", (U32)rc); + } + COVER_ctx_destroy(&ctx); + COVER_map_destroy(&activeDmers); + return rc; +} + +/** + * COVER_best_t is used for two purposes: + * 1. Synchronizing threads. + * 2. Saving the best parameters and dictionary. + * + * All of the methods are thread safe if `ZSTD_PTHREAD` is defined. + */ +typedef struct COVER_best_s { +#ifdef ZSTD_PTHREAD + pthread_mutex_t mutex; + pthread_cond_t cond; + size_t liveJobs; +#endif + void *dict; + size_t dictSize; + COVER_params_t parameters; + size_t compressedSize; +} COVER_best_t; + +/** + * Initialize the `COVER_best_t`. + */ +static void COVER_best_init(COVER_best_t *best) { + if (!best) { + return; + } +#ifdef ZSTD_PTHREAD + pthread_mutex_init(&best->mutex, NULL); + pthread_cond_init(&best->cond, NULL); + best->liveJobs = 0; +#endif + best->dict = NULL; + best->dictSize = 0; + best->compressedSize = (size_t)-1; + memset(&best->parameters, 0, sizeof(best->parameters)); +} + +/** + * Wait until liveJobs == 0. + */ +static void COVER_best_wait(COVER_best_t *best) { + if (!best) { + return; + } +#ifdef ZSTD_PTHREAD + pthread_mutex_lock(&best->mutex); + while (best->liveJobs != 0) { + pthread_cond_wait(&best->cond, &best->mutex); + } + pthread_mutex_unlock(&best->mutex); +#endif +} + +/** + * Call COVER_best_wait() and then destroy the COVER_best_t. + */ +static void COVER_best_destroy(COVER_best_t *best) { + if (!best) { + return; + } + COVER_best_wait(best); + if (best->dict) { + free(best->dict); + } +#ifdef ZSTD_PTHREAD + pthread_mutex_destroy(&best->mutex); + pthread_cond_destroy(&best->cond); +#endif +} + +/** + * Called when a thread is about to be launched. + * Increments liveJobs. + */ +static void COVER_best_start(COVER_best_t *best) { + if (!best) { + return; + } +#ifdef ZSTD_PTHREAD + pthread_mutex_lock(&best->mutex); + ++best->liveJobs; + pthread_mutex_unlock(&best->mutex); +#endif +} + +/** + * Called when a thread finishes executing, both on error or success. + * Decrements liveJobs and signals any waiting threads if liveJobs == 0. + * If this dictionary is the best so far save it and its parameters. + */ +static void COVER_best_finish(COVER_best_t *best, size_t compressedSize, + COVER_params_t parameters, void *dict, + size_t dictSize) { + if (!best) { + return; + } + { +#ifdef ZSTD_PTHREAD + size_t liveJobs; + pthread_mutex_lock(&best->mutex); + --best->liveJobs; + liveJobs = best->liveJobs; +#endif + /* If the new dictionary is better */ + if (compressedSize < best->compressedSize) { + /* Allocate space if necessary */ + if (!best->dict || best->dictSize < dictSize) { + if (best->dict) { + free(best->dict); + } + best->dict = malloc(dictSize); + if (!best->dict) { + best->compressedSize = ERROR(GENERIC); + best->dictSize = 0; + return; + } + } + /* Save the dictionary, parameters, and size */ + memcpy(best->dict, dict, dictSize); + best->dictSize = dictSize; + best->parameters = parameters; + best->compressedSize = compressedSize; + } +#ifdef ZSTD_PTHREAD + pthread_mutex_unlock(&best->mutex); + if (liveJobs == 0) { + pthread_cond_broadcast(&best->cond); + } +#endif + } +} + +/** + * Parameters for COVER_tryParameters(). + */ +typedef struct COVER_tryParameters_data_s { + const COVER_ctx_t *ctx; + COVER_best_t *best; + size_t dictBufferCapacity; + COVER_params_t parameters; +} COVER_tryParameters_data_t; + +/** + * Tries a set of parameters and upates the COVER_best_t with the results. + * This function is thread safe if ZSTD_PTHREAD is defined. + * It takes its parameters as an *OWNING* opaque pointer to support threading. + */ +static void COVER_tryParameters(void *opaque) { + /* Save parameters as local variables */ + COVER_tryParameters_data_t *data = (COVER_tryParameters_data_t *)opaque; + const COVER_ctx_t *ctx = data->ctx; + COVER_params_t parameters = data->parameters; + size_t dictBufferCapacity = data->dictBufferCapacity; + size_t totalCompressedSize = ERROR(GENERIC); + /* Allocate space for hash table, dict, and freqs */ + COVER_map_t activeDmers; + BYTE *const dict = (BYTE * const)malloc(dictBufferCapacity); + U32 *freqs = (U32 *)malloc(ctx->suffixSize * sizeof(U32)); + if (!COVER_map_init(&activeDmers, parameters.kMax - parameters.d + 1)) { + DISPLAYLEVEL(1, "Failed to allocate dmer map: out of memory\n"); + goto _cleanup; + } + if (!dict || !freqs) { + DISPLAYLEVEL(1, "Failed to allocate dictionary buffer\n"); + goto _cleanup; + } + /* Copy the frequencies because we need to modify them */ + memcpy(freqs, ctx->freqs, ctx->suffixSize * sizeof(U32)); + /* Build the dictionary */ + { + const size_t tail = COVER_buildDictionary(ctx, freqs, &activeDmers, dict, + dictBufferCapacity, parameters); + ZDICT_params_t zdictParams = COVER_translateParams(parameters); + dictBufferCapacity = ZDICT_finalizeDictionary( + dict, dictBufferCapacity, dict + tail, dictBufferCapacity - tail, + ctx->samples, ctx->samplesSizes, (unsigned)ctx->nbSamples, zdictParams); + if (ZDICT_isError(dictBufferCapacity)) { + DISPLAYLEVEL(1, "Failed to finalize dictionary\n"); + goto _cleanup; + } + } + /* Check total compressed size */ + { + /* Pointers */ + ZSTD_CCtx *cctx; + ZSTD_CDict *cdict; + void *dst; + /* Local variables */ + size_t dstCapacity; + size_t i; + /* Allocate dst with enough space to compress the maximum sized sample */ + { + size_t maxSampleSize = 0; + for (i = 0; i < ctx->nbSamples; ++i) { + maxSampleSize = MAX(ctx->samplesSizes[i], maxSampleSize); + } + dstCapacity = ZSTD_compressBound(maxSampleSize); + dst = malloc(dstCapacity); + } + /* Create the cctx and cdict */ + cctx = ZSTD_createCCtx(); + cdict = + ZSTD_createCDict(dict, dictBufferCapacity, parameters.compressionLevel); + if (!dst || !cctx || !cdict) { + goto _compressCleanup; + } + /* Compress each sample and sum their sizes (or error) */ + totalCompressedSize = 0; + for (i = 0; i < ctx->nbSamples; ++i) { + const size_t size = ZSTD_compress_usingCDict( + cctx, dst, dstCapacity, ctx->samples + ctx->offsets[i], + ctx->samplesSizes[i], cdict); + if (ZSTD_isError(size)) { + totalCompressedSize = ERROR(GENERIC); + goto _compressCleanup; + } + totalCompressedSize += size; + } + _compressCleanup: + ZSTD_freeCCtx(cctx); + ZSTD_freeCDict(cdict); + if (dst) { + free(dst); + } + } + +_cleanup: + COVER_best_finish(data->best, totalCompressedSize, parameters, dict, + dictBufferCapacity); + free(data); + COVER_map_destroy(&activeDmers); + if (dict) { + free(dict); + } + if (freqs) { + free(freqs); + } +} + +ZDICTLIB_API size_t COVER_optimizeTrainFromBuffer(void *dictBuffer, + size_t dictBufferCapacity, + const void *samplesBuffer, + const size_t *samplesSizes, + unsigned nbSamples, + COVER_params_t *parameters) { + /* constants */ + const unsigned dMin = parameters->d == 0 ? 6 : parameters->d; + const unsigned dMax = parameters->d == 0 ? 16 : parameters->d; + const unsigned min = parameters->kMin == 0 ? 32 : parameters->kMin; + const unsigned max = parameters->kMax == 0 ? 1024 : parameters->kMax; + const unsigned kStep = parameters->kStep == 0 ? 8 : parameters->kStep; + const unsigned step = MAX((max - min) / kStep, 1); + /* Local variables */ + unsigned iteration = 1; + const unsigned iterations = + (1 + (dMax - dMin) / 2) * (((1 + kStep) * (2 + kStep)) / 2) * 4; + const int displayLevel = parameters->notificationLevel; + unsigned d; + COVER_best_t best; + COVER_best_init(&best); + /* Turn down display level to clean up display at level 2 and below */ + g_displayLevel = parameters->notificationLevel - 1; + /* Loop through d first because each new value needs a new context */ + LOCALDISPLAYLEVEL(displayLevel, 3, "Trying %u different sets of parameters\n", + iterations); + for (d = dMin; d <= dMax; d += 2) { + unsigned kMin; + /* Initialize the context for this value of d */ + COVER_ctx_t ctx; + LOCALDISPLAYLEVEL(displayLevel, 3, "d=%u\n", d); + if (!COVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, d)) { + LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to initialize context\n"); + COVER_best_destroy(&best); + return ERROR(GENERIC); + } + /* Loop through the rest of the parameters reusing the same context */ + for (kMin = min; kMin <= max; kMin += step) { + unsigned kMax; + LOCALDISPLAYLEVEL(displayLevel, 3, "kMin=%u\n", kMin); + for (kMax = kMin; kMax <= max; kMax += step) { + unsigned smoothing; + LOCALDISPLAYLEVEL(displayLevel, 3, "kMax=%u\n", kMax); + for (smoothing = kMin / 4; smoothing <= kMin * 2; smoothing *= 2) { + /* Prepare the arguments */ + COVER_tryParameters_data_t *data = + (COVER_tryParameters_data_t *)malloc( + sizeof(COVER_tryParameters_data_t)); + LOCALDISPLAYLEVEL(displayLevel, 3, "smoothing=%u\n", smoothing); + if (!data) { + LOCALDISPLAYLEVEL(displayLevel, 1, + "Failed to allocate parameters\n"); + COVER_best_destroy(&best); + COVER_ctx_destroy(&ctx); + return ERROR(GENERIC); + } + data->ctx = &ctx; + data->best = &best; + data->dictBufferCapacity = dictBufferCapacity; + data->parameters = *parameters; + data->parameters.d = d; + data->parameters.kMin = kMin; + data->parameters.kStep = kStep; + data->parameters.kMax = kMax; + data->parameters.smoothing = smoothing; + /* Call the function and pass ownership of data to it */ + COVER_best_start(&best); + COVER_tryParameters(data); + /* Print status */ + LOCALDISPLAYUPDATE(displayLevel, 2, "\r%u%% ", + (U32)((iteration * 100) / iterations)); + ++iteration; + } + } + } + COVER_best_wait(&best); + COVER_ctx_destroy(&ctx); + } + LOCALDISPLAYLEVEL(displayLevel, 2, "\r%79s\r", ""); + /* Fill the output buffer and parameters with output of the best parameters */ + { + const size_t dictSize = best.dictSize; + if (ZSTD_isError(best.compressedSize)) { + COVER_best_destroy(&best); + return best.compressedSize; + } + *parameters = best.parameters; + memcpy(dictBuffer, best.dict, dictSize); + COVER_best_destroy(&best); + return dictSize; + } +} diff --git a/lib/dictBuilder/zdict.h b/lib/dictBuilder/zdict.h index 63b8f0722..4a2e79448 100644 --- a/lib/dictBuilder/zdict.h +++ b/lib/dictBuilder/zdict.h @@ -86,6 +86,58 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_advanced(void* dictBuffer, size_t dict const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, ZDICT_params_t parameters); +/*! COVER_params_t : + For all values 0 means default. + kMin and d are the only required parameters. +*/ +typedef struct { + unsigned d; /* dmer size : constraint: <= kMin : Should probably be in the range [6, 16]. */ + unsigned kMin; /* Minimum segment size : constraint: > 0 */ + unsigned kStep; /* Try kStep segment lengths uniformly distributed in the range [kMin, kMax] : 0 (default) only if kMax == 0 */ + unsigned kMax; /* Maximum segment size : 0 = kMin (default) : constraint : 0 or >= kMin */ + unsigned smoothing; /* Higher smoothing => larger segments are selected. Only useful if kMax > kMin. */ + + unsigned notificationLevel; /* Write to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */ + unsigned dictID; /* 0 means auto mode (32-bits random value); other : force dictID value */ + int compressionLevel; /* 0 means default; target a specific zstd compression level */ +} COVER_params_t; + + +/*! COVER_trainFromBuffer() : + Train a dictionary from an array of samples using the COVER algorithm. + Samples must be stored concatenated in a single flat buffer `samplesBuffer`, + supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order. + The resulting dictionary will be saved into `dictBuffer`. + @return : size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) + or an error code, which can be tested with ZDICT_isError(). + Tips : In general, a reasonable dictionary has a size of ~ 100 KB. + It's obviously possible to target smaller or larger ones, just by specifying different `dictBufferCapacity`. + In general, it's recommended to provide a few thousands samples, but this can vary a lot. + It's recommended that total size of all samples be about ~x100 times the target size of dictionary. +*/ +ZDICTLIB_API size_t COVER_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, + COVER_params_t parameters); + +/*! COVER_optimizeTrainFromBuffer() : + The same requirements as above hold for all the parameters except `parameters`. + This function tries many parameter combinations and picks the best parameters. + `*parameters` is filled with the best parameters found, and the dictionary + constructed with those parameters is stored in `dictBuffer`. + + All of the {d, kMin, kStep, kMax} are optional, and smoothing is ignored. + If d is non-zero then we don't check multiple values of d, otherwise we check d = {6, 8, 10, 12, 14, 16}. + If kStep is non-zero then it is used, otherwise we pick 8. + If kMin and kMax are non-zero, then they limit the search space for kMin and kMax, + otherwise we check kMin and kMax values in the range [32, 1024]. + + @return : size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) + or an error code, which can be tested with ZDICT_isError(). + On success `*parameters` contains the parameters selected. +*/ +ZDICTLIB_API size_t COVER_optimizeTrainFromBuffer(void* dictBuffer, size_t dictBufferCapacity, + const void* samplesBuffer, const size_t *samplesSizes, unsigned nbSamples, + COVER_params_t *parameters); /*! ZDICT_finalizeDictionary() : From 9103ed6c8bf01531602d09efe6a0fa21c5012b54 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Sat, 31 Dec 2016 21:08:12 -0800 Subject: [PATCH 2/6] Add cover.c to build files --- build/VS2005/fuzzer/fuzzer.vcproj | 4 ++++ build/VS2005/zstd/zstd.vcproj | 4 ++++ build/VS2005/zstdlib/zstdlib.vcproj | 4 ++++ build/VS2008/fuzzer/fuzzer.vcproj | 4 ++++ build/VS2008/zstd/zstd.vcproj | 4 ++++ build/VS2008/zstdlib/zstdlib.vcproj | 4 ++++ build/VS2010/fuzzer/fuzzer.vcxproj | 1 + build/VS2010/libzstd-dll/libzstd-dll.vcxproj | 1 + build/VS2010/libzstd/libzstd.vcxproj | 1 + build/VS2010/zstd/zstd.vcxproj | 1 + build/cmake/lib/CMakeLists.txt | 1 + 11 files changed, 29 insertions(+) diff --git a/build/VS2005/fuzzer/fuzzer.vcproj b/build/VS2005/fuzzer/fuzzer.vcproj index b1ac81365..d6ec14d16 100644 --- a/build/VS2005/fuzzer/fuzzer.vcproj +++ b/build/VS2005/fuzzer/fuzzer.vcproj @@ -331,6 +331,10 @@ RelativePath="..\..\..\programs\datagen.c" > + + diff --git a/build/VS2005/zstd/zstd.vcproj b/build/VS2005/zstd/zstd.vcproj index 9f49e3cb6..5ef7a98f8 100644 --- a/build/VS2005/zstd/zstd.vcproj +++ b/build/VS2005/zstd/zstd.vcproj @@ -343,6 +343,10 @@ RelativePath="..\..\..\programs\dibio.c" > + + diff --git a/build/VS2005/zstdlib/zstdlib.vcproj b/build/VS2005/zstdlib/zstdlib.vcproj index d95212b33..828cc8285 100644 --- a/build/VS2005/zstdlib/zstdlib.vcproj +++ b/build/VS2005/zstdlib/zstdlib.vcproj @@ -327,6 +327,10 @@ Filter="cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx" UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}" > + + diff --git a/build/VS2008/fuzzer/fuzzer.vcproj b/build/VS2008/fuzzer/fuzzer.vcproj index 311b79909..f6912b8d9 100644 --- a/build/VS2008/fuzzer/fuzzer.vcproj +++ b/build/VS2008/fuzzer/fuzzer.vcproj @@ -332,6 +332,10 @@ RelativePath="..\..\..\programs\datagen.c" > + + diff --git a/build/VS2008/zstd/zstd.vcproj b/build/VS2008/zstd/zstd.vcproj index ad64f8696..3bd5ed536 100644 --- a/build/VS2008/zstd/zstd.vcproj +++ b/build/VS2008/zstd/zstd.vcproj @@ -344,6 +344,10 @@ RelativePath="..\..\..\programs\dibio.c" > + + diff --git a/build/VS2008/zstdlib/zstdlib.vcproj b/build/VS2008/zstdlib/zstdlib.vcproj index b1c103e32..69b742d19 100644 --- a/build/VS2008/zstdlib/zstdlib.vcproj +++ b/build/VS2008/zstdlib/zstdlib.vcproj @@ -328,6 +328,10 @@ Filter="cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx" UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}" > + + diff --git a/build/VS2010/fuzzer/fuzzer.vcxproj b/build/VS2010/fuzzer/fuzzer.vcxproj index 7227c7ec0..623a9ca43 100644 --- a/build/VS2010/fuzzer/fuzzer.vcxproj +++ b/build/VS2010/fuzzer/fuzzer.vcxproj @@ -165,6 +165,7 @@ + diff --git a/build/VS2010/libzstd-dll/libzstd-dll.vcxproj b/build/VS2010/libzstd-dll/libzstd-dll.vcxproj index f1ea5c822..f0feecb26 100644 --- a/build/VS2010/libzstd-dll/libzstd-dll.vcxproj +++ b/build/VS2010/libzstd-dll/libzstd-dll.vcxproj @@ -32,6 +32,7 @@ + diff --git a/build/VS2010/libzstd/libzstd.vcxproj b/build/VS2010/libzstd/libzstd.vcxproj index 228e83da4..c8d21dd4a 100644 --- a/build/VS2010/libzstd/libzstd.vcxproj +++ b/build/VS2010/libzstd/libzstd.vcxproj @@ -32,6 +32,7 @@ + diff --git a/build/VS2010/zstd/zstd.vcxproj b/build/VS2010/zstd/zstd.vcxproj index 5b5852689..2b30966ab 100644 --- a/build/VS2010/zstd/zstd.vcxproj +++ b/build/VS2010/zstd/zstd.vcxproj @@ -29,6 +29,7 @@ + diff --git a/build/cmake/lib/CMakeLists.txt b/build/cmake/lib/CMakeLists.txt index 41fe2733f..34a639cdb 100644 --- a/build/cmake/lib/CMakeLists.txt +++ b/build/cmake/lib/CMakeLists.txt @@ -67,6 +67,7 @@ SET(Sources ${LIBRARY_DIR}/compress/zstd_compress.c ${LIBRARY_DIR}/decompress/huf_decompress.c ${LIBRARY_DIR}/decompress/zstd_decompress.c + ${LIBRARY_DIR}/dictBuilder/cover.c ${LIBRARY_DIR}/dictBuilder/divsufsort.c ${LIBRARY_DIR}/dictBuilder/zdict.c ${LIBRARY_DIR}/deprecated/zbuff_common.c From df8415c50275cd0a4f2ab2e57fc2d6a928de2995 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Sat, 31 Dec 2016 21:08:24 -0800 Subject: [PATCH 3/6] Add COVER to the zstd cli --- programs/dibio.c | 61 ++++++++++++++++++++++++++++++++++++++----- programs/dibio.h | 4 +-- programs/zstdcli.c | 64 +++++++++++++++++++++++++++++++++++++++++----- 3 files changed, 114 insertions(+), 15 deletions(-) diff --git a/programs/dibio.c b/programs/dibio.c index b95bab34e..ba15d2106 100644 --- a/programs/dibio.c +++ b/programs/dibio.c @@ -42,6 +42,7 @@ #define SAMPLESIZE_MAX (128 KB) #define MEMMULT 11 /* rough estimation : memory cost to analyze 1 byte of sample */ +#define COVER_MEMMULT 9 /* rough estimation : memory cost to analyze 1 byte of sample */ static const size_t maxMemory = (sizeof(size_t) == 4) ? (2 GB - 64 MB) : ((size_t)(512 MB) << sizeof(size_t)); #define NOISELENGTH 32 @@ -118,10 +119,36 @@ static unsigned DiB_loadFiles(void* buffer, size_t* bufferSizePtr, fileSizes[n] = fileSize; fclose(f); } } + DISPLAYLEVEL(2, "\r%79s\r", ""); *bufferSizePtr = pos; return n; } +#define DiB_rotl32(x,r) ((x << r) | (x >> (32 - r))) +static U32 DiB_rand(U32* src) +{ + static const U32 prime1 = 2654435761U; + static const U32 prime2 = 2246822519U; + U32 rand32 = *src; + rand32 *= prime1; + rand32 ^= prime2; + rand32 = DiB_rotl32(rand32, 13); + *src = rand32; + return rand32 >> 5; +} + +static void DiB_shuffle(const char** fileNamesTable, unsigned nbFiles) { + /* Initialize the pseudorandom number generator */ + U32 seed = 0xFD2FB528; + unsigned i; + for (i = nbFiles - 1; i > 0; --i) { + unsigned const j = DiB_rand(&seed) % (i + 1); + const char* tmp = fileNamesTable[j]; + fileNamesTable[j] = fileNamesTable[i]; + fileNamesTable[i] = tmp; + } +} + /*-******************************************************** * Dictionary training functions @@ -202,7 +229,8 @@ size_t ZDICT_trainFromBuffer_unsafe(void* dictBuffer, size_t dictBufferCapacity, int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize, const char** fileNamesTable, unsigned nbFiles, - ZDICT_params_t params) + ZDICT_params_t *params, COVER_params_t *coverParams, + int optimizeCover) { void* const dictBuffer = malloc(maxDictSize); size_t* const fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t)); @@ -213,8 +241,10 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize, int result = 0; /* Checks */ + if (params) g_displayLevel = params->notificationLevel; + else if (coverParams) g_displayLevel = coverParams->notificationLevel; + else EXM_THROW(13, "Neither dictionary algorith selected"); /* should not happen */ if ((!fileSizes) || (!srcBuffer) || (!dictBuffer)) EXM_THROW(12, "not enough memory for DiB_trainFiles"); /* should not happen */ - g_displayLevel = params.notificationLevel; if (g_tooLargeSamples) { DISPLAYLEVEL(2, "! Warning : some samples are very large \n"); DISPLAYLEVEL(2, "! Note that dictionary is only useful for small files or beginning of large files. \n"); @@ -233,12 +263,31 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize, DISPLAYLEVEL(1, "Not enough memory; training on %u MB only...\n", (unsigned)(benchedSize >> 20)); /* Load input buffer */ + DISPLAYLEVEL(3, "Shuffling input files\n"); + DiB_shuffle(fileNamesTable, nbFiles); nbFiles = DiB_loadFiles(srcBuffer, &benchedSize, fileSizes, fileNamesTable, nbFiles); - DiB_fillNoise((char*)srcBuffer + benchedSize, NOISELENGTH); /* guard band, for end of buffer condition */ - { size_t const dictSize = ZDICT_trainFromBuffer_unsafe(dictBuffer, maxDictSize, - srcBuffer, fileSizes, nbFiles, - params); + { + size_t dictSize; + if (params) { + DiB_fillNoise((char*)srcBuffer + benchedSize, NOISELENGTH); /* guard band, for end of buffer condition */ + dictSize = ZDICT_trainFromBuffer_unsafe(dictBuffer, maxDictSize, + srcBuffer, fileSizes, nbFiles, + *params); + } else if (optimizeCover) { + dictSize = COVER_optimizeTrainFromBuffer( + dictBuffer, maxDictSize, srcBuffer, fileSizes, nbFiles, + coverParams); + if (!ZDICT_isError(dictSize)) { + DISPLAYLEVEL(2, "smoothing=%d\nkMin=%d\nkStep=%d\nkMax=%d\nd=%d\n", + coverParams->smoothing, coverParams->kMin, + coverParams->kStep, coverParams->kMax, coverParams->d); + } + } else { + dictSize = COVER_trainFromBuffer(dictBuffer, maxDictSize, + srcBuffer, fileSizes, nbFiles, + *coverParams); + } if (ZDICT_isError(dictSize)) { DISPLAYLEVEL(1, "dictionary training failed : %s \n", ZDICT_getErrorName(dictSize)); /* should not happen */ result = 1; diff --git a/programs/dibio.h b/programs/dibio.h index 6780d8698..e61d0042c 100644 --- a/programs/dibio.h +++ b/programs/dibio.h @@ -32,7 +32,7 @@ */ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize, const char** fileNamesTable, unsigned nbFiles, - ZDICT_params_t parameters); - + ZDICT_params_t *params, COVER_params_t *coverParams, + int optimizeCover); #endif diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 978ffcfe0..f4d33d3f8 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -127,6 +127,8 @@ static int usage_advanced(const char* programName) DISPLAY( "\n"); DISPLAY( "Dictionary builder :\n"); DISPLAY( "--train ## : create a dictionary from a training set of files \n"); + DISPLAY( "--cover=k=#,d=# : use the cover algorithm with parameters k and d \n"); + DISPLAY( "--optimize-cover[=steps=#,k=#,d=#] : optimize cover parameters with optional parameters\n"); DISPLAY( " -o file : `file` is dictionary name (default: %s) \n", g_defaultDictName); DISPLAY( "--maxdict ## : limit dictionary to specified size (default : %u) \n", g_defaultMaxDictSize); DISPLAY( " -s# : dictionary selectivity level (default: %u)\n", g_defaultSelectivityLevel); @@ -192,6 +194,29 @@ static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) } +#ifndef ZSTD_NODICT +/** + * parseCoverParameters() : + * reads cover parameters from *stringPtr (e.g. "--cover=smoothing=100,kmin=48,kstep=4,kmax=64,d=8") into *params + * @return 1 means that cover parameters were correct + * @return 0 in case of malformed parameters + */ +static unsigned parseCoverParameters(const char* stringPtr, COVER_params_t *params) +{ + memset(params, 0, sizeof(*params)); + for (; ;) { + if (longCommandWArg(&stringPtr, "smoothing=")) { params->smoothing = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } + if (longCommandWArg(&stringPtr, "k=") || longCommandWArg(&stringPtr, "kMin=") || longCommandWArg(&stringPtr, "kmin=")) { params->kMin = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } + if (longCommandWArg(&stringPtr, "kStep=") || longCommandWArg(&stringPtr, "kstep=")) { params->kStep = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } + if (longCommandWArg(&stringPtr, "kMax=") || longCommandWArg(&stringPtr, "kmax=")) { params->kMax = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } + if (longCommandWArg(&stringPtr, "d=")) { params->d = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } + return 0; + } + if (stringPtr[0] != 0) return 0; + DISPLAYLEVEL(4, "smoothing=%d\nkMin=%d\nkStep=%d\nkMax=%d\nd=%d\n", params->smoothing, params->kMin, params->kStep, params->kMax, params->d); + return 1; +} +#endif /** parseCompressionParameters() : * reads compression parameters from *stringPtr (e.g. "--zstd=wlog=23,clog=23,hlog=22,slog=6,slen=3,tlen=48,strat=6") into *params * @return 1 means that compression parameters were correct @@ -254,6 +279,10 @@ int main(int argCount, const char* argv[]) char* fileNamesBuf = NULL; unsigned fileNamesNb; #endif +#ifndef ZSTD_NODICT + COVER_params_t coverParams; + int cover = 0; +#endif /* init */ (void)recursive; (void)cLevelLast; /* not used when ZSTD_NOBENCH set */ @@ -318,6 +347,20 @@ int main(int argCount, const char* argv[]) if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; } /* long commands with arguments */ +#ifndef ZSTD_NODICT + if (longCommandWArg(&argument, "--cover=")) { + cover=1; if (!parseCoverParameters(argument, &coverParams)) CLEAN_RETURN(badusage(programName)); + continue; + } + if (longCommandWArg(&argument, "--optimize-cover")) { + cover=2; + /* Allow optional arguments following an = */ + if (*argument == 0) { memset(&coverParams, 0, sizeof(coverParams)); } + else if (*argument++ != '=') { CLEAN_RETURN(badusage(programName)); } + else if (!parseCoverParameters(argument, &coverParams)) { CLEAN_RETURN(badusage(programName)); } + continue; + } +#endif if (longCommandWArg(&argument, "--memlimit=")) { memLimit = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "--memory=")) { memLimit = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "--memlimit-decompress=")) { memLimit = readU32FromChar(&argument); continue; } @@ -520,13 +563,20 @@ int main(int argCount, const char* argv[]) /* Check if dictionary builder is selected */ if (operation==zom_train) { #ifndef ZSTD_NODICT - ZDICT_params_t dictParams; - memset(&dictParams, 0, sizeof(dictParams)); - dictParams.compressionLevel = dictCLevel; - dictParams.selectivityLevel = dictSelect; - dictParams.notificationLevel = displayLevel; - dictParams.dictID = dictID; - DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, dictParams); + if (cover) { + coverParams.compressionLevel = dictCLevel; + coverParams.notificationLevel = displayLevel; + coverParams.dictID = dictID; + DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, NULL, &coverParams, cover - 1); + } else { + ZDICT_params_t dictParams; + memset(&dictParams, 0, sizeof(dictParams)); + dictParams.compressionLevel = dictCLevel; + dictParams.selectivityLevel = dictSelect; + dictParams.notificationLevel = displayLevel; + dictParams.dictID = dictID; + DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, &dictParams, NULL, 0); + } #endif goto _end; } From cbb3ce376b3ecbe2a1a45d8ddcdeb0d6f2d7deb8 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Sun, 1 Jan 2017 01:59:51 -0500 Subject: [PATCH 4/6] Add cover cli to playtests --- tests/playTests.sh | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/playTests.sh b/tests/playTests.sh index dfc90c338..a94446e7e 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -255,6 +255,27 @@ rm -rf dirTestDict rm tmp* +$ECHO "\n**** cover dictionary tests **** " + +TESTFILE=../programs/zstdcli.c +./datagen > tmpDict +$ECHO "- Create first dictionary" +$ZSTD --train --cover=k=46,d=8 *.c ../programs/*.c -o tmpDict +cp $TESTFILE tmp +$ZSTD -f tmp -D tmpDict +$ZSTD -d tmp.zst -D tmpDict -fo result +$DIFF $TESTFILE result +$ECHO "- Create second (different) dictionary" +$ZSTD --train --cover=kmin=46,kstep=2,kmax=64,d=6,smoothing=23 *.c ../programs/*.c ../programs/*.h -o tmpDictC +$ZSTD -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!" +$ECHO "- Create dictionary with short dictID" +$ZSTD --train --cover=k=46,d=8 *.c ../programs/*.c --dictID 1 -o tmpDict1 +cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" +$ECHO "- Create dictionary with size limit" +$ZSTD --train --optimize-cover=kstep=2,d=8 *.c ../programs/*.c -o tmpDict2 --maxdict 4K +rm tmp* + + $ECHO "\n**** integrity tests **** " $ECHO "test one file (tmp1.zst) " From 3a1fefcf006484bda63d8948146d13ef2b7d2428 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Mon, 2 Jan 2017 12:40:43 -0800 Subject: [PATCH 5/6] Simplify COVER parameters --- lib/dictBuilder/cover.c | 340 +++++++++++++++++++--------------------- lib/dictBuilder/zdict.h | 15 +- programs/dibio.c | 4 +- programs/zstdcli.c | 8 +- tests/playTests.sh | 4 +- 5 files changed, 172 insertions(+), 199 deletions(-) diff --git a/lib/dictBuilder/cover.c b/lib/dictBuilder/cover.c index 1bad01869..c8bee4e26 100644 --- a/lib/dictBuilder/cover.c +++ b/lib/dictBuilder/cover.c @@ -361,137 +361,103 @@ typedef struct { * Segments of are scored according to the function: * * Let F(d) be the frequency of dmer d. - * Let L(S) be the length of segment S. - * Let S_i be the dmer at position i of segment S. + * Let S_i be the dmer at position i of segment S which has length k. * - * F(S_1) + F(S_2) + ... + F(S_{L(S)-d+1}) - * Score(S) = -------------------------------------- - * smoothing + L(S) + * Score(S) = F(S_1) + F(S_2) + ... + F(S_{k-d+1}) * - * We try kStep segment lengths in the range [kMin, kMax]. - * For each segment length we find the best segment according to Score. - * We then take the best segment overall according to Score and return it. - * - * The difference from the paper is that we try multiple segment lengths. - * We want to fit the segment length closer to the length of the useful part. - * Longer segments allow longer matches, so they are worth more than shorter - * ones. However, if the extra length isn't high frequency it hurts us. - * We add the smoothing in to give an advantage to longer segments. - * The larger smoothing is, the more longer matches are favored. + * Once the dmer d is in the dictionay we set F(d) = 0. */ static COVER_segment_t COVER_selectSegment(const COVER_ctx_t *ctx, U32 *freqs, COVER_map_t *activeDmers, U32 begin, U32 end, COVER_params_t parameters) { - /* Saves the best segment of any length tried */ - COVER_segment_t globalBestSegment = {0, 0, 0}; - /* For each segment length */ - U32 k; - U32 step = MAX((parameters.kMax - parameters.kMin) / parameters.kStep, 1); - for (k = parameters.kMin; k <= parameters.kMax; k += step) { - /* Save the best segment of this length */ - COVER_segment_t bestSegment = {0, 0, 0}; - COVER_segment_t activeSegment; - const size_t dmersInK = k - ctx->d + 1; - /* Reset the activeDmers in the segment */ - COVER_map_clear(activeDmers); - activeSegment.begin = begin; - activeSegment.end = begin; - activeSegment.score = 0; - /* Slide the active segment through the whole epoch. - * Save the best segment in bestSegment. - */ - while (activeSegment.end < end) { - /* The dmerId for the dmer at the next position */ - U32 newDmer = ctx->dmerAt[activeSegment.end]; - /* The entry in activeDmers for this dmerId */ - U32 *newDmerOcc = COVER_map_at(activeDmers, newDmer); - /* If the dmer isn't already present in the segment add its score. */ - if (*newDmerOcc == 0) { - /* The paper suggest using the L-0.5 norm, but experiments show that it - * doesn't help. - */ - activeSegment.score += freqs[newDmer]; - } - /* Add the dmer to the segment */ - activeSegment.end += 1; - *newDmerOcc += 1; + /* Constants */ + const U32 k = parameters.k; + const U32 d = parameters.d; + const U32 dmersInK = k - d + 1; + /* Try each segment (activeSegment) and save the best (bestSegment) */ + COVER_segment_t bestSegment = {0, 0, 0}; + COVER_segment_t activeSegment; + /* Reset the activeDmers in the segment */ + COVER_map_clear(activeDmers); + /* The activeSegment starts at the beginning of the epoch. */ + activeSegment.begin = begin; + activeSegment.end = begin; + activeSegment.score = 0; + /* Slide the activeSegment through the whole epoch. + * Save the best segment in bestSegment. + */ + while (activeSegment.end < end) { + /* The dmerId for the dmer at the next position */ + U32 newDmer = ctx->dmerAt[activeSegment.end]; + /* The entry in activeDmers for this dmerId */ + U32 *newDmerOcc = COVER_map_at(activeDmers, newDmer); + /* If the dmer isn't already present in the segment add its score. */ + if (*newDmerOcc == 0) { + /* The paper suggest using the L-0.5 norm, but experiments show that it + * doesn't help. + */ + activeSegment.score += freqs[newDmer]; + } + /* Add the dmer to the segment */ + activeSegment.end += 1; + *newDmerOcc += 1; - /* If the window is now too large, drop the first position */ - if (activeSegment.end - activeSegment.begin == dmersInK + 1) { - U32 delDmer = ctx->dmerAt[activeSegment.begin]; - U32 *delDmerOcc = COVER_map_at(activeDmers, delDmer); - activeSegment.begin += 1; - *delDmerOcc -= 1; - /* If this is the last occurence of the dmer, subtract its score */ - if (*delDmerOcc == 0) { - COVER_map_remove(activeDmers, delDmer); - activeSegment.score -= freqs[delDmer]; - } - } - - /* If this segment is the best so far save it */ - if (activeSegment.score > bestSegment.score) { - bestSegment = activeSegment; + /* If the window is now too large, drop the first position */ + if (activeSegment.end - activeSegment.begin == dmersInK + 1) { + U32 delDmer = ctx->dmerAt[activeSegment.begin]; + U32 *delDmerOcc = COVER_map_at(activeDmers, delDmer); + activeSegment.begin += 1; + *delDmerOcc -= 1; + /* If this is the last occurence of the dmer, subtract its score */ + if (*delDmerOcc == 0) { + COVER_map_remove(activeDmers, delDmer); + activeSegment.score -= freqs[delDmer]; } } - { - /* Trim off the zero frequency head and tail from the segment. */ - U32 newBegin = bestSegment.end; - U32 newEnd = bestSegment.begin; - U32 pos; - for (pos = bestSegment.begin; pos != bestSegment.end; ++pos) { - U32 freq = freqs[ctx->dmerAt[pos]]; - if (freq != 0) { - newBegin = MIN(newBegin, pos); - newEnd = pos + 1; - } - } - bestSegment.begin = newBegin; - bestSegment.end = newEnd; - /* Calculate the final score normalizing for segment length */ - bestSegment.score /= - (parameters.smoothing + (bestSegment.end - bestSegment.begin)); - } - /* If this segment is the best so far for any length save it */ - if (bestSegment.score > globalBestSegment.score) { - globalBestSegment = bestSegment; + + /* If this segment is the best so far save it */ + if (activeSegment.score > bestSegment.score) { + bestSegment = activeSegment; } } + { + /* Trim off the zero frequency head and tail from the segment. */ + U32 newBegin = bestSegment.end; + U32 newEnd = bestSegment.begin; + U32 pos; + for (pos = bestSegment.begin; pos != bestSegment.end; ++pos) { + U32 freq = freqs[ctx->dmerAt[pos]]; + if (freq != 0) { + newBegin = MIN(newBegin, pos); + newEnd = pos + 1; + } + } + bestSegment.begin = newBegin; + bestSegment.end = newEnd; + } { /* Zero out the frequency of each dmer covered by the chosen segment. */ - size_t pos; - for (pos = globalBestSegment.begin; pos != globalBestSegment.end; ++pos) { + U32 pos; + for (pos = bestSegment.begin; pos != bestSegment.end; ++pos) { freqs[ctx->dmerAt[pos]] = 0; } } - return globalBestSegment; + return bestSegment; } /** * Check the validity of the parameters. - * If the parameters are valid and any are default, set them to the correct - * values. - * Returns 1 on success, 0 on failure. + * Returns non-zero if the parameters are valid and 0 otherwise. */ -static int COVER_defaultParameters(COVER_params_t *parameters) { - /* kMin and d are required parameters */ - if (parameters->d == 0 || parameters->kMin == 0) { +static int COVER_checkParameters(COVER_params_t parameters) { + /* k and d are required parameters */ + if (parameters.d == 0 || parameters.k == 0) { return 0; } - /* d <= kMin */ - if (parameters->d > parameters->kMin) { + /* d <= k */ + if (parameters.d > parameters.k) { return 0; } - /* If kMax is set (non-zero) then kMin <= kMax */ - if (parameters->kMax != 0 && parameters->kMax < parameters->kMin) { - return 0; - } - /* If kMax is set, then kStep must be as well */ - if (parameters->kMax != 0 && parameters->kStep == 0) { - return 0; - } - parameters->kMax = MAX(parameters->kMin, parameters->kMax); - parameters->kStep = MAX(1, parameters->kStep); return 1; } @@ -607,17 +573,22 @@ static size_t COVER_buildDictionary(const COVER_ctx_t *ctx, U32 *freqs, /* Divide the data up into epochs of equal size. * We will select at least one segment from each epoch. */ - const U32 epochs = (U32)(dictBufferCapacity / parameters.kMax); + const U32 epochs = (U32)(dictBufferCapacity / parameters.k); const U32 epochSize = (U32)(ctx->suffixSize / epochs); size_t epoch; DISPLAYLEVEL(2, "Breaking content into %u epochs of size %u\n", epochs, epochSize); + /* Loop through the epochs until there are no more segments or the dictionary + * is full. + */ for (epoch = 0; tail > 0; epoch = (epoch + 1) % epochs) { const U32 epochBegin = (U32)(epoch * epochSize); const U32 epochEnd = epochBegin + epochSize; size_t segmentSize; + /* Select a segment */ COVER_segment_t segment = COVER_selectSegment( ctx, freqs, activeDmers, epochBegin, epochEnd, parameters); + /* Trim the segment if necessary and if it is empty then we are done */ segmentSize = MIN(segment.end - segment.begin + parameters.d - 1, tail); if (segmentSize == 0) { break; @@ -661,9 +632,8 @@ ZDICTLIB_API size_t COVER_trainFromBuffer( BYTE *const dict = (BYTE *)dictBuffer; COVER_ctx_t ctx; COVER_map_t activeDmers; - size_t rc; /* Checks */ - if (!COVER_defaultParameters(¶meters)) { + if (!COVER_checkParameters(parameters)) { DISPLAYLEVEL(1, "Cover parameters incorrect\n"); return ERROR(GENERIC); } @@ -672,6 +642,8 @@ ZDICTLIB_API size_t COVER_trainFromBuffer( return ERROR(GENERIC); } if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) { + DISPLAYLEVEL(1, "dictBufferCapacity must be at least %u\n", + ZDICT_DICTSIZE_MIN); return ERROR(dstSize_tooSmall); } /* Initialize global data */ @@ -682,7 +654,7 @@ ZDICTLIB_API size_t COVER_trainFromBuffer( DISPLAYLEVEL(1, "Failed to initialize context\n"); return ERROR(GENERIC); } - if (!COVER_map_init(&activeDmers, parameters.kMax - parameters.d + 1)) { + if (!COVER_map_init(&activeDmers, parameters.k - parameters.d + 1)) { DISPLAYLEVEL(1, "Failed to allocate dmer map: out of memory\n"); COVER_ctx_destroy(&ctx); return ERROR(GENERIC); @@ -694,18 +666,17 @@ ZDICTLIB_API size_t COVER_trainFromBuffer( COVER_buildDictionary(&ctx, ctx.freqs, &activeDmers, dictBuffer, dictBufferCapacity, parameters); ZDICT_params_t zdictParams = COVER_translateParams(parameters); - DISPLAYLEVEL(2, "Dictionary content size: %u", - (U32)(dictBufferCapacity - tail)); - rc = ZDICT_finalizeDictionary(dict, dictBufferCapacity, dict + tail, - dictBufferCapacity - tail, samplesBuffer, - samplesSizes, nbSamples, zdictParams); + const size_t dictionarySize = ZDICT_finalizeDictionary( + dict, dictBufferCapacity, dict + tail, dictBufferCapacity - tail, + samplesBuffer, samplesSizes, nbSamples, zdictParams); + if (!ZSTD_isError(dictionarySize)) { + DISPLAYLEVEL(2, "Constructed dictionary of size %u\n", + (U32)dictionarySize); + } + COVER_ctx_destroy(&ctx); + COVER_map_destroy(&activeDmers); + return dictionarySize; } - if (!ZSTD_isError(rc)) { - DISPLAYLEVEL(2, "Constructed dictionary of size %u\n", (U32)rc); - } - COVER_ctx_destroy(&ctx); - COVER_map_destroy(&activeDmers); - return rc; } /** @@ -713,7 +684,8 @@ ZDICTLIB_API size_t COVER_trainFromBuffer( * 1. Synchronizing threads. * 2. Saving the best parameters and dictionary. * - * All of the methods are thread safe if `ZSTD_PTHREAD` is defined. + * All of the methods except COVER_best_init() are thread safe if zstd is + * compiled with multithreaded support. */ typedef struct COVER_best_s { #ifdef ZSTD_PTHREAD @@ -852,26 +824,26 @@ typedef struct COVER_tryParameters_data_s { /** * Tries a set of parameters and upates the COVER_best_t with the results. - * This function is thread safe if ZSTD_PTHREAD is defined. + * This function is thread safe if zstd is compiled with multithreaded support. * It takes its parameters as an *OWNING* opaque pointer to support threading. */ static void COVER_tryParameters(void *opaque) { /* Save parameters as local variables */ - COVER_tryParameters_data_t *data = (COVER_tryParameters_data_t *)opaque; - const COVER_ctx_t *ctx = data->ctx; - COVER_params_t parameters = data->parameters; + COVER_tryParameters_data_t *const data = (COVER_tryParameters_data_t *)opaque; + const COVER_ctx_t *const ctx = data->ctx; + const COVER_params_t parameters = data->parameters; size_t dictBufferCapacity = data->dictBufferCapacity; size_t totalCompressedSize = ERROR(GENERIC); /* Allocate space for hash table, dict, and freqs */ COVER_map_t activeDmers; BYTE *const dict = (BYTE * const)malloc(dictBufferCapacity); U32 *freqs = (U32 *)malloc(ctx->suffixSize * sizeof(U32)); - if (!COVER_map_init(&activeDmers, parameters.kMax - parameters.d + 1)) { + if (!COVER_map_init(&activeDmers, parameters.k - parameters.d + 1)) { DISPLAYLEVEL(1, "Failed to allocate dmer map: out of memory\n"); goto _cleanup; } if (!dict || !freqs) { - DISPLAYLEVEL(1, "Failed to allocate dictionary buffer\n"); + DISPLAYLEVEL(1, "Failed to allocate buffers: out of memory\n"); goto _cleanup; } /* Copy the frequencies because we need to modify them */ @@ -880,7 +852,7 @@ static void COVER_tryParameters(void *opaque) { { const size_t tail = COVER_buildDictionary(ctx, freqs, &activeDmers, dict, dictBufferCapacity, parameters); - ZDICT_params_t zdictParams = COVER_translateParams(parameters); + const ZDICT_params_t zdictParams = COVER_translateParams(parameters); dictBufferCapacity = ZDICT_finalizeDictionary( dict, dictBufferCapacity, dict + tail, dictBufferCapacity - tail, ctx->samples, ctx->samplesSizes, (unsigned)ctx->nbSamples, zdictParams); @@ -954,27 +926,42 @@ ZDICTLIB_API size_t COVER_optimizeTrainFromBuffer(void *dictBuffer, unsigned nbSamples, COVER_params_t *parameters) { /* constants */ - const unsigned dMin = parameters->d == 0 ? 6 : parameters->d; - const unsigned dMax = parameters->d == 0 ? 16 : parameters->d; - const unsigned min = parameters->kMin == 0 ? 32 : parameters->kMin; - const unsigned max = parameters->kMax == 0 ? 1024 : parameters->kMax; - const unsigned kStep = parameters->kStep == 0 ? 8 : parameters->kStep; - const unsigned step = MAX((max - min) / kStep, 1); + const unsigned kMinD = parameters->d == 0 ? 6 : parameters->d; + const unsigned kMaxD = parameters->d == 0 ? 16 : parameters->d; + const unsigned kMinK = parameters->k == 0 ? kMaxD : parameters->k; + const unsigned kMaxK = parameters->k == 0 ? 2048 : parameters->k; + const unsigned kSteps = parameters->steps == 0 ? 256 : parameters->steps; + const unsigned kStepSize = MAX((kMaxK - kMinK) / kSteps, 1); + const unsigned kIterations = + (1 + (kMaxD - kMinD) / 2) * (1 + (kMaxK - kMinK) / kStepSize); /* Local variables */ - unsigned iteration = 1; - const unsigned iterations = - (1 + (dMax - dMin) / 2) * (((1 + kStep) * (2 + kStep)) / 2) * 4; const int displayLevel = parameters->notificationLevel; + unsigned iteration = 1; unsigned d; + unsigned k; COVER_best_t best; + /* Checks */ + if (kMinK < kMaxD || kMaxK < kMinK) { + LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect parameters\n"); + return ERROR(GENERIC); + } + if (nbSamples == 0) { + DISPLAYLEVEL(1, "Cover must have at least one input file\n"); + return ERROR(GENERIC); + } + if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) { + DISPLAYLEVEL(1, "dictBufferCapacity must be at least %u\n", + ZDICT_DICTSIZE_MIN); + return ERROR(dstSize_tooSmall); + } + /* Initialization */ COVER_best_init(&best); - /* Turn down display level to clean up display at level 2 and below */ + /* Turn down global display level to clean up display at level 2 and below */ g_displayLevel = parameters->notificationLevel - 1; /* Loop through d first because each new value needs a new context */ - LOCALDISPLAYLEVEL(displayLevel, 3, "Trying %u different sets of parameters\n", - iterations); - for (d = dMin; d <= dMax; d += 2) { - unsigned kMin; + LOCALDISPLAYLEVEL(displayLevel, 2, "Trying %u different sets of parameters\n", + kIterations); + for (d = kMinD; d <= kMaxD; d += 2) { /* Initialize the context for this value of d */ COVER_ctx_t ctx; LOCALDISPLAYLEVEL(displayLevel, 3, "d=%u\n", d); @@ -983,44 +970,37 @@ ZDICTLIB_API size_t COVER_optimizeTrainFromBuffer(void *dictBuffer, COVER_best_destroy(&best); return ERROR(GENERIC); } - /* Loop through the rest of the parameters reusing the same context */ - for (kMin = min; kMin <= max; kMin += step) { - unsigned kMax; - LOCALDISPLAYLEVEL(displayLevel, 3, "kMin=%u\n", kMin); - for (kMax = kMin; kMax <= max; kMax += step) { - unsigned smoothing; - LOCALDISPLAYLEVEL(displayLevel, 3, "kMax=%u\n", kMax); - for (smoothing = kMin / 4; smoothing <= kMin * 2; smoothing *= 2) { - /* Prepare the arguments */ - COVER_tryParameters_data_t *data = - (COVER_tryParameters_data_t *)malloc( - sizeof(COVER_tryParameters_data_t)); - LOCALDISPLAYLEVEL(displayLevel, 3, "smoothing=%u\n", smoothing); - if (!data) { - LOCALDISPLAYLEVEL(displayLevel, 1, - "Failed to allocate parameters\n"); - COVER_best_destroy(&best); - COVER_ctx_destroy(&ctx); - return ERROR(GENERIC); - } - data->ctx = &ctx; - data->best = &best; - data->dictBufferCapacity = dictBufferCapacity; - data->parameters = *parameters; - data->parameters.d = d; - data->parameters.kMin = kMin; - data->parameters.kStep = kStep; - data->parameters.kMax = kMax; - data->parameters.smoothing = smoothing; - /* Call the function and pass ownership of data to it */ - COVER_best_start(&best); - COVER_tryParameters(data); - /* Print status */ - LOCALDISPLAYUPDATE(displayLevel, 2, "\r%u%% ", - (U32)((iteration * 100) / iterations)); - ++iteration; - } + /* Loop through k reusing the same context */ + for (k = kMinK; k <= kMaxK; k += kStepSize) { + /* Prepare the arguments */ + COVER_tryParameters_data_t *data = (COVER_tryParameters_data_t *)malloc( + sizeof(COVER_tryParameters_data_t)); + LOCALDISPLAYLEVEL(displayLevel, 3, "k=%u\n", k); + if (!data) { + LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to allocate parameters\n"); + COVER_best_destroy(&best); + COVER_ctx_destroy(&ctx); + return ERROR(GENERIC); } + data->ctx = &ctx; + data->best = &best; + data->dictBufferCapacity = dictBufferCapacity; + data->parameters = *parameters; + data->parameters.k = k; + data->parameters.d = d; + data->parameters.steps = kSteps; + /* Check the parameters */ + if (!COVER_checkParameters(data->parameters)) { + DISPLAYLEVEL(1, "Cover parameters incorrect\n"); + continue; + } + /* Call the function and pass ownership of data to it */ + COVER_best_start(&best); + COVER_tryParameters(data); + /* Print status */ + LOCALDISPLAYUPDATE(displayLevel, 2, "\r%u%% ", + (U32)((iteration * 100) / kIterations)); + ++iteration; } COVER_best_wait(&best); COVER_ctx_destroy(&ctx); diff --git a/lib/dictBuilder/zdict.h b/lib/dictBuilder/zdict.h index 4a2e79448..b99990c37 100644 --- a/lib/dictBuilder/zdict.h +++ b/lib/dictBuilder/zdict.h @@ -91,11 +91,9 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_advanced(void* dictBuffer, size_t dict kMin and d are the only required parameters. */ typedef struct { - unsigned d; /* dmer size : constraint: <= kMin : Should probably be in the range [6, 16]. */ - unsigned kMin; /* Minimum segment size : constraint: > 0 */ - unsigned kStep; /* Try kStep segment lengths uniformly distributed in the range [kMin, kMax] : 0 (default) only if kMax == 0 */ - unsigned kMax; /* Maximum segment size : 0 = kMin (default) : constraint : 0 or >= kMin */ - unsigned smoothing; /* Higher smoothing => larger segments are selected. Only useful if kMax > kMin. */ + unsigned k; /* Segment size : constraint: 0 < k : Reasonable range [16, 2048+] */ + unsigned d; /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */ + unsigned steps; /* Number of steps : Only used for optimization : 0 means default (256) : Higher means more parameters checked */ unsigned notificationLevel; /* Write to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */ unsigned dictID; /* 0 means auto mode (32-bits random value); other : force dictID value */ @@ -125,11 +123,10 @@ ZDICTLIB_API size_t COVER_trainFromBuffer(void* dictBuffer, size_t dictBufferCap `*parameters` is filled with the best parameters found, and the dictionary constructed with those parameters is stored in `dictBuffer`. - All of the {d, kMin, kStep, kMax} are optional, and smoothing is ignored. + All of the parameters d, k, steps are optional. If d is non-zero then we don't check multiple values of d, otherwise we check d = {6, 8, 10, 12, 14, 16}. - If kStep is non-zero then it is used, otherwise we pick 8. - If kMin and kMax are non-zero, then they limit the search space for kMin and kMax, - otherwise we check kMin and kMax values in the range [32, 1024]. + if steps is zero it defaults to its default value. + If k is non-zero then we don't check multiple values of k, otherwise we check steps values in [16, 2048]. @return : size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) or an error code, which can be tested with ZDICT_isError(). diff --git a/programs/dibio.c b/programs/dibio.c index ba15d2106..33c0250c5 100644 --- a/programs/dibio.c +++ b/programs/dibio.c @@ -279,9 +279,7 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize, dictBuffer, maxDictSize, srcBuffer, fileSizes, nbFiles, coverParams); if (!ZDICT_isError(dictSize)) { - DISPLAYLEVEL(2, "smoothing=%d\nkMin=%d\nkStep=%d\nkMax=%d\nd=%d\n", - coverParams->smoothing, coverParams->kMin, - coverParams->kStep, coverParams->kMax, coverParams->d); + DISPLAYLEVEL(2, "k=%u\nd=%u\nsteps=%u\n", coverParams->k, coverParams->d, coverParams->steps); } } else { dictSize = COVER_trainFromBuffer(dictBuffer, maxDictSize, diff --git a/programs/zstdcli.c b/programs/zstdcli.c index f4d33d3f8..54c66a320 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -205,15 +205,13 @@ static unsigned parseCoverParameters(const char* stringPtr, COVER_params_t *para { memset(params, 0, sizeof(*params)); for (; ;) { - if (longCommandWArg(&stringPtr, "smoothing=")) { params->smoothing = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } - if (longCommandWArg(&stringPtr, "k=") || longCommandWArg(&stringPtr, "kMin=") || longCommandWArg(&stringPtr, "kmin=")) { params->kMin = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } - if (longCommandWArg(&stringPtr, "kStep=") || longCommandWArg(&stringPtr, "kstep=")) { params->kStep = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } - if (longCommandWArg(&stringPtr, "kMax=") || longCommandWArg(&stringPtr, "kmax=")) { params->kMax = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } + if (longCommandWArg(&stringPtr, "k=")) { params->k = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } if (longCommandWArg(&stringPtr, "d=")) { params->d = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } + if (longCommandWArg(&stringPtr, "steps=")) { params->steps = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } return 0; } if (stringPtr[0] != 0) return 0; - DISPLAYLEVEL(4, "smoothing=%d\nkMin=%d\nkStep=%d\nkMax=%d\nd=%d\n", params->smoothing, params->kMin, params->kStep, params->kMax, params->d); + DISPLAYLEVEL(4, "k=%u\nd=%u\nsteps=%u\n", params->k, params->d, params->steps); return 1; } #endif diff --git a/tests/playTests.sh b/tests/playTests.sh index a94446e7e..5bb882aa4 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -266,13 +266,13 @@ $ZSTD -f tmp -D tmpDict $ZSTD -d tmp.zst -D tmpDict -fo result $DIFF $TESTFILE result $ECHO "- Create second (different) dictionary" -$ZSTD --train --cover=kmin=46,kstep=2,kmax=64,d=6,smoothing=23 *.c ../programs/*.c ../programs/*.h -o tmpDictC +$ZSTD --train --cover=k=56,d=8 *.c ../programs/*.c ../programs/*.h -o tmpDictC $ZSTD -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!" $ECHO "- Create dictionary with short dictID" $ZSTD --train --cover=k=46,d=8 *.c ../programs/*.c --dictID 1 -o tmpDict1 cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" $ECHO "- Create dictionary with size limit" -$ZSTD --train --optimize-cover=kstep=2,d=8 *.c ../programs/*.c -o tmpDict2 --maxdict 4K +$ZSTD --train --optimize-cover=steps=8 *.c ../programs/*.c -o tmpDict2 --maxdict 4K rm tmp* From a8b4fe04819241ec15567f1e6913cbf73daf4720 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Mon, 2 Jan 2017 18:45:19 -0800 Subject: [PATCH 6/6] Add COVER dictionary builder to fuzzer unit tests --- tests/fuzzer.c | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 86d4c6beb..37fa8dced 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -28,6 +28,7 @@ #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue, ZSTD_compressBlock */ #include "zstd.h" /* ZSTD_VERSION_STRING */ #include "zstd_errors.h" /* ZSTD_getErrorCode */ +#define ZDICT_STATIC_LINKING_ONLY #include "zdict.h" /* ZDICT_trainFromBuffer */ #include "datagen.h" /* RDG_genBuffer */ #include "mem.h" @@ -317,6 +318,61 @@ static int basicUnitTests(U32 seed, double compressibility) free(samplesSizes); } + /* COVER dictionary builder tests */ + { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + size_t dictSize = 16 KB; + size_t optDictSize = dictSize; + void* dictBuffer = malloc(dictSize); + size_t const totalSampleSize = 1 MB; + size_t const sampleUnitSize = 8 KB; + U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize); + size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t)); + COVER_params_t params; + U32 dictID; + + if (dictBuffer==NULL || samplesSizes==NULL) { + free(dictBuffer); + free(samplesSizes); + goto _output_error; + } + + DISPLAYLEVEL(4, "test%3i : COVER_trainFromBuffer : ", testNb++); + { U32 u; for (u=0; u