From 7f3f70f76621f4e488080d27f09614167c7b9a4b Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Wed, 25 Jul 2018 16:34:07 -0700 Subject: [PATCH 1/9] Add Fast Cover Dictionary Builder --- .../fastCover/Makefile | 54 ++ .../fastCover/README.md | 24 + .../fastCover/fastCover.c | 738 ++++++++++++++++++ .../fastCover/fastCover.h | 47 ++ .../fastCover/main.c | 177 +++++ .../fastCover/test.sh | 14 + 6 files changed, 1054 insertions(+) create mode 100644 contrib/experimental_dict_builders/fastCover/Makefile create mode 100644 contrib/experimental_dict_builders/fastCover/README.md create mode 100644 contrib/experimental_dict_builders/fastCover/fastCover.c create mode 100644 contrib/experimental_dict_builders/fastCover/fastCover.h create mode 100644 contrib/experimental_dict_builders/fastCover/main.c create mode 100644 contrib/experimental_dict_builders/fastCover/test.sh diff --git a/contrib/experimental_dict_builders/fastCover/Makefile b/contrib/experimental_dict_builders/fastCover/Makefile new file mode 100644 index 000000000..9c56013d8 --- /dev/null +++ b/contrib/experimental_dict_builders/fastCover/Makefile @@ -0,0 +1,54 @@ +ARG := + +CC ?= gcc +CFLAGS ?= -O3 +INCLUDES := -I ../../../programs -I ../randomDictBuilder -I ../../../lib/common -I ../../../lib -I ../../../lib/dictBuilder + +IO_FILE := ../randomDictBuilder/io.c + +TEST_INPUT := ../../../lib +TEST_OUTPUT := fastCoverDict + +all: main run clean + +.PHONY: test +test: main testrun testshell clean + +.PHONY: run +run: + echo "Building a fastCover dictionary with given arguments" + ./main $(ARG) + +main: main.o io.o fastCover.o libzstd.a + $(CC) $(CFLAGS) main.o io.o fastCover.o libzstd.a -o main + +main.o: main.c + $(CC) $(CFLAGS) $(INCLUDES) -c main.c + +fastCover.o: fastCover.c + $(CC) $(CFLAGS) $(INCLUDES) -c fastCover.c + +io.o: $(IO_FILE) + $(CC) $(CFLAGS) $(INCLUDES) -c $(IO_FILE) + +libzstd.a: + $(MAKE) -C ../../../lib libzstd.a + mv ../../../lib/libzstd.a . + +.PHONY: testrun +testrun: main + echo "Run with $(TEST_INPUT) and $(TEST_OUTPUT) " + ./main in=$(TEST_INPUT) out=$(TEST_OUTPUT) + zstd -be3 -D $(TEST_OUTPUT) -r $(TEST_INPUT) -q + rm -f $(TEST_OUTPUT) + +.PHONY: testshell +testshell: test.sh + sh test.sh + echo "Finish running test.sh" + +.PHONY: clean +clean: + rm -f *.o main libzstd.a + $(MAKE) -C ../../../lib clean + echo "Cleaning is completed" diff --git a/contrib/experimental_dict_builders/fastCover/README.md b/contrib/experimental_dict_builders/fastCover/README.md new file mode 100644 index 000000000..088e38be7 --- /dev/null +++ b/contrib/experimental_dict_builders/fastCover/README.md @@ -0,0 +1,24 @@ +FastCover Dictionary Builder + +### Permitted Arguments: +Input File/Directory (in=fileName): required; file/directory used to build dictionary; if directory, will operate recursively for files inside directory; can include multiple files/directories, each following "in=" +Output Dictionary (out=dictName): if not provided, default to fastCoverDict +Dictionary ID (dictID=#): nonnegative number; if not provided, default to 0 +Maximum Dictionary Size (maxdict=#): positive number; in bytes, if not provided, default to 110KB +Size of Selected Segment (k=#): positive number; in bytes; if not provided, default to 200 +Size of Dmer (d=#): positive number; in bytes; if not provided, default to 8 +Number of steps (steps=#): positive number, if not provided, default to 32 +Percentage of samples used for training(split=#): positive number; if not provided, default to 100 + + +###Running Test: +make test + + +###Usage: +To build a random dictionary with the provided arguments: make ARG= followed by arguments + + +### Examples: +make ARG="in=../../../lib/dictBuilder out=dict100 dictID=520" +make ARG="in=../../../lib/dictBuilder in=../../../lib/compress" diff --git a/contrib/experimental_dict_builders/fastCover/fastCover.c b/contrib/experimental_dict_builders/fastCover/fastCover.c new file mode 100644 index 000000000..6d3ad90ab --- /dev/null +++ b/contrib/experimental_dict_builders/fastCover/fastCover.c @@ -0,0 +1,738 @@ +/*-************************************* +* Dependencies +***************************************/ +#include /* fprintf */ +#include /* malloc, free, qsort */ +#include /* memset */ +#include /* clock */ +#include "mem.h" /* read */ +#include "pool.h" +#include "threading.h" +#include "fastCover.h" +#include "zstd_internal.h" /* includes zstd.h */ +#include "zdict.h" + + +/*-************************************* +* Constants +***************************************/ +#define FASTCOVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((U32)-1) : ((U32)1 GB)) +#define FASTCOVER_MAX_F 32 +#define DEFAULT_SPLITPOINT 1.0 + +/*-************************************* +* Console display +***************************************/ +static int g_displayLevel = 2; +#define DISPLAY(...) \ + { \ + fprintf(stderr, __VA_ARGS__); \ + fflush(stderr); \ + } +#define LOCALDISPLAYLEVEL(displayLevel, l, ...) \ + if (displayLevel >= l) { \ + DISPLAY(__VA_ARGS__); \ + } /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */ +#define DISPLAYLEVEL(l, ...) LOCALDISPLAYLEVEL(g_displayLevel, l, __VA_ARGS__) + +#define LOCALDISPLAYUPDATE(displayLevel, l, ...) \ + if (displayLevel >= l) { \ + if ((clock() - g_time > refreshRate) || (displayLevel >= 4)) { \ + g_time = clock(); \ + DISPLAY(__VA_ARGS__); \ + } \ + } +#define DISPLAYUPDATE(l, ...) LOCALDISPLAYUPDATE(g_displayLevel, l, __VA_ARGS__) +static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100; +static clock_t g_time = 0; + + +/*-************************************* +* Hash Function +***************************************/ +static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL; +static size_t ZSTD_hash8(U64 u, U32 h) { return (size_t)(((u) * prime8bytes) >> (64-h)) ; } +static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(MEM_readLE64(p), h); } + +/** + * Hash the 8-byte value pointed to by p and mod 2^f + */ +static size_t FASTCOVER_hash8PtrToIndex(const void* p, U32 h) { + return ZSTD_hash8Ptr(p, h) & ((1 << h) - 1); +} + + +/*-************************************* +* Context +***************************************/ +typedef struct { + const BYTE *samples; + size_t *offsets; + const size_t *samplesSizes; + size_t nbSamples; + size_t nbTrainSamples; + size_t nbTestSamples; + size_t nbDmers; + U32 *freqs; + unsigned d; +} FASTCOVER_ctx_t; + + +/*-************************************* +* Helper functions +***************************************/ +/** + * Returns the sum of the sample sizes. + */ +static size_t FASTCOVER_sum(const size_t *samplesSizes, unsigned nbSamples) { + size_t sum = 0; + unsigned i; + for (i = 0; i < nbSamples; ++i) { + sum += samplesSizes[i]; + } + return sum; +} + + +/*-************************************* +* fast functions +***************************************/ +/** + * A segment is a range in the source as well as the score of the segment. + */ +typedef struct { + U32 begin; + U32 end; + U32 score; +} FASTCOVER_segment_t; + + +/** + * Selects the best segment in an epoch. + * Segments of are scored according to the function: + * + * Let F(d) be the frequency of all dmers with hash value d. + * Let S_i be hash value of the dmer at position i of segment S which has length k. + * + * Score(S) = F(S_1) + F(S_2) + ... + F(S_{k-d+1}) + * + * Once the dmer with hash value d is in the dictionay we set F(d) = F(d)/2. + */ +static FASTCOVER_segment_t FASTCOVER_selectSegment(const FASTCOVER_ctx_t *ctx, + U32 *freqs, U32 begin,U32 end, + ZDICT_fastCover_params_t parameters) { + /* Constants */ + const U32 k = parameters.k; + const U32 d = parameters.d; + const U32 dmersInK = k - d + 1; + /* Try each segment (activeSegment) and save the best (bestSegment) */ + FASTCOVER_segment_t bestSegment = {0, 0, 0}; + FASTCOVER_segment_t activeSegment; + /* Reset the activeDmers in the segment */ + /* The activeSegment starts at the beginning of the epoch. */ + activeSegment.begin = begin; + activeSegment.end = begin; + activeSegment.score = 0; + /* Slide the activeSegment through the whole epoch. + * Save the best segment in bestSegment. + */ + while (activeSegment.end < end) { + /* Get hash value of current dmer */ + size_t index = FASTCOVER_hash8PtrToIndex(ctx->samples + activeSegment.end, parameters.f); + /* Add frequency of this index to score */ + activeSegment.score += freqs[index]; + /* Increment end of segment */ + activeSegment.end += 1; + /* If the window is now too large, drop the first position */ + if (activeSegment.end - activeSegment.begin == dmersInK + 1) { + /* Get hash value of the dmer to be eliminated from active segment */ + size_t delIndex = FASTCOVER_hash8PtrToIndex(ctx->samples + activeSegment.begin, parameters.f); + /* Subtract frequency of this index from score */ + activeSegment.score -= freqs[delIndex]; + /* Increment start of segment */ + activeSegment.begin += 1; + } + /* If this segment is the best so far save it */ + if (activeSegment.score > bestSegment.score) { + bestSegment = activeSegment; + } + } + { + /* Trim off the zero frequency head and tail from the segment. */ + U32 newBegin = bestSegment.end; + U32 newEnd = bestSegment.begin; + U32 pos; + for (pos = bestSegment.begin; pos != bestSegment.end; ++pos) { + size_t index = FASTCOVER_hash8PtrToIndex(ctx->samples + pos, parameters.f); + U32 freq = freqs[index]; + if (freq != 0) { + newBegin = MIN(newBegin, pos); + newEnd = pos + 1; + } + } + bestSegment.begin = newBegin; + bestSegment.end = newEnd; + } + { + /* Half the frequency of hash value of each dmer covered by the chosen segment. */ + U32 pos; + for (pos = bestSegment.begin; pos != bestSegment.end; ++pos) { + size_t i = FASTCOVER_hash8PtrToIndex(ctx->samples + pos, parameters.f); + freqs[i] = freqs[i]/2; + } + } + return bestSegment; +} + +/** + * Check the validity of the parameters. + * Returns non-zero if the parameters are valid and 0 otherwise. + */ +static int FASTCOVER_checkParameters(ZDICT_fastCover_params_t parameters, + size_t maxDictSize) { + /* k, d, and f are required parameters */ + if (parameters.d == 0 || parameters.k == 0 || parameters.f == 0) { + return 0; + } + /* 0 < f <= FASTCOVER_MAX_F */ + if (parameters.f > FASTCOVER_MAX_F) { + return 0; + } + /* k <= maxDictSize */ + if (parameters.k > maxDictSize) { + return 0; + } + /* d <= k */ + if (parameters.d > parameters.k) { + return 0; + } + /* 0 < splitPoint <= 1 */ + if (parameters.splitPoint <= 0 || parameters.splitPoint > 1) { + return 0; + } + return 1; +} + + +/** + * Clean up a context initialized with `FASTCOVER_ctx_init()`. + */ +static void FASTCOVER_ctx_destroy(FASTCOVER_ctx_t *ctx) { + if (!ctx) { + return; + } + if (ctx->freqs) { + free(ctx->freqs); + ctx->freqs = NULL; + } + if (ctx->offsets) { + free(ctx->offsets); + ctx->offsets = NULL; + } +} + +/** + * Calculate for frequency of hash value of each dmer in ctx->samples + */ +static void FASTCOVER_getFrequency(U32 *freqs, unsigned f, FASTCOVER_ctx_t *ctx){ + /* inCurrSample keeps track of this hash value has already be seen in previous dmers in the same sample*/ + size_t* inCurrSample = (size_t *)malloc((1<nbTrainSamples; i++) { + memset(inCurrSample, 0, (1 << f)); /* Reset inCurrSample for each sample */ + size_t currSampleStart = ctx->offsets[i]; + size_t currSampleEnd = ctx->offsets[i+1]; + start = currSampleStart; + while (start + f < currSampleEnd) { + size_t dmerIndex = FASTCOVER_hash8PtrToIndex(ctx->samples + start, f); + /* if no dmer with same hash value has been seen in current sample */ + if (inCurrSample[dmerIndex] == 0) { + inCurrSample[dmerIndex]++; + freqs[dmerIndex]++; + } + start++; + } + } + free(inCurrSample); +} + +/** + * Prepare a context for dictionary building. + * The context is only dependent on the parameter `d` and can used multiple + * times. + * Returns 1 on success or zero on error. + * The context must be destroyed with `FASTCOVER_ctx_destroy()`. + */ +static int FASTCOVER_ctx_init(FASTCOVER_ctx_t *ctx, const void *samplesBuffer, + const size_t *samplesSizes, unsigned nbSamples, + unsigned d, double splitPoint, unsigned f) { + const BYTE *const samples = (const BYTE *)samplesBuffer; + const size_t totalSamplesSize = FASTCOVER_sum(samplesSizes, nbSamples); + /* Split samples into testing and training sets */ + const unsigned nbTrainSamples = splitPoint < 1.0 ? (unsigned)((double)nbSamples * splitPoint) : nbSamples; + const unsigned nbTestSamples = splitPoint < 1.0 ? nbSamples - nbTrainSamples : nbSamples; + const size_t trainingSamplesSize = splitPoint < 1.0 ? FASTCOVER_sum(samplesSizes, nbTrainSamples) : totalSamplesSize; + const size_t testSamplesSize = splitPoint < 1.0 ? FASTCOVER_sum(samplesSizes + nbTrainSamples, nbTestSamples) : totalSamplesSize; + /* Checks */ + if (totalSamplesSize < MAX(d, sizeof(U64)) || + totalSamplesSize >= (size_t)FASTCOVER_MAX_SAMPLES_SIZE) { + DISPLAYLEVEL(1, "Total samples size is too large (%u MB), maximum size is %u MB\n", + (U32)(totalSamplesSize>>20), (FASTCOVER_MAX_SAMPLES_SIZE >> 20)); + return 0; + } + /* Check if there are at least 5 training samples */ + if (nbTrainSamples < 5) { + DISPLAYLEVEL(1, "Total number of training samples is %u and is invalid.", nbTrainSamples); + return 0; + } + /* Check if there's testing sample */ + if (nbTestSamples < 1) { + DISPLAYLEVEL(1, "Total number of testing samples is %u and is invalid.", nbTestSamples); + return 0; + } + /* Zero the context */ + memset(ctx, 0, sizeof(*ctx)); + DISPLAYLEVEL(2, "Training on %u samples of total size %u\n", nbTrainSamples, + (U32)trainingSamplesSize); + DISPLAYLEVEL(2, "Testing on %u samples of total size %u\n", nbTestSamples, + (U32)testSamplesSize); + + ctx->samples = samples; + ctx->samplesSizes = samplesSizes; + ctx->nbSamples = nbSamples; + ctx->nbTrainSamples = nbTrainSamples; + ctx->nbTestSamples = nbTestSamples; + ctx->nbDmers = trainingSamplesSize - d + 1; + ctx->d = d; + + /* The offsets of each file */ + ctx->offsets = (size_t *)malloc((nbSamples + 1) * sizeof(size_t)); + if (!ctx->offsets) { + DISPLAYLEVEL(1, "Failed to allocate scratch buffers\n"); + FASTCOVER_ctx_destroy(ctx); + return 0; + } + + /* Fill offsets from the samplesSizes */ + { + U32 i; + ctx->offsets[0] = 0; + for (i = 1; i <= nbSamples; ++i) { + ctx->offsets[i] = ctx->offsets[i - 1] + samplesSizes[i - 1]; + } + } + + /* Initialize frequency array of size 2^f */ + ctx->freqs =(U32 *)malloc((1 << f) * sizeof(U32)); + memset(ctx->freqs, 0, (1 << f) * sizeof(U32)); + + DISPLAYLEVEL(2, "Computing frequencies\n"); + FASTCOVER_getFrequency(ctx->freqs, f, ctx); + + return 1; +} + + +/** + * Given the prepared context build the dictionary. + */ +static size_t FASTCOVER_buildDictionary(const FASTCOVER_ctx_t *ctx, U32 *freqs, + void *dictBuffer, + size_t dictBufferCapacity, + ZDICT_fastCover_params_t parameters){ + BYTE *const dict = (BYTE *)dictBuffer; + size_t tail = dictBufferCapacity; + /* Divide the data up into epochs of equal size. + * We will select at least one segment from each epoch. + */ + const U32 epochs = MAX(1, (U32)(dictBufferCapacity / parameters.k)); + const U32 epochSize = (U32)(ctx->nbDmers / epochs); + size_t epoch; + DISPLAYLEVEL(2, "Breaking content into %u epochs of size %u\n", epochs, + epochSize); + /* Loop through the epochs until there are no more segments or the dictionary + * is full. + */ + for (epoch = 0; tail > 0; epoch = (epoch + 1) % epochs) { + const U32 epochBegin = (U32)(epoch * epochSize); + const U32 epochEnd = epochBegin + epochSize; + size_t segmentSize; + /* Select a segment */ + FASTCOVER_segment_t segment = FASTCOVER_selectSegment( + ctx, freqs, epochBegin, epochEnd, parameters); + + /* If the segment covers no dmers, then we are out of content */ + if (segment.score == 0) { + break; + } + + /* Trim the segment if necessary and if it is too small then we are done */ + segmentSize = MIN(segment.end - segment.begin + parameters.d - 1, tail); + if (segmentSize < parameters.d) { + break; + } + + /* We fill the dictionary from the back to allow the best segments to be + * referenced with the smallest offsets. + */ + tail -= segmentSize; + memcpy(dict + tail, ctx->samples + segment.begin, segmentSize); + DISPLAYUPDATE( + 2, "\r%u%% ", + (U32)(((dictBufferCapacity - tail) * 100) / dictBufferCapacity)); + } + DISPLAYLEVEL(2, "\r%79s\r", ""); + return tail; +} + + +/** + * FASTCOVER_best_t is used for two purposes: + * 1. Synchronizing threads. + * 2. Saving the best parameters and dictionary. + * + * All of the methods except FASTCOVER_best_init() are thread safe if zstd is + * compiled with multithreaded support. + */ +typedef struct fast_best_s { + ZSTD_pthread_mutex_t mutex; + ZSTD_pthread_cond_t cond; + size_t liveJobs; + void *dict; + size_t dictSize; + ZDICT_fastCover_params_t parameters; + size_t compressedSize; +} FASTCOVER_best_t; + +/** + * Initialize the `FASTCOVER_best_t`. + */ +static void FASTCOVER_best_init(FASTCOVER_best_t *best) { + if (best==NULL) return; /* compatible with init on NULL */ + (void)ZSTD_pthread_mutex_init(&best->mutex, NULL); + (void)ZSTD_pthread_cond_init(&best->cond, NULL); + best->liveJobs = 0; + best->dict = NULL; + best->dictSize = 0; + best->compressedSize = (size_t)-1; + memset(&best->parameters, 0, sizeof(best->parameters)); +} + +/** + * Wait until liveJobs == 0. + */ +static void FASTCOVER_best_wait(FASTCOVER_best_t *best) { + if (!best) { + return; + } + ZSTD_pthread_mutex_lock(&best->mutex); + while (best->liveJobs != 0) { + ZSTD_pthread_cond_wait(&best->cond, &best->mutex); + } + ZSTD_pthread_mutex_unlock(&best->mutex); +} + +/** + * Call FASTCOVER_best_wait() and then destroy the FASTCOVER_best_t. + */ +static void FASTCOVER_best_destroy(FASTCOVER_best_t *best) { + if (!best) { + return; + } + FASTCOVER_best_wait(best); + if (best->dict) { + free(best->dict); + } + ZSTD_pthread_mutex_destroy(&best->mutex); + ZSTD_pthread_cond_destroy(&best->cond); +} + +/** + * Called when a thread is about to be launched. + * Increments liveJobs. + */ +static void FASTCOVER_best_start(FASTCOVER_best_t *best) { + if (!best) { + return; + } + ZSTD_pthread_mutex_lock(&best->mutex); + ++best->liveJobs; + ZSTD_pthread_mutex_unlock(&best->mutex); +} + +/** + * Called when a thread finishes executing, both on error or success. + * Decrements liveJobs and signals any waiting threads if liveJobs == 0. + * If this dictionary is the best so far save it and its parameters. + */ +static void FASTCOVER_best_finish(FASTCOVER_best_t *best, size_t compressedSize, + ZDICT_fastCover_params_t parameters, void *dict, + size_t dictSize) { + if (!best) { + return; + } + { + size_t liveJobs; + ZSTD_pthread_mutex_lock(&best->mutex); + --best->liveJobs; + liveJobs = best->liveJobs; + /* If the new dictionary is better */ + if (compressedSize < best->compressedSize) { + /* Allocate space if necessary */ + if (!best->dict || best->dictSize < dictSize) { + if (best->dict) { + free(best->dict); + } + best->dict = malloc(dictSize); + if (!best->dict) { + best->compressedSize = ERROR(GENERIC); + best->dictSize = 0; + return; + } + } + /* Save the dictionary, parameters, and size */ + memcpy(best->dict, dict, dictSize); + best->dictSize = dictSize; + best->parameters = parameters; + best->compressedSize = compressedSize; + } + ZSTD_pthread_mutex_unlock(&best->mutex); + if (liveJobs == 0) { + ZSTD_pthread_cond_broadcast(&best->cond); + } + } +} + +/** + * Parameters for FASTCOVER_tryParameters(). + */ +typedef struct FASTCOVER_tryParameters_data_s { + const FASTCOVER_ctx_t *ctx; + FASTCOVER_best_t *best; + size_t dictBufferCapacity; + ZDICT_fastCover_params_t parameters; +} FASTCOVER_tryParameters_data_t; + +/** + * Tries a set of parameters and updates the FASTCOVER_best_t with the results. + * This function is thread safe if zstd is compiled with multithreaded support. + * It takes its parameters as an *OWNING* opaque pointer to support threading. + */ +static void FASTCOVER_tryParameters(void *opaque) { + /* Save parameters as local variables */ + FASTCOVER_tryParameters_data_t *const data = (FASTCOVER_tryParameters_data_t *)opaque; + const FASTCOVER_ctx_t *const ctx = data->ctx; + const ZDICT_fastCover_params_t parameters = data->parameters; + size_t dictBufferCapacity = data->dictBufferCapacity; + size_t totalCompressedSize = ERROR(GENERIC); + /* Allocate space for hash table, dict, and freqs */ + BYTE *const dict = (BYTE * const)malloc(dictBufferCapacity); + U32 *freqs = (U32*) malloc((1 << parameters.f) * sizeof(U32)); + if (!dict || !freqs) { + DISPLAYLEVEL(1, "Failed to allocate buffers: out of memory\n"); + goto _cleanup; + } + /* Copy the frequencies because we need to modify them */ + memcpy(freqs, ctx->freqs, (1 << parameters.f) * sizeof(U32)); + /* Build the dictionary */ + { + const size_t tail = FASTCOVER_buildDictionary(ctx, freqs, dict, + dictBufferCapacity, parameters); + + dictBufferCapacity = ZDICT_finalizeDictionary( + dict, dictBufferCapacity, dict + tail, dictBufferCapacity - tail, + ctx->samples, ctx->samplesSizes, (unsigned)ctx->nbTrainSamples, + parameters.zParams); + if (ZDICT_isError(dictBufferCapacity)) { + DISPLAYLEVEL(1, "Failed to finalize dictionary\n"); + goto _cleanup; + } + } + /* Check total compressed size */ + { + /* Pointers */ + ZSTD_CCtx *cctx; + ZSTD_CDict *cdict; + void *dst; + /* Local variables */ + size_t dstCapacity; + size_t i; + /* Allocate dst with enough space to compress the maximum sized sample */ + { + size_t maxSampleSize = 0; + i = parameters.splitPoint < 1.0 ? ctx->nbTrainSamples : 0; + for (; i < ctx->nbSamples; ++i) { + maxSampleSize = MAX(ctx->samplesSizes[i], maxSampleSize); + } + dstCapacity = ZSTD_compressBound(maxSampleSize); + dst = malloc(dstCapacity); + } + /* Create the cctx and cdict */ + cctx = ZSTD_createCCtx(); + cdict = ZSTD_createCDict(dict, dictBufferCapacity, + parameters.zParams.compressionLevel); + if (!dst || !cctx || !cdict) { + goto _compressCleanup; + } + /* Compress each sample and sum their sizes (or error) */ + totalCompressedSize = dictBufferCapacity; + i = parameters.splitPoint < 1.0 ? ctx->nbTrainSamples : 0; + for (; i < ctx->nbSamples; ++i) { + const size_t size = ZSTD_compress_usingCDict( + cctx, dst, dstCapacity, ctx->samples + ctx->offsets[i], + ctx->samplesSizes[i], cdict); + if (ZSTD_isError(size)) { + totalCompressedSize = ERROR(GENERIC); + goto _compressCleanup; + } + totalCompressedSize += size; + } + _compressCleanup: + ZSTD_freeCCtx(cctx); + ZSTD_freeCDict(cdict); + if (dst) { + free(dst); + } + } + +_cleanup: + FASTCOVER_best_finish(data->best, totalCompressedSize, parameters, dict, + dictBufferCapacity); + free(data); + if (dict) { + free(dict); + } + if (freqs) { + free(freqs); + } +} + +ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_fastCover( + void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer, + const size_t *samplesSizes, unsigned nbSamples, + ZDICT_fastCover_params_t *parameters) { + /* constants */ + const unsigned nbThreads = parameters->nbThreads; + const double splitPoint = + parameters->splitPoint <= 0.0 ? DEFAULT_SPLITPOINT : parameters->splitPoint; + const unsigned kMinD = parameters->d == 0 ? 8 : parameters->d; + const unsigned kMaxD = parameters->d == 0 ? 8 : parameters->d; + const unsigned kMinK = parameters->k == 0 ? 50 : parameters->k; + const unsigned kMaxK = parameters->k == 0 ? 2000 : parameters->k; + const unsigned kSteps = parameters->steps == 0 ? 40 : parameters->steps; + const unsigned kStepSize = MAX((kMaxK - kMinK) / kSteps, 1); + const unsigned kIterations = + (1 + (kMaxD - kMinD) / 2) * (1 + (kMaxK - kMinK) / kStepSize); + const unsigned f = parameters->f == 0 ? 23 : parameters->f; + + /* Local variables */ + const int displayLevel = parameters->zParams.notificationLevel; + unsigned iteration = 1; + unsigned d; + unsigned k; + FASTCOVER_best_t best; + POOL_ctx *pool = NULL; + + /* Checks */ + if (splitPoint <= 0 || splitPoint > 1) { + LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect parameters\n"); + return ERROR(GENERIC); + } + if (kMinK < kMaxD || kMaxK < kMinK) { + LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect parameters\n"); + return ERROR(GENERIC); + } + if (nbSamples == 0) { + DISPLAYLEVEL(1, "fast must have at least one input file\n"); + return ERROR(GENERIC); + } + if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) { + DISPLAYLEVEL(1, "dictBufferCapacity must be at least %u\n", + ZDICT_DICTSIZE_MIN); + return ERROR(dstSize_tooSmall); + } + if (nbThreads > 1) { + pool = POOL_create(nbThreads, 1); + if (!pool) { + return ERROR(memory_allocation); + } + } + /* Initialization */ + FASTCOVER_best_init(&best); + /* Turn down global display level to clean up display at level 2 and below */ + g_displayLevel = displayLevel == 0 ? 0 : displayLevel - 1; + /* Loop through d first because each new value needs a new context */ + LOCALDISPLAYLEVEL(displayLevel, 2, "Trying %u different sets of parameters\n", + kIterations); + for (d = kMinD; d <= kMaxD; d += 2) { + /* Initialize the context for this value of d */ + FASTCOVER_ctx_t ctx; + LOCALDISPLAYLEVEL(displayLevel, 3, "d=%u\n", d); + if (!FASTCOVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, d, splitPoint, f)) { + LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to initialize context\n"); + FASTCOVER_best_destroy(&best); + POOL_free(pool); + return ERROR(GENERIC); + } + /* Loop through k reusing the same context */ + for (k = kMinK; k <= kMaxK; k += kStepSize) { + /* Prepare the arguments */ + FASTCOVER_tryParameters_data_t *data = (FASTCOVER_tryParameters_data_t *)malloc( + sizeof(FASTCOVER_tryParameters_data_t)); + LOCALDISPLAYLEVEL(displayLevel, 3, "k=%u\n", k); + if (!data) { + LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to allocate parameters\n"); + FASTCOVER_best_destroy(&best); + FASTCOVER_ctx_destroy(&ctx); + POOL_free(pool); + return ERROR(GENERIC); + } + data->ctx = &ctx; + data->best = &best; + data->dictBufferCapacity = dictBufferCapacity; + data->parameters = *parameters; + data->parameters.k = k; + data->parameters.d = d; + data->parameters.f = f; + data->parameters.splitPoint = splitPoint; + data->parameters.steps = kSteps; + data->parameters.zParams.notificationLevel = g_displayLevel; + /* Check the parameters */ + if (!FASTCOVER_checkParameters(data->parameters, dictBufferCapacity)) { + DISPLAYLEVEL(1, "fastCover parameters incorrect\n"); + free(data); + continue; + } + /* Call the function and pass ownership of data to it */ + FASTCOVER_best_start(&best); + if (pool) { + POOL_add(pool, &FASTCOVER_tryParameters, data); + } else { + FASTCOVER_tryParameters(data); + } + /* Print status */ + LOCALDISPLAYUPDATE(displayLevel, 2, "\r%u%% ", + (U32)((iteration * 100) / kIterations)); + ++iteration; + } + FASTCOVER_best_wait(&best); + FASTCOVER_ctx_destroy(&ctx); + } + LOCALDISPLAYLEVEL(displayLevel, 2, "\r%79s\r", ""); + /* Fill the output buffer and parameters with output of the best parameters */ + { + const size_t dictSize = best.dictSize; + if (ZSTD_isError(best.compressedSize)) { + const size_t compressedSize = best.compressedSize; + FASTCOVER_best_destroy(&best); + POOL_free(pool); + return compressedSize; + } + *parameters = best.parameters; + memcpy(dictBuffer, best.dict, dictSize); + FASTCOVER_best_destroy(&best); + POOL_free(pool); + return dictSize; + } + +} diff --git a/contrib/experimental_dict_builders/fastCover/fastCover.h b/contrib/experimental_dict_builders/fastCover/fastCover.h new file mode 100644 index 000000000..eca04baab --- /dev/null +++ b/contrib/experimental_dict_builders/fastCover/fastCover.h @@ -0,0 +1,47 @@ +#include /* fprintf */ +#include /* malloc, free, qsort */ +#include /* memset */ +#include /* clock */ +#include "mem.h" /* read */ +#include "pool.h" +#include "threading.h" +#include "zstd_internal.h" /* includes zstd.h */ +#ifndef ZDICT_STATIC_LINKING_ONLY +#define ZDICT_STATIC_LINKING_ONLY +#endif +#include "zdict.h" + + + + + +typedef struct { + unsigned k; /* Segment size : constraint: 0 < k : Reasonable range [16, 2048+] */ + unsigned d; /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */ + unsigned f; /* log of size of frequency array */ + unsigned steps; /* Number of steps : Only used for optimization : 0 means default (32) : Higher means more parameters checked */ + unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */ + double splitPoint; /* Percentage of samples used for training: the first nbSamples * splitPoint samples will be used to training, the last nbSamples * (1 - splitPoint) samples will be used for testing, 0 means default (1.0), 1.0 when all samples are used for both training and testing */ + ZDICT_params_t zParams; +} ZDICT_fastCover_params_t; + + + +/*! ZDICT_optimizeTrainFromBuffer_fastCover(): + * Train a dictionary from an array of samples using a modified version of the COVER algorithm. + * Samples must be stored concatenated in a single flat buffer `samplesBuffer`, + * supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order. + * The resulting dictionary will be saved into `dictBuffer`. + * All of the parameters except for f are optional. + * If d is non-zero then we don't check multiple values of d, otherwise we check d = {6, 8, 10, 12, 14, 16}. + * if steps is zero it defaults to its default value. + * If k is non-zero then we don't check multiple values of k, otherwise we check steps values in [16, 2048]. + * + * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) + * or an error code, which can be tested with ZDICT_isError(). + * On success `*parameters` contains the parameters selected. + */ +ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_fastCover( + void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer, + const size_t *samplesSizes, unsigned nbSamples, + ZDICT_fastCover_params_t *parameters); diff --git a/contrib/experimental_dict_builders/fastCover/main.c b/contrib/experimental_dict_builders/fastCover/main.c new file mode 100644 index 000000000..260eeb281 --- /dev/null +++ b/contrib/experimental_dict_builders/fastCover/main.c @@ -0,0 +1,177 @@ +#include /* fprintf */ +#include /* malloc, free, qsort */ +#include /* strcmp, strlen */ +#include /* errno */ +#include +#include "fastCover.h" +#include "io.h" +#include "util.h" +#include "zdict.h" + + +/*-************************************* +* Console display +***************************************/ +#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } + +static const U64 g_refreshRate = SEC_TO_MICRO / 6; +static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; + +#define DISPLAYUPDATE(l, ...) { if (displayLevel>=l) { \ + if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (displayLevel>=4)) \ + { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \ + if (displayLevel>=4) fflush(stderr); } } } + + +/*-************************************* +* Exceptions +***************************************/ +#ifndef DEBUG +# define DEBUG 0 +#endif +#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__); +#define EXM_THROW(error, ...) \ +{ \ + DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \ + DISPLAY("Error %i : ", error); \ + DISPLAY(__VA_ARGS__); \ + DISPLAY("\n"); \ + exit(error); \ +} + + +/*-************************************* +* Constants +***************************************/ +static const unsigned g_defaultMaxDictSize = 110 KB; +#define DEFAULT_CLEVEL 3 + + +/*-************************************* +* FASTCOVER +***************************************/ +int FASTCOVER_trainFromFiles(const char* dictFileName, sampleInfo *info, + unsigned maxDictSize, + ZDICT_fastCover_params_t *params) { + unsigned const displayLevel = params->zParams.notificationLevel; + void* const dictBuffer = malloc(maxDictSize); + + int result = 0; + + /* Checks */ + if (!dictBuffer) + EXM_THROW(12, "not enough memory for trainFromFiles"); /* should not happen */ + + { size_t dictSize; + dictSize = ZDICT_optimizeTrainFromBuffer_fastCover(dictBuffer, maxDictSize, info->srcBuffer, + info->samplesSizes, info->nbSamples, params); + DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\n", params->k, params->d, params->f, params->steps, (unsigned)(params->splitPoint*100)); + if (ZDICT_isError(dictSize)) { + DISPLAYLEVEL(1, "dictionary training failed : %s \n", ZDICT_getErrorName(dictSize)); /* should not happen */ + result = 1; + goto _done; + } + /* save dict */ + DISPLAYLEVEL(2, "Save dictionary of size %u into file %s \n", (U32)dictSize, dictFileName); + saveDict(dictFileName, dictBuffer, dictSize); + } + + /* clean up */ +_done: + free(dictBuffer); + return result; +} + + + +int main(int argCount, const char* argv[]) +{ + int displayLevel = 2; + const char* programName = argv[0]; + int operationResult = 0; + + /* Initialize arguments to default values */ + unsigned k = 200; + unsigned d = 8; + unsigned f = 23; + unsigned steps = 32; + unsigned nbThreads = 1; + unsigned split = 100; + const char* outputFile = "fastCoverDict"; + unsigned dictID = 0; + unsigned maxDictSize = g_defaultMaxDictSize; + + /* Initialize table to store input files */ + const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); + unsigned filenameIdx = 0; + + char* fileNamesBuf = NULL; + unsigned fileNamesNb = filenameIdx; + int followLinks = 0; /* follow directory recursively */ + const char** extendedFileList = NULL; + + /* Parse arguments */ + for (int i = 1; i < argCount; i++) { + const char* argument = argv[i]; + if (longCommandWArg(&argument, "k=")) { k = readU32FromChar(&argument); continue; } + if (longCommandWArg(&argument, "d=")) { d = readU32FromChar(&argument); continue; } + if (longCommandWArg(&argument, "f=")) { f = readU32FromChar(&argument); continue; } + if (longCommandWArg(&argument, "steps=")) { steps = readU32FromChar(&argument); continue; } + if (longCommandWArg(&argument, "split=")) { split = readU32FromChar(&argument); continue; } + if (longCommandWArg(&argument, "dictID=")) { dictID = readU32FromChar(&argument); continue; } + if (longCommandWArg(&argument, "maxdict=")) { maxDictSize = readU32FromChar(&argument); continue; } + if (longCommandWArg(&argument, "in=")) { + filenameTable[filenameIdx] = argument; + filenameIdx++; + continue; + } + if (longCommandWArg(&argument, "out=")) { + outputFile = argument; + continue; + } + DISPLAYLEVEL(1, "Incorrect parameters\n"); + operationResult = 1; + return operationResult; + } + + /* Get the list of all files recursively (because followLinks==0)*/ + extendedFileList = UTIL_createFileList(filenameTable, filenameIdx, &fileNamesBuf, + &fileNamesNb, followLinks); + if (extendedFileList) { + unsigned u; + for (u=0; u Date: Wed, 25 Jul 2018 16:54:08 -0700 Subject: [PATCH 2/9] Make hash value const --- .../experimental_dict_builders/fastCover/fastCover.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/contrib/experimental_dict_builders/fastCover/fastCover.c b/contrib/experimental_dict_builders/fastCover/fastCover.c index 6d3ad90ab..32a15a4bf 100644 --- a/contrib/experimental_dict_builders/fastCover/fastCover.c +++ b/contrib/experimental_dict_builders/fastCover/fastCover.c @@ -138,7 +138,7 @@ static FASTCOVER_segment_t FASTCOVER_selectSegment(const FASTCOVER_ctx_t *ctx, */ while (activeSegment.end < end) { /* Get hash value of current dmer */ - size_t index = FASTCOVER_hash8PtrToIndex(ctx->samples + activeSegment.end, parameters.f); + const size_t index = FASTCOVER_hash8PtrToIndex(ctx->samples + activeSegment.end, parameters.f); /* Add frequency of this index to score */ activeSegment.score += freqs[index]; /* Increment end of segment */ @@ -146,7 +146,7 @@ static FASTCOVER_segment_t FASTCOVER_selectSegment(const FASTCOVER_ctx_t *ctx, /* If the window is now too large, drop the first position */ if (activeSegment.end - activeSegment.begin == dmersInK + 1) { /* Get hash value of the dmer to be eliminated from active segment */ - size_t delIndex = FASTCOVER_hash8PtrToIndex(ctx->samples + activeSegment.begin, parameters.f); + const size_t delIndex = FASTCOVER_hash8PtrToIndex(ctx->samples + activeSegment.begin, parameters.f); /* Subtract frequency of this index from score */ activeSegment.score -= freqs[delIndex]; /* Increment start of segment */ @@ -163,7 +163,7 @@ static FASTCOVER_segment_t FASTCOVER_selectSegment(const FASTCOVER_ctx_t *ctx, U32 newEnd = bestSegment.begin; U32 pos; for (pos = bestSegment.begin; pos != bestSegment.end; ++pos) { - size_t index = FASTCOVER_hash8PtrToIndex(ctx->samples + pos, parameters.f); + const size_t index = FASTCOVER_hash8PtrToIndex(ctx->samples + pos, parameters.f); U32 freq = freqs[index]; if (freq != 0) { newBegin = MIN(newBegin, pos); @@ -177,7 +177,7 @@ static FASTCOVER_segment_t FASTCOVER_selectSegment(const FASTCOVER_ctx_t *ctx, /* Half the frequency of hash value of each dmer covered by the chosen segment. */ U32 pos; for (pos = bestSegment.begin; pos != bestSegment.end; ++pos) { - size_t i = FASTCOVER_hash8PtrToIndex(ctx->samples + pos, parameters.f); + const size_t i = FASTCOVER_hash8PtrToIndex(ctx->samples + pos, parameters.f); freqs[i] = freqs[i]/2; } } @@ -244,7 +244,7 @@ static void FASTCOVER_getFrequency(U32 *freqs, unsigned f, FASTCOVER_ctx_t *ctx) size_t currSampleEnd = ctx->offsets[i+1]; start = currSampleStart; while (start + f < currSampleEnd) { - size_t dmerIndex = FASTCOVER_hash8PtrToIndex(ctx->samples + start, f); + const size_t dmerIndex = FASTCOVER_hash8PtrToIndex(ctx->samples + start, f); /* if no dmer with same hash value has been seen in current sample */ if (inCurrSample[dmerIndex] == 0) { inCurrSample[dmerIndex]++; From d1fc507ef998f511f6f1da7edc57670bb6b3404f Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Wed, 25 Jul 2018 17:05:54 -0700 Subject: [PATCH 3/9] Initial benchmarking result for fastCover --- .../benchmarkDictBuilder/Makefile | 10 +++-- .../benchmarkDictBuilder/README.md | 40 ++++++++++-------- .../benchmarkDictBuilder/benchmark.c | 42 +++++++++++++++---- 3 files changed, 62 insertions(+), 30 deletions(-) diff --git a/contrib/experimental_dict_builders/benchmarkDictBuilder/Makefile b/contrib/experimental_dict_builders/benchmarkDictBuilder/Makefile index 72ce04f2a..681494888 100644 --- a/contrib/experimental_dict_builders/benchmarkDictBuilder/Makefile +++ b/contrib/experimental_dict_builders/benchmarkDictBuilder/Makefile @@ -2,9 +2,10 @@ ARG := CC ?= gcc CFLAGS ?= -O3 -INCLUDES := -I ../randomDictBuilder -I ../../../programs -I ../../../lib/common -I ../../../lib -I ../../../lib/dictBuilder +INCLUDES := -I ../randomDictBuilder -I ../fastCover -I ../../../programs -I ../../../lib/common -I ../../../lib -I ../../../lib/dictBuilder RANDOM_FILE := ../randomDictBuilder/random.c +FAST_FILE := ../fastCover/fastCover.c IO_FILE := ../randomDictBuilder/io.c all: run clean @@ -21,8 +22,8 @@ test: benchmarkTest clean benchmarkTest: benchmark test.sh sh test.sh -benchmark: benchmark.o io.o random.o libzstd.a - $(CC) $(CFLAGS) benchmark.o io.o random.o libzstd.a -o benchmark +benchmark: benchmark.o io.o random.o fastCover.o libzstd.a + $(CC) $(CFLAGS) benchmark.o io.o random.o fastCover.o libzstd.a -o benchmark benchmark.o: benchmark.c $(CC) $(CFLAGS) $(INCLUDES) -c benchmark.c @@ -30,6 +31,9 @@ benchmark.o: benchmark.c random.o: $(RANDOM_FILE) $(CC) $(CFLAGS) $(INCLUDES) -c $(RANDOM_FILE) +fastCover.o: $(FAST_FILE) + $(CC) $(CFLAGS) $(INCLUDES) -c $(FAST_FILE) + io.o: $(IO_FILE) $(CC) $(CFLAGS) $(INCLUDES) -c $(IO_FILE) diff --git a/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md b/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md index de783a0ec..e02d592c4 100644 --- a/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md +++ b/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md @@ -18,30 +18,34 @@ github: | Algorithm | Speed(sec) | Compression Ratio | | ------------- |:-------------:| ------------------:| | nodict | 0.000004 | 2.999642 | -| random | 0.180238 | 8.786957 | -| cover | 33.891987 | 10.430999 | -| legacy | 1.077569 | 8.989482 | +| random | 0.135459 | 8.786957 | +| cover | 50.341079 | 10.641263 | +| legacy | 0.866283 | 8.989482 | +| fastCover | 13.450947 | 10.215174 | hg-commands | Algorithm | Speed(sec) | Compression Ratio | | ------------- |:-------------:| ------------------:| -| nodict | 0.000006 | 2.425291 | -| random | 0.088735 | 3.489515 | -| cover | 35.447300 | 4.030274 | -| legacy | 1.048509 | 3.911896 | +| nodict | 0.000020 | 2.425291 | +| random | 0.088828 | 3.489515 | +| cover | 60.028672 | 4.131136 | +| legacy | 0.852481 | 3.911896 | +| fastCover | 9.524284 | 3.977229 | + +hg-changelog +| Algorithm | Speed(sec) | Compression Ratio | +| ------------- |:-------------:| ------------------:| +| nodict | 0.000004 | 1.377613 | +| random | 0.621812 | 2.096785 | +| cover | 217.510962 | 2.188654 | +| legacy | 2.559194 | 2.058273 | +| fastCover | 51.132516 | 2.124185 | hg-manifest | Algorithm | Speed(sec) | Compression Ratio | | ------------- |:-------------:| ------------------:| | nodict | 0.000005 | 1.866385 | -| random | 1.148231 | 2.309485 | -| cover | 509.685257 | 2.575331 | -| legacy | 10.705866 | 2.506775 | - -hg-changelog -| Algorithm | Speed(sec) | Compression Ratio | -| ------------- |:-------------:| ------------------:| -| nodict | 0.000005 | 1.377613 | -| random | 0.706434 | 2.096785 | -| cover | 122.815783 | 2.175706 | -| legacy | 3.010318 | 2.058273 | +| random | 1.035220 | 2.309485 | +| cover | 930.480173 | 2.582597 | +| legacy | 8.916513 | 2.506775 | +| fastCover | 116.871089 | 2.525689 | diff --git a/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c b/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c index 640419649..865ecb34d 100644 --- a/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c +++ b/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c @@ -5,6 +5,7 @@ #include #include #include "random.h" +#include "fastCover.h" #include "dictBuilder.h" #include "zstd_internal.h" /* includes zstd.h */ #include "io.h" @@ -71,10 +72,11 @@ typedef struct { */ dictInfo* createDictFromFiles(sampleInfo *info, unsigned maxDictSize, ZDICT_random_params_t *randomParams, ZDICT_cover_params_t *coverParams, - ZDICT_legacy_params_t *legacyParams) { + ZDICT_legacy_params_t *legacyParams, ZDICT_fastCover_params_t *fastParams) { unsigned const displayLevel = randomParams ? randomParams->zParams.notificationLevel : coverParams ? coverParams->zParams.notificationLevel : legacyParams ? legacyParams->zParams.notificationLevel : + fastParams ? fastParams->zParams.notificationLevel : DEFAULT_DISPLAYLEVEL; /* no dict */ void* const dictBuffer = malloc(maxDictSize); @@ -94,6 +96,9 @@ dictInfo* createDictFromFiles(sampleInfo *info, unsigned maxDictSize, } else if(legacyParams) { dictSize = ZDICT_trainFromBuffer_legacy(dictBuffer, maxDictSize, info->srcBuffer, info->samplesSizes, info->nbSamples, *legacyParams); + } else if(fastParams) { + dictSize = ZDICT_optimizeTrainFromBuffer_fastCover(dictBuffer, maxDictSize, info->srcBuffer, + info->samplesSizes, info->nbSamples, fastParams); } else { dictSize = 0; } @@ -216,25 +221,29 @@ void freeDictInfo(dictInfo* info) { * @return 0 if benchmark successfully, 1 otherwise */ int benchmarkDictBuilder(sampleInfo *srcInfo, unsigned maxDictSize, ZDICT_random_params_t *randomParam, - ZDICT_cover_params_t *coverParam, ZDICT_legacy_params_t *legacyParam) { + ZDICT_cover_params_t *coverParam, ZDICT_legacy_params_t *legacyParam, + ZDICT_fastCover_params_t *fastParam) { /* Local variables */ const unsigned displayLevel = randomParam ? randomParam->zParams.notificationLevel : coverParam ? coverParam->zParams.notificationLevel : legacyParam ? legacyParam->zParams.notificationLevel : + fastParam ? fastParam->zParams.notificationLevel: DEFAULT_DISPLAYLEVEL; /* no dict */ const char* name = randomParam ? "RANDOM" : coverParam ? "COVER" : legacyParam ? "LEGACY" : + fastParam ? "FAST": "NODICT"; /* no dict */ const unsigned cLevel = randomParam ? randomParam->zParams.compressionLevel : coverParam ? coverParam->zParams.compressionLevel : legacyParam ? legacyParam->zParams.compressionLevel : + fastParam ? fastParam->zParams.compressionLevel: DEFAULT_CLEVEL; /* no dict */ int result = 0; /* Calculate speed */ const UTIL_time_t begin = UTIL_getTime(); - dictInfo* dInfo = createDictFromFiles(srcInfo, maxDictSize, randomParam, coverParam, legacyParam); + dictInfo* dInfo = createDictFromFiles(srcInfo, maxDictSize, randomParam, coverParam, legacyParam, fastParam); const U64 timeMicro = UTIL_clockSpanMicro(begin); const double timeSec = timeMicro / (double)SEC_TO_MICRO; if (!dInfo) { @@ -269,7 +278,6 @@ int main(int argCount, const char* argv[]) /* Initialize arguments to default values */ const unsigned k = 200; - const unsigned d = 6; const unsigned cLevel = DEFAULT_CLEVEL; const unsigned dictID = 0; const unsigned maxDictSize = g_defaultMaxDictSize; @@ -319,7 +327,7 @@ int main(int argCount, const char* argv[]) /* with no dict */ { - const int noDictResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL); + const int noDictResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, NULL); if(noDictResult) { result = 1; goto _cleanup; @@ -331,7 +339,7 @@ int main(int argCount, const char* argv[]) ZDICT_random_params_t randomParam; randomParam.zParams = zParams; randomParam.k = k; - const int randomResult = benchmarkDictBuilder(srcInfo, maxDictSize, &randomParam, NULL, NULL); + const int randomResult = benchmarkDictBuilder(srcInfo, maxDictSize, &randomParam, NULL, NULL, NULL); if(randomResult) { result = 1; goto _cleanup; @@ -344,10 +352,9 @@ int main(int argCount, const char* argv[]) memset(&coverParam, 0, sizeof(coverParam)); coverParam.zParams = zParams; coverParam.splitPoint = 1.0; - coverParam.d = d; coverParam.steps = 40; coverParam.nbThreads = 1; - const int coverOptResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, &coverParam, NULL); + const int coverOptResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, &coverParam, NULL, NULL); if(coverOptResult) { result = 1; goto _cleanup; @@ -359,13 +366,30 @@ int main(int argCount, const char* argv[]) ZDICT_legacy_params_t legacyParam; legacyParam.zParams = zParams; legacyParam.selectivityLevel = 9; - const int legacyResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, &legacyParam); + const int legacyResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, &legacyParam, NULL); if(legacyResult) { result = 1; goto _cleanup; } } + /* for fastCover */ + { + ZDICT_fastCover_params_t fastParam; + memset(&fastParam, 0, sizeof(fastParam)); + fastParam.zParams = zParams; + fastParam.splitPoint = 1.0; + fastParam.d = 8; + fastParam.f = 23; + fastParam.steps = 40; + fastParam.nbThreads = 1; + const int fastOptResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, &fastParam); + if(fastOptResult) { + result = 1; + goto _cleanup; + } + } + /* Free allocated memory */ _cleanup: UTIL_freeFileList(extendedFileList, fileNamesBuf); From 1e85f314d859c5295f88c98fcd0dc9fa03f68b12 Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Wed, 25 Jul 2018 17:53:38 -0700 Subject: [PATCH 4/9] Benchmark fast cover optimize vs k=200 --- .../benchmarkDictBuilder/README.md | 60 ++++++++++--------- .../benchmarkDictBuilder/benchmark.c | 22 ++++++- 2 files changed, 53 insertions(+), 29 deletions(-) diff --git a/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md b/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md index e02d592c4..478d8793e 100644 --- a/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md +++ b/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md @@ -15,37 +15,41 @@ make ARG="in=../../../lib/dictBuilder in=../../../lib/compress" ###Benchmarking Result: github: -| Algorithm | Speed(sec) | Compression Ratio | -| ------------- |:-------------:| ------------------:| -| nodict | 0.000004 | 2.999642 | -| random | 0.135459 | 8.786957 | -| cover | 50.341079 | 10.641263 | -| legacy | 0.866283 | 8.989482 | -| fastCover | 13.450947 | 10.215174 | +| Algorithm | Speed(sec) | Compression Ratio | +| ------------------|:-------------:| ------------------:| +| nodict | 0.000004 | 2.999642 | +| random | 0.148247 | 8.786957 | +| cover | 56.331553 | 10.641263 | +| legacy | 0.917595 | 8.989482 | +| fastCover(opt) | 13.169979 | 10.215174 | +| fastCover(k=200) | 2.692406 | 8.657219 | hg-commands -| Algorithm | Speed(sec) | Compression Ratio | -| ------------- |:-------------:| ------------------:| -| nodict | 0.000020 | 2.425291 | -| random | 0.088828 | 3.489515 | -| cover | 60.028672 | 4.131136 | -| legacy | 0.852481 | 3.911896 | -| fastCover | 9.524284 | 3.977229 | +| Algorithm | Speed(sec) | Compression Ratio | +| ----------------- |:-------------:| ------------------:| +| nodict | 0.000007 | 2.425291 | +| random | 0.093990 | 3.489515 | +| cover | 58.602385 | 4.131136 | +| legacy | 0.865683 | 3.911896 | +| fastCover(opt) | 9.404134 | 3.977229 | +| fastCover(k=200) | 1.037434 | 3.810326 | hg-changelog -| Algorithm | Speed(sec) | Compression Ratio | -| ------------- |:-------------:| ------------------:| -| nodict | 0.000004 | 1.377613 | -| random | 0.621812 | 2.096785 | -| cover | 217.510962 | 2.188654 | -| legacy | 2.559194 | 2.058273 | -| fastCover | 51.132516 | 2.124185 | +| Algorithm | Speed(sec) | Compression Ratio | +| ----------------- |:-------------:| ------------------:| +| nodict | 0.000022 | 1.377613 | +| random | 0.551539 | 2.096785 | +| cover | 221.370056 | 2.188654 | +| legacy | 2.405923 | 2.058273 | +| fastCover(opt) | 49.526246 | 2.124185 | +| fastCover(k=200) | 9.746872 | 2.114674 | hg-manifest -| Algorithm | Speed(sec) | Compression Ratio | -| ------------- |:-------------:| ------------------:| -| nodict | 0.000005 | 1.866385 | -| random | 1.035220 | 2.309485 | -| cover | 930.480173 | 2.582597 | -| legacy | 8.916513 | 2.506775 | -| fastCover | 116.871089 | 2.525689 | +| Algorithm | Speed(sec) | Compression Ratio | +| ----------------- |:-------------:| ------------------:| +| nodict | 0.000019 | 1.866385 | +| random | 1.083536 | 2.309485 | +| cover | 928.894887 | 2.582597 | +| legacy | 9.110371 | 2.506775 | +| fastCover(opt) | 116.508270 | 2.525689 | +| fastCover(k=200) | 12.176555 | 2.472221 | diff --git a/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c b/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c index 865ecb34d..62135436b 100644 --- a/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c +++ b/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c @@ -373,7 +373,8 @@ int main(int argCount, const char* argv[]) } } - /* for fastCover */ + + /* for fastCover (optimizing k) */ { ZDICT_fastCover_params_t fastParam; memset(&fastParam, 0, sizeof(fastParam)); @@ -390,6 +391,25 @@ int main(int argCount, const char* argv[]) } } + /* for fastCover (with k provided) */ + { + ZDICT_fastCover_params_t fastParam; + memset(&fastParam, 0, sizeof(fastParam)); + fastParam.zParams = zParams; + fastParam.splitPoint = 1.0; + fastParam.d = 8; + fastParam.f = 23; + fastParam.k = 200; + fastParam.steps = 40; + fastParam.nbThreads = 1; + const int fastOptResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, &fastParam); + if(fastOptResult) { + result = 1; + goto _cleanup; + } + } + + /* Free allocated memory */ _cleanup: UTIL_freeFileList(extendedFileList, fileNamesBuf); From 2333ecb173077edaf34f032baadfcc63531928c1 Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Wed, 25 Jul 2018 18:10:09 -0700 Subject: [PATCH 5/9] Allow d=6 --- .../fastCover/README.md | 2 +- .../fastCover/fastCover.c | 27 +++++++++++++------ .../fastCover/test.sh | 3 ++- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/contrib/experimental_dict_builders/fastCover/README.md b/contrib/experimental_dict_builders/fastCover/README.md index 088e38be7..66e00ee04 100644 --- a/contrib/experimental_dict_builders/fastCover/README.md +++ b/contrib/experimental_dict_builders/fastCover/README.md @@ -6,7 +6,7 @@ Output Dictionary (out=dictName): if not provided, default to fastCoverDict Dictionary ID (dictID=#): nonnegative number; if not provided, default to 0 Maximum Dictionary Size (maxdict=#): positive number; in bytes, if not provided, default to 110KB Size of Selected Segment (k=#): positive number; in bytes; if not provided, default to 200 -Size of Dmer (d=#): positive number; in bytes; if not provided, default to 8 +Size of Dmer (d=#): either 6 or 8; if not provided, default to 8 Number of steps (steps=#): positive number, if not provided, default to 32 Percentage of samples used for training(split=#): positive number; if not provided, default to 100 diff --git a/contrib/experimental_dict_builders/fastCover/fastCover.c b/contrib/experimental_dict_builders/fastCover/fastCover.c index 32a15a4bf..abd592cd8 100644 --- a/contrib/experimental_dict_builders/fastCover/fastCover.c +++ b/contrib/experimental_dict_builders/fastCover/fastCover.c @@ -50,14 +50,21 @@ static clock_t g_time = 0; /*-************************************* * Hash Function ***************************************/ +static const U64 prime6bytes = 227718039650203ULL; +static size_t ZSTD_hash6(U64 u, U32 h) { return (size_t)(((u << (64-48)) * prime6bytes) >> (64-h)) ; } +static size_t ZSTD_hash6Ptr(const void* p, U32 h) { return ZSTD_hash6(MEM_readLE64(p), h); } + static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL; static size_t ZSTD_hash8(U64 u, U32 h) { return (size_t)(((u) * prime8bytes) >> (64-h)) ; } static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(MEM_readLE64(p), h); } /** - * Hash the 8-byte value pointed to by p and mod 2^f + * Hash the d-byte value pointed to by p and mod 2^f */ -static size_t FASTCOVER_hash8PtrToIndex(const void* p, U32 h) { +static size_t FASTCOVER_hashPtrToIndex(const void* p, U32 h, unsigned d) { + if (d == 6) { + return ZSTD_hash6Ptr(p, h) & ((1 << h) - 1); + } return ZSTD_hash8Ptr(p, h) & ((1 << h) - 1); } @@ -138,7 +145,7 @@ static FASTCOVER_segment_t FASTCOVER_selectSegment(const FASTCOVER_ctx_t *ctx, */ while (activeSegment.end < end) { /* Get hash value of current dmer */ - const size_t index = FASTCOVER_hash8PtrToIndex(ctx->samples + activeSegment.end, parameters.f); + const size_t index = FASTCOVER_hashPtrToIndex(ctx->samples + activeSegment.end, parameters.f, ctx->d); /* Add frequency of this index to score */ activeSegment.score += freqs[index]; /* Increment end of segment */ @@ -146,7 +153,7 @@ static FASTCOVER_segment_t FASTCOVER_selectSegment(const FASTCOVER_ctx_t *ctx, /* If the window is now too large, drop the first position */ if (activeSegment.end - activeSegment.begin == dmersInK + 1) { /* Get hash value of the dmer to be eliminated from active segment */ - const size_t delIndex = FASTCOVER_hash8PtrToIndex(ctx->samples + activeSegment.begin, parameters.f); + const size_t delIndex = FASTCOVER_hashPtrToIndex(ctx->samples + activeSegment.begin, parameters.f, ctx->d); /* Subtract frequency of this index from score */ activeSegment.score -= freqs[delIndex]; /* Increment start of segment */ @@ -163,7 +170,7 @@ static FASTCOVER_segment_t FASTCOVER_selectSegment(const FASTCOVER_ctx_t *ctx, U32 newEnd = bestSegment.begin; U32 pos; for (pos = bestSegment.begin; pos != bestSegment.end; ++pos) { - const size_t index = FASTCOVER_hash8PtrToIndex(ctx->samples + pos, parameters.f); + const size_t index = FASTCOVER_hashPtrToIndex(ctx->samples + pos, parameters.f, ctx->d); U32 freq = freqs[index]; if (freq != 0) { newBegin = MIN(newBegin, pos); @@ -177,7 +184,7 @@ static FASTCOVER_segment_t FASTCOVER_selectSegment(const FASTCOVER_ctx_t *ctx, /* Half the frequency of hash value of each dmer covered by the chosen segment. */ U32 pos; for (pos = bestSegment.begin; pos != bestSegment.end; ++pos) { - const size_t i = FASTCOVER_hash8PtrToIndex(ctx->samples + pos, parameters.f); + const size_t i = FASTCOVER_hashPtrToIndex(ctx->samples + pos, parameters.f, ctx->d); freqs[i] = freqs[i]/2; } } @@ -194,6 +201,10 @@ static int FASTCOVER_checkParameters(ZDICT_fastCover_params_t parameters, if (parameters.d == 0 || parameters.k == 0 || parameters.f == 0) { return 0; } + /* d has to be 6 or 8 */ + if (parameters.d != 6 && parameters.d != 8) { + return 0; + } /* 0 < f <= FASTCOVER_MAX_F */ if (parameters.f > FASTCOVER_MAX_F) { return 0; @@ -244,7 +255,7 @@ static void FASTCOVER_getFrequency(U32 *freqs, unsigned f, FASTCOVER_ctx_t *ctx) size_t currSampleEnd = ctx->offsets[i+1]; start = currSampleStart; while (start + f < currSampleEnd) { - const size_t dmerIndex = FASTCOVER_hash8PtrToIndex(ctx->samples + start, f); + const size_t dmerIndex = FASTCOVER_hashPtrToIndex(ctx->samples + start, f, ctx->d); /* if no dmer with same hash value has been seen in current sample */ if (inCurrSample[dmerIndex] == 0) { inCurrSample[dmerIndex]++; @@ -615,7 +626,7 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_fastCover( const unsigned nbThreads = parameters->nbThreads; const double splitPoint = parameters->splitPoint <= 0.0 ? DEFAULT_SPLITPOINT : parameters->splitPoint; - const unsigned kMinD = parameters->d == 0 ? 8 : parameters->d; + const unsigned kMinD = parameters->d == 0 ? 6 : parameters->d; const unsigned kMaxD = parameters->d == 0 ? 8 : parameters->d; const unsigned kMinK = parameters->k == 0 ? 50 : parameters->k; const unsigned kMaxK = parameters->k == 0 ? 2000 : parameters->k; diff --git a/contrib/experimental_dict_builders/fastCover/test.sh b/contrib/experimental_dict_builders/fastCover/test.sh index b5570fef1..91d4f4923 100644 --- a/contrib/experimental_dict_builders/fastCover/test.sh +++ b/contrib/experimental_dict_builders/fastCover/test.sh @@ -11,4 +11,5 @@ echo "Removing dict1 dict2 dict3" rm -f dict1 dict2 dict3 echo "Testing with invalid parameters, should fail" -! ./main r=10 +! ./main in=../../../lib/common r=10 +! ./main in=../../../lib/common d=10 From 3b163e0b5b5f9eec427b87001483c3b627c95a8f Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Thu, 26 Jul 2018 13:53:13 -0700 Subject: [PATCH 6/9] Add array to keep track of frequency within active segment, fix malloc bug, update benchmarking result --- .../benchmarkDictBuilder/README.md | 60 ++++++++-------- .../fastCover/fastCover.c | 69 +++++++++++-------- .../fastCover/main.c | 2 +- .../randomDictBuilder/main.c | 2 +- 4 files changed, 75 insertions(+), 58 deletions(-) diff --git a/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md b/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md index 478d8793e..07d65b08c 100644 --- a/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md +++ b/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md @@ -14,42 +14,46 @@ make ARG="in=../../../lib/dictBuilder in=../../../lib/compress" ###Benchmarking Result: +d=8 +f=23 +freq[i] = 0 when dmer added to best segment + github: | Algorithm | Speed(sec) | Compression Ratio | -| ------------------|:-------------:| ------------------:| -| nodict | 0.000004 | 2.999642 | -| random | 0.148247 | 8.786957 | -| cover | 56.331553 | 10.641263 | -| legacy | 0.917595 | 8.989482 | -| fastCover(opt) | 13.169979 | 10.215174 | -| fastCover(k=200) | 2.692406 | 8.657219 | +| ----------------- | ------------- | ------------------ | +| nodict | 0.000007 | 2.999642 | +| random | 0.150258 | 8.786957 | +| cover | 60.388853 | 10.641263 | +| legacy | 0.965050 | 8.989482 | +| fastCover(opt) | 84.968131 | 10.614747 | +| fastCover(k=200) | 6.465490 | 9.484150 | hg-commands | Algorithm | Speed(sec) | Compression Ratio | -| ----------------- |:-------------:| ------------------:| -| nodict | 0.000007 | 2.425291 | -| random | 0.093990 | 3.489515 | -| cover | 58.602385 | 4.131136 | -| legacy | 0.865683 | 3.911896 | -| fastCover(opt) | 9.404134 | 3.977229 | -| fastCover(k=200) | 1.037434 | 3.810326 | +| ----------------- | ------------- | ------------------ | +| nodict | 0.000005 | 2.425291 | +| random | 0.084348 | 3.489515 | +| cover | 60.144894 | 4.131136 | +| legacy | 0.831981 | 3.911896 | +| fastCover(opt) | 59.030437 | 4.157595 | +| fastCover(k=200) | 3.702932 | 4.134222 | hg-changelog | Algorithm | Speed(sec) | Compression Ratio | -| ----------------- |:-------------:| ------------------:| -| nodict | 0.000022 | 1.377613 | -| random | 0.551539 | 2.096785 | -| cover | 221.370056 | 2.188654 | -| legacy | 2.405923 | 2.058273 | -| fastCover(opt) | 49.526246 | 2.124185 | -| fastCover(k=200) | 9.746872 | 2.114674 | +| ----------------- | ------------- | ------------------ | +| nodict | 0.000004 | 1.377613 | +| random | 0.555964 | 2.096785 | +| cover | 214.423753 | 2.188654 | +| legacy | 2.180249 | 2.058273 | +| fastCover(opt) | 102.261452 | 2.180347 | +| fastCover(k=200) | 11.81039 | 2.170673 | hg-manifest | Algorithm | Speed(sec) | Compression Ratio | -| ----------------- |:-------------:| ------------------:| -| nodict | 0.000019 | 1.866385 | -| random | 1.083536 | 2.309485 | -| cover | 928.894887 | 2.582597 | -| legacy | 9.110371 | 2.506775 | -| fastCover(opt) | 116.508270 | 2.525689 | -| fastCover(k=200) | 12.176555 | 2.472221 | +| ----------------- | ------------- | ------------------ | +| nodict | 0.000006 | 1.866385 | +| random | 1.063974 | 2.309485 | +| cover | 909.101849 | 2.582597 | +| legacy | 8.706580 | 2.506775 | +| fastCover(opt) | 188.598079 | 2.596761 | +| fastCover(k=200) | 13.392734 | 2.592985 | diff --git a/contrib/experimental_dict_builders/fastCover/fastCover.c b/contrib/experimental_dict_builders/fastCover/fastCover.c index abd592cd8..6f990e0c2 100644 --- a/contrib/experimental_dict_builders/fastCover/fastCover.c +++ b/contrib/experimental_dict_builders/fastCover/fastCover.c @@ -48,7 +48,7 @@ static clock_t g_time = 0; /*-************************************* -* Hash Function +* Hash Functions ***************************************/ static const U64 prime6bytes = 227718039650203ULL; static size_t ZSTD_hash6(U64 u, U32 h) { return (size_t)(((u << (64-48)) * prime6bytes) >> (64-h)) ; } @@ -58,6 +58,7 @@ static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL; static size_t ZSTD_hash8(U64 u, U32 h) { return (size_t)(((u) * prime8bytes) >> (64-h)) ; } static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(MEM_readLE64(p), h); } + /** * Hash the d-byte value pointed to by p and mod 2^f */ @@ -140,29 +141,41 @@ static FASTCOVER_segment_t FASTCOVER_selectSegment(const FASTCOVER_ctx_t *ctx, activeSegment.begin = begin; activeSegment.end = begin; activeSegment.score = 0; - /* Slide the activeSegment through the whole epoch. - * Save the best segment in bestSegment. - */ - while (activeSegment.end < end) { - /* Get hash value of current dmer */ - const size_t index = FASTCOVER_hashPtrToIndex(ctx->samples + activeSegment.end, parameters.f, ctx->d); - /* Add frequency of this index to score */ - activeSegment.score += freqs[index]; - /* Increment end of segment */ - activeSegment.end += 1; - /* If the window is now too large, drop the first position */ - if (activeSegment.end - activeSegment.begin == dmersInK + 1) { - /* Get hash value of the dmer to be eliminated from active segment */ - const size_t delIndex = FASTCOVER_hashPtrToIndex(ctx->samples + activeSegment.begin, parameters.f, ctx->d); - /* Subtract frequency of this index from score */ - activeSegment.score -= freqs[delIndex]; - /* Increment start of segment */ - activeSegment.begin += 1; - } - /* If this segment is the best so far save it */ - if (activeSegment.score > bestSegment.score) { - bestSegment = activeSegment; + { + /* Keep track of number of times an index has been seen in current segment */ + U16* currfreqs =(U16 *)malloc((1 << parameters.f) * sizeof(U16)); + memset(currfreqs, 0, (1 << parameters.f) * sizeof(*currfreqs)); + /* Slide the activeSegment through the whole epoch. + * Save the best segment in bestSegment. + */ + while (activeSegment.end < end) { + /* Get hash value of current dmer */ + const size_t index = FASTCOVER_hashPtrToIndex(ctx->samples + activeSegment.end, parameters.f, ctx->d); + /* Add frequency of this index to score if this is the first occurence of index in active segment */ + if (currfreqs[index] == 0) { + activeSegment.score += freqs[index]; + } + currfreqs[index] += 1; + /* Increment end of segment */ + activeSegment.end += 1; + /* If the window is now too large, drop the first position */ + if (activeSegment.end - activeSegment.begin == dmersInK + 1) { + /* Get hash value of the dmer to be eliminated from active segment */ + const size_t delIndex = FASTCOVER_hashPtrToIndex(ctx->samples + activeSegment.begin, parameters.f, ctx->d); + currfreqs[delIndex] -= 1; + /* Subtract frequency of this index from score if this is the last occurrence of this index in active segment */ + if (currfreqs[delIndex] == 0) { + activeSegment.score -= freqs[delIndex]; + } + /* Increment start of segment */ + activeSegment.begin += 1; + } + /* If this segment is the best so far save it */ + if (activeSegment.score > bestSegment.score) { + bestSegment = activeSegment; + } } + free(currfreqs); } { /* Trim off the zero frequency head and tail from the segment. */ @@ -185,7 +198,7 @@ static FASTCOVER_segment_t FASTCOVER_selectSegment(const FASTCOVER_ctx_t *ctx, U32 pos; for (pos = bestSegment.begin; pos != bestSegment.end; ++pos) { const size_t i = FASTCOVER_hashPtrToIndex(ctx->samples + pos, parameters.f, ctx->d); - freqs[i] = freqs[i]/2; + freqs[i] = 0; } } return bestSegment; @@ -245,12 +258,12 @@ static void FASTCOVER_ctx_destroy(FASTCOVER_ctx_t *ctx) { /** * Calculate for frequency of hash value of each dmer in ctx->samples */ -static void FASTCOVER_getFrequency(U32 *freqs, unsigned f, FASTCOVER_ctx_t *ctx){ +static void FASTCOVER_computeFrequency(U32 *freqs, unsigned f, FASTCOVER_ctx_t *ctx){ /* inCurrSample keeps track of this hash value has already be seen in previous dmers in the same sample*/ - size_t* inCurrSample = (size_t *)malloc((1<nbTrainSamples; i++) { - memset(inCurrSample, 0, (1 << f)); /* Reset inCurrSample for each sample */ + memset(inCurrSample, 0, (1 << f) * sizeof(*inCurrSample)); /* Reset inCurrSample for each sample */ size_t currSampleStart = ctx->offsets[i]; size_t currSampleEnd = ctx->offsets[i+1]; start = currSampleStart; @@ -338,7 +351,7 @@ static int FASTCOVER_ctx_init(FASTCOVER_ctx_t *ctx, const void *samplesBuffer, memset(ctx->freqs, 0, (1 << f) * sizeof(U32)); DISPLAYLEVEL(2, "Computing frequencies\n"); - FASTCOVER_getFrequency(ctx->freqs, f, ctx); + FASTCOVER_computeFrequency(ctx->freqs, f, ctx); return 1; } diff --git a/contrib/experimental_dict_builders/fastCover/main.c b/contrib/experimental_dict_builders/fastCover/main.c index 260eeb281..f286b0506 100644 --- a/contrib/experimental_dict_builders/fastCover/main.c +++ b/contrib/experimental_dict_builders/fastCover/main.c @@ -165,7 +165,7 @@ int main(int argCount, const char* argv[]) params.splitPoint = (double)split/100; /* Build dictionary */ - sampleInfo* info= getSampleInfo(filenameTable, + sampleInfo* info = getSampleInfo(filenameTable, filenameIdx, blockSize, maxDictSize, zParams.notificationLevel); operationResult = FASTCOVER_trainFromFiles(outputFile, info, maxDictSize, ¶ms); diff --git a/contrib/experimental_dict_builders/randomDictBuilder/main.c b/contrib/experimental_dict_builders/randomDictBuilder/main.c index 3f3a6ca70..3ad885746 100644 --- a/contrib/experimental_dict_builders/randomDictBuilder/main.c +++ b/contrib/experimental_dict_builders/randomDictBuilder/main.c @@ -149,7 +149,7 @@ int main(int argCount, const char* argv[]) params.zParams = zParams; params.k = k; - sampleInfo* info= getSampleInfo(filenameTable, + sampleInfo* info = getSampleInfo(filenameTable, filenameIdx, blockSize, maxDictSize, zParams.notificationLevel); operationResult = RANDOM_trainFromFiles(outputFile, info, maxDictSize, ¶ms); From 3d7941ce41d33bbbedb15fa9794c9fbcb1713384 Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Thu, 26 Jul 2018 16:24:13 -0700 Subject: [PATCH 7/9] Benchmark different f values --- .../benchmarkDictBuilder/README.md | 131 +++++++++++++----- .../benchmarkDictBuilder/benchmark.c | 104 +++++++------- 2 files changed, 152 insertions(+), 83 deletions(-) diff --git a/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md b/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md index 07d65b08c..1ee4b19ba 100644 --- a/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md +++ b/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md @@ -14,46 +14,107 @@ make ARG="in=../../../lib/dictBuilder in=../../../lib/compress" ###Benchmarking Result: -d=8 -f=23 -freq[i] = 0 when dmer added to best segment +For every f value for fast, the first one is optimize and the second one has k=200 github: -| Algorithm | Speed(sec) | Compression Ratio | -| ----------------- | ------------- | ------------------ | -| nodict | 0.000007 | 2.999642 | -| random | 0.150258 | 8.786957 | -| cover | 60.388853 | 10.641263 | -| legacy | 0.965050 | 8.989482 | -| fastCover(opt) | 84.968131 | 10.614747 | -| fastCover(k=200) | 6.465490 | 9.484150 | +NODICT 0.000023 2.999642 +RANDOM 0.149020 8.786957 +LEGACY 0.854277 8.989482 +FAST15 8.764078 10.609015 +FAST15 0.232610 9.135669 +FAST16 9.597777 10.474574 +FAST16 0.243698 9.346482 +FAST17 9.385449 10.611737 +FAST17 0.268376 9.605798 +FAST18 9.988885 10.626382 +FAST18 0.311769 9.130565 +FAST19 10.737259 10.411729 +FAST19 0.331885 9.271814 +FAST20 10.479782 10.388895 +FAST20 0.498416 9.194115 +FAST21 21.189883 10.376394 +FAST21 1.098532 9.244456 +FAST22 39.849935 10.432555 +FAST22 2.590561 9.410930 +FAST23 75.832399 10.614747 +FAST23 6.108487 9.484150 +FAST24 139.782714 10.611753 +FAST24 13.029406 9.379030 +COVER 55.118542 10.641263 hg-commands -| Algorithm | Speed(sec) | Compression Ratio | -| ----------------- | ------------- | ------------------ | -| nodict | 0.000005 | 2.425291 | -| random | 0.084348 | 3.489515 | -| cover | 60.144894 | 4.131136 | -| legacy | 0.831981 | 3.911896 | -| fastCover(opt) | 59.030437 | 4.157595 | -| fastCover(k=200) | 3.702932 | 4.134222 | +NODICT 0.000012 2.425291 +RANDOM 0.083071 3.489515 +LEGACY 0.835195 3.911896 +FAST15 0.163980 3.808375 +FAST16 6.373850 4.010783 +FAST16 0.160299 3.966604 +FAST17 6.668799 4.091602 +FAST17 0.172480 4.062773 +FAST18 6.266105 4.130824 +FAST18 0.171554 4.094666 +FAST19 6.869651 4.158180 +FAST19 0.209468 4.111289 +FAST20 8.267766 4.149707 +FAST20 0.331680 4.119873 +FAST21 18.824296 4.171784 +FAST21 0.783961 4.120884 +FAST22 33.321252 4.152035 +FAST22 1.854215 4.126626 +FAST23 60.775388 4.157595 +FAST23 4.040395 4.134222 +FAST24 110.910038 4.163091 +FAST24 8.505828 4.143533 +COVER 61.654796 4.131136 hg-changelog -| Algorithm | Speed(sec) | Compression Ratio | -| ----------------- | ------------- | ------------------ | -| nodict | 0.000004 | 1.377613 | -| random | 0.555964 | 2.096785 | -| cover | 214.423753 | 2.188654 | -| legacy | 2.180249 | 2.058273 | -| fastCover(opt) | 102.261452 | 2.180347 | -| fastCover(k=200) | 11.81039 | 2.170673 | +NODICT 0.000004 1.377613 +RANDOM 0.582067 2.096785 +LEGACY 2.739515 2.058273 +FAST15 35.682665 2.127596 +FAST15 0.931621 2.115299 +FAST16 36.557988 2.141787 +FAST16 1.008155 2.136080 +FAST17 36.272242 2.155332 +FAST17 0.906803 2.154596 +FAST18 35.542043 2.171997 +FAST18 1.063101 2.167723 +FAST19 37.756934 2.180893 +FAST19 1.257291 2.173768 +FAST20 40.273755 2.179442 +FAST20 1.630522 2.170072 +FAST21 54.606548 2.181400 +FAST21 2.321266 2.171643 +FAST22 72.454066 2.178774 +FAST22 5.092888 2.168885 +FAST23 106.753208 2.180347 +FAST23 14.722222 2.170673 +FAST24 171.083201 2.183426 +FAST24 27.575575 2.170623 +COVER 227.219660 2.188654 hg-manifest -| Algorithm | Speed(sec) | Compression Ratio | -| ----------------- | ------------- | ------------------ | -| nodict | 0.000006 | 1.866385 | -| random | 1.063974 | 2.309485 | -| cover | 909.101849 | 2.582597 | -| legacy | 8.706580 | 2.506775 | -| fastCover(opt) | 188.598079 | 2.596761 | -| fastCover(k=200) | 13.392734 | 2.592985 | +NODICT 0.000007 1.866385 +RANDOM 1.086571 2.309485 +LEGACY 9.567507 2.506775 +FAST15 77.811380 2.380461 +FAST15 1.969718 2.317727 +FAST16 75.789019 2.469144 +FAST16 2.051283 2.375815 +FAST17 79.659040 2.539069 +FAST17 1.995394 2.501047 +FAST18 76.281105 2.578095 +FAST18 2.059272 2.564840 +FAST19 79.395382 2.590433 +FAST19 2.354158 2.591024 +FAST20 87.937568 2.597813 +FAST20 2.922189 2.597104 +FAST21 121.760549 2.598408 +FAST21 4.798981 2.600269 +FAST22 155.878461 2.594560 +FAST22 8.151807 2.601047 +FAST23 194.238003 2.596761 +FAST23 15.160578 2.592985 +FAST24 267.425904 2.597657 +FAST24 29.513286 2.600363 +COVER 930.675322 2.582597 diff --git a/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c b/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c index 62135436b..9feaae592 100644 --- a/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c +++ b/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c @@ -340,12 +340,67 @@ int main(int argCount, const char* argv[]) randomParam.zParams = zParams; randomParam.k = k; const int randomResult = benchmarkDictBuilder(srcInfo, maxDictSize, &randomParam, NULL, NULL, NULL); + DISPLAYLEVEL(2, "k=%u\n", randomParam.k); if(randomResult) { result = 1; goto _cleanup; } } + /* for legacy */ + { + ZDICT_legacy_params_t legacyParam; + legacyParam.zParams = zParams; + legacyParam.selectivityLevel = 9; + const int legacyResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, &legacyParam, NULL); + DISPLAYLEVEL(2, "selectivityLevel=%u\n", legacyParam.selectivityLevel); + if(legacyResult) { + result = 1; + goto _cleanup; + } + } + + /* for fastCover */ + for (unsigned f = 15; f < 25; f++){ + DISPLAYLEVEL(2, "current f is %u\n", f); + /* for fastCover (optimizing k) */ + { + ZDICT_fastCover_params_t fastParam; + memset(&fastParam, 0, sizeof(fastParam)); + fastParam.zParams = zParams; + fastParam.splitPoint = 1.0; + fastParam.d = 8; + fastParam.f = f; + fastParam.steps = 40; + fastParam.nbThreads = 1; + const int fastOptResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, &fastParam); + DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\n", fastParam.k, fastParam.d, fastParam.f, fastParam.steps, (unsigned)(fastParam.splitPoint * 100)); + if(fastOptResult) { + result = 1; + goto _cleanup; + } + } + + /* for fastCover (with k provided) */ + { + ZDICT_fastCover_params_t fastParam; + memset(&fastParam, 0, sizeof(fastParam)); + fastParam.zParams = zParams; + fastParam.splitPoint = 1.0; + fastParam.d = 8; + fastParam.f = f; + fastParam.k = 200; + fastParam.steps = 40; + fastParam.nbThreads = 1; + const int fastOptResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, &fastParam); + DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\n", fastParam.k, fastParam.d, fastParam.f, fastParam.steps, (unsigned)(fastParam.splitPoint * 100)); + if(fastOptResult) { + result = 1; + goto _cleanup; + } + } + } + /* for cover */ { ZDICT_cover_params_t coverParam; @@ -355,60 +410,13 @@ int main(int argCount, const char* argv[]) coverParam.steps = 40; coverParam.nbThreads = 1; const int coverOptResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, &coverParam, NULL, NULL); + DISPLAYLEVEL(2, "k=%u\nd=%u\nsteps=%u\nsplit=%u\n", coverParam.k, coverParam.d, coverParam.steps, (unsigned)(coverParam.splitPoint * 100)); if(coverOptResult) { result = 1; goto _cleanup; } } - /* for legacy */ - { - ZDICT_legacy_params_t legacyParam; - legacyParam.zParams = zParams; - legacyParam.selectivityLevel = 9; - const int legacyResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, &legacyParam, NULL); - if(legacyResult) { - result = 1; - goto _cleanup; - } - } - - - /* for fastCover (optimizing k) */ - { - ZDICT_fastCover_params_t fastParam; - memset(&fastParam, 0, sizeof(fastParam)); - fastParam.zParams = zParams; - fastParam.splitPoint = 1.0; - fastParam.d = 8; - fastParam.f = 23; - fastParam.steps = 40; - fastParam.nbThreads = 1; - const int fastOptResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, &fastParam); - if(fastOptResult) { - result = 1; - goto _cleanup; - } - } - - /* for fastCover (with k provided) */ - { - ZDICT_fastCover_params_t fastParam; - memset(&fastParam, 0, sizeof(fastParam)); - fastParam.zParams = zParams; - fastParam.splitPoint = 1.0; - fastParam.d = 8; - fastParam.f = 23; - fastParam.k = 200; - fastParam.steps = 40; - fastParam.nbThreads = 1; - const int fastOptResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, &fastParam); - if(fastOptResult) { - result = 1; - goto _cleanup; - } - } - /* Free allocated memory */ _cleanup: From 759c543312fd722c6f351513411d6d57742c7e4e Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Thu, 26 Jul 2018 19:03:01 -0700 Subject: [PATCH 8/9] Rerun cover and fastCover with optimized values --- .../benchmarkDictBuilder/README.md | 197 +++++++++--------- .../benchmarkDictBuilder/benchmark.c | 109 ++++++---- .../fastCover/fastCover.c | 2 +- 3 files changed, 169 insertions(+), 139 deletions(-) diff --git a/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md b/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md index 1ee4b19ba..04866b7e6 100644 --- a/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md +++ b/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md @@ -13,108 +13,113 @@ Benchmark given input files: make ARG= followed by permitted arguments make ARG="in=../../../lib/dictBuilder in=../../../lib/compress" ###Benchmarking Result: - -For every f value for fast, the first one is optimize and the second one has k=200 +First Cover is optimize cover, second Cover uses optimized d and k from first one. +For every f value of fastCover, the first one is optimize fastCover and the second one uses optimized d and k from first one. github: -NODICT 0.000023 2.999642 -RANDOM 0.149020 8.786957 -LEGACY 0.854277 8.989482 -FAST15 8.764078 10.609015 -FAST15 0.232610 9.135669 -FAST16 9.597777 10.474574 -FAST16 0.243698 9.346482 -FAST17 9.385449 10.611737 -FAST17 0.268376 9.605798 -FAST18 9.988885 10.626382 -FAST18 0.311769 9.130565 -FAST19 10.737259 10.411729 -FAST19 0.331885 9.271814 -FAST20 10.479782 10.388895 -FAST20 0.498416 9.194115 -FAST21 21.189883 10.376394 -FAST21 1.098532 9.244456 -FAST22 39.849935 10.432555 -FAST22 2.590561 9.410930 -FAST23 75.832399 10.614747 -FAST23 6.108487 9.484150 -FAST24 139.782714 10.611753 -FAST24 13.029406 9.379030 -COVER 55.118542 10.641263 +NODICT 0.000004 2.999642 +RANDOM 0.146096 8.786957 +LEGACY 0.956888 8.989482 +COVER 56.596152 10.641263 +COVER 4.937047 10.641263 +FAST15 17.722269 10.586461 +FAST15 0.239135 10.586461 +FAST16 18.276179 10.492503 +FAST16 0.265285 10.492503 +FAST17 18.077916 10.611737 +FAST17 0.236573 10.611737 +FAST18 19.510150 10.621586 +FAST18 0.278683 10.621586 +FAST19 18.794350 10.629626 +FAST19 0.307943 10.629626 +FAST20 19.671099 10.610308 +FAST20 0.428814 10.610308 +FAST21 36.527238 10.625733 +FAST21 0.716384 10.625733 +FAST22 83.803521 10.625281 +FAST22 1.290246 10.625281 +FAST23 158.287924 10.602342 +FAST23 3.084848 10.602342 +FAST24 283.630941 10.603379 +FAST24 8.088933 10.603379 hg-commands -NODICT 0.000012 2.425291 -RANDOM 0.083071 3.489515 -LEGACY 0.835195 3.911896 -FAST15 0.163980 3.808375 -FAST16 6.373850 4.010783 -FAST16 0.160299 3.966604 -FAST17 6.668799 4.091602 -FAST17 0.172480 4.062773 -FAST18 6.266105 4.130824 -FAST18 0.171554 4.094666 -FAST19 6.869651 4.158180 -FAST19 0.209468 4.111289 -FAST20 8.267766 4.149707 -FAST20 0.331680 4.119873 -FAST21 18.824296 4.171784 -FAST21 0.783961 4.120884 -FAST22 33.321252 4.152035 -FAST22 1.854215 4.126626 -FAST23 60.775388 4.157595 -FAST23 4.040395 4.134222 -FAST24 110.910038 4.163091 -FAST24 8.505828 4.143533 -COVER 61.654796 4.131136 +NODICT 0.000007 2.425291 +RANDOM 0.084010 3.489515 +LEGACY 0.926763 3.911896 +COVER 62.036915 4.131136 +COVER 2.194398 4.131136 +FAST15 12.169025 3.903719 +FAST15 0.156552 3.903719 +FAST16 11.886255 4.005077 +FAST16 0.155506 4.005077 +FAST17 11.886955 4.097811 +FAST17 0.176327 4.097811 +FAST18 12.544698 4.136081 +FAST18 0.171796 4.136081 +FAST19 12.920868 4.166021 +FAST19 0.207029 4.166021 +FAST20 15.771429 4.163740 +FAST20 0.258685 4.163740 +FAST21 33.165829 4.157057 +FAST21 0.663088 4.157057 +FAST22 68.779201 4.158195 +FAST22 1.568439 4.158195 +FAST23 121.921931 4.161450 +FAST23 2.498972 4.161450 +FAST24 221.990451 4.159658 +FAST24 5.793594 4.159658 hg-changelog NODICT 0.000004 1.377613 -RANDOM 0.582067 2.096785 -LEGACY 2.739515 2.058273 -FAST15 35.682665 2.127596 -FAST15 0.931621 2.115299 -FAST16 36.557988 2.141787 -FAST16 1.008155 2.136080 -FAST17 36.272242 2.155332 -FAST17 0.906803 2.154596 -FAST18 35.542043 2.171997 -FAST18 1.063101 2.167723 -FAST19 37.756934 2.180893 -FAST19 1.257291 2.173768 -FAST20 40.273755 2.179442 -FAST20 1.630522 2.170072 -FAST21 54.606548 2.181400 -FAST21 2.321266 2.171643 -FAST22 72.454066 2.178774 -FAST22 5.092888 2.168885 -FAST23 106.753208 2.180347 -FAST23 14.722222 2.170673 -FAST24 171.083201 2.183426 -FAST24 27.575575 2.170623 -COVER 227.219660 2.188654 +RANDOM 0.549307 2.096785 +LEGACY 2.273818 2.058273 +COVER 219.640608 2.188654 +COVER 6.055391 2.188654 +FAST15 67.820700 2.127194 +FAST15 0.824624 2.127194 +FAST16 69.774209 2.145401 +FAST16 0.889737 2.145401 +FAST17 70.027355 2.157544 +FAST17 0.869004 2.157544 +FAST18 68.229652 2.173127 +FAST18 0.930689 2.173127 +FAST19 70.696241 2.179527 +FAST19 1.385515 2.179527 +FAST20 80.618172 2.183233 +FAST20 1.699632 2.183233 +FAST21 96.366254 2.180920 +FAST21 2.606553 2.180920 +FAST22 139.440758 2.184297 +FAST22 5.962606 2.184297 +FAST23 207.791930 2.187666 +FAST23 14.823301 2.187666 +FAST24 322.050385 2.189889 +FAST24 29.294918 2.189889 hg-manifest -NODICT 0.000007 1.866385 -RANDOM 1.086571 2.309485 -LEGACY 9.567507 2.506775 -FAST15 77.811380 2.380461 -FAST15 1.969718 2.317727 -FAST16 75.789019 2.469144 -FAST16 2.051283 2.375815 -FAST17 79.659040 2.539069 -FAST17 1.995394 2.501047 -FAST18 76.281105 2.578095 -FAST18 2.059272 2.564840 -FAST19 79.395382 2.590433 -FAST19 2.354158 2.591024 -FAST20 87.937568 2.597813 -FAST20 2.922189 2.597104 -FAST21 121.760549 2.598408 -FAST21 4.798981 2.600269 -FAST22 155.878461 2.594560 -FAST22 8.151807 2.601047 -FAST23 194.238003 2.596761 -FAST23 15.160578 2.592985 -FAST24 267.425904 2.597657 -FAST24 29.513286 2.600363 -COVER 930.675322 2.582597 +NODICT 0.000008 1.866385 +RANDOM 1.075766 2.309485 +LEGACY 8.688387 2.506775 +COVER 926.024689 2.582597 +COVER 33.630695 2.582597 +FAST15 152.845945 2.377689 +FAST15 2.206285 2.377689 +FAST16 147.772371 2.464814 +FAST16 1.937997 2.464814 +FAST17 147.729498 2.539834 +FAST17 1.966577 2.539834 +FAST18 144.156821 2.576924 +FAST18 1.954106 2.576924 +FAST19 145.678760 2.592479 +FAST19 2.096876 2.592479 +FAST20 159.634674 2.594551 +FAST20 2.568766 2.594551 +FAST21 228.116552 2.597128 +FAST21 4.634508 2.597128 +FAST22 288.890644 2.596971 +FAST22 6.618204 2.596971 +FAST23 377.196211 2.601416 +FAST23 13.497286 2.601416 +FAST24 503.208577 2.602830 +FAST24 29.538585 2.602830 diff --git a/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c b/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c index 9feaae592..a775eae3a 100644 --- a/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c +++ b/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c @@ -277,7 +277,8 @@ int main(int argCount, const char* argv[]) int result = 0; /* Initialize arguments to default values */ - const unsigned k = 200; + unsigned k = 200; + unsigned d = 8; const unsigned cLevel = DEFAULT_CLEVEL; const unsigned dictID = 0; const unsigned maxDictSize = g_defaultMaxDictSize; @@ -360,47 +361,6 @@ int main(int argCount, const char* argv[]) } } - /* for fastCover */ - for (unsigned f = 15; f < 25; f++){ - DISPLAYLEVEL(2, "current f is %u\n", f); - /* for fastCover (optimizing k) */ - { - ZDICT_fastCover_params_t fastParam; - memset(&fastParam, 0, sizeof(fastParam)); - fastParam.zParams = zParams; - fastParam.splitPoint = 1.0; - fastParam.d = 8; - fastParam.f = f; - fastParam.steps = 40; - fastParam.nbThreads = 1; - const int fastOptResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, &fastParam); - DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\n", fastParam.k, fastParam.d, fastParam.f, fastParam.steps, (unsigned)(fastParam.splitPoint * 100)); - if(fastOptResult) { - result = 1; - goto _cleanup; - } - } - - /* for fastCover (with k provided) */ - { - ZDICT_fastCover_params_t fastParam; - memset(&fastParam, 0, sizeof(fastParam)); - fastParam.zParams = zParams; - fastParam.splitPoint = 1.0; - fastParam.d = 8; - fastParam.f = f; - fastParam.k = 200; - fastParam.steps = 40; - fastParam.nbThreads = 1; - const int fastOptResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, &fastParam); - DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\n", fastParam.k, fastParam.d, fastParam.f, fastParam.steps, (unsigned)(fastParam.splitPoint * 100)); - if(fastOptResult) { - result = 1; - goto _cleanup; - } - } - } - /* for cover */ { ZDICT_cover_params_t coverParam; @@ -415,8 +375,73 @@ int main(int argCount, const char* argv[]) result = 1; goto _cleanup; } + + k = coverParam.k; + d = coverParam.d; + + /* for COVER with k and d provided */ + ZDICT_cover_params_t covernParam; + memset(&covernParam, 0, sizeof(covernParam)); + covernParam.zParams = zParams; + covernParam.splitPoint = 1.0; + covernParam.steps = 40; + covernParam.nbThreads = 1; + covernParam.k = k; + covernParam.d = d; + const int coverResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, &covernParam, NULL, NULL); + DISPLAYLEVEL(2, "k=%u\nd=%u\nsteps=%u\nsplit=%u\n", covernParam.k, covernParam.d, covernParam.steps, (unsigned)(covernParam.splitPoint * 100)); + if(coverResult) { + result = 1; + goto _cleanup; + } } + /* for fastCover */ + for (unsigned f = 15; f < 25; f++){ + DISPLAYLEVEL(2, "current f is %u\n", f); + /* for fastCover (optimizing k and d) */ + { + ZDICT_fastCover_params_t fastParam; + memset(&fastParam, 0, sizeof(fastParam)); + fastParam.zParams = zParams; + fastParam.splitPoint = 1.0; + fastParam.f = f; + fastParam.steps = 40; + fastParam.nbThreads = 1; + const int fastOptResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, &fastParam); + DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\n", fastParam.k, fastParam.d, fastParam.f, fastParam.steps, (unsigned)(fastParam.splitPoint * 100)); + if(fastOptResult) { + result = 1; + goto _cleanup; + } + + k = fastParam.k; + d = fastParam.d; + } + + + /* for fastCover (with k and d provided) */ + { + ZDICT_fastCover_params_t fastParam; + memset(&fastParam, 0, sizeof(fastParam)); + fastParam.zParams = zParams; + fastParam.splitPoint = 1.0; + fastParam.d = d; + fastParam.f = f; + fastParam.k = k; + fastParam.steps = 40; + fastParam.nbThreads = 1; + const int fastOptResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, &fastParam); + DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\n", fastParam.k, fastParam.d, fastParam.f, fastParam.steps, (unsigned)(fastParam.splitPoint * 100)); + if(fastOptResult) { + result = 1; + goto _cleanup; + } + } + } + + + /* Free allocated memory */ _cleanup: diff --git a/contrib/experimental_dict_builders/fastCover/fastCover.c b/contrib/experimental_dict_builders/fastCover/fastCover.c index 6f990e0c2..d6b3254ec 100644 --- a/contrib/experimental_dict_builders/fastCover/fastCover.c +++ b/contrib/experimental_dict_builders/fastCover/fastCover.c @@ -267,7 +267,7 @@ static void FASTCOVER_computeFrequency(U32 *freqs, unsigned f, FASTCOVER_ctx_t * size_t currSampleStart = ctx->offsets[i]; size_t currSampleEnd = ctx->offsets[i+1]; start = currSampleStart; - while (start + f < currSampleEnd) { + while (start + ctx->d <= currSampleEnd) { const size_t dmerIndex = FASTCOVER_hashPtrToIndex(ctx->samples + start, f, ctx->d); /* if no dmer with same hash value has been seen in current sample */ if (inCurrSample[dmerIndex] == 0) { From 49b398e93f5357c4311b678a7e4b4d875035f379 Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Fri, 27 Jul 2018 13:39:19 -0700 Subject: [PATCH 9/9] Use same param after optimizing cover and fastCover and record k and d for benchmarking --- .../benchmarkDictBuilder/README.md | 211 +++++++++--------- .../benchmarkDictBuilder/benchmark.c | 74 ++---- 2 files changed, 129 insertions(+), 156 deletions(-) diff --git a/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md b/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md index 04866b7e6..654ca4095 100644 --- a/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md +++ b/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md @@ -13,113 +13,114 @@ Benchmark given input files: make ARG= followed by permitted arguments make ARG="in=../../../lib/dictBuilder in=../../../lib/compress" ###Benchmarking Result: -First Cover is optimize cover, second Cover uses optimized d and k from first one. -For every f value of fastCover, the first one is optimize fastCover and the second one uses optimized d and k from first one. +- First Cover is optimize cover, second Cover uses optimized d and k from first one. +- For every f value of fastCover, the first one is optimize fastCover and the second one uses optimized d and k from first one. +- Fourth column is chosen d and fifth column is chosen k github: -NODICT 0.000004 2.999642 -RANDOM 0.146096 8.786957 -LEGACY 0.956888 8.989482 -COVER 56.596152 10.641263 -COVER 4.937047 10.641263 -FAST15 17.722269 10.586461 -FAST15 0.239135 10.586461 -FAST16 18.276179 10.492503 -FAST16 0.265285 10.492503 -FAST17 18.077916 10.611737 -FAST17 0.236573 10.611737 -FAST18 19.510150 10.621586 -FAST18 0.278683 10.621586 -FAST19 18.794350 10.629626 -FAST19 0.307943 10.629626 -FAST20 19.671099 10.610308 -FAST20 0.428814 10.610308 -FAST21 36.527238 10.625733 -FAST21 0.716384 10.625733 -FAST22 83.803521 10.625281 -FAST22 1.290246 10.625281 -FAST23 158.287924 10.602342 -FAST23 3.084848 10.602342 -FAST24 283.630941 10.603379 -FAST24 8.088933 10.603379 +NODICT 0.000004 2.999642 +RANDOM 0.146096 8.786957 +LEGACY 0.956888 8.989482 +COVER 56.596152 10.641263 8 1298 +COVER 4.937047 10.641263 8 1298 +FAST15 17.722269 10.586461 8 1778 +FAST15 0.239135 10.586461 8 1778 +FAST16 18.276179 10.492503 6 1778 +FAST16 0.265285 10.492503 6 1778 +FAST17 18.077916 10.611737 8 1778 +FAST17 0.236573 10.611737 8 1778 +FAST18 19.510150 10.621586 8 1778 +FAST18 0.278683 10.621586 8 1778 +FAST19 18.794350 10.629626 8 1778 +FAST19 0.307943 10.629626 8 1778 +FAST20 19.671099 10.610308 8 1778 +FAST20 0.428814 10.610308 8 1778 +FAST21 36.527238 10.625733 8 1778 +FAST21 0.716384 10.625733 8 1778 +FAST22 83.803521 10.625281 8 1778 +FAST22 1.290246 10.625281 8 1778 +FAST23 158.287924 10.602342 8 1778 +FAST23 3.084848 10.602342 8 1778 +FAST24 283.630941 10.603379 8 1778 +FAST24 8.088933 10.603379 8 1778 -hg-commands -NODICT 0.000007 2.425291 -RANDOM 0.084010 3.489515 -LEGACY 0.926763 3.911896 -COVER 62.036915 4.131136 -COVER 2.194398 4.131136 -FAST15 12.169025 3.903719 -FAST15 0.156552 3.903719 -FAST16 11.886255 4.005077 -FAST16 0.155506 4.005077 -FAST17 11.886955 4.097811 -FAST17 0.176327 4.097811 -FAST18 12.544698 4.136081 -FAST18 0.171796 4.136081 -FAST19 12.920868 4.166021 -FAST19 0.207029 4.166021 -FAST20 15.771429 4.163740 -FAST20 0.258685 4.163740 -FAST21 33.165829 4.157057 -FAST21 0.663088 4.157057 -FAST22 68.779201 4.158195 -FAST22 1.568439 4.158195 -FAST23 121.921931 4.161450 -FAST23 2.498972 4.161450 -FAST24 221.990451 4.159658 -FAST24 5.793594 4.159658 +hg-commands: +NODICT 0.000007 2.425291 +RANDOM 0.084010 3.489515 +LEGACY 0.926763 3.911896 +COVER 62.036915 4.131136 8 386 +COVER 2.194398 4.131136 8 386 +FAST15 12.169025 3.903719 6 1106 +FAST15 0.156552 3.903719 6 1106 +FAST16 11.886255 4.005077 8 530 +FAST16 0.155506 4.005077 8 530 +FAST17 11.886955 4.097811 8 818 +FAST17 0.176327 4.097811 8 818 +FAST18 12.544698 4.136081 8 770 +FAST18 0.171796 4.136081 8 770 +FAST19 12.920868 4.166021 8 530 +FAST19 0.207029 4.166021 8 530 +FAST20 15.771429 4.163740 8 482 +FAST20 0.258685 4.163740 8 482 +FAST21 33.165829 4.157057 8 434 +FAST21 0.663088 4.157057 8 434 +FAST22 68.779201 4.158195 8 290 +FAST22 1.568439 4.158195 8 290 +FAST23 121.921931 4.161450 8 434 +FAST23 2.498972 4.161450 8 434 +FAST24 221.990451 4.159658 8 338 +FAST24 5.793594 4.159658 8 338 -hg-changelog -NODICT 0.000004 1.377613 -RANDOM 0.549307 2.096785 -LEGACY 2.273818 2.058273 -COVER 219.640608 2.188654 -COVER 6.055391 2.188654 -FAST15 67.820700 2.127194 -FAST15 0.824624 2.127194 -FAST16 69.774209 2.145401 -FAST16 0.889737 2.145401 -FAST17 70.027355 2.157544 -FAST17 0.869004 2.157544 -FAST18 68.229652 2.173127 -FAST18 0.930689 2.173127 -FAST19 70.696241 2.179527 -FAST19 1.385515 2.179527 -FAST20 80.618172 2.183233 -FAST20 1.699632 2.183233 -FAST21 96.366254 2.180920 -FAST21 2.606553 2.180920 -FAST22 139.440758 2.184297 -FAST22 5.962606 2.184297 -FAST23 207.791930 2.187666 -FAST23 14.823301 2.187666 -FAST24 322.050385 2.189889 -FAST24 29.294918 2.189889 +hg-changelog: +NODICT 0.000004 1.377613 +RANDOM 0.549307 2.096785 +LEGACY 2.273818 2.058273 +COVER 219.640608 2.188654 8 98 +COVER 6.055391 2.188654 8 98 +FAST15 67.820700 2.127194 8 866 +FAST15 0.824624 2.127194 8 866 +FAST16 69.774209 2.145401 8 338 +FAST16 0.889737 2.145401 8 338 +FAST17 70.027355 2.157544 8 194 +FAST17 0.869004 2.157544 8 194 +FAST18 68.229652 2.173127 8 98 +FAST18 0.930689 2.173127 8 98 +FAST19 70.696241 2.179527 8 98 +FAST19 1.385515 2.179527 8 98 +FAST20 80.618172 2.183233 6 98 +FAST20 1.699632 2.183233 6 98 +FAST21 96.366254 2.180920 8 98 +FAST21 2.606553 2.180920 8 98 +FAST22 139.440758 2.184297 8 98 +FAST22 5.962606 2.184297 8 98 +FAST23 207.791930 2.187666 6 98 +FAST23 14.823301 2.187666 6 98 +FAST24 322.050385 2.189889 6 98 +FAST24 29.294918 2.189889 6 98 -hg-manifest -NODICT 0.000008 1.866385 -RANDOM 1.075766 2.309485 -LEGACY 8.688387 2.506775 -COVER 926.024689 2.582597 -COVER 33.630695 2.582597 -FAST15 152.845945 2.377689 -FAST15 2.206285 2.377689 -FAST16 147.772371 2.464814 -FAST16 1.937997 2.464814 -FAST17 147.729498 2.539834 -FAST17 1.966577 2.539834 -FAST18 144.156821 2.576924 -FAST18 1.954106 2.576924 -FAST19 145.678760 2.592479 -FAST19 2.096876 2.592479 -FAST20 159.634674 2.594551 -FAST20 2.568766 2.594551 -FAST21 228.116552 2.597128 -FAST21 4.634508 2.597128 -FAST22 288.890644 2.596971 -FAST22 6.618204 2.596971 -FAST23 377.196211 2.601416 -FAST23 13.497286 2.601416 -FAST24 503.208577 2.602830 -FAST24 29.538585 2.602830 +hg-manifest: +NODICT 0.000008 1.866385 +RANDOM 1.075766 2.309485 +LEGACY 8.688387 2.506775 +COVER 926.024689 2.582597 8 434 +COVER 33.630695 2.582597 8 434 +FAST15 152.845945 2.377689 8 1682 +FAST15 2.206285 2.377689 8 1682 +FAST16 147.772371 2.464814 8 1538 +FAST16 1.937997 2.464814 8 1538 +FAST17 147.729498 2.539834 6 1826 +FAST17 1.966577 2.539834 6 1826 +FAST18 144.156821 2.576924 8 1922 +FAST18 1.954106 2.576924 8 1922 +FAST19 145.678760 2.592479 6 290 +FAST19 2.096876 2.592479 6 290 +FAST20 159.634674 2.594551 8 194 +FAST20 2.568766 2.594551 8 194 +FAST21 228.116552 2.597128 6 194 +FAST21 4.634508 2.597128 6 194 +FAST22 288.890644 2.596971 6 386 +FAST22 6.618204 2.596971 6 386 +FAST23 377.196211 2.601416 8 194 +FAST23 13.497286 2.601416 8 194 +FAST24 503.208577 2.602830 6 194 +FAST24 29.538585 2.602830 6 194 diff --git a/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c b/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c index a775eae3a..75008a087 100644 --- a/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c +++ b/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c @@ -251,7 +251,7 @@ int benchmarkDictBuilder(sampleInfo *srcInfo, unsigned maxDictSize, ZDICT_random result = 1; goto _cleanup; } - DISPLAYLEVEL(2, "%s took %f seconds to execute \n", name, timeSec); + DISPLAYLEVEL(1, "%s took %f seconds to execute \n", name, timeSec); /* Calculate compression ratio */ const double cRatio = compressWithDict(srcInfo, dInfo, cLevel, displayLevel); @@ -261,7 +261,7 @@ int benchmarkDictBuilder(sampleInfo *srcInfo, unsigned maxDictSize, ZDICT_random goto _cleanup; } - DISPLAYLEVEL(2, "Compression ratio with %s dictionary is %f\n", name, cRatio); + DISPLAYLEVEL(1, "Compression ratio with %s dictionary is %f\n", name, cRatio); _cleanup: freeDictInfo(dInfo); @@ -376,73 +376,45 @@ int main(int argCount, const char* argv[]) goto _cleanup; } - k = coverParam.k; - d = coverParam.d; - - /* for COVER with k and d provided */ - ZDICT_cover_params_t covernParam; - memset(&covernParam, 0, sizeof(covernParam)); - covernParam.zParams = zParams; - covernParam.splitPoint = 1.0; - covernParam.steps = 40; - covernParam.nbThreads = 1; - covernParam.k = k; - covernParam.d = d; - const int coverResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, &covernParam, NULL, NULL); - DISPLAYLEVEL(2, "k=%u\nd=%u\nsteps=%u\nsplit=%u\n", covernParam.k, covernParam.d, covernParam.steps, (unsigned)(covernParam.splitPoint * 100)); + const int coverResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, &coverParam, NULL, NULL); + DISPLAYLEVEL(2, "k=%u\nd=%u\nsteps=%u\nsplit=%u\n", coverParam.k, coverParam.d, coverParam.steps, (unsigned)(coverParam.splitPoint * 100)); if(coverResult) { result = 1; goto _cleanup; } + } /* for fastCover */ for (unsigned f = 15; f < 25; f++){ DISPLAYLEVEL(2, "current f is %u\n", f); /* for fastCover (optimizing k and d) */ - { - ZDICT_fastCover_params_t fastParam; - memset(&fastParam, 0, sizeof(fastParam)); - fastParam.zParams = zParams; - fastParam.splitPoint = 1.0; - fastParam.f = f; - fastParam.steps = 40; - fastParam.nbThreads = 1; - const int fastOptResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, &fastParam); - DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\n", fastParam.k, fastParam.d, fastParam.f, fastParam.steps, (unsigned)(fastParam.splitPoint * 100)); - if(fastOptResult) { - result = 1; - goto _cleanup; - } - - k = fastParam.k; - d = fastParam.d; + ZDICT_fastCover_params_t fastParam; + memset(&fastParam, 0, sizeof(fastParam)); + fastParam.zParams = zParams; + fastParam.splitPoint = 1.0; + fastParam.f = f; + fastParam.steps = 40; + fastParam.nbThreads = 1; + const int fastOptResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, &fastParam); + DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\n", fastParam.k, fastParam.d, fastParam.f, fastParam.steps, (unsigned)(fastParam.splitPoint * 100)); + if(fastOptResult) { + result = 1; + goto _cleanup; } /* for fastCover (with k and d provided) */ - { - ZDICT_fastCover_params_t fastParam; - memset(&fastParam, 0, sizeof(fastParam)); - fastParam.zParams = zParams; - fastParam.splitPoint = 1.0; - fastParam.d = d; - fastParam.f = f; - fastParam.k = k; - fastParam.steps = 40; - fastParam.nbThreads = 1; - const int fastOptResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, &fastParam); - DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\n", fastParam.k, fastParam.d, fastParam.f, fastParam.steps, (unsigned)(fastParam.splitPoint * 100)); - if(fastOptResult) { - result = 1; - goto _cleanup; - } + const int fastResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, &fastParam); + DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\n", fastParam.k, fastParam.d, fastParam.f, fastParam.steps, (unsigned)(fastParam.splitPoint * 100)); + if(fastResult) { + result = 1; + goto _cleanup; } + } - - /* Free allocated memory */ _cleanup: UTIL_freeFileList(extendedFileList, fileNamesBuf);