Merge remote-tracking branch 'refs/remotes/facebook/dev' into dev11
This commit is contained in:
Przemyslaw Skibinski
@@ -88,7 +88,7 @@ travis-install:
|
||||
$(MAKE) install PREFIX=~/install_test_dir
|
||||
|
||||
gpptest: clean
|
||||
$(MAKE) -C programs all CC=g++ CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
|
||||
CC=g++ $(MAKE) -C programs all CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
|
||||
|
||||
gcc5test: clean
|
||||
gcc-5 -v
|
||||
|
||||
@@ -331,6 +331,10 @@
|
||||
RelativePath="..\..\..\programs\datagen.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\cover.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\divsufsort.c"
|
||||
>
|
||||
|
||||
@@ -343,6 +343,10 @@
|
||||
RelativePath="..\..\..\programs\dibio.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\cover.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\divsufsort.c"
|
||||
>
|
||||
|
||||
@@ -327,6 +327,10 @@
|
||||
Filter="cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx"
|
||||
UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}"
|
||||
>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\cover.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\divsufsort.c"
|
||||
>
|
||||
|
||||
@@ -332,6 +332,10 @@
|
||||
RelativePath="..\..\..\programs\datagen.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\cover.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\divsufsort.c"
|
||||
>
|
||||
|
||||
@@ -344,6 +344,10 @@
|
||||
RelativePath="..\..\..\programs\dibio.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\cover.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\divsufsort.c"
|
||||
>
|
||||
|
||||
@@ -328,6 +328,10 @@
|
||||
Filter="cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx"
|
||||
UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}"
|
||||
>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\cover.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\divsufsort.c"
|
||||
>
|
||||
|
||||
@@ -165,6 +165,7 @@
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\cover.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\zdict.c" />
|
||||
<ClCompile Include="..\..\..\programs\datagen.c" />
|
||||
|
||||
@@ -32,6 +32,7 @@
|
||||
<ClCompile Include="..\..\..\lib\deprecated\zbuff_common.c" />
|
||||
<ClCompile Include="..\..\..\lib\deprecated\zbuff_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\deprecated\zbuff_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\cover.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\zdict.c" />
|
||||
<ClCompile Include="..\..\..\lib\legacy\zstd_v01.c" />
|
||||
|
||||
@@ -32,6 +32,7 @@
|
||||
<ClCompile Include="..\..\..\lib\deprecated\zbuff_common.c" />
|
||||
<ClCompile Include="..\..\..\lib\deprecated\zbuff_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\deprecated\zbuff_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\cover.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\zdict.c" />
|
||||
<ClCompile Include="..\..\..\lib\legacy\zstd_v01.c" />
|
||||
|
||||
@@ -29,6 +29,7 @@
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\cover.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\zdict.c" />
|
||||
<ClCompile Include="..\..\..\lib\legacy\zstd_v01.c" />
|
||||
|
||||
@@ -67,6 +67,7 @@ SET(Sources
|
||||
${LIBRARY_DIR}/compress/zstd_compress.c
|
||||
${LIBRARY_DIR}/decompress/huf_decompress.c
|
||||
${LIBRARY_DIR}/decompress/zstd_decompress.c
|
||||
${LIBRARY_DIR}/dictBuilder/cover.c
|
||||
${LIBRARY_DIR}/dictBuilder/divsufsort.c
|
||||
${LIBRARY_DIR}/dictBuilder/zdict.c
|
||||
${LIBRARY_DIR}/deprecated/zbuff_common.c
|
||||
|
||||
@@ -2970,6 +2970,7 @@ size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict)
|
||||
ZSTD_parameters const params = ZSTD_getParamsFromCDict(cdict);
|
||||
size_t const initError = ZSTD_initCStream_advanced(zcs, NULL, 0, params, 0);
|
||||
zcs->cdict = cdict;
|
||||
zcs->cctx->dictID = params.fParams.noDictIDFlag ? 0 : cdict->refContext->dictID;
|
||||
return initError;
|
||||
}
|
||||
|
||||
|
||||
@@ -1444,7 +1444,7 @@ size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
|
||||
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
|
||||
if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, dict, dictSize);
|
||||
#endif
|
||||
ZSTD_decompressBegin_usingDict(dctx, dict, dictSize);
|
||||
CHECK_F(ZSTD_decompressBegin_usingDict(dctx, dict, dictSize));
|
||||
ZSTD_checkContinuity(dctx, dst);
|
||||
return ZSTD_decompressFrame(dctx, dst, dstCapacity, src, srcSize);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,1023 @@
|
||||
/**
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under the BSD-style license found in the
|
||||
* LICENSE file in the root directory of this source tree. An additional grant
|
||||
* of patent rights can be found in the PATENTS file in the same directory.
|
||||
*/
|
||||
|
||||
/*-*************************************
|
||||
* Dependencies
|
||||
***************************************/
|
||||
#include <stdio.h> /* fprintf */
|
||||
#include <stdlib.h> /* malloc, free, qsort */
|
||||
#include <string.h> /* memset */
|
||||
#include <time.h> /* clock */
|
||||
#ifdef ZSTD_PTHREAD
|
||||
#include "threading.h"
|
||||
#endif
|
||||
|
||||
#include "mem.h" /* read */
|
||||
#include "zstd_internal.h" /* includes zstd.h */
|
||||
#ifndef ZDICT_STATIC_LINKING_ONLY
|
||||
#define ZDICT_STATIC_LINKING_ONLY
|
||||
#endif
|
||||
#include "zdict.h"
|
||||
|
||||
/*-*************************************
|
||||
* Constants
|
||||
***************************************/
|
||||
#define COVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((U32)-1) : ((U32)1 GB))
|
||||
|
||||
/*-*************************************
|
||||
* Console display
|
||||
***************************************/
|
||||
static int g_displayLevel = 2;
|
||||
#define DISPLAY(...) \
|
||||
{ \
|
||||
fprintf(stderr, __VA_ARGS__); \
|
||||
fflush(stderr); \
|
||||
}
|
||||
#define LOCALDISPLAYLEVEL(displayLevel, l, ...) \
|
||||
if (displayLevel >= l) { \
|
||||
DISPLAY(__VA_ARGS__); \
|
||||
} /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */
|
||||
#define DISPLAYLEVEL(l, ...) LOCALDISPLAYLEVEL(g_displayLevel, l, __VA_ARGS__)
|
||||
|
||||
#define LOCALDISPLAYUPDATE(displayLevel, l, ...) \
|
||||
if (displayLevel >= l) { \
|
||||
if ((clock() - g_time > refreshRate) || (displayLevel >= 4)) { \
|
||||
g_time = clock(); \
|
||||
DISPLAY(__VA_ARGS__); \
|
||||
if (displayLevel >= 4) \
|
||||
fflush(stdout); \
|
||||
} \
|
||||
}
|
||||
#define DISPLAYUPDATE(l, ...) LOCALDISPLAYUPDATE(g_displayLevel, l, __VA_ARGS__)
|
||||
static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100;
|
||||
static clock_t g_time = 0;
|
||||
|
||||
/*-*************************************
|
||||
* Hash table
|
||||
***************************************
|
||||
* A small specialized hash map for storing activeDmers.
|
||||
* The map does not resize, so if it becomes full it will loop forever.
|
||||
* Thus, the map must be large enough to store every value.
|
||||
* The map implements linear probing and keeps its load less than 0.5.
|
||||
*/
|
||||
|
||||
#define MAP_EMPTY_VALUE ((U32)-1)
|
||||
typedef struct COVER_map_pair_t_s {
|
||||
U32 key;
|
||||
U32 value;
|
||||
} COVER_map_pair_t;
|
||||
|
||||
typedef struct COVER_map_s {
|
||||
COVER_map_pair_t *data;
|
||||
U32 sizeLog;
|
||||
U32 size;
|
||||
U32 sizeMask;
|
||||
} COVER_map_t;
|
||||
|
||||
/**
|
||||
* Clear the map.
|
||||
*/
|
||||
static void COVER_map_clear(COVER_map_t *map) {
|
||||
memset(map->data, MAP_EMPTY_VALUE, map->size * sizeof(COVER_map_pair_t));
|
||||
}
|
||||
|
||||
/**
|
||||
* Initializes a map of the given size.
|
||||
* Returns 1 on success and 0 on failure.
|
||||
* The map must be destroyed with COVER_map_destroy().
|
||||
* The map is only guaranteed to be large enough to hold size elements.
|
||||
*/
|
||||
static int COVER_map_init(COVER_map_t *map, U32 size) {
|
||||
map->sizeLog = ZSTD_highbit32(size) + 2;
|
||||
map->size = (U32)1 << map->sizeLog;
|
||||
map->sizeMask = map->size - 1;
|
||||
map->data = (COVER_map_pair_t *)malloc(map->size * sizeof(COVER_map_pair_t));
|
||||
if (!map->data) {
|
||||
map->sizeLog = 0;
|
||||
map->size = 0;
|
||||
return 0;
|
||||
}
|
||||
COVER_map_clear(map);
|
||||
return 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Internal hash function
|
||||
*/
|
||||
static const U32 prime4bytes = 2654435761U;
|
||||
static U32 COVER_map_hash(COVER_map_t *map, U32 key) {
|
||||
return (key * prime4bytes) >> (32 - map->sizeLog);
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper function that returns the index that a key should be placed into.
|
||||
*/
|
||||
static U32 COVER_map_index(COVER_map_t *map, U32 key) {
|
||||
const U32 hash = COVER_map_hash(map, key);
|
||||
U32 i;
|
||||
for (i = hash;; i = (i + 1) & map->sizeMask) {
|
||||
COVER_map_pair_t *pos = &map->data[i];
|
||||
if (pos->value == MAP_EMPTY_VALUE) {
|
||||
return i;
|
||||
}
|
||||
if (pos->key == key) {
|
||||
return i;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the pointer to the value for key.
|
||||
* If key is not in the map, it is inserted and the value is set to 0.
|
||||
* The map must not be full.
|
||||
*/
|
||||
static U32 *COVER_map_at(COVER_map_t *map, U32 key) {
|
||||
COVER_map_pair_t *pos = &map->data[COVER_map_index(map, key)];
|
||||
if (pos->value == MAP_EMPTY_VALUE) {
|
||||
pos->key = key;
|
||||
pos->value = 0;
|
||||
}
|
||||
return &pos->value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes key from the map if present.
|
||||
*/
|
||||
static void COVER_map_remove(COVER_map_t *map, U32 key) {
|
||||
U32 i = COVER_map_index(map, key);
|
||||
COVER_map_pair_t *del = &map->data[i];
|
||||
U32 shift = 1;
|
||||
if (del->value == MAP_EMPTY_VALUE) {
|
||||
return;
|
||||
}
|
||||
for (i = (i + 1) & map->sizeMask;; i = (i + 1) & map->sizeMask) {
|
||||
COVER_map_pair_t *const pos = &map->data[i];
|
||||
/* If the position is empty we are done */
|
||||
if (pos->value == MAP_EMPTY_VALUE) {
|
||||
del->value = MAP_EMPTY_VALUE;
|
||||
return;
|
||||
}
|
||||
/* If pos can be moved to del do so */
|
||||
if (((i - COVER_map_hash(map, pos->key)) & map->sizeMask) >= shift) {
|
||||
del->key = pos->key;
|
||||
del->value = pos->value;
|
||||
del = pos;
|
||||
shift = 1;
|
||||
} else {
|
||||
++shift;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Destroyes a map that is inited with COVER_map_init().
|
||||
*/
|
||||
static void COVER_map_destroy(COVER_map_t *map) {
|
||||
if (map->data) {
|
||||
free(map->data);
|
||||
}
|
||||
map->data = NULL;
|
||||
map->size = 0;
|
||||
}
|
||||
|
||||
/*-*************************************
|
||||
* Context
|
||||
***************************************/
|
||||
|
||||
typedef struct {
|
||||
const BYTE *samples;
|
||||
size_t *offsets;
|
||||
const size_t *samplesSizes;
|
||||
size_t nbSamples;
|
||||
U32 *suffix;
|
||||
size_t suffixSize;
|
||||
U32 *freqs;
|
||||
U32 *dmerAt;
|
||||
unsigned d;
|
||||
} COVER_ctx_t;
|
||||
|
||||
/* We need a global context for qsort... */
|
||||
static COVER_ctx_t *g_ctx = NULL;
|
||||
|
||||
/*-*************************************
|
||||
* Helper functions
|
||||
***************************************/
|
||||
|
||||
/**
|
||||
* Returns the sum of the sample sizes.
|
||||
*/
|
||||
static size_t COVER_sum(const size_t *samplesSizes, unsigned nbSamples) {
|
||||
size_t sum = 0;
|
||||
size_t i;
|
||||
for (i = 0; i < nbSamples; ++i) {
|
||||
sum += samplesSizes[i];
|
||||
}
|
||||
return sum;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns -1 if the dmer at lp is less than the dmer at rp.
|
||||
* Return 0 if the dmers at lp and rp are equal.
|
||||
* Returns 1 if the dmer at lp is greater than the dmer at rp.
|
||||
*/
|
||||
static int COVER_cmp(COVER_ctx_t *ctx, const void *lp, const void *rp) {
|
||||
const U32 lhs = *(const U32 *)lp;
|
||||
const U32 rhs = *(const U32 *)rp;
|
||||
return memcmp(ctx->samples + lhs, ctx->samples + rhs, ctx->d);
|
||||
}
|
||||
|
||||
/**
|
||||
* Same as COVER_cmp() except ties are broken by pointer value
|
||||
* NOTE: g_ctx must be set to call this function. A global is required because
|
||||
* qsort doesn't take an opaque pointer.
|
||||
*/
|
||||
static int COVER_strict_cmp(const void *lp, const void *rp) {
|
||||
int result = COVER_cmp(g_ctx, lp, rp);
|
||||
if (result == 0) {
|
||||
result = lp < rp ? -1 : 1;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the first pointer in [first, last) whose element does not compare
|
||||
* less than value. If no such element exists it returns last.
|
||||
*/
|
||||
static const size_t *COVER_lower_bound(const size_t *first, const size_t *last,
|
||||
size_t value) {
|
||||
size_t count = last - first;
|
||||
while (count != 0) {
|
||||
size_t step = count / 2;
|
||||
const size_t *ptr = first;
|
||||
ptr += step;
|
||||
if (*ptr < value) {
|
||||
first = ++ptr;
|
||||
count -= step + 1;
|
||||
} else {
|
||||
count = step;
|
||||
}
|
||||
}
|
||||
return first;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generic groupBy function.
|
||||
* Groups an array sorted by cmp into groups with equivalent values.
|
||||
* Calls grp for each group.
|
||||
*/
|
||||
static void
|
||||
COVER_groupBy(const void *data, size_t count, size_t size, COVER_ctx_t *ctx,
|
||||
int (*cmp)(COVER_ctx_t *, const void *, const void *),
|
||||
void (*grp)(COVER_ctx_t *, const void *, const void *)) {
|
||||
const BYTE *ptr = (const BYTE *)data;
|
||||
size_t num = 0;
|
||||
while (num < count) {
|
||||
const BYTE *grpEnd = ptr + size;
|
||||
++num;
|
||||
while (num < count && cmp(ctx, ptr, grpEnd) == 0) {
|
||||
grpEnd += size;
|
||||
++num;
|
||||
}
|
||||
grp(ctx, ptr, grpEnd);
|
||||
ptr = grpEnd;
|
||||
}
|
||||
}
|
||||
|
||||
/*-*************************************
|
||||
* Cover functions
|
||||
***************************************/
|
||||
|
||||
/**
|
||||
* Called on each group of positions with the same dmer.
|
||||
* Counts the frequency of each dmer and saves it in the suffix array.
|
||||
* Fills `ctx->dmerAt`.
|
||||
*/
|
||||
static void COVER_group(COVER_ctx_t *ctx, const void *group,
|
||||
const void *groupEnd) {
|
||||
/* The group consists of all the positions with the same first d bytes. */
|
||||
const U32 *grpPtr = (const U32 *)group;
|
||||
const U32 *grpEnd = (const U32 *)groupEnd;
|
||||
/* The dmerId is how we will reference this dmer.
|
||||
* This allows us to map the whole dmer space to a much smaller space, the
|
||||
* size of the suffix array.
|
||||
*/
|
||||
const U32 dmerId = (U32)(grpPtr - ctx->suffix);
|
||||
/* Count the number of samples this dmer shows up in */
|
||||
U32 freq = 0;
|
||||
/* Details */
|
||||
const size_t *curOffsetPtr = ctx->offsets;
|
||||
const size_t *offsetsEnd = ctx->offsets + ctx->nbSamples;
|
||||
/* Once *grpPtr >= curSampleEnd this occurrence of the dmer is in a
|
||||
* different sample than the last.
|
||||
*/
|
||||
size_t curSampleEnd = ctx->offsets[0];
|
||||
for (; grpPtr != grpEnd; ++grpPtr) {
|
||||
/* Save the dmerId for this position so we can get back to it. */
|
||||
ctx->dmerAt[*grpPtr] = dmerId;
|
||||
/* Dictionaries only help for the first reference to the dmer.
|
||||
* After that zstd can reference the match from the previous reference.
|
||||
* So only count each dmer once for each sample it is in.
|
||||
*/
|
||||
if (*grpPtr < curSampleEnd) {
|
||||
continue;
|
||||
}
|
||||
freq += 1;
|
||||
/* Binary search to find the end of the sample *grpPtr is in.
|
||||
* In the common case that grpPtr + 1 == grpEnd we can skip the binary
|
||||
* search because the loop is over.
|
||||
*/
|
||||
if (grpPtr + 1 != grpEnd) {
|
||||
const size_t *sampleEndPtr =
|
||||
COVER_lower_bound(curOffsetPtr, offsetsEnd, *grpPtr);
|
||||
curSampleEnd = *sampleEndPtr;
|
||||
curOffsetPtr = sampleEndPtr + 1;
|
||||
}
|
||||
}
|
||||
/* At this point we are never going to look at this segment of the suffix
|
||||
* array again. We take advantage of this fact to save memory.
|
||||
* We store the frequency of the dmer in the first position of the group,
|
||||
* which is dmerId.
|
||||
*/
|
||||
ctx->suffix[dmerId] = freq;
|
||||
}
|
||||
|
||||
/**
|
||||
* A segment is a range in the source as well as the score of the segment.
|
||||
*/
|
||||
typedef struct {
|
||||
U32 begin;
|
||||
U32 end;
|
||||
double score;
|
||||
} COVER_segment_t;
|
||||
|
||||
/**
|
||||
* Selects the best segment in an epoch.
|
||||
* Segments of are scored according to the function:
|
||||
*
|
||||
* Let F(d) be the frequency of dmer d.
|
||||
* Let S_i be the dmer at position i of segment S which has length k.
|
||||
*
|
||||
* Score(S) = F(S_1) + F(S_2) + ... + F(S_{k-d+1})
|
||||
*
|
||||
* Once the dmer d is in the dictionay we set F(d) = 0.
|
||||
*/
|
||||
static COVER_segment_t COVER_selectSegment(const COVER_ctx_t *ctx, U32 *freqs,
|
||||
COVER_map_t *activeDmers, U32 begin,
|
||||
U32 end, COVER_params_t parameters) {
|
||||
/* Constants */
|
||||
const U32 k = parameters.k;
|
||||
const U32 d = parameters.d;
|
||||
const U32 dmersInK = k - d + 1;
|
||||
/* Try each segment (activeSegment) and save the best (bestSegment) */
|
||||
COVER_segment_t bestSegment = {0, 0, 0};
|
||||
COVER_segment_t activeSegment;
|
||||
/* Reset the activeDmers in the segment */
|
||||
COVER_map_clear(activeDmers);
|
||||
/* The activeSegment starts at the beginning of the epoch. */
|
||||
activeSegment.begin = begin;
|
||||
activeSegment.end = begin;
|
||||
activeSegment.score = 0;
|
||||
/* Slide the activeSegment through the whole epoch.
|
||||
* Save the best segment in bestSegment.
|
||||
*/
|
||||
while (activeSegment.end < end) {
|
||||
/* The dmerId for the dmer at the next position */
|
||||
U32 newDmer = ctx->dmerAt[activeSegment.end];
|
||||
/* The entry in activeDmers for this dmerId */
|
||||
U32 *newDmerOcc = COVER_map_at(activeDmers, newDmer);
|
||||
/* If the dmer isn't already present in the segment add its score. */
|
||||
if (*newDmerOcc == 0) {
|
||||
/* The paper suggest using the L-0.5 norm, but experiments show that it
|
||||
* doesn't help.
|
||||
*/
|
||||
activeSegment.score += freqs[newDmer];
|
||||
}
|
||||
/* Add the dmer to the segment */
|
||||
activeSegment.end += 1;
|
||||
*newDmerOcc += 1;
|
||||
|
||||
/* If the window is now too large, drop the first position */
|
||||
if (activeSegment.end - activeSegment.begin == dmersInK + 1) {
|
||||
U32 delDmer = ctx->dmerAt[activeSegment.begin];
|
||||
U32 *delDmerOcc = COVER_map_at(activeDmers, delDmer);
|
||||
activeSegment.begin += 1;
|
||||
*delDmerOcc -= 1;
|
||||
/* If this is the last occurence of the dmer, subtract its score */
|
||||
if (*delDmerOcc == 0) {
|
||||
COVER_map_remove(activeDmers, delDmer);
|
||||
activeSegment.score -= freqs[delDmer];
|
||||
}
|
||||
}
|
||||
|
||||
/* If this segment is the best so far save it */
|
||||
if (activeSegment.score > bestSegment.score) {
|
||||
bestSegment = activeSegment;
|
||||
}
|
||||
}
|
||||
{
|
||||
/* Trim off the zero frequency head and tail from the segment. */
|
||||
U32 newBegin = bestSegment.end;
|
||||
U32 newEnd = bestSegment.begin;
|
||||
U32 pos;
|
||||
for (pos = bestSegment.begin; pos != bestSegment.end; ++pos) {
|
||||
U32 freq = freqs[ctx->dmerAt[pos]];
|
||||
if (freq != 0) {
|
||||
newBegin = MIN(newBegin, pos);
|
||||
newEnd = pos + 1;
|
||||
}
|
||||
}
|
||||
bestSegment.begin = newBegin;
|
||||
bestSegment.end = newEnd;
|
||||
}
|
||||
{
|
||||
/* Zero out the frequency of each dmer covered by the chosen segment. */
|
||||
U32 pos;
|
||||
for (pos = bestSegment.begin; pos != bestSegment.end; ++pos) {
|
||||
freqs[ctx->dmerAt[pos]] = 0;
|
||||
}
|
||||
}
|
||||
return bestSegment;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check the validity of the parameters.
|
||||
* Returns non-zero if the parameters are valid and 0 otherwise.
|
||||
*/
|
||||
static int COVER_checkParameters(COVER_params_t parameters) {
|
||||
/* k and d are required parameters */
|
||||
if (parameters.d == 0 || parameters.k == 0) {
|
||||
return 0;
|
||||
}
|
||||
/* d <= k */
|
||||
if (parameters.d > parameters.k) {
|
||||
return 0;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up a context initialized with `COVER_ctx_init()`.
|
||||
*/
|
||||
static void COVER_ctx_destroy(COVER_ctx_t *ctx) {
|
||||
if (!ctx) {
|
||||
return;
|
||||
}
|
||||
if (ctx->suffix) {
|
||||
free(ctx->suffix);
|
||||
ctx->suffix = NULL;
|
||||
}
|
||||
if (ctx->freqs) {
|
||||
free(ctx->freqs);
|
||||
ctx->freqs = NULL;
|
||||
}
|
||||
if (ctx->dmerAt) {
|
||||
free(ctx->dmerAt);
|
||||
ctx->dmerAt = NULL;
|
||||
}
|
||||
if (ctx->offsets) {
|
||||
free(ctx->offsets);
|
||||
ctx->offsets = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Prepare a context for dictionary building.
|
||||
* The context is only dependent on the parameter `d` and can used multiple
|
||||
* times.
|
||||
* Returns 1 on success or zero on error.
|
||||
* The context must be destroyed with `COVER_ctx_destroy()`.
|
||||
*/
|
||||
static int COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer,
|
||||
const size_t *samplesSizes, unsigned nbSamples,
|
||||
unsigned d) {
|
||||
const BYTE *const samples = (const BYTE *)samplesBuffer;
|
||||
const size_t totalSamplesSize = COVER_sum(samplesSizes, nbSamples);
|
||||
/* Checks */
|
||||
if (totalSamplesSize < d ||
|
||||
totalSamplesSize >= (size_t)COVER_MAX_SAMPLES_SIZE) {
|
||||
DISPLAYLEVEL(1, "Total samples size is too large, maximum size is %u MB\n",
|
||||
(COVER_MAX_SAMPLES_SIZE >> 20));
|
||||
return 0;
|
||||
}
|
||||
/* Zero the context */
|
||||
memset(ctx, 0, sizeof(*ctx));
|
||||
DISPLAYLEVEL(2, "Training on %u samples of total size %u\n", nbSamples,
|
||||
(U32)totalSamplesSize);
|
||||
ctx->samples = samples;
|
||||
ctx->samplesSizes = samplesSizes;
|
||||
ctx->nbSamples = nbSamples;
|
||||
/* Partial suffix array */
|
||||
ctx->suffixSize = totalSamplesSize - d + 1;
|
||||
ctx->suffix = (U32 *)malloc(ctx->suffixSize * sizeof(U32));
|
||||
/* Maps index to the dmerID */
|
||||
ctx->dmerAt = (U32 *)malloc(ctx->suffixSize * sizeof(U32));
|
||||
/* The offsets of each file */
|
||||
ctx->offsets = (size_t *)malloc((nbSamples + 1) * sizeof(size_t));
|
||||
if (!ctx->suffix || !ctx->dmerAt || !ctx->offsets) {
|
||||
DISPLAYLEVEL(1, "Failed to allocate scratch buffers\n");
|
||||
COVER_ctx_destroy(ctx);
|
||||
return 0;
|
||||
}
|
||||
ctx->freqs = NULL;
|
||||
ctx->d = d;
|
||||
|
||||
/* Fill offsets from the samlesSizes */
|
||||
{
|
||||
U32 i;
|
||||
ctx->offsets[0] = 0;
|
||||
for (i = 1; i <= nbSamples; ++i) {
|
||||
ctx->offsets[i] = ctx->offsets[i - 1] + samplesSizes[i - 1];
|
||||
}
|
||||
}
|
||||
DISPLAYLEVEL(2, "Constructing partial suffix array\n");
|
||||
{
|
||||
/* suffix is a partial suffix array.
|
||||
* It only sorts suffixes by their first parameters.d bytes.
|
||||
* The sort is stable, so each dmer group is sorted by position in input.
|
||||
*/
|
||||
U32 i;
|
||||
for (i = 0; i < ctx->suffixSize; ++i) {
|
||||
ctx->suffix[i] = i;
|
||||
}
|
||||
/* qsort doesn't take an opaque pointer, so pass as a global */
|
||||
g_ctx = ctx;
|
||||
qsort(ctx->suffix, ctx->suffixSize, sizeof(U32), &COVER_strict_cmp);
|
||||
}
|
||||
DISPLAYLEVEL(2, "Computing frequencies\n");
|
||||
/* For each dmer group (group of positions with the same first d bytes):
|
||||
* 1. For each position we set dmerAt[position] = dmerID. The dmerID is
|
||||
* (groupBeginPtr - suffix). This allows us to go from position to
|
||||
* dmerID so we can look up values in freq.
|
||||
* 2. We calculate how many samples the dmer occurs in and save it in
|
||||
* freqs[dmerId].
|
||||
*/
|
||||
COVER_groupBy(ctx->suffix, ctx->suffixSize, sizeof(U32), ctx, &COVER_cmp,
|
||||
&COVER_group);
|
||||
ctx->freqs = ctx->suffix;
|
||||
ctx->suffix = NULL;
|
||||
return 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Given the prepared context build the dictionary.
|
||||
*/
|
||||
static size_t COVER_buildDictionary(const COVER_ctx_t *ctx, U32 *freqs,
|
||||
COVER_map_t *activeDmers, void *dictBuffer,
|
||||
size_t dictBufferCapacity,
|
||||
COVER_params_t parameters) {
|
||||
BYTE *const dict = (BYTE *)dictBuffer;
|
||||
size_t tail = dictBufferCapacity;
|
||||
/* Divide the data up into epochs of equal size.
|
||||
* We will select at least one segment from each epoch.
|
||||
*/
|
||||
const U32 epochs = (U32)(dictBufferCapacity / parameters.k);
|
||||
const U32 epochSize = (U32)(ctx->suffixSize / epochs);
|
||||
size_t epoch;
|
||||
DISPLAYLEVEL(2, "Breaking content into %u epochs of size %u\n", epochs,
|
||||
epochSize);
|
||||
/* Loop through the epochs until there are no more segments or the dictionary
|
||||
* is full.
|
||||
*/
|
||||
for (epoch = 0; tail > 0; epoch = (epoch + 1) % epochs) {
|
||||
const U32 epochBegin = (U32)(epoch * epochSize);
|
||||
const U32 epochEnd = epochBegin + epochSize;
|
||||
size_t segmentSize;
|
||||
/* Select a segment */
|
||||
COVER_segment_t segment = COVER_selectSegment(
|
||||
ctx, freqs, activeDmers, epochBegin, epochEnd, parameters);
|
||||
/* Trim the segment if necessary and if it is empty then we are done */
|
||||
segmentSize = MIN(segment.end - segment.begin + parameters.d - 1, tail);
|
||||
if (segmentSize == 0) {
|
||||
break;
|
||||
}
|
||||
/* We fill the dictionary from the back to allow the best segments to be
|
||||
* referenced with the smallest offsets.
|
||||
*/
|
||||
tail -= segmentSize;
|
||||
memcpy(dict + tail, ctx->samples + segment.begin, segmentSize);
|
||||
DISPLAYUPDATE(
|
||||
2, "\r%u%% ",
|
||||
(U32)(((dictBufferCapacity - tail) * 100) / dictBufferCapacity));
|
||||
}
|
||||
DISPLAYLEVEL(2, "\r%79s\r", "");
|
||||
return tail;
|
||||
}
|
||||
|
||||
/**
|
||||
* Translate from COVER_params_t to ZDICT_params_t required for finalizing the
|
||||
* dictionary.
|
||||
*/
|
||||
static ZDICT_params_t COVER_translateParams(COVER_params_t parameters) {
|
||||
ZDICT_params_t zdictParams;
|
||||
memset(&zdictParams, 0, sizeof(zdictParams));
|
||||
zdictParams.notificationLevel = 1;
|
||||
zdictParams.dictID = parameters.dictID;
|
||||
zdictParams.compressionLevel = parameters.compressionLevel;
|
||||
return zdictParams;
|
||||
}
|
||||
|
||||
/**
|
||||
* Constructs a dictionary using a heuristic based on the following paper:
|
||||
*
|
||||
* Liao, Petri, Moffat, Wirth
|
||||
* Effective Construction of Relative Lempel-Ziv Dictionaries
|
||||
* Published in WWW 2016.
|
||||
*/
|
||||
ZDICTLIB_API size_t COVER_trainFromBuffer(
|
||||
void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer,
|
||||
const size_t *samplesSizes, unsigned nbSamples, COVER_params_t parameters) {
|
||||
BYTE *const dict = (BYTE *)dictBuffer;
|
||||
COVER_ctx_t ctx;
|
||||
COVER_map_t activeDmers;
|
||||
/* Checks */
|
||||
if (!COVER_checkParameters(parameters)) {
|
||||
DISPLAYLEVEL(1, "Cover parameters incorrect\n");
|
||||
return ERROR(GENERIC);
|
||||
}
|
||||
if (nbSamples == 0) {
|
||||
DISPLAYLEVEL(1, "Cover must have at least one input file\n");
|
||||
return ERROR(GENERIC);
|
||||
}
|
||||
if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) {
|
||||
DISPLAYLEVEL(1, "dictBufferCapacity must be at least %u\n",
|
||||
ZDICT_DICTSIZE_MIN);
|
||||
return ERROR(dstSize_tooSmall);
|
||||
}
|
||||
/* Initialize global data */
|
||||
g_displayLevel = parameters.notificationLevel;
|
||||
/* Initialize context and activeDmers */
|
||||
if (!COVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples,
|
||||
parameters.d)) {
|
||||
return ERROR(GENERIC);
|
||||
}
|
||||
if (!COVER_map_init(&activeDmers, parameters.k - parameters.d + 1)) {
|
||||
DISPLAYLEVEL(1, "Failed to allocate dmer map: out of memory\n");
|
||||
COVER_ctx_destroy(&ctx);
|
||||
return ERROR(GENERIC);
|
||||
}
|
||||
|
||||
DISPLAYLEVEL(2, "Building dictionary\n");
|
||||
{
|
||||
const size_t tail =
|
||||
COVER_buildDictionary(&ctx, ctx.freqs, &activeDmers, dictBuffer,
|
||||
dictBufferCapacity, parameters);
|
||||
ZDICT_params_t zdictParams = COVER_translateParams(parameters);
|
||||
const size_t dictionarySize = ZDICT_finalizeDictionary(
|
||||
dict, dictBufferCapacity, dict + tail, dictBufferCapacity - tail,
|
||||
samplesBuffer, samplesSizes, nbSamples, zdictParams);
|
||||
if (!ZSTD_isError(dictionarySize)) {
|
||||
DISPLAYLEVEL(2, "Constructed dictionary of size %u\n",
|
||||
(U32)dictionarySize);
|
||||
}
|
||||
COVER_ctx_destroy(&ctx);
|
||||
COVER_map_destroy(&activeDmers);
|
||||
return dictionarySize;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* COVER_best_t is used for two purposes:
|
||||
* 1. Synchronizing threads.
|
||||
* 2. Saving the best parameters and dictionary.
|
||||
*
|
||||
* All of the methods except COVER_best_init() are thread safe if zstd is
|
||||
* compiled with multithreaded support.
|
||||
*/
|
||||
typedef struct COVER_best_s {
|
||||
#ifdef ZSTD_PTHREAD
|
||||
pthread_mutex_t mutex;
|
||||
pthread_cond_t cond;
|
||||
size_t liveJobs;
|
||||
#endif
|
||||
void *dict;
|
||||
size_t dictSize;
|
||||
COVER_params_t parameters;
|
||||
size_t compressedSize;
|
||||
} COVER_best_t;
|
||||
|
||||
/**
|
||||
* Initialize the `COVER_best_t`.
|
||||
*/
|
||||
static void COVER_best_init(COVER_best_t *best) {
|
||||
if (!best) {
|
||||
return;
|
||||
}
|
||||
#ifdef ZSTD_PTHREAD
|
||||
pthread_mutex_init(&best->mutex, NULL);
|
||||
pthread_cond_init(&best->cond, NULL);
|
||||
best->liveJobs = 0;
|
||||
#endif
|
||||
best->dict = NULL;
|
||||
best->dictSize = 0;
|
||||
best->compressedSize = (size_t)-1;
|
||||
memset(&best->parameters, 0, sizeof(best->parameters));
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait until liveJobs == 0.
|
||||
*/
|
||||
static void COVER_best_wait(COVER_best_t *best) {
|
||||
if (!best) {
|
||||
return;
|
||||
}
|
||||
#ifdef ZSTD_PTHREAD
|
||||
pthread_mutex_lock(&best->mutex);
|
||||
while (best->liveJobs != 0) {
|
||||
pthread_cond_wait(&best->cond, &best->mutex);
|
||||
}
|
||||
pthread_mutex_unlock(&best->mutex);
|
||||
#endif
|
||||
}
|
||||
|
||||
/**
|
||||
* Call COVER_best_wait() and then destroy the COVER_best_t.
|
||||
*/
|
||||
static void COVER_best_destroy(COVER_best_t *best) {
|
||||
if (!best) {
|
||||
return;
|
||||
}
|
||||
COVER_best_wait(best);
|
||||
if (best->dict) {
|
||||
free(best->dict);
|
||||
}
|
||||
#ifdef ZSTD_PTHREAD
|
||||
pthread_mutex_destroy(&best->mutex);
|
||||
pthread_cond_destroy(&best->cond);
|
||||
#endif
|
||||
}
|
||||
|
||||
/**
|
||||
* Called when a thread is about to be launched.
|
||||
* Increments liveJobs.
|
||||
*/
|
||||
static void COVER_best_start(COVER_best_t *best) {
|
||||
if (!best) {
|
||||
return;
|
||||
}
|
||||
#ifdef ZSTD_PTHREAD
|
||||
pthread_mutex_lock(&best->mutex);
|
||||
++best->liveJobs;
|
||||
pthread_mutex_unlock(&best->mutex);
|
||||
#endif
|
||||
}
|
||||
|
||||
/**
|
||||
* Called when a thread finishes executing, both on error or success.
|
||||
* Decrements liveJobs and signals any waiting threads if liveJobs == 0.
|
||||
* If this dictionary is the best so far save it and its parameters.
|
||||
*/
|
||||
static void COVER_best_finish(COVER_best_t *best, size_t compressedSize,
|
||||
COVER_params_t parameters, void *dict,
|
||||
size_t dictSize) {
|
||||
if (!best) {
|
||||
return;
|
||||
}
|
||||
{
|
||||
#ifdef ZSTD_PTHREAD
|
||||
size_t liveJobs;
|
||||
pthread_mutex_lock(&best->mutex);
|
||||
--best->liveJobs;
|
||||
liveJobs = best->liveJobs;
|
||||
#endif
|
||||
/* If the new dictionary is better */
|
||||
if (compressedSize < best->compressedSize) {
|
||||
/* Allocate space if necessary */
|
||||
if (!best->dict || best->dictSize < dictSize) {
|
||||
if (best->dict) {
|
||||
free(best->dict);
|
||||
}
|
||||
best->dict = malloc(dictSize);
|
||||
if (!best->dict) {
|
||||
best->compressedSize = ERROR(GENERIC);
|
||||
best->dictSize = 0;
|
||||
return;
|
||||
}
|
||||
}
|
||||
/* Save the dictionary, parameters, and size */
|
||||
memcpy(best->dict, dict, dictSize);
|
||||
best->dictSize = dictSize;
|
||||
best->parameters = parameters;
|
||||
best->compressedSize = compressedSize;
|
||||
}
|
||||
#ifdef ZSTD_PTHREAD
|
||||
pthread_mutex_unlock(&best->mutex);
|
||||
if (liveJobs == 0) {
|
||||
pthread_cond_broadcast(&best->cond);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parameters for COVER_tryParameters().
|
||||
*/
|
||||
typedef struct COVER_tryParameters_data_s {
|
||||
const COVER_ctx_t *ctx;
|
||||
COVER_best_t *best;
|
||||
size_t dictBufferCapacity;
|
||||
COVER_params_t parameters;
|
||||
} COVER_tryParameters_data_t;
|
||||
|
||||
/**
|
||||
* Tries a set of parameters and upates the COVER_best_t with the results.
|
||||
* This function is thread safe if zstd is compiled with multithreaded support.
|
||||
* It takes its parameters as an *OWNING* opaque pointer to support threading.
|
||||
*/
|
||||
static void COVER_tryParameters(void *opaque) {
|
||||
/* Save parameters as local variables */
|
||||
COVER_tryParameters_data_t *const data = (COVER_tryParameters_data_t *)opaque;
|
||||
const COVER_ctx_t *const ctx = data->ctx;
|
||||
const COVER_params_t parameters = data->parameters;
|
||||
size_t dictBufferCapacity = data->dictBufferCapacity;
|
||||
size_t totalCompressedSize = ERROR(GENERIC);
|
||||
/* Allocate space for hash table, dict, and freqs */
|
||||
COVER_map_t activeDmers;
|
||||
BYTE *const dict = (BYTE * const)malloc(dictBufferCapacity);
|
||||
U32 *freqs = (U32 *)malloc(ctx->suffixSize * sizeof(U32));
|
||||
if (!COVER_map_init(&activeDmers, parameters.k - parameters.d + 1)) {
|
||||
DISPLAYLEVEL(1, "Failed to allocate dmer map: out of memory\n");
|
||||
goto _cleanup;
|
||||
}
|
||||
if (!dict || !freqs) {
|
||||
DISPLAYLEVEL(1, "Failed to allocate buffers: out of memory\n");
|
||||
goto _cleanup;
|
||||
}
|
||||
/* Copy the frequencies because we need to modify them */
|
||||
memcpy(freqs, ctx->freqs, ctx->suffixSize * sizeof(U32));
|
||||
/* Build the dictionary */
|
||||
{
|
||||
const size_t tail = COVER_buildDictionary(ctx, freqs, &activeDmers, dict,
|
||||
dictBufferCapacity, parameters);
|
||||
const ZDICT_params_t zdictParams = COVER_translateParams(parameters);
|
||||
dictBufferCapacity = ZDICT_finalizeDictionary(
|
||||
dict, dictBufferCapacity, dict + tail, dictBufferCapacity - tail,
|
||||
ctx->samples, ctx->samplesSizes, (unsigned)ctx->nbSamples, zdictParams);
|
||||
if (ZDICT_isError(dictBufferCapacity)) {
|
||||
DISPLAYLEVEL(1, "Failed to finalize dictionary\n");
|
||||
goto _cleanup;
|
||||
}
|
||||
}
|
||||
/* Check total compressed size */
|
||||
{
|
||||
/* Pointers */
|
||||
ZSTD_CCtx *cctx;
|
||||
ZSTD_CDict *cdict;
|
||||
void *dst;
|
||||
/* Local variables */
|
||||
size_t dstCapacity;
|
||||
size_t i;
|
||||
/* Allocate dst with enough space to compress the maximum sized sample */
|
||||
{
|
||||
size_t maxSampleSize = 0;
|
||||
for (i = 0; i < ctx->nbSamples; ++i) {
|
||||
maxSampleSize = MAX(ctx->samplesSizes[i], maxSampleSize);
|
||||
}
|
||||
dstCapacity = ZSTD_compressBound(maxSampleSize);
|
||||
dst = malloc(dstCapacity);
|
||||
}
|
||||
/* Create the cctx and cdict */
|
||||
cctx = ZSTD_createCCtx();
|
||||
cdict =
|
||||
ZSTD_createCDict(dict, dictBufferCapacity, parameters.compressionLevel);
|
||||
if (!dst || !cctx || !cdict) {
|
||||
goto _compressCleanup;
|
||||
}
|
||||
/* Compress each sample and sum their sizes (or error) */
|
||||
totalCompressedSize = 0;
|
||||
for (i = 0; i < ctx->nbSamples; ++i) {
|
||||
const size_t size = ZSTD_compress_usingCDict(
|
||||
cctx, dst, dstCapacity, ctx->samples + ctx->offsets[i],
|
||||
ctx->samplesSizes[i], cdict);
|
||||
if (ZSTD_isError(size)) {
|
||||
totalCompressedSize = ERROR(GENERIC);
|
||||
goto _compressCleanup;
|
||||
}
|
||||
totalCompressedSize += size;
|
||||
}
|
||||
_compressCleanup:
|
||||
ZSTD_freeCCtx(cctx);
|
||||
ZSTD_freeCDict(cdict);
|
||||
if (dst) {
|
||||
free(dst);
|
||||
}
|
||||
}
|
||||
|
||||
_cleanup:
|
||||
COVER_best_finish(data->best, totalCompressedSize, parameters, dict,
|
||||
dictBufferCapacity);
|
||||
free(data);
|
||||
COVER_map_destroy(&activeDmers);
|
||||
if (dict) {
|
||||
free(dict);
|
||||
}
|
||||
if (freqs) {
|
||||
free(freqs);
|
||||
}
|
||||
}
|
||||
|
||||
ZDICTLIB_API size_t COVER_optimizeTrainFromBuffer(void *dictBuffer,
|
||||
size_t dictBufferCapacity,
|
||||
const void *samplesBuffer,
|
||||
const size_t *samplesSizes,
|
||||
unsigned nbSamples,
|
||||
COVER_params_t *parameters) {
|
||||
/* constants */
|
||||
const unsigned kMinD = parameters->d == 0 ? 6 : parameters->d;
|
||||
const unsigned kMaxD = parameters->d == 0 ? 16 : parameters->d;
|
||||
const unsigned kMinK = parameters->k == 0 ? kMaxD : parameters->k;
|
||||
const unsigned kMaxK = parameters->k == 0 ? 2048 : parameters->k;
|
||||
const unsigned kSteps = parameters->steps == 0 ? 256 : parameters->steps;
|
||||
const unsigned kStepSize = MAX((kMaxK - kMinK) / kSteps, 1);
|
||||
const unsigned kIterations =
|
||||
(1 + (kMaxD - kMinD) / 2) * (1 + (kMaxK - kMinK) / kStepSize);
|
||||
/* Local variables */
|
||||
const int displayLevel = parameters->notificationLevel;
|
||||
unsigned iteration = 1;
|
||||
unsigned d;
|
||||
unsigned k;
|
||||
COVER_best_t best;
|
||||
/* Checks */
|
||||
if (kMinK < kMaxD || kMaxK < kMinK) {
|
||||
LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect parameters\n");
|
||||
return ERROR(GENERIC);
|
||||
}
|
||||
if (nbSamples == 0) {
|
||||
DISPLAYLEVEL(1, "Cover must have at least one input file\n");
|
||||
return ERROR(GENERIC);
|
||||
}
|
||||
if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) {
|
||||
DISPLAYLEVEL(1, "dictBufferCapacity must be at least %u\n",
|
||||
ZDICT_DICTSIZE_MIN);
|
||||
return ERROR(dstSize_tooSmall);
|
||||
}
|
||||
/* Initialization */
|
||||
COVER_best_init(&best);
|
||||
/* Turn down global display level to clean up display at level 2 and below */
|
||||
g_displayLevel = parameters->notificationLevel - 1;
|
||||
/* Loop through d first because each new value needs a new context */
|
||||
LOCALDISPLAYLEVEL(displayLevel, 2, "Trying %u different sets of parameters\n",
|
||||
kIterations);
|
||||
for (d = kMinD; d <= kMaxD; d += 2) {
|
||||
/* Initialize the context for this value of d */
|
||||
COVER_ctx_t ctx;
|
||||
LOCALDISPLAYLEVEL(displayLevel, 3, "d=%u\n", d);
|
||||
if (!COVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, d)) {
|
||||
LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to initialize context\n");
|
||||
COVER_best_destroy(&best);
|
||||
return ERROR(GENERIC);
|
||||
}
|
||||
/* Loop through k reusing the same context */
|
||||
for (k = kMinK; k <= kMaxK; k += kStepSize) {
|
||||
/* Prepare the arguments */
|
||||
COVER_tryParameters_data_t *data = (COVER_tryParameters_data_t *)malloc(
|
||||
sizeof(COVER_tryParameters_data_t));
|
||||
LOCALDISPLAYLEVEL(displayLevel, 3, "k=%u\n", k);
|
||||
if (!data) {
|
||||
LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to allocate parameters\n");
|
||||
COVER_best_destroy(&best);
|
||||
COVER_ctx_destroy(&ctx);
|
||||
return ERROR(GENERIC);
|
||||
}
|
||||
data->ctx = &ctx;
|
||||
data->best = &best;
|
||||
data->dictBufferCapacity = dictBufferCapacity;
|
||||
data->parameters = *parameters;
|
||||
data->parameters.k = k;
|
||||
data->parameters.d = d;
|
||||
data->parameters.steps = kSteps;
|
||||
/* Check the parameters */
|
||||
if (!COVER_checkParameters(data->parameters)) {
|
||||
DISPLAYLEVEL(1, "Cover parameters incorrect\n");
|
||||
continue;
|
||||
}
|
||||
/* Call the function and pass ownership of data to it */
|
||||
COVER_best_start(&best);
|
||||
COVER_tryParameters(data);
|
||||
/* Print status */
|
||||
LOCALDISPLAYUPDATE(displayLevel, 2, "\r%u%% ",
|
||||
(U32)((iteration * 100) / kIterations));
|
||||
++iteration;
|
||||
}
|
||||
COVER_best_wait(&best);
|
||||
COVER_ctx_destroy(&ctx);
|
||||
}
|
||||
LOCALDISPLAYLEVEL(displayLevel, 2, "\r%79s\r", "");
|
||||
/* Fill the output buffer and parameters with output of the best parameters */
|
||||
{
|
||||
const size_t dictSize = best.dictSize;
|
||||
if (ZSTD_isError(best.compressedSize)) {
|
||||
COVER_best_destroy(&best);
|
||||
return best.compressedSize;
|
||||
}
|
||||
*parameters = best.parameters;
|
||||
memcpy(dictBuffer, best.dict, dictSize);
|
||||
COVER_best_destroy(&best);
|
||||
return dictSize;
|
||||
}
|
||||
}
|
||||
@@ -86,6 +86,57 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_advanced(void* dictBuffer, size_t dict
|
||||
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
|
||||
ZDICT_params_t parameters);
|
||||
|
||||
/*! COVER_params_t :
|
||||
For all values 0 means default.
|
||||
kMin and d are the only required parameters.
|
||||
*/
|
||||
typedef struct {
|
||||
unsigned k; /* Segment size : constraint: 0 < k : Reasonable range [16, 2048+] */
|
||||
unsigned d; /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */
|
||||
unsigned steps; /* Number of steps : Only used for optimization : 0 means default (256) : Higher means more parameters checked */
|
||||
|
||||
unsigned notificationLevel; /* Write to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */
|
||||
unsigned dictID; /* 0 means auto mode (32-bits random value); other : force dictID value */
|
||||
int compressionLevel; /* 0 means default; target a specific zstd compression level */
|
||||
} COVER_params_t;
|
||||
|
||||
|
||||
/*! COVER_trainFromBuffer() :
|
||||
Train a dictionary from an array of samples using the COVER algorithm.
|
||||
Samples must be stored concatenated in a single flat buffer `samplesBuffer`,
|
||||
supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order.
|
||||
The resulting dictionary will be saved into `dictBuffer`.
|
||||
@return : size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
|
||||
or an error code, which can be tested with ZDICT_isError().
|
||||
Note : COVER_trainFromBuffer() requires about 9 bytes of memory for each input byte.
|
||||
Tips : In general, a reasonable dictionary has a size of ~ 100 KB.
|
||||
It's obviously possible to target smaller or larger ones, just by specifying different `dictBufferCapacity`.
|
||||
In general, it's recommended to provide a few thousands samples, but this can vary a lot.
|
||||
It's recommended that total size of all samples be about ~x100 times the target size of dictionary.
|
||||
*/
|
||||
ZDICTLIB_API size_t COVER_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity,
|
||||
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
|
||||
COVER_params_t parameters);
|
||||
|
||||
/*! COVER_optimizeTrainFromBuffer() :
|
||||
The same requirements as above hold for all the parameters except `parameters`.
|
||||
This function tries many parameter combinations and picks the best parameters.
|
||||
`*parameters` is filled with the best parameters found, and the dictionary
|
||||
constructed with those parameters is stored in `dictBuffer`.
|
||||
|
||||
All of the parameters d, k, steps are optional.
|
||||
If d is non-zero then we don't check multiple values of d, otherwise we check d = {6, 8, 10, 12, 14, 16}.
|
||||
if steps is zero it defaults to its default value.
|
||||
If k is non-zero then we don't check multiple values of k, otherwise we check steps values in [16, 2048].
|
||||
|
||||
@return : size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
|
||||
or an error code, which can be tested with ZDICT_isError().
|
||||
On success `*parameters` contains the parameters selected.
|
||||
Note : COVER_optimizeTrainFromBuffer() requires about 9 bytes of memory for each input byte.
|
||||
*/
|
||||
ZDICTLIB_API size_t COVER_optimizeTrainFromBuffer(void* dictBuffer, size_t dictBufferCapacity,
|
||||
const void* samplesBuffer, const size_t *samplesSizes, unsigned nbSamples,
|
||||
COVER_params_t *parameters);
|
||||
|
||||
/*! ZDICT_finalizeDictionary() :
|
||||
|
||||
|
||||
+55
-7
@@ -42,6 +42,7 @@
|
||||
|
||||
#define SAMPLESIZE_MAX (128 KB)
|
||||
#define MEMMULT 11 /* rough estimation : memory cost to analyze 1 byte of sample */
|
||||
#define COVER_MEMMULT 9 /* rough estimation : memory cost to analyze 1 byte of sample */
|
||||
static const size_t maxMemory = (sizeof(size_t) == 4) ? (2 GB - 64 MB) : ((size_t)(512 MB) << sizeof(size_t));
|
||||
|
||||
#define NOISELENGTH 32
|
||||
@@ -118,10 +119,36 @@ static unsigned DiB_loadFiles(void* buffer, size_t* bufferSizePtr,
|
||||
fileSizes[n] = fileSize;
|
||||
fclose(f);
|
||||
} }
|
||||
DISPLAYLEVEL(2, "\r%79s\r", "");
|
||||
*bufferSizePtr = pos;
|
||||
return n;
|
||||
}
|
||||
|
||||
#define DiB_rotl32(x,r) ((x << r) | (x >> (32 - r)))
|
||||
static U32 DiB_rand(U32* src)
|
||||
{
|
||||
static const U32 prime1 = 2654435761U;
|
||||
static const U32 prime2 = 2246822519U;
|
||||
U32 rand32 = *src;
|
||||
rand32 *= prime1;
|
||||
rand32 ^= prime2;
|
||||
rand32 = DiB_rotl32(rand32, 13);
|
||||
*src = rand32;
|
||||
return rand32 >> 5;
|
||||
}
|
||||
|
||||
static void DiB_shuffle(const char** fileNamesTable, unsigned nbFiles) {
|
||||
/* Initialize the pseudorandom number generator */
|
||||
U32 seed = 0xFD2FB528;
|
||||
unsigned i;
|
||||
for (i = nbFiles - 1; i > 0; --i) {
|
||||
unsigned const j = DiB_rand(&seed) % (i + 1);
|
||||
const char* tmp = fileNamesTable[j];
|
||||
fileNamesTable[j] = fileNamesTable[i];
|
||||
fileNamesTable[i] = tmp;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*-********************************************************
|
||||
* Dictionary training functions
|
||||
@@ -202,19 +229,23 @@ size_t ZDICT_trainFromBuffer_unsafe(void* dictBuffer, size_t dictBufferCapacity,
|
||||
|
||||
int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
|
||||
const char** fileNamesTable, unsigned nbFiles,
|
||||
ZDICT_params_t params)
|
||||
ZDICT_params_t *params, COVER_params_t *coverParams,
|
||||
int optimizeCover)
|
||||
{
|
||||
void* const dictBuffer = malloc(maxDictSize);
|
||||
size_t* const fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t));
|
||||
unsigned long long const totalSizeToLoad = DiB_getTotalCappedFileSize(fileNamesTable, nbFiles);
|
||||
size_t const maxMem = DiB_findMaxMem(totalSizeToLoad * MEMMULT) / MEMMULT;
|
||||
size_t const memMult = params ? MEMMULT : COVER_MEMMULT;
|
||||
size_t const maxMem = DiB_findMaxMem(totalSizeToLoad * memMult) / memMult;
|
||||
size_t benchedSize = (size_t) MIN ((unsigned long long)maxMem, totalSizeToLoad);
|
||||
void* const srcBuffer = malloc(benchedSize+NOISELENGTH);
|
||||
int result = 0;
|
||||
|
||||
/* Checks */
|
||||
if (params) g_displayLevel = params->notificationLevel;
|
||||
else if (coverParams) g_displayLevel = coverParams->notificationLevel;
|
||||
else EXM_THROW(13, "Neither dictionary algorith selected"); /* should not happen */
|
||||
if ((!fileSizes) || (!srcBuffer) || (!dictBuffer)) EXM_THROW(12, "not enough memory for DiB_trainFiles"); /* should not happen */
|
||||
g_displayLevel = params.notificationLevel;
|
||||
if (g_tooLargeSamples) {
|
||||
DISPLAYLEVEL(2, "! Warning : some samples are very large \n");
|
||||
DISPLAYLEVEL(2, "! Note that dictionary is only useful for small files or beginning of large files. \n");
|
||||
@@ -233,12 +264,29 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
|
||||
DISPLAYLEVEL(1, "Not enough memory; training on %u MB only...\n", (unsigned)(benchedSize >> 20));
|
||||
|
||||
/* Load input buffer */
|
||||
DISPLAYLEVEL(3, "Shuffling input files\n");
|
||||
DiB_shuffle(fileNamesTable, nbFiles);
|
||||
nbFiles = DiB_loadFiles(srcBuffer, &benchedSize, fileSizes, fileNamesTable, nbFiles);
|
||||
DiB_fillNoise((char*)srcBuffer + benchedSize, NOISELENGTH); /* guard band, for end of buffer condition */
|
||||
|
||||
{ size_t const dictSize = ZDICT_trainFromBuffer_unsafe(dictBuffer, maxDictSize,
|
||||
srcBuffer, fileSizes, nbFiles,
|
||||
params);
|
||||
{
|
||||
size_t dictSize;
|
||||
if (params) {
|
||||
DiB_fillNoise((char*)srcBuffer + benchedSize, NOISELENGTH); /* guard band, for end of buffer condition */
|
||||
dictSize = ZDICT_trainFromBuffer_unsafe(dictBuffer, maxDictSize,
|
||||
srcBuffer, fileSizes, nbFiles,
|
||||
*params);
|
||||
} else if (optimizeCover) {
|
||||
dictSize = COVER_optimizeTrainFromBuffer(
|
||||
dictBuffer, maxDictSize, srcBuffer, fileSizes, nbFiles,
|
||||
coverParams);
|
||||
if (!ZDICT_isError(dictSize)) {
|
||||
DISPLAYLEVEL(2, "k=%u\nd=%u\nsteps=%u\n", coverParams->k, coverParams->d, coverParams->steps);
|
||||
}
|
||||
} else {
|
||||
dictSize = COVER_trainFromBuffer(dictBuffer, maxDictSize,
|
||||
srcBuffer, fileSizes, nbFiles,
|
||||
*coverParams);
|
||||
}
|
||||
if (ZDICT_isError(dictSize)) {
|
||||
DISPLAYLEVEL(1, "dictionary training failed : %s \n", ZDICT_getErrorName(dictSize)); /* should not happen */
|
||||
result = 1;
|
||||
|
||||
+2
-2
@@ -32,7 +32,7 @@
|
||||
*/
|
||||
int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
|
||||
const char** fileNamesTable, unsigned nbFiles,
|
||||
ZDICT_params_t parameters);
|
||||
|
||||
ZDICT_params_t *params, COVER_params_t *coverParams,
|
||||
int optimizeCover);
|
||||
|
||||
#endif
|
||||
|
||||
+6
-3
@@ -69,6 +69,8 @@ from standard input if it is a terminal.
|
||||
.PP
|
||||
Unless
|
||||
.B \-\-stdout
|
||||
or
|
||||
.B \-o
|
||||
is specified,
|
||||
.I files
|
||||
are written to a new file whose name is derived from the source
|
||||
@@ -159,7 +161,8 @@ No files are created or removed.
|
||||
# compression level [1-19] (default:3)
|
||||
.TP
|
||||
.BR \--ultra
|
||||
unlocks high compression levels 20+ (maximum 22), using a lot more memory
|
||||
unlocks high compression levels 20+ (maximum 22), using a lot more memory.
|
||||
Note that decompression will also require more memory when using these levels.
|
||||
.TP
|
||||
.B \-D file
|
||||
use `file` as Dictionary to compress or decompress FILE(s)
|
||||
@@ -293,7 +296,7 @@ There are 8 strategies numbered from 0 to 7, from faster to stronger:
|
||||
.PD
|
||||
Specify the maximum number of bits for a match distance.
|
||||
.IP ""
|
||||
The higher number of bits increases the chance to find a match what usually improves compression ratio.
|
||||
The higher number of bits increases the chance to find a match what usually improves compression ratio.
|
||||
It also increases memory requirements for compressor and decompressor.
|
||||
.IP ""
|
||||
The minimum \fIwlog\fR is 10 (1 KiB) and the maximum is 25 (32 MiB) for 32-bit compilation and 27 (128 MiB) for 64-bit compilation.
|
||||
@@ -319,7 +322,7 @@ The minimum \fIhlog\fR is 6 (64 B) and the maximum is 25 (32 MiB) for 32-bit com
|
||||
.PD
|
||||
Specify the maximum number of bits for a hash chain or a binary tree.
|
||||
.IP ""
|
||||
The higher number of bits increases the chance to find a match what usually improves compression ratio.
|
||||
The higher number of bits increases the chance to find a match what usually improves compression ratio.
|
||||
It also slows down compression speed and increases memory requirements for compression.
|
||||
This option is ignored for the ZSTD_fast strategy.
|
||||
.IP ""
|
||||
|
||||
+55
-7
@@ -127,6 +127,8 @@ static int usage_advanced(const char* programName)
|
||||
DISPLAY( "\n");
|
||||
DISPLAY( "Dictionary builder :\n");
|
||||
DISPLAY( "--train ## : create a dictionary from a training set of files \n");
|
||||
DISPLAY( "--cover=k=#,d=# : use the cover algorithm with parameters k and d \n");
|
||||
DISPLAY( "--optimize-cover[=steps=#,k=#,d=#] : optimize cover parameters with optional parameters\n");
|
||||
DISPLAY( " -o file : `file` is dictionary name (default: %s) \n", g_defaultDictName);
|
||||
DISPLAY( "--maxdict ## : limit dictionary to specified size (default : %u) \n", g_defaultMaxDictSize);
|
||||
DISPLAY( " -s# : dictionary selectivity level (default: %u)\n", g_defaultSelectivityLevel);
|
||||
@@ -192,6 +194,27 @@ static unsigned longCommandWArg(const char** stringPtr, const char* longCommand)
|
||||
}
|
||||
|
||||
|
||||
#ifndef ZSTD_NODICT
|
||||
/**
|
||||
* parseCoverParameters() :
|
||||
* reads cover parameters from *stringPtr (e.g. "--cover=smoothing=100,kmin=48,kstep=4,kmax=64,d=8") into *params
|
||||
* @return 1 means that cover parameters were correct
|
||||
* @return 0 in case of malformed parameters
|
||||
*/
|
||||
static unsigned parseCoverParameters(const char* stringPtr, COVER_params_t *params)
|
||||
{
|
||||
memset(params, 0, sizeof(*params));
|
||||
for (; ;) {
|
||||
if (longCommandWArg(&stringPtr, "k=")) { params->k = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
|
||||
if (longCommandWArg(&stringPtr, "d=")) { params->d = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
|
||||
if (longCommandWArg(&stringPtr, "steps=")) { params->steps = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
|
||||
return 0;
|
||||
}
|
||||
if (stringPtr[0] != 0) return 0;
|
||||
DISPLAYLEVEL(4, "k=%u\nd=%u\nsteps=%u\n", params->k, params->d, params->steps);
|
||||
return 1;
|
||||
}
|
||||
#endif
|
||||
/** parseCompressionParameters() :
|
||||
* reads compression parameters from *stringPtr (e.g. "--zstd=wlog=23,clog=23,hlog=22,slog=6,slen=3,tlen=48,strat=6") into *params
|
||||
* @return 1 means that compression parameters were correct
|
||||
@@ -254,6 +277,10 @@ int main(int argCount, const char* argv[])
|
||||
char* fileNamesBuf = NULL;
|
||||
unsigned fileNamesNb;
|
||||
#endif
|
||||
#ifndef ZSTD_NODICT
|
||||
COVER_params_t coverParams;
|
||||
int cover = 0;
|
||||
#endif
|
||||
|
||||
/* init */
|
||||
(void)recursive; (void)cLevelLast; /* not used when ZSTD_NOBENCH set */
|
||||
@@ -318,6 +345,20 @@ int main(int argCount, const char* argv[])
|
||||
if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; }
|
||||
|
||||
/* long commands with arguments */
|
||||
#ifndef ZSTD_NODICT
|
||||
if (longCommandWArg(&argument, "--cover=")) {
|
||||
cover=1; if (!parseCoverParameters(argument, &coverParams)) CLEAN_RETURN(badusage(programName));
|
||||
continue;
|
||||
}
|
||||
if (longCommandWArg(&argument, "--optimize-cover")) {
|
||||
cover=2;
|
||||
/* Allow optional arguments following an = */
|
||||
if (*argument == 0) { memset(&coverParams, 0, sizeof(coverParams)); }
|
||||
else if (*argument++ != '=') { CLEAN_RETURN(badusage(programName)); }
|
||||
else if (!parseCoverParameters(argument, &coverParams)) { CLEAN_RETURN(badusage(programName)); }
|
||||
continue;
|
||||
}
|
||||
#endif
|
||||
if (longCommandWArg(&argument, "--memlimit=")) { memLimit = readU32FromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "--memory=")) { memLimit = readU32FromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "--memlimit-decompress=")) { memLimit = readU32FromChar(&argument); continue; }
|
||||
@@ -520,13 +561,20 @@ int main(int argCount, const char* argv[])
|
||||
/* Check if dictionary builder is selected */
|
||||
if (operation==zom_train) {
|
||||
#ifndef ZSTD_NODICT
|
||||
ZDICT_params_t dictParams;
|
||||
memset(&dictParams, 0, sizeof(dictParams));
|
||||
dictParams.compressionLevel = dictCLevel;
|
||||
dictParams.selectivityLevel = dictSelect;
|
||||
dictParams.notificationLevel = displayLevel;
|
||||
dictParams.dictID = dictID;
|
||||
DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, dictParams);
|
||||
if (cover) {
|
||||
coverParams.compressionLevel = dictCLevel;
|
||||
coverParams.notificationLevel = displayLevel;
|
||||
coverParams.dictID = dictID;
|
||||
DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, NULL, &coverParams, cover - 1);
|
||||
} else {
|
||||
ZDICT_params_t dictParams;
|
||||
memset(&dictParams, 0, sizeof(dictParams));
|
||||
dictParams.compressionLevel = dictCLevel;
|
||||
dictParams.selectivityLevel = dictSelect;
|
||||
dictParams.notificationLevel = displayLevel;
|
||||
dictParams.dictID = dictID;
|
||||
DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, &dictParams, NULL, 0);
|
||||
}
|
||||
#endif
|
||||
goto _end;
|
||||
}
|
||||
|
||||
+2
-6
@@ -20,10 +20,6 @@
|
||||
# zstreamtest32: Same as zstreamtest, but forced to compile in 32-bits mode
|
||||
# ##########################################################################
|
||||
|
||||
DESTDIR?=
|
||||
PREFIX ?= /usr/local
|
||||
BINDIR = $(PREFIX)/bin
|
||||
MANDIR = $(PREFIX)/share/man/man1
|
||||
ZSTDDIR = ../lib
|
||||
PRGDIR = ../programs
|
||||
PYTHON ?= python3
|
||||
@@ -125,10 +121,10 @@ zbufftest-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/datagen.c zbufftest.c
|
||||
$(MAKE) -C $(ZSTDDIR) libzstd
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@$(EXT)
|
||||
|
||||
zstreamtest : $(ZSTD_FILES) $(PRGDIR)/datagen.c zstreamtest.c
|
||||
zstreamtest : $(ZSTD_FILES) $(ZDICT_FILES) $(PRGDIR)/datagen.c zstreamtest.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
zstreamtest32 : $(ZSTD_FILES) $(PRGDIR)/datagen.c zstreamtest.c
|
||||
zstreamtest32 : $(ZSTD_FILES) $(ZDICT_FILES) $(PRGDIR)/datagen.c zstreamtest.c
|
||||
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
zstreamtest-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd
|
||||
|
||||
@@ -28,6 +28,7 @@
|
||||
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue, ZSTD_compressBlock */
|
||||
#include "zstd.h" /* ZSTD_VERSION_STRING */
|
||||
#include "zstd_errors.h" /* ZSTD_getErrorCode */
|
||||
#define ZDICT_STATIC_LINKING_ONLY
|
||||
#include "zdict.h" /* ZDICT_trainFromBuffer */
|
||||
#include "datagen.h" /* RDG_genBuffer */
|
||||
#include "mem.h"
|
||||
@@ -311,6 +312,70 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
if (r != CNBuffSize) goto _output_error);
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : dictionary containing only header should return error : ", testNb++);
|
||||
{
|
||||
const size_t ret = ZSTD_decompress_usingDict(
|
||||
dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize,
|
||||
"\x37\xa4\x30\xec\x11\x22\x33\x44", 8);
|
||||
if (ZSTD_getErrorCode(ret) != ZSTD_error_dictionary_corrupted) goto _output_error;
|
||||
}
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
ZSTD_freeCCtx(cctx);
|
||||
ZSTD_freeDCtx(dctx);
|
||||
free(dictBuffer);
|
||||
free(samplesSizes);
|
||||
}
|
||||
|
||||
/* COVER dictionary builder tests */
|
||||
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
|
||||
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
|
||||
size_t dictSize = 16 KB;
|
||||
size_t optDictSize = dictSize;
|
||||
void* dictBuffer = malloc(dictSize);
|
||||
size_t const totalSampleSize = 1 MB;
|
||||
size_t const sampleUnitSize = 8 KB;
|
||||
U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize);
|
||||
size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t));
|
||||
COVER_params_t params;
|
||||
U32 dictID;
|
||||
|
||||
if (dictBuffer==NULL || samplesSizes==NULL) {
|
||||
free(dictBuffer);
|
||||
free(samplesSizes);
|
||||
goto _output_error;
|
||||
}
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : COVER_trainFromBuffer : ", testNb++);
|
||||
{ U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
|
||||
memset(¶ms, 0, sizeof(params));
|
||||
params.d = 1 + (FUZ_rand(&seed) % 16);
|
||||
params.k = params.d + (FUZ_rand(&seed) % 256);
|
||||
dictSize = COVER_trainFromBuffer(dictBuffer, dictSize,
|
||||
CNBuffer, samplesSizes, nbSamples,
|
||||
params);
|
||||
if (ZDICT_isError(dictSize)) goto _output_error;
|
||||
DISPLAYLEVEL(4, "OK, created dictionary of size %u \n", (U32)dictSize);
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : check dictID : ", testNb++);
|
||||
dictID = ZDICT_getDictID(dictBuffer, dictSize);
|
||||
if (dictID==0) goto _output_error;
|
||||
DISPLAYLEVEL(4, "OK : %u \n", dictID);
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : COVER_optimizeTrainFromBuffer : ", testNb++);
|
||||
memset(¶ms, 0, sizeof(params));
|
||||
params.steps = 4;
|
||||
optDictSize = COVER_optimizeTrainFromBuffer(dictBuffer, optDictSize,
|
||||
CNBuffer, samplesSizes, nbSamples,
|
||||
¶ms);
|
||||
if (ZDICT_isError(optDictSize)) goto _output_error;
|
||||
DISPLAYLEVEL(4, "OK, created dictionary of size %u \n", (U32)optDictSize);
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : check dictID : ", testNb++);
|
||||
dictID = ZDICT_getDictID(dictBuffer, optDictSize);
|
||||
if (dictID==0) goto _output_error;
|
||||
DISPLAYLEVEL(4, "OK : %u \n", dictID);
|
||||
|
||||
ZSTD_freeCCtx(cctx);
|
||||
ZSTD_freeDCtx(dctx);
|
||||
free(dictBuffer);
|
||||
|
||||
@@ -255,6 +255,27 @@ rm -rf dirTestDict
|
||||
rm tmp*
|
||||
|
||||
|
||||
$ECHO "\n**** cover dictionary tests **** "
|
||||
|
||||
TESTFILE=../programs/zstdcli.c
|
||||
./datagen > tmpDict
|
||||
$ECHO "- Create first dictionary"
|
||||
$ZSTD --train --cover=k=46,d=8 *.c ../programs/*.c -o tmpDict
|
||||
cp $TESTFILE tmp
|
||||
$ZSTD -f tmp -D tmpDict
|
||||
$ZSTD -d tmp.zst -D tmpDict -fo result
|
||||
$DIFF $TESTFILE result
|
||||
$ECHO "- Create second (different) dictionary"
|
||||
$ZSTD --train --cover=k=56,d=8 *.c ../programs/*.c ../programs/*.h -o tmpDictC
|
||||
$ZSTD -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!"
|
||||
$ECHO "- Create dictionary with short dictID"
|
||||
$ZSTD --train --cover=k=46,d=8 *.c ../programs/*.c --dictID 1 -o tmpDict1
|
||||
cmp tmpDict tmpDict1 && die "dictionaries should have different ID !"
|
||||
$ECHO "- Create dictionary with size limit"
|
||||
$ZSTD --train --optimize-cover=steps=8 *.c ../programs/*.c -o tmpDict2 --maxdict 4K
|
||||
rm tmp*
|
||||
|
||||
|
||||
$ECHO "\n**** integrity tests **** "
|
||||
|
||||
$ECHO "test one file (tmp1.zst) "
|
||||
|
||||
+62
-10
@@ -26,9 +26,10 @@
|
||||
#include <time.h> /* clock_t, clock() */
|
||||
#include <string.h> /* strcmp */
|
||||
#include "mem.h"
|
||||
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel, ZSTD_customMem */
|
||||
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel, ZSTD_customMem, ZSTD_getDictID_fromFrame */
|
||||
#include "zstd.h" /* ZSTD_compressBound */
|
||||
#include "zstd_errors.h" /* ZSTD_error_srcSize_wrong */
|
||||
#include "zdict.h" /* ZDICT_trainFromBuffer */
|
||||
#include "datagen.h" /* RDG_genBuffer */
|
||||
#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
|
||||
#include "xxhash.h" /* XXH64_* */
|
||||
@@ -44,8 +45,7 @@
|
||||
static const U32 nbTestsDefault = 10000;
|
||||
#define COMPRESSIBLE_NOISE_LENGTH (10 MB)
|
||||
#define FUZ_COMPRESSIBILITY_DEFAULT 50
|
||||
static const U32 prime1 = 2654435761U;
|
||||
static const U32 prime2 = 2246822519U;
|
||||
static const U32 prime32 = 2654435761U;
|
||||
|
||||
|
||||
/*-************************************
|
||||
@@ -81,8 +81,9 @@ static clock_t FUZ_GetClockSpan(clock_t clockStart)
|
||||
#define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
|
||||
unsigned int FUZ_rand(unsigned int* seedPtr)
|
||||
{
|
||||
static const U32 prime2 = 2246822519U;
|
||||
U32 rand32 = *seedPtr;
|
||||
rand32 *= prime1;
|
||||
rand32 *= prime32;
|
||||
rand32 += prime2;
|
||||
rand32 = FUZ_rotl32(rand32, 13);
|
||||
*seedPtr = rand32;
|
||||
@@ -107,6 +108,41 @@ static void freeFunction(void* opaque, void* address)
|
||||
* Basic Unit tests
|
||||
======================================================*/
|
||||
|
||||
typedef struct {
|
||||
void* start;
|
||||
size_t size;
|
||||
size_t filled;
|
||||
} buffer_t;
|
||||
|
||||
static const buffer_t g_nullBuffer = { NULL, 0 , 0 };
|
||||
|
||||
static buffer_t FUZ_createDictionary(const void* src, size_t srcSize, size_t blockSize, size_t requestedDictSize)
|
||||
{
|
||||
buffer_t dict = { NULL, 0, 0 };
|
||||
size_t const nbBlocks = (srcSize + (blockSize-1)) / blockSize;
|
||||
size_t* const blockSizes = (size_t*) malloc(nbBlocks * sizeof(size_t));
|
||||
if (!blockSizes) return dict;
|
||||
dict.start = malloc(requestedDictSize);
|
||||
if (!dict.start) { free(blockSizes); return dict; }
|
||||
{ size_t nb;
|
||||
for (nb=0; nb<nbBlocks-1; nb++) blockSizes[nb] = blockSize;
|
||||
blockSizes[nbBlocks-1] = srcSize - (blockSize * (nbBlocks-1));
|
||||
}
|
||||
{ size_t const dictSize = ZDICT_trainFromBuffer(dict.start, requestedDictSize, src, blockSizes, (unsigned)nbBlocks);
|
||||
free(blockSizes);
|
||||
if (ZDICT_isError(dictSize)) { free(dict.start); return (buffer_t){ NULL, 0, 0 }; }
|
||||
dict.size = requestedDictSize;
|
||||
dict.filled = dictSize;
|
||||
return dict; /* how to return dictSize ? */
|
||||
}
|
||||
}
|
||||
|
||||
static void FUZ_freeDictionary(buffer_t dict)
|
||||
{
|
||||
free(dict.start);
|
||||
}
|
||||
|
||||
|
||||
static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem customMem)
|
||||
{
|
||||
size_t const CNBufferSize = COMPRESSIBLE_NOISE_LENGTH;
|
||||
@@ -123,14 +159,25 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
|
||||
ZSTD_DStream* zd = ZSTD_createDStream_advanced(customMem);
|
||||
ZSTD_inBuffer inBuff, inBuff2;
|
||||
ZSTD_outBuffer outBuff;
|
||||
buffer_t dictionary = g_nullBuffer;
|
||||
unsigned dictID = 0;
|
||||
|
||||
/* Create compressible test buffer */
|
||||
if (!CNBuffer || !compressedBuffer || !decodedBuffer || !zc || !zd) {
|
||||
DISPLAY("Not enough memory, aborting\n");
|
||||
DISPLAY("Not enough memory, aborting \n");
|
||||
goto _output_error;
|
||||
}
|
||||
RDG_genBuffer(CNBuffer, CNBufferSize, compressibility, 0., seed);
|
||||
|
||||
/* Create dictionary */
|
||||
MEM_STATIC_ASSERT(COMPRESSIBLE_NOISE_LENGTH >= 4 MB);
|
||||
dictionary = FUZ_createDictionary(CNBuffer, 4 MB, 4 KB, 40 KB);
|
||||
if (!dictionary.start) {
|
||||
DISPLAY("Error creating dictionary, aborting \n");
|
||||
goto _output_error;
|
||||
}
|
||||
dictID = ZDICT_getDictID(dictionary.start, dictionary.filled);
|
||||
|
||||
/* generate skippable frame */
|
||||
MEM_writeLE32(compressedBuffer, ZSTD_MAGIC_SKIPPABLE_START);
|
||||
MEM_writeLE32(((char*)compressedBuffer)+4, (U32)skippableFrameSize);
|
||||
@@ -260,8 +307,6 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
|
||||
{ size_t const r = ZSTD_endStream(zc, &outBuff);
|
||||
if (r != 0) goto _output_error; } /* error, or some data not flushed */
|
||||
{ unsigned long long origSize = ZSTD_getDecompressedSize(outBuff.dst, outBuff.pos);
|
||||
DISPLAY("outBuff.pos : %u \n", (U32)outBuff.pos);
|
||||
DISPLAY("origSize = %u \n", (U32)origSize);
|
||||
if ((size_t)origSize != CNBufferSize) goto _output_error; } /* exact original size must be present */
|
||||
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
|
||||
|
||||
@@ -320,7 +365,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
|
||||
|
||||
/* CDict scenario */
|
||||
DISPLAYLEVEL(4, "test%3i : digested dictionary : ", testNb++);
|
||||
{ ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, 128 KB, 1);
|
||||
{ ZSTD_CDict* const cdict = ZSTD_createCDict(dictionary.start, dictionary.filled, 1);
|
||||
size_t const initError = ZSTD_initCStream_usingCDict(zc, cdict);
|
||||
if (ZSTD_isError(initError)) goto _output_error;
|
||||
cSize = 0;
|
||||
@@ -346,9 +391,15 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
|
||||
DISPLAYLEVEL(4, "OK (%u bytes) \n", (U32)s);
|
||||
}
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : check Dictionary ID : ", testNb++);
|
||||
{ unsigned const dID = ZSTD_getDictID_fromFrame(compressedBuffer, cSize);
|
||||
if (dID != dictID) goto _output_error;
|
||||
DISPLAYLEVEL(4, "OK (%u) \n", dID);
|
||||
}
|
||||
|
||||
/* DDict scenario */
|
||||
DISPLAYLEVEL(4, "test%3i : decompress %u bytes with digested dictionary : ", testNb++, (U32)CNBufferSize);
|
||||
{ ZSTD_DDict* const ddict = ZSTD_createDDict(CNBuffer, 128 KB);
|
||||
{ ZSTD_DDict* const ddict = ZSTD_createDDict(dictionary.start, dictionary.filled);
|
||||
size_t const initError = ZSTD_initDStream_usingDDict(zd, ddict);
|
||||
if (ZSTD_isError(initError)) goto _output_error;
|
||||
inBuff.src = compressedBuffer;
|
||||
@@ -388,6 +439,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
|
||||
|
||||
|
||||
_end:
|
||||
FUZ_freeDictionary(dictionary);
|
||||
ZSTD_freeCStream(zc);
|
||||
ZSTD_freeDStream(zd);
|
||||
free(CNBuffer);
|
||||
@@ -492,7 +544,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
|
||||
if (nbTests >= testNb) { DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); }
|
||||
else { DISPLAYUPDATE(2, "\r%6u ", testNb); }
|
||||
FUZ_rand(&coreSeed);
|
||||
lseed = coreSeed ^ prime1;
|
||||
lseed = coreSeed ^ prime32;
|
||||
|
||||
/* states full reset (deliberately not synchronized) */
|
||||
/* some issues can only happen when reusing states */
|
||||
|
||||
Reference in New Issue
Block a user