From ab6e038482f78d5c3a31a0e74fc0856e1ee91bf5 Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Wed, 1 Aug 2018 17:39:15 -0700 Subject: [PATCH 1/5] Minor fix --- contrib/experimental_dict_builders/fastCover/fastCover.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/experimental_dict_builders/fastCover/fastCover.c b/contrib/experimental_dict_builders/fastCover/fastCover.c index 84d841b10..02c155a81 100644 --- a/contrib/experimental_dict_builders/fastCover/fastCover.c +++ b/contrib/experimental_dict_builders/fastCover/fastCover.c @@ -197,7 +197,7 @@ static FASTCOVER_segment_t FASTCOVER_selectSegment(const FASTCOVER_ctx_t *ctx, bestSegment.end = newEnd; } { - /* Half the frequency of hash value of each dmer covered by the chosen segment. */ + /* Zero the frequency of hash value of each dmer covered by the chosen segment. */ U32 pos; for (pos = bestSegment.begin; pos != bestSegment.end; ++pos) { const size_t i = FASTCOVER_hashPtrToIndex(ctx->samples + pos, parameters.f, ctx->d); @@ -300,7 +300,7 @@ static int FASTCOVER_ctx_init(FASTCOVER_ctx_t *ctx, const void *samplesBuffer, if (totalSamplesSize < MAX(d, sizeof(U64)) || totalSamplesSize >= (size_t)FASTCOVER_MAX_SAMPLES_SIZE) { DISPLAYLEVEL(1, "Total samples size is too large (%u MB), maximum size is %u MB\n", - (U32)(totalSamplesSize>>20), (FASTCOVER_MAX_SAMPLES_SIZE >> 20)); + (U32)(totalSamplesSize >> 20), (FASTCOVER_MAX_SAMPLES_SIZE >> 20)); return 0; } /* Check if there are at least 5 training samples */ From 0feed8e1f8f1c76e3fb1b81316e31eb11decddb5 Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Mon, 6 Aug 2018 17:54:04 -0700 Subject: [PATCH 2/5] Run non-optimize FASTCOVER 5 times in benchmark --- .../benchmarkDictBuilder/benchmark.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c b/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c index d92e8d5cb..7dade2da3 100644 --- a/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c +++ b/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c @@ -416,11 +416,13 @@ int main(int argCount, const char* argv[]) } /* for fastCover (with k and d provided) */ - const int fastResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, &fastParam); - DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\n", fastParam.k, fastParam.d, fastParam.f, fastParam.steps, (unsigned)(fastParam.splitPoint * 100)); - if(fastResult) { - result = 1; - goto _cleanup; + for (int i = 0; i < 5; i++) { + const int fastResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, &fastParam); + DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\n", fastParam.k, fastParam.d, fastParam.f, fastParam.steps, (unsigned)(fastParam.splitPoint * 100)); + if(fastResult) { + result = 1; + goto _cleanup; + } } } From 16db0337b18124adbc9872a1c21c526b23bce632 Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Thu, 30 Aug 2018 14:53:59 -0700 Subject: [PATCH 3/5] Always use splitPoint=1.0 for non-optimize cover and fastcover --- lib/dictBuilder/cover.c | 2 +- lib/dictBuilder/fastcover.c | 2 +- lib/dictBuilder/zdict.h | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/dictBuilder/cover.c b/lib/dictBuilder/cover.c index 74b70ef82..6b4af69d2 100644 --- a/lib/dictBuilder/cover.c +++ b/lib/dictBuilder/cover.c @@ -684,7 +684,7 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover( BYTE* const dict = (BYTE*)dictBuffer; COVER_ctx_t ctx; COVER_map_t activeDmers; - parameters.splitPoint = parameters.splitPoint <= 0.0 ? DEFAULT_SPLITPOINT : parameters.splitPoint; + parameters.splitPoint = 1.0; /* Initialize global data */ g_displayLevel = parameters.zParams.notificationLevel; /* Checks */ diff --git a/lib/dictBuilder/fastcover.c b/lib/dictBuilder/fastcover.c index 9a41ac2f9..fbf5770ba 100644 --- a/lib/dictBuilder/fastcover.c +++ b/lib/dictBuilder/fastcover.c @@ -510,7 +510,7 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_fastCover( /* Initialize global data */ g_displayLevel = parameters.zParams.notificationLevel; /* Assign splitPoint and f if not provided */ - parameters.splitPoint = parameters.splitPoint <= 0.0 ? DEFAULT_SPLITPOINT : parameters.splitPoint; + parameters.splitPoint = 1.0; parameters.f = parameters.f == 0 ? DEFAULT_F : parameters.f; parameters.accel = parameters.accel == 0 ? DEFAULT_ACCEL : parameters.accel; /* Convert to cover parameter */ diff --git a/lib/dictBuilder/zdict.h b/lib/dictBuilder/zdict.h index c838fd4a7..a22e91782 100644 --- a/lib/dictBuilder/zdict.h +++ b/lib/dictBuilder/zdict.h @@ -87,7 +87,7 @@ typedef struct { unsigned d; /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */ unsigned steps; /* Number of steps : Only used for optimization : 0 means default (32) : Higher means more parameters checked */ unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */ - double splitPoint; /* Percentage of samples used for training: the first nbSamples * splitPoint samples will be used to training, the last nbSamples * (1 - splitPoint) samples will be used for testing, 0 means default (1.0), 1.0 when all samples are used for both training and testing */ + double splitPoint; /* Percentage of samples used for training: Only used for optimization : the first nbSamples * splitPoint samples will be used to training, the last nbSamples * (1 - splitPoint) samples will be used for testing, 0 means default (1.0), 1.0 when all samples are used for both training and testing */ ZDICT_params_t zParams; } ZDICT_cover_params_t; @@ -97,7 +97,7 @@ typedef struct { unsigned f; /* log of size of frequency array : constraint: 0 < f <= 31 : 1 means default(18)*/ unsigned steps; /* Number of steps : Only used for optimization : 0 means default (32) : Higher means more parameters checked */ unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */ - double splitPoint; /* Percentage of samples used for training: the first nbSamples * splitPoint samples will be used to training, the last nbSamples * (1 - splitPoint) samples will be used for testing, 0 means default (0.75), 1.0 when all samples are used for both training and testing */ + double splitPoint; /* Percentage of samples used for training: Only used for optimization : the first nbSamples * splitPoint samples will be used to training, the last nbSamples * (1 - splitPoint) samples will be used for testing, 0 means default (0.75), 1.0 when all samples are used for both training and testing */ unsigned accel; /* Acceleration level: constraint: 0 < accel <= 10, higher means faster and less accurate, 0 means default(1) */ ZDICT_params_t zParams; } ZDICT_fastCover_params_t; From 944c9986e0502bd01cade3ed90511970fb9a6e91 Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Thu, 30 Aug 2018 15:37:29 -0700 Subject: [PATCH 4/5] Update comment on default steps of cover and fastcover --- lib/dictBuilder/zdict.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/dictBuilder/zdict.h b/lib/dictBuilder/zdict.h index a22e91782..cf5981779 100644 --- a/lib/dictBuilder/zdict.h +++ b/lib/dictBuilder/zdict.h @@ -85,7 +85,7 @@ typedef struct { typedef struct { unsigned k; /* Segment size : constraint: 0 < k : Reasonable range [16, 2048+] */ unsigned d; /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */ - unsigned steps; /* Number of steps : Only used for optimization : 0 means default (32) : Higher means more parameters checked */ + unsigned steps; /* Number of steps : Only used for optimization : 0 means default (40) : Higher means more parameters checked */ unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */ double splitPoint; /* Percentage of samples used for training: Only used for optimization : the first nbSamples * splitPoint samples will be used to training, the last nbSamples * (1 - splitPoint) samples will be used for testing, 0 means default (1.0), 1.0 when all samples are used for both training and testing */ ZDICT_params_t zParams; @@ -95,7 +95,7 @@ typedef struct { unsigned k; /* Segment size : constraint: 0 < k : Reasonable range [16, 2048+] */ unsigned d; /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */ unsigned f; /* log of size of frequency array : constraint: 0 < f <= 31 : 1 means default(18)*/ - unsigned steps; /* Number of steps : Only used for optimization : 0 means default (32) : Higher means more parameters checked */ + unsigned steps; /* Number of steps : Only used for optimization : 0 means default (40) : Higher means more parameters checked */ unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */ double splitPoint; /* Percentage of samples used for training: Only used for optimization : the first nbSamples * splitPoint samples will be used to training, the last nbSamples * (1 - splitPoint) samples will be used for testing, 0 means default (0.75), 1.0 when all samples are used for both training and testing */ unsigned accel; /* Acceleration level: constraint: 0 < accel <= 10, higher means faster and less accurate, 0 means default(1) */ From f87383507d3756fac86799ea9aad7931d898dba8 Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Thu, 30 Aug 2018 15:46:39 -0700 Subject: [PATCH 5/5] Update comment about default dictionary builder --- programs/zstd.1 | 2 +- programs/zstd.1.md | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/programs/zstd.1 b/programs/zstd.1 index 51825af74..526701a2c 100644 --- a/programs/zstd.1 +++ b/programs/zstd.1 @@ -194,7 +194,7 @@ All arguments after \fB\-\-\fR are treated as files Use FILEs as training set to create a dictionary\. The training set should contain a lot of small files (> 100), and weight typically 100x the target dictionary size (for example, 10 MB for a 100 KB dictionary)\. . .IP -Supports multithreading if \fBzstd\fR is compiled with threading support\. Additional parameters can be specified with \fB\-\-train\-cover\fR\ or \fB\-\-train\-fastcover\fR\. The legacy dictionary builder can be accessed with \fB\-\-train\-legacy\fR\. Equivalent to \fB\-\-train\-fastCover=d=8,steps=4\fR\. +Supports multithreading if \fBzstd\fR is compiled with threading support\. Additional parameters can be specified with \fB\-\-train\-fastcover\fR\. The legacy dictionary builder can be accessed with \fB\-\-train\-legacy\fR\. The cover dictionary builder can be accessed with \fB\-\-train\-cover\fR\. Equivalent to \fB\-\-train\-fastCover=d=8,steps=4\fR\. . .TP \fB\-o file\fR diff --git a/programs/zstd.1.md b/programs/zstd.1.md index 324b765b7..14c262650 100644 --- a/programs/zstd.1.md +++ b/programs/zstd.1.md @@ -200,9 +200,10 @@ Compression of small files similar to the sample set will be greatly improved. (for example, 10 MB for a 100 KB dictionary). Supports multithreading if `zstd` is compiled with threading support. - Additional parameters can be specified with `--train-cover`. + Additional parameters can be specified with `--train-fastcover`. The legacy dictionary builder can be accessed with `--train-legacy`. - Equivalent to `--train-cover=d=8,steps=4`. + The cover dictionary builder can be accessed with `--train-cover`. + Equivalent to `--train-fastcover=d=8,steps=4`. * `-o file`: Dictionary saved into `file` (default name: dictionary). * `--maxdict=#`: