ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Initial benchmarking result for fastCover

Browse Source
This commit is contained in:
Jennifer Liu
2018-07-25 17:05:54 -07:00
parent f5407e398a
commit d1fc507ef9
3 changed files with 62 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
contrib/experimental_dict_builders/benchmarkDictBuilder/Makefile
+7 -3
View File
@@ -2,9 +2,10 @@ ARG :=
CC ?= gcc CC ?= gcc
CFLAGS ?= -O3 CFLAGS ?= -O3
INCLUDES := -I ../randomDictBuilder -I ../../../programs -I ../../../lib/common -I ../../../lib -I ../../../lib/dictBuilder INCLUDES := -I ../randomDictBuilder -I ../fastCover -I ../../../programs -I ../../../lib/common -I ../../../lib -I ../../../lib/dictBuilder
RANDOM_FILE := ../randomDictBuilder/random.c RANDOM_FILE := ../randomDictBuilder/random.c
FAST_FILE := ../fastCover/fastCover.c
IO_FILE := ../randomDictBuilder/io.c IO_FILE := ../randomDictBuilder/io.c
all: run clean all: run clean
@@ -21,8 +22,8 @@ test: benchmarkTest clean
benchmarkTest: benchmark test.sh benchmarkTest: benchmark test.sh
sh test.sh sh test.sh
benchmark: benchmark.o io.o random.o libzstd.a benchmark: benchmark.o io.o random.o fastCover.o libzstd.a
$(CC) $(CFLAGS) benchmark.o io.o random.o libzstd.a -o benchmark $(CC) $(CFLAGS) benchmark.o io.o random.o fastCover.o libzstd.a -o benchmark
benchmark.o: benchmark.c benchmark.o: benchmark.c
$(CC) $(CFLAGS) $(INCLUDES) -c benchmark.c $(CC) $(CFLAGS) $(INCLUDES) -c benchmark.c
@@ -30,6 +31,9 @@ benchmark.o: benchmark.c
random.o: $(RANDOM_FILE) random.o: $(RANDOM_FILE)
$(CC) $(CFLAGS) $(INCLUDES) -c $(RANDOM_FILE) $(CC) $(CFLAGS) $(INCLUDES) -c $(RANDOM_FILE)
fastCover.o: $(FAST_FILE)
$(CC) $(CFLAGS) $(INCLUDES) -c $(FAST_FILE)
io.o: $(IO_FILE) io.o: $(IO_FILE)
$(CC) $(CFLAGS) $(INCLUDES) -c $(IO_FILE) $(CC) $(CFLAGS) $(INCLUDES) -c $(IO_FILE)
contrib/experimental_dict_builders/benchmarkDictBuilder/README.md
+22 -18
View File
@@ -18,30 +18,34 @@ github:
| Algorithm | Speed(sec) | Compression Ratio | | Algorithm | Speed(sec) | Compression Ratio |
| ------------- |:-------------:| ------------------:| | ------------- |:-------------:| ------------------:|
| nodict | 0.000004 | 2.999642 | | nodict | 0.000004 | 2.999642 |
| random | 0.180238 | 8.786957 | | random | 0.135459 | 8.786957 |
| cover | 33.891987 | 10.430999 | | cover | 50.341079 | 10.641263 |
| legacy | 1.077569 | 8.989482 | | legacy | 0.866283 | 8.989482 |
| fastCover | 13.450947 | 10.215174 |
hg-commands hg-commands
| Algorithm | Speed(sec) | Compression Ratio | | Algorithm | Speed(sec) | Compression Ratio |
| ------------- |:-------------:| ------------------:| | ------------- |:-------------:| ------------------:|
| nodict | 0.000006 | 2.425291 | | nodict | 0.000020 | 2.425291 |
| random | 0.088735 | 3.489515 | | random | 0.088828 | 3.489515 |
| cover | 35.447300 | 4.030274 | | cover | 60.028672 | 4.131136 |
| legacy | 1.048509 | 3.911896 | | legacy | 0.852481 | 3.911896 |
| fastCover | 9.524284 | 3.977229 |
hg-changelog
| Algorithm | Speed(sec) | Compression Ratio |
| ------------- |:-------------:| ------------------:|
| nodict | 0.000004 | 1.377613 |
| random | 0.621812 | 2.096785 |
| cover | 217.510962 | 2.188654 |
| legacy | 2.559194 | 2.058273 |
| fastCover | 51.132516 | 2.124185 |
hg-manifest hg-manifest
| Algorithm | Speed(sec) | Compression Ratio | | Algorithm | Speed(sec) | Compression Ratio |
| ------------- |:-------------:| ------------------:| | ------------- |:-------------:| ------------------:|
| nodict | 0.000005 | 1.866385 | | nodict | 0.000005 | 1.866385 |
| random | 1.148231 | 2.309485 | | random | 1.035220 | 2.309485 |
| cover | 509.685257 | 2.575331 | | cover | 930.480173 | 2.582597 |
| legacy | 10.705866 | 2.506775 | | legacy | 8.916513 | 2.506775 |
| fastCover | 116.871089 | 2.525689 |
hg-changelog
| Algorithm | Speed(sec) | Compression Ratio |
| ------------- |:-------------:| ------------------:|
| nodict | 0.000005 | 1.377613 |
| random | 0.706434 | 2.096785 |
| cover | 122.815783 | 2.175706 |
| legacy | 3.010318 | 2.058273 |
contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c
+33 -9
View File
@@ -5,6 +5,7 @@
#include <ctype.h> #include <ctype.h>
#include <time.h> #include <time.h>
#include "random.h" #include "random.h"
#include "fastCover.h"
#include "dictBuilder.h" #include "dictBuilder.h"
#include "zstd_internal.h" /* includes zstd.h */ #include "zstd_internal.h" /* includes zstd.h */
#include "io.h" #include "io.h"
@@ -71,10 +72,11 @@ typedef struct {
*/ */
dictInfo* createDictFromFiles(sampleInfo *info, unsigned maxDictSize, dictInfo* createDictFromFiles(sampleInfo *info, unsigned maxDictSize,
ZDICT_random_params_t *randomParams, ZDICT_cover_params_t *coverParams, ZDICT_random_params_t *randomParams, ZDICT_cover_params_t *coverParams,
ZDICT_legacy_params_t *legacyParams) { ZDICT_legacy_params_t *legacyParams, ZDICT_fastCover_params_t *fastParams) {
unsigned const displayLevel = randomParams ? randomParams->zParams.notificationLevel : unsigned const displayLevel = randomParams ? randomParams->zParams.notificationLevel :
coverParams ? coverParams->zParams.notificationLevel : coverParams ? coverParams->zParams.notificationLevel :
legacyParams ? legacyParams->zParams.notificationLevel : legacyParams ? legacyParams->zParams.notificationLevel :
fastParams ? fastParams->zParams.notificationLevel :
DEFAULT_DISPLAYLEVEL; /* no dict */ DEFAULT_DISPLAYLEVEL; /* no dict */
void* const dictBuffer = malloc(maxDictSize); void* const dictBuffer = malloc(maxDictSize);
@@ -94,6 +96,9 @@ dictInfo* createDictFromFiles(sampleInfo *info, unsigned maxDictSize,
} else if(legacyParams) { } else if(legacyParams) {
dictSize = ZDICT_trainFromBuffer_legacy(dictBuffer, maxDictSize, info->srcBuffer, dictSize = ZDICT_trainFromBuffer_legacy(dictBuffer, maxDictSize, info->srcBuffer,
info->samplesSizes, info->nbSamples, *legacyParams); info->samplesSizes, info->nbSamples, *legacyParams);
} else if(fastParams) {
dictSize = ZDICT_optimizeTrainFromBuffer_fastCover(dictBuffer, maxDictSize, info->srcBuffer,
info->samplesSizes, info->nbSamples, fastParams);
} else { } else {
dictSize = 0; dictSize = 0;
} }
@@ -216,25 +221,29 @@ void freeDictInfo(dictInfo* info) {
* @return 0 if benchmark successfully, 1 otherwise * @return 0 if benchmark successfully, 1 otherwise
*/ */
int benchmarkDictBuilder(sampleInfo *srcInfo, unsigned maxDictSize, ZDICT_random_params_t *randomParam, int benchmarkDictBuilder(sampleInfo *srcInfo, unsigned maxDictSize, ZDICT_random_params_t *randomParam,
ZDICT_cover_params_t *coverParam, ZDICT_legacy_params_t *legacyParam) { ZDICT_cover_params_t *coverParam, ZDICT_legacy_params_t *legacyParam,
ZDICT_fastCover_params_t *fastParam) {
/* Local variables */ /* Local variables */
const unsigned displayLevel = randomParam ? randomParam->zParams.notificationLevel : const unsigned displayLevel = randomParam ? randomParam->zParams.notificationLevel :
coverParam ? coverParam->zParams.notificationLevel : coverParam ? coverParam->zParams.notificationLevel :
legacyParam ? legacyParam->zParams.notificationLevel : legacyParam ? legacyParam->zParams.notificationLevel :
fastParam ? fastParam->zParams.notificationLevel:
DEFAULT_DISPLAYLEVEL; /* no dict */ DEFAULT_DISPLAYLEVEL; /* no dict */
const char* name = randomParam ? "RANDOM" : const char* name = randomParam ? "RANDOM" :
coverParam ? "COVER" : coverParam ? "COVER" :
legacyParam ? "LEGACY" : legacyParam ? "LEGACY" :
fastParam ? "FAST":
"NODICT"; /* no dict */ "NODICT"; /* no dict */
const unsigned cLevel = randomParam ? randomParam->zParams.compressionLevel : const unsigned cLevel = randomParam ? randomParam->zParams.compressionLevel :
coverParam ? coverParam->zParams.compressionLevel : coverParam ? coverParam->zParams.compressionLevel :
legacyParam ? legacyParam->zParams.compressionLevel : legacyParam ? legacyParam->zParams.compressionLevel :
fastParam ? fastParam->zParams.compressionLevel:
DEFAULT_CLEVEL; /* no dict */ DEFAULT_CLEVEL; /* no dict */
int result = 0; int result = 0;
/* Calculate speed */ /* Calculate speed */
const UTIL_time_t begin = UTIL_getTime(); const UTIL_time_t begin = UTIL_getTime();
dictInfo* dInfo = createDictFromFiles(srcInfo, maxDictSize, randomParam, coverParam, legacyParam); dictInfo* dInfo = createDictFromFiles(srcInfo, maxDictSize, randomParam, coverParam, legacyParam, fastParam);
const U64 timeMicro = UTIL_clockSpanMicro(begin); const U64 timeMicro = UTIL_clockSpanMicro(begin);
const double timeSec = timeMicro / (double)SEC_TO_MICRO; const double timeSec = timeMicro / (double)SEC_TO_MICRO;
if (!dInfo) { if (!dInfo) {
@@ -269,7 +278,6 @@ int main(int argCount, const char* argv[])
/* Initialize arguments to default values */ /* Initialize arguments to default values */
const unsigned k = 200; const unsigned k = 200;
const unsigned d = 6;
const unsigned cLevel = DEFAULT_CLEVEL; const unsigned cLevel = DEFAULT_CLEVEL;
const unsigned dictID = 0; const unsigned dictID = 0;
const unsigned maxDictSize = g_defaultMaxDictSize; const unsigned maxDictSize = g_defaultMaxDictSize;
@@ -319,7 +327,7 @@ int main(int argCount, const char* argv[])
/* with no dict */ /* with no dict */
{ {
const int noDictResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL); const int noDictResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, NULL);
if(noDictResult) { if(noDictResult) {
result = 1; result = 1;
goto _cleanup; goto _cleanup;
@@ -331,7 +339,7 @@ int main(int argCount, const char* argv[])
ZDICT_random_params_t randomParam; ZDICT_random_params_t randomParam;
randomParam.zParams = zParams; randomParam.zParams = zParams;
randomParam.k = k; randomParam.k = k;
const int randomResult = benchmarkDictBuilder(srcInfo, maxDictSize, &randomParam, NULL, NULL); const int randomResult = benchmarkDictBuilder(srcInfo, maxDictSize, &randomParam, NULL, NULL, NULL);
if(randomResult) { if(randomResult) {
result = 1; result = 1;
goto _cleanup; goto _cleanup;
@@ -344,10 +352,9 @@ int main(int argCount, const char* argv[])
memset(&coverParam, 0, sizeof(coverParam)); memset(&coverParam, 0, sizeof(coverParam));
coverParam.zParams = zParams; coverParam.zParams = zParams;
coverParam.splitPoint = 1.0; coverParam.splitPoint = 1.0;
coverParam.d = d;
coverParam.steps = 40; coverParam.steps = 40;
coverParam.nbThreads = 1; coverParam.nbThreads = 1;
const int coverOptResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, &coverParam, NULL); const int coverOptResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, &coverParam, NULL, NULL);
if(coverOptResult) { if(coverOptResult) {
result = 1; result = 1;
goto _cleanup; goto _cleanup;
@@ -359,13 +366,30 @@ int main(int argCount, const char* argv[])
ZDICT_legacy_params_t legacyParam; ZDICT_legacy_params_t legacyParam;
legacyParam.zParams = zParams; legacyParam.zParams = zParams;
legacyParam.selectivityLevel = 9; legacyParam.selectivityLevel = 9;
const int legacyResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, &legacyParam); const int legacyResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, &legacyParam, NULL);
if(legacyResult) { if(legacyResult) {
result = 1; result = 1;
goto _cleanup; goto _cleanup;
} }
} }
/* for fastCover */
{
ZDICT_fastCover_params_t fastParam;
memset(&fastParam, 0, sizeof(fastParam));
fastParam.zParams = zParams;
fastParam.splitPoint = 1.0;
fastParam.d = 8;
fastParam.f = 23;
fastParam.steps = 40;
fastParam.nbThreads = 1;
const int fastOptResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, &fastParam);
if(fastOptResult) {
result = 1;
goto _cleanup;
}
}
/* Free allocated memory */ /* Free allocated memory */
_cleanup: _cleanup:
UTIL_freeFileList(extendedFileList, fileNamesBuf); UTIL_freeFileList(extendedFileList, fileNamesBuf);