ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add array to keep track of frequency within active segment, fix malloc bug, update benchmarking result

Browse Source
This commit is contained in:
Jennifer Liu
2018-07-26 13:53:13 -07:00
parent 2333ecb173
commit 3b163e0b5b
4 changed files with 75 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files
contrib/experimental_dict_builders/benchmarkDictBuilder/README.md
+32 -28
View File
@@ -14,42 +14,46 @@ make ARG="in=../../../lib/dictBuilder in=../../../lib/compress"
###Benchmarking Result: ###Benchmarking Result:
d=8
f=23
freq[i] = 0 when dmer added to best segment
github: github:
| Algorithm | Speed(sec) | Compression Ratio | | Algorithm | Speed(sec) | Compression Ratio |
| ------------------|:-------------:| ------------------:| | ----------------- | ------------- | ------------------ |
| nodict | 0.000004 | 2.999642 | | nodict | 0.000007 | 2.999642 |
| random | 0.148247 | 8.786957 | | random | 0.150258 | 8.786957 |
| cover | 56.331553 | 10.641263 | | cover | 60.388853 | 10.641263 |
| legacy | 0.917595 | 8.989482 | | legacy | 0.965050 | 8.989482 |
| fastCover(opt) | 13.169979 | 10.215174 | | fastCover(opt) | 84.968131 | 10.614747 |
| fastCover(k=200) | 2.692406 | 8.657219 | | fastCover(k=200) | 6.465490 | 9.484150 |
hg-commands hg-commands
| Algorithm | Speed(sec) | Compression Ratio | | Algorithm | Speed(sec) | Compression Ratio |
| ----------------- |:-------------:| ------------------:| | ----------------- | ------------- | ------------------ |
| nodict | 0.000007 | 2.425291 | | nodict | 0.000005 | 2.425291 |
| random | 0.093990 | 3.489515 | | random | 0.084348 | 3.489515 |
| cover | 58.602385 | 4.131136 | | cover | 60.144894 | 4.131136 |
| legacy | 0.865683 | 3.911896 | | legacy | 0.831981 | 3.911896 |
| fastCover(opt) | 9.404134 | 3.977229 | | fastCover(opt) | 59.030437 | 4.157595 |
| fastCover(k=200) | 1.037434 | 3.810326 | | fastCover(k=200) | 3.702932 | 4.134222 |
hg-changelog hg-changelog
| Algorithm | Speed(sec) | Compression Ratio | | Algorithm | Speed(sec) | Compression Ratio |
| ----------------- |:-------------:| ------------------:| | ----------------- | ------------- | ------------------ |
| nodict | 0.000022 | 1.377613 | | nodict | 0.000004 | 1.377613 |
| random | 0.551539 | 2.096785 | | random | 0.555964 | 2.096785 |
| cover | 221.370056 | 2.188654 | | cover | 214.423753 | 2.188654 |
| legacy | 2.405923 | 2.058273 | | legacy | 2.180249 | 2.058273 |
| fastCover(opt) | 49.526246 | 2.124185 | | fastCover(opt) | 102.261452 | 2.180347 |
| fastCover(k=200) | 9.746872 | 2.114674 | | fastCover(k=200) | 11.81039 | 2.170673 |
hg-manifest hg-manifest
| Algorithm | Speed(sec) | Compression Ratio | | Algorithm | Speed(sec) | Compression Ratio |
| ----------------- |:-------------:| ------------------:| | ----------------- | ------------- | ------------------ |
| nodict | 0.000019 | 1.866385 | | nodict | 0.000006 | 1.866385 |
| random | 1.083536 | 2.309485 | | random | 1.063974 | 2.309485 |
| cover | 928.894887 | 2.582597 | | cover | 909.101849 | 2.582597 |
| legacy | 9.110371 | 2.506775 | | legacy | 8.706580 | 2.506775 |
| fastCover(opt) | 116.508270 | 2.525689 | | fastCover(opt) | 188.598079 | 2.596761 |
| fastCover(k=200) | 12.176555 | 2.472221 | | fastCover(k=200) | 13.392734 | 2.592985 |
contrib/experimental_dict_builders/fastCover/fastCover.c
+41 -28
View File
@@ -48,7 +48,7 @@ static clock_t g_time = 0;
/*-************************************* /*-*************************************
* Hash Function * Hash Functions
***************************************/ ***************************************/
static const U64 prime6bytes = 227718039650203ULL; static const U64 prime6bytes = 227718039650203ULL;
static size_t ZSTD_hash6(U64 u, U32 h) { return (size_t)(((u << (64-48)) * prime6bytes) >> (64-h)) ; } static size_t ZSTD_hash6(U64 u, U32 h) { return (size_t)(((u << (64-48)) * prime6bytes) >> (64-h)) ; }
@@ -58,6 +58,7 @@ static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL;
static size_t ZSTD_hash8(U64 u, U32 h) { return (size_t)(((u) * prime8bytes) >> (64-h)) ; } static size_t ZSTD_hash8(U64 u, U32 h) { return (size_t)(((u) * prime8bytes) >> (64-h)) ; }
static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(MEM_readLE64(p), h); } static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(MEM_readLE64(p), h); }
/** /**
* Hash the d-byte value pointed to by p and mod 2^f * Hash the d-byte value pointed to by p and mod 2^f
*/ */
@@ -140,29 +141,41 @@ static FASTCOVER_segment_t FASTCOVER_selectSegment(const FASTCOVER_ctx_t *ctx,
activeSegment.begin = begin; activeSegment.begin = begin;
activeSegment.end = begin; activeSegment.end = begin;
activeSegment.score = 0; activeSegment.score = 0;
/* Slide the activeSegment through the whole epoch. {
* Save the best segment in bestSegment. /* Keep track of number of times an index has been seen in current segment */
*/ U16* currfreqs =(U16 *)malloc((1 << parameters.f) * sizeof(U16));
while (activeSegment.end < end) { memset(currfreqs, 0, (1 << parameters.f) * sizeof(*currfreqs));
/* Get hash value of current dmer */ /* Slide the activeSegment through the whole epoch.
const size_t index = FASTCOVER_hashPtrToIndex(ctx->samples + activeSegment.end, parameters.f, ctx->d); * Save the best segment in bestSegment.
/* Add frequency of this index to score */ */
activeSegment.score += freqs[index]; while (activeSegment.end < end) {
/* Increment end of segment */ /* Get hash value of current dmer */
activeSegment.end += 1; const size_t index = FASTCOVER_hashPtrToIndex(ctx->samples + activeSegment.end, parameters.f, ctx->d);
/* If the window is now too large, drop the first position */ /* Add frequency of this index to score if this is the first occurence of index in active segment */
if (activeSegment.end - activeSegment.begin == dmersInK + 1) { if (currfreqs[index] == 0) {
/* Get hash value of the dmer to be eliminated from active segment */ activeSegment.score += freqs[index];
const size_t delIndex = FASTCOVER_hashPtrToIndex(ctx->samples + activeSegment.begin, parameters.f, ctx->d); }
/* Subtract frequency of this index from score */ currfreqs[index] += 1;
activeSegment.score -= freqs[delIndex]; /* Increment end of segment */
/* Increment start of segment */ activeSegment.end += 1;
activeSegment.begin += 1; /* If the window is now too large, drop the first position */
} if (activeSegment.end - activeSegment.begin == dmersInK + 1) {
/* If this segment is the best so far save it */ /* Get hash value of the dmer to be eliminated from active segment */
if (activeSegment.score > bestSegment.score) { const size_t delIndex = FASTCOVER_hashPtrToIndex(ctx->samples + activeSegment.begin, parameters.f, ctx->d);
bestSegment = activeSegment; currfreqs[delIndex] -= 1;
/* Subtract frequency of this index from score if this is the last occurrence of this index in active segment */
if (currfreqs[delIndex] == 0) {
activeSegment.score -= freqs[delIndex];
}
/* Increment start of segment */
activeSegment.begin += 1;
}
/* If this segment is the best so far save it */
if (activeSegment.score > bestSegment.score) {
bestSegment = activeSegment;
}
} }
free(currfreqs);
} }
{ {
/* Trim off the zero frequency head and tail from the segment. */ /* Trim off the zero frequency head and tail from the segment. */
@@ -185,7 +198,7 @@ static FASTCOVER_segment_t FASTCOVER_selectSegment(const FASTCOVER_ctx_t *ctx,
U32 pos; U32 pos;
for (pos = bestSegment.begin; pos != bestSegment.end; ++pos) { for (pos = bestSegment.begin; pos != bestSegment.end; ++pos) {
const size_t i = FASTCOVER_hashPtrToIndex(ctx->samples + pos, parameters.f, ctx->d); const size_t i = FASTCOVER_hashPtrToIndex(ctx->samples + pos, parameters.f, ctx->d);
freqs[i] = freqs[i]/2; freqs[i] = 0;
} }
} }
return bestSegment; return bestSegment;
@@ -245,12 +258,12 @@ static void FASTCOVER_ctx_destroy(FASTCOVER_ctx_t *ctx) {
/** /**
* Calculate for frequency of hash value of each dmer in ctx->samples * Calculate for frequency of hash value of each dmer in ctx->samples
*/ */
static void FASTCOVER_getFrequency(U32 *freqs, unsigned f, FASTCOVER_ctx_t *ctx){ static void FASTCOVER_computeFrequency(U32 *freqs, unsigned f, FASTCOVER_ctx_t *ctx){
/* inCurrSample keeps track of this hash value has already be seen in previous dmers in the same sample*/ /* inCurrSample keeps track of this hash value has already be seen in previous dmers in the same sample*/
size_t* inCurrSample = (size_t *)malloc((1<<f)*sizeof(size_t)); BYTE* inCurrSample = (BYTE *)malloc((1 << f) * sizeof(BYTE));
size_t start; /* start of current dmer */ size_t start; /* start of current dmer */
for (unsigned i = 0; i < ctx->nbTrainSamples; i++) { for (unsigned i = 0; i < ctx->nbTrainSamples; i++) {
memset(inCurrSample, 0, (1 << f)); /* Reset inCurrSample for each sample */ memset(inCurrSample, 0, (1 << f) * sizeof(*inCurrSample)); /* Reset inCurrSample for each sample */
size_t currSampleStart = ctx->offsets[i]; size_t currSampleStart = ctx->offsets[i];
size_t currSampleEnd = ctx->offsets[i+1]; size_t currSampleEnd = ctx->offsets[i+1];
start = currSampleStart; start = currSampleStart;
@@ -338,7 +351,7 @@ static int FASTCOVER_ctx_init(FASTCOVER_ctx_t *ctx, const void *samplesBuffer,
memset(ctx->freqs, 0, (1 << f) * sizeof(U32)); memset(ctx->freqs, 0, (1 << f) * sizeof(U32));
DISPLAYLEVEL(2, "Computing frequencies\n"); DISPLAYLEVEL(2, "Computing frequencies\n");
FASTCOVER_getFrequency(ctx->freqs, f, ctx); FASTCOVER_computeFrequency(ctx->freqs, f, ctx);
return 1; return 1;
} }
contrib/experimental_dict_builders/fastCover/main.c
+1 -1
View File
@@ -165,7 +165,7 @@ int main(int argCount, const char* argv[])
params.splitPoint = (double)split/100; params.splitPoint = (double)split/100;
/* Build dictionary */ /* Build dictionary */
sampleInfo* info= getSampleInfo(filenameTable, sampleInfo* info = getSampleInfo(filenameTable,
filenameIdx, blockSize, maxDictSize, zParams.notificationLevel); filenameIdx, blockSize, maxDictSize, zParams.notificationLevel);
operationResult = FASTCOVER_trainFromFiles(outputFile, info, maxDictSize, &params); operationResult = FASTCOVER_trainFromFiles(outputFile, info, maxDictSize, &params);
contrib/experimental_dict_builders/randomDictBuilder/main.c
+1 -1
View File
@@ -149,7 +149,7 @@ int main(int argCount, const char* argv[])
params.zParams = zParams; params.zParams = zParams;
params.k = k; params.k = k;
sampleInfo* info= getSampleInfo(filenameTable, sampleInfo* info = getSampleInfo(filenameTable,
filenameIdx, blockSize, maxDictSize, zParams.notificationLevel); filenameIdx, blockSize, maxDictSize, zParams.notificationLevel);
operationResult = RANDOM_trainFromFiles(outputFile, info, maxDictSize, &params); operationResult = RANDOM_trainFromFiles(outputFile, info, maxDictSize, &params);