ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fixed dictBuilder issue

Browse Source 
dictionary loading would fail during entropy analysis
This commit is contained in:
Yann Collet
2017-03-26 02:50:00 -07:00
parent ecee9f2ef8
commit 858f72eeb8
1 changed files with 24 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/dictBuilder/zdict.c
+13 -16
View File
@@ -393,7 +393,7 @@ static U32 ZDICT_checkMerge(dictItem* table, dictItem elt, U32 eltNbToSkip, cons
table[u].length += addedLength; table[u].length += addedLength;
table[u].pos = elt.pos; table[u].pos = elt.pos;
table[u].savings += elt.savings * addedLength / elt.length; /* rough approx */ table[u].savings += elt.savings * addedLength / elt.length; /* rough approx */
table[u].savings += elt.length / 8; /* rough approx bonus */ // table[u].savings += elt.length / 8; /* rough approx bonus */
elt = table[u]; elt = table[u];
/* sort : improve rank */ /* sort : improve rank */
while ((u>1) && (table[u-1].savings < elt.savings)) while ((u>1) && (table[u-1].savings < elt.savings))
@@ -422,8 +422,8 @@ static U32 ZDICT_checkMerge(dictItem* table, dictItem elt, U32 eltNbToSkip, cons
return u; return u;
} }
if (MEM_read64(buf + table[u].pos) == MEM_read64(buf + elt.pos + 1)) { if ( (MEM_read64(buf + table[u].pos) == MEM_read64(buf + elt.pos + 1))
if (isIncluded(buf + table[u].pos, buf + elt.pos + 1, table[u].length)) { && (isIncluded(buf + table[u].pos, buf + elt.pos + 1, table[u].length)) ) {
size_t const addedLength = MAX( (int)elt.length - (int)table[u].length , 1 ); size_t const addedLength = MAX( (int)elt.length - (int)table[u].length , 1 );
table[u].pos = elt.pos; table[u].pos = elt.pos;
table[u].savings += (U32)(elt.savings * addedLength / elt.length); table[u].savings += (U32)(elt.savings * addedLength / elt.length);
@@ -431,7 +431,6 @@ static U32 ZDICT_checkMerge(dictItem* table, dictItem elt, U32 eltNbToSkip, cons
return u; return u;
} }
} }
}
return 0; return 0;
} }
@@ -718,8 +717,8 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
params = ZSTD_getParams(compressionLevel, averageSampleSize, dictBufferSize); params = ZSTD_getParams(compressionLevel, averageSampleSize, dictBufferSize);
{ size_t const beginResult = ZSTD_compressBegin_advanced(esr.ref, dictBuffer, dictBufferSize, params, 0); { size_t const beginResult = ZSTD_compressBegin_advanced(esr.ref, dictBuffer, dictBufferSize, params, 0);
if (ZSTD_isError(beginResult)) { if (ZSTD_isError(beginResult)) {
DISPLAYLEVEL(1, "error : ZSTD_compressBegin_advanced() failed : %s \n", ZSTD_getErrorName(beginResult));
eSize = ERROR(GENERIC); eSize = ERROR(GENERIC);
DISPLAYLEVEL(1, "error : ZSTD_compressBegin_advanced failed \n");
goto _cleanup; goto _cleanup;
} } } }
@@ -901,20 +900,11 @@ size_t ZDICT_addEntropyTablesFromBuffer_advanced(void* dictBuffer, size_t dictCo
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
ZDICT_params_t params) ZDICT_params_t params)
{ {
size_t hSize;
int const compressionLevel = (params.compressionLevel <= 0) ? g_compressionLevel_default : params.compressionLevel; int const compressionLevel = (params.compressionLevel <= 0) ? g_compressionLevel_default : params.compressionLevel;
U32 const notificationLevel = params.notificationLevel; U32 const notificationLevel = params.notificationLevel;
size_t hSize = 8;
/* dictionary header */ /* calculate entropy tables */
MEM_writeLE32(dictBuffer, ZSTD_DICT_MAGIC);
{ U64 const randomID = XXH64((char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize, 0);
U32 const compliantID = (randomID % ((1U<<31)-32768)) + 32768;
U32 const dictID = params.dictID ? params.dictID : compliantID;
MEM_writeLE32((char*)dictBuffer+4, dictID);
}
hSize = 8;
/* entropy tables */
DISPLAYLEVEL(2, "\r%70s\r", ""); /* clean display line */ DISPLAYLEVEL(2, "\r%70s\r", ""); /* clean display line */
DISPLAYLEVEL(2, "statistics ... \n"); DISPLAYLEVEL(2, "statistics ... \n");
{ size_t const eSize = ZDICT_analyzeEntropy((char*)dictBuffer+hSize, dictBufferCapacity-hSize, { size_t const eSize = ZDICT_analyzeEntropy((char*)dictBuffer+hSize, dictBufferCapacity-hSize,
@@ -926,6 +916,13 @@ size_t ZDICT_addEntropyTablesFromBuffer_advanced(void* dictBuffer, size_t dictCo
hSize += eSize; hSize += eSize;
} }
/* add dictionary header (after entropy tables) */
MEM_writeLE32(dictBuffer, ZSTD_DICT_MAGIC);
{ U64 const randomID = XXH64((char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize, 0);
U32 const compliantID = (randomID % ((1U<<31)-32768)) + 32768;
U32 const dictID = params.dictID ? params.dictID : compliantID;
MEM_writeLE32((char*)dictBuffer+4, dictID);
}
if (hSize + dictContentSize < dictBufferCapacity) if (hSize + dictContentSize < dictBufferCapacity)
memmove((char*)dictBuffer + hSize, (char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize); memmove((char*)dictBuffer + hSize, (char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize);