ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

patch-from speed optimization (#3545)

Browse Source 
* patch-from speed optimization: only load portion of dictionary into normal matchfinders

* test regression for x8 multiplier

* fix off-by-one error for bit shift bound

* restrict patchfrom speed optimization to strategy < ZSTD_btultra

* update results.csv

* update regression test
This commit is contained in:
daniellerozenblit
2023-03-14 20:36:56 -04:00
committed by GitHub
parent 488e45f38b
commit 53bad103ce
2 changed files with 145 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+19 -9
View File
@@ -4693,7 +4693,8 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms,
ip = iend - maxDictSize;
src = ip;
srcSize = maxDictSize;
} }
}
}
if (srcSize > ZSTD_CHUNKSIZE_MAX) {
/* We must have cleared our windows when our source is this large. */
@@ -4702,22 +4703,31 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms,
}
DEBUGLOG(4, "ZSTD_loadDictionaryContent(): useRowMatchFinder=%d", (int)params->useRowMatchFinder);
if (loadLdmDict) { /* Load the entire dict into LDM matchfinders. */
ZSTD_window_update(&ls->window, src, srcSize, /* forceNonContiguous */ 0);
ls->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ls->window.base);
ZSTD_ldm_fillHashTable(ls, ip, iend, &params->ldmParams);
}
/* If the dict is larger than we can reasonably index in our tables, only load the suffix. */
if (params->cParams.strategy < ZSTD_btultra) {
U32 maxDictSize = 8U << MIN(MAX(params->cParams.hashLog, params->cParams.chainLog), 27);
if (srcSize > maxDictSize) {
ip = iend - maxDictSize;
src = ip;
srcSize = maxDictSize;
}
}
ZSTD_window_update(&ms->window, src, srcSize, /* forceNonContiguous */ 0);
ms->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ms->window.base);
ms->forceNonContiguous = params->deterministicRefPrefix;
if (loadLdmDict) {
ZSTD_window_update(&ls->window, src, srcSize, /* forceNonContiguous */ 0);
ls->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ls->window.base);
}
if (srcSize <= HASH_READ_SIZE) return 0;
ZSTD_overflowCorrectIfNeeded(ms, ws, params, ip, iend);
if (loadLdmDict)
ZSTD_ldm_fillHashTable(ls, ip, iend, &params->ldmParams);
switch(params->cParams.strategy)
{
case ZSTD_fast: