ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Bugfixes for the External Matchfinder API (#3433)

Browse Source 
* external matchfinder bugfixes + tests

* small doc fix
This commit is contained in:
Elliot Gorokhovsky
2023-01-19 10:41:24 -05:00
committed by GitHub
parent dc1c6cc5df
commit bce0382c82
3 changed files with 88 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+22 -11
View File
@@ -1599,7 +1599,7 @@ static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal(
size_t const maxNbExternalSeq = ZSTD_sequenceBound(blockSize);
size_t const externalSeqSpace = useExternalMatchFinder
? ZSTD_cwksp_alloc_size(maxNbExternalSeq * sizeof(ZSTD_Sequence))
? ZSTD_cwksp_aligned_alloc_size(maxNbExternalSeq * sizeof(ZSTD_Sequence))
: 0;
size_t const neededSpace =
@@ -3158,7 +3158,6 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->appliedParams.cParams.strategy,
zc->appliedParams.useRowMatchFinder,
dictMode);
assert(zc->externalMatchCtx.mFinder == NULL);
ms->ldmSeqStore = NULL;
lastLLSize = blockCompressor(ms, &zc->seqStore, zc->blockState.nextCBlock->rep, src, srcSize);
}
@@ -5945,6 +5944,13 @@ static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx,
params.maxBlockSize = ZSTD_resolveMaxBlockSize(params.maxBlockSize);
#ifdef ZSTD_MULTITHREAD
/* If external matchfinder is enabled, make sure to fail before checking job size (for consistency) */
RETURN_ERROR_IF(
params.useExternalMatchFinder == 1 && params.nbWorkers >= 1,
parameter_combination_unsupported,
"External matchfinder isn't supported with nbWorkers >= 1"
);
if ((cctx->pledgedSrcSizePlusOne-1) <= ZSTDMT_JOBSIZE_MIN) {
params.nbWorkers = 0; /* do not invoke multi-threading when src size is too small */
}
@@ -6774,14 +6780,19 @@ void ZSTD_registerExternalMatchFinder(
ZSTD_CCtx* zc, void* mState,
ZSTD_externalMatchFinder_F* mFinder
) {
ZSTD_externalMatchCtx emctx = {
mState,
mFinder,
if (mFinder != NULL) {
ZSTD_externalMatchCtx emctx = {
mState,
mFinder,
/* seqBuffer is allocated later (from the cwskp) */
NULL, /* seqBuffer */
0 /* seqBufferCapacity */
};
zc->externalMatchCtx = emctx;
zc->requestedParams.useExternalMatchFinder = 1;
/* seqBuffer is allocated later (from the cwskp) */
NULL, /* seqBuffer */
0 /* seqBufferCapacity */
};
zc->externalMatchCtx = emctx;
zc->requestedParams.useExternalMatchFinder = 1;
} else {
ZSTD_memset(&zc->externalMatchCtx, 0, sizeof(zc->externalMatchCtx));
zc->requestedParams.useExternalMatchFinder = 0;
}
}